Python 3 support for mcsema-disass. (#699)

* Python 3 support for mcsema-disass.

* Conditional Python3 installation libraries

* Add conditional check for Python2 also

* Update CMakeLists.txt

Make the install options for py2/3 always available.

* Fix CI tool name

* Update flow.py

Co-authored-by: Eric Kilmer <eric.d.kilmer@gmail.com>

.github/workflows/ci.yml

@@ -36,7 +36,7 @@ jobs:
     - name: Test final Docker image
       run: |
         docker run --rm docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --version
-        docker run --rm --entrypoint=mcsema-disass docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --help
+        docker run --rm --entrypoint=mcsema-disass-2 docker.pkg.github.com/lifting-bits/mcsema/mcsema-llvm${{ matrix.llvm }}-ubuntu${{ matrix.ubuntu }}-amd64:latest --help
     - name: Push Image for LLVM ${{ matrix.llvm }} on ${{ matrix.ubuntu }}
       if: github.event_name == 'push' && github.ref == 'refs/heads/master'
       run: |

CMakeLists.txt

@@ -29,6 +29,20 @@ set(MCSEMA_SOURCE_DIR "${PROJECT_SOURCE_DIR}")
 
 option(MCSEMA_ENABLE_RUNTIME "Should runtimes for re-compilation of bitcode be produced?" ON)
 
+find_package(Python2 QUIET)
+if(Python2_FOUND)
+  option(MCSEMA_INSTALL_PYTHON2_LIBS "Install Python 2 libraries" ON)
+else()
+  option(MCSEMA_INSTALL_PYTHON2_LIBS "Install Python 2 libraries")
+endif()
+
+find_package(Python3 QUIET)
+if(Python3_FOUND)
+  option(MCSEMA_INSTALL_PYTHON3_LIBS "Install Python 3 libraries" ON)
+else()
+  option(MCSEMA_INSTALL_PYTHON3_LIBS "Install Python 3 libraries")
+endif()
+
 # warnings and compiler settings
 if(NOT DEFINED WIN32)
   set(PROJECT_CXXFLAGS
@@ -114,6 +128,7 @@ add_executable(${MCSEMA_LIFT}
   tools/mcsema_lift/Lift.cpp
 )
 
+
 # Copy mcsema-disass in
 add_custom_command(
   TARGET protobuf_python_module_ida POST_BUILD
@@ -153,6 +168,61 @@ if(NOT TARGET anvill-${REMILL_LLVM_VERSION})
 endif()
 list(APPEND PROJECT_LIBRARIES anvill-${REMILL_LLVM_VERSION})
 
+# mcsema-disass
+
+set(MCSEMA_PYTHON_SOURCES
+    tools/setup.py
+    tools/mcsema_disass/__init__.py
+    tools/mcsema_disass/__main__.py
+    tools/mcsema_disass/ida7/__init__.py
+    tools/mcsema_disass/ida7/anvill_compat.py
+    tools/mcsema_disass/ida7/arm_util.py
+    tools/mcsema_disass/ida7/CFG_pb2.py
+    tools/mcsema_disass/ida7/disass.py
+    tools/mcsema_disass/ida7/exception.py
+    tools/mcsema_disass/ida7/flow.py
+    tools/mcsema_disass/ida7/get_cfg.py
+    tools/mcsema_disass/ida7/refs.py
+    tools/mcsema_disass/ida7/segment.py
+    tools/mcsema_disass/ida7/table.py
+    tools/mcsema_disass/ida7/util.py
+    tools/mcsema_disass/ida7/x86_util.py
+)
+
+if(MCSEMA_INSTALL_PYTHON2_LIBS)
+  add_custom_target(build_mcsema_disass_python2
+    DEPENDS ${MCSEMA_PYTHON_SOURCES})
+
+  add_custom_command(
+      TARGET build_mcsema_disass_python2 POST_BUILD
+      COMMAND which python2 && python2 setup.py build --force
+      COMMENT "Building McSema Python 2 mcsema-disass"
+      WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tools")
+
+  add_dependencies(${MCSEMA_LIFT}
+    build_mcsema_disass_python2)
+
+  add_dependencies(build_mcsema_disass_python2
+    protobuf_python_module_ida)
+endif()
+
+if(MCSEMA_INSTALL_PYTHON3_LIBS)
+  add_custom_target(build_mcsema_disass_python3
+    DEPENDS ${MCSEMA_PYTHON_SOURCES})
+
+  add_custom_command(
+      TARGET build_mcsema_disass_python3 POST_BUILD
+      COMMAND which python3 && python3 setup.py build --force
+      COMMENT "Building McSema Python 3 mcsema-disass"
+      WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tools")
+
+  add_dependencies(${MCSEMA_LIFT}
+    build_mcsema_disass_python3)
+
+  add_dependencies(build_mcsema_disass_python3
+    protobuf_python_module_ida)
+endif()
+
 #
 # target settings
 #
@@ -213,30 +283,34 @@ install(
   LIBRARY DESTINATION "${install_folder}/lib"
 )
 
-set(python_package_installer "${CMAKE_CURRENT_SOURCE_DIR}/tools/setup_launcher")
+# target for dyninst frontend
+if (DEFINED BUILD_MCSEMA_DYNINST_DISASS)
+    if(${BUILD_MCSEMA_DYNINST_DISASS} EQUAL 1)
+        add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/tools/mcsema_disass/dyninst)
+    endif()
+endif()
 
+set(python_package_installer "${CMAKE_CURRENT_SOURCE_DIR}/tools/setup_launcher")
 if(DEFINED WIN32)
   string(REPLACE "/" "\\" python_package_install_path "${install_folder}")
-  set(python_package_installer "${python_package_installer}.bat")
+  set(python_package_installer2 "${python_package_installer}.bat")
+  set(python_package_installer3 "${python_package_installer}.bat")
   set(optional_interpreter "cmd.exe /C")
 else()
   set(python_package_install_path "${install_folder}")
-  set(python_package_installer "${python_package_installer}.sh")
+  set(python_package_installer2 "${python_package_installer}_py2.sh")
+  set(python_package_installer3 "${python_package_installer}_py3.sh")
 endif()
 
-# target for dyninst frontend
-if (DEFINED BUILD_MCSEMA_DYNINST_DISASS)
-    if(${BUILD_MCSEMA_DYNINST_DISASS} EQUAL 1)
-        add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/tools/mcsema_disass/dyninst)
-    endif()
+if(MCSEMA_INSTALL_PYTHON2_LIBS)
+  install(CODE
+    "execute_process(COMMAND ${optional_interpreter} \"${python_package_installer2}\" \"${python_package_install_path}\"\nWORKING_DIRECTORY \"${PROJECT_SOURCE_DIR}/tools\"\nRESULT_VARIABLE exit_code2)\n   if(NOT exit_code2 EQUAL 0)\n       message(FATAL_ERROR \"Failed to install the Python 2 package\")\n     endif()")
 endif()
 
-install(CODE
-  "execute_process(COMMAND ${optional_interpreter} \"${python_package_installer}\" \"${python_package_install_path}\" WORKING_DIRECTORY \"${PROJECT_SOURCE_DIR}/tools\" RESULT_VARIABLE exit_code)
-   if(NOT exit_code EQUAL 0)
-     message(FATAL_ERROR \"Failed to install the Python package\")
-   endif()"
-)
+if(MCSEMA_INSTALL_PYTHON3_LIBS)
+  install(CODE
+    "execute_process(COMMAND ${optional_interpreter} \"${python_package_installer3}\" \"${python_package_install_path}\"\nWORKING_DIRECTORY \"${PROJECT_SOURCE_DIR}/tools\"\nRESULT_VARIABLE exit_code3)\n   if(NOT exit_code3 EQUAL 0)\n       message(FATAL_ERROR \"Failed to install the Python 3 package\")\n     endif()")
+endif()
 
 add_subdirectory(examples)
 

tools/mcsema_disass/ida7/arm_util.py

@@ -82,7 +82,8 @@ def fixup_function_return_address(inst, next_ea):
 
 def is_ELF_thunk_by_structure(ea):
   """Try to manually identify an ELF thunk by its structure."""
-  from util import *
+  from util import decode_instruction, is_direct_jump, is_indirect_jump
+  from util import is_invalid_ea, get_reference_target
   global _INVALID_THUNK_ADDR
   inst = None
 
@@ -111,7 +112,6 @@ def is_ELF_thunk_by_structure(ea):
 _ARM_REF_CANDIDATES = set()
 
 def _get_arm_ref_candidate(mask, op_val, op_str, all_refs):
-  from util import *
   global _BAD_ARM_REF_OFF, _ARM_REF_CANDIDATES
 
   try:
@@ -154,7 +154,7 @@ def _get_arm_ref_candidate(mask, op_val, op_str, all_refs):
 def try_get_ref_addr(inst, op, op_val, all_refs, _NOT_A_REF):
   global _BAD_ARM_REF_OFF
 
-  from util import *
+  from util import is_invalid_ea
 
   #if op.type not in (idc.o_imm, idc.o_displ):
     # This is a reference type that the other ref tracking code
@@ -167,7 +167,7 @@ def try_get_ref_addr(inst, op, op_val, all_refs, _NOT_A_REF):
     return _get_arm_ref_candidate(4095, op_val, op_str, all_refs)
 
   elif '@PAGE' in op_str:
-    return _get_arm_ref_candidate(-4096L, op_val, op_str, all_refs)
+    return _get_arm_ref_candidate(-4096, op_val, op_str, all_refs)
 
   elif not is_invalid_ea(op_val) and inst.get_canon_mnem().lower() == "adr":
     return op_val, 0, 0