From b805ac18ce52d5add8f7e3aa5691d96317e9009f Mon Sep 17 00:00:00 2001 From: Aleksey Nogin Date: Thu, 22 Feb 2024 11:47:25 -0800 Subject: [PATCH] Update to angr==9.2.77 (#417) * Use python3.8 in docker images * Require pytest<8.0 This is needed becase of https://github.com/pytest-dev/pytest/issues/11890 https://github.com/TvoroG/pytest-lazy-fixture/issues/65 * Update changelog * Revert "Update changelog" This reverts commit 500ee9b1c6140848a6ff102bf4a6180a77b85d8e. Making changes before having coffee :( * Update to latest angr==9.2.89, which also necessitates Python >= 3.8 and capstone==5.0.0.post1 * Apply Edward's attempted fix to angr test failure * Add a note on recommending Python 3.8 * Add a note on recommending Python 3.8 * Document the requirement of Python 3.8+ * Switch to angr 9.2.77 * `ofrak_core` also needs `pytest<8.0` * ignore DataWord in test due to angr bug * add another now missing block * black linting * Attempt to fix a capstone error * Dropping the .altinstr_replacement section from the toolchain (#414) * Dropping the .altinstr_replacement section from the toolchain * Updated CHANGELOG * Set the fallback font to monospace (#422) * Set the fallback font to monospace * Update CHANGELOG * Display strings with numbers primarily as strings (#423) * Display strings with numbers primarily as strings * Update CHANGELOG * Add typing support to ofrak_ghidra package (#421) * Add typing to ofrak_ghidra package * Add changelog --------- Co-authored-by: Paul Noalhyt * Increase time limit on `test_comment_content` * Fix a spurious "no current event loop" test error * Explain 3.7 vs 3.8 better in the docs * Cite specific versions of angr in comment * Update docs/environment-setup.md * Update docs/getting-started.md --------- Co-authored-by: Edward Larson Co-authored-by: rbs-alexr <122491504+rbs-alexr@users.noreply.github.com> Co-authored-by: Jacob Strieb <99368685+rbs-jacob@users.noreply.github.com> Co-authored-by: Paul Noalhyt Co-authored-by: Paul Noalhyt Co-authored-by: Wyatt <53830972+whyitfor@users.noreply.github.com> --- disassemblers/ofrak_angr/CHANGELOG.md | 3 +++ .../ofrak_angr/components/blocks/unpackers.py | 2 +- .../ofrak_angr_test/test_unpackers.py | 19 ++++++++++++++++++- disassemblers/ofrak_angr/requirements.txt | 2 +- disassemblers/ofrak_angr/setup.py | 2 +- disassemblers/ofrak_capstone/CHANGELOG.md | 3 +++ disassemblers/ofrak_capstone/requirements.txt | 2 +- docs/environment-setup.md | 2 +- docs/getting-started.md | 2 +- .../patterns/basic_block_unpacker.py | 8 ++++++++ .../patterns/complex_block_unpacker.py | 6 +++--- 11 files changed, 41 insertions(+), 10 deletions(-) diff --git a/disassemblers/ofrak_angr/CHANGELOG.md b/disassemblers/ofrak_angr/CHANGELOG.md index eb673296c..d4ba9f23a 100644 --- a/disassemblers/ofrak_angr/CHANGELOG.md +++ b/disassemblers/ofrak_angr/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +### Changed +- Update to latest angr==9.2.77, which also necessitates Python >= 3.8. + ### Fixed - Add `importlib-resources` dependency as workaround for z3-solver dependency issue. ([#401](https://github.com/redballoonsecurity/ofrak/pull/401)) diff --git a/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py b/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py index e9c34d525..15b7c803b 100644 --- a/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr/components/blocks/unpackers.py @@ -228,7 +228,7 @@ def _angr_get_dword_blocks( if xref is None or not any(xref in bb_range for bb_range in valid_data_xref_ranges): continue - LOGGER.debug(f"Creating DataWord for {cb_data_xref.content} @ {cb_data_xref_addr:#x}") + LOGGER.debug(f"Creating DataWord for {cb_data_xref.content!r} @ {cb_data_xref_addr:#x}") format_string = endian_flag + dword_size_map[word_size] diff --git a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py index 4f09974e3..7e9279d31 100755 --- a/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py +++ b/disassemblers/ofrak_angr/ofrak_angr_test/test_unpackers.py @@ -54,7 +54,7 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe 0x110, 0x110, 0x130, - keep_same_is_exit_point=True, + keep_same_is_exit_point=False, ) return self._fixup_test_case_for_pie( @@ -62,6 +62,23 @@ async def expected_results(self, unpack_verify_test_case: ComplexBlockUnpackerTe pie_base_vaddr=0x400000, ) + elif unpack_verify_test_case.binary_md5_digest == "c79d1bea0398d7a9d0faa1ba68786f5e": + # Unlike angr 9.2.6, angr 9.2.77 and 9.2.91 miss this DataWord now + # = the ref to it does not appear in the list of xrefs + + missing_data_words = {0x8030, 0x8060} + + fixed_up_results = { + vaddr: [ + block + for block in original_expected_blocks + if block.virtual_address not in missing_data_words + ] + for vaddr, original_expected_blocks in unpack_verify_test_case.expected_results.items() + } + + return fixed_up_results + return unpack_verify_test_case.expected_results def _split_bb( diff --git a/disassemblers/ofrak_angr/requirements.txt b/disassemblers/ofrak_angr/requirements.txt index a529c3fff..c6a33a3dd 100644 --- a/disassemblers/ofrak_angr/requirements.txt +++ b/disassemblers/ofrak_angr/requirements.txt @@ -1,2 +1,2 @@ -angr==9.2.6 +angr==9.2.77 importlib-resources # A workaround for https://github.com/redballoonsecurity/ofrak/issues/398 diff --git a/disassemblers/ofrak_angr/setup.py b/disassemblers/ofrak_angr/setup.py index a106800d8..481de0e1b 100644 --- a/disassemblers/ofrak_angr/setup.py +++ b/disassemblers/ofrak_angr/setup.py @@ -69,7 +69,7 @@ def read_requirements(requirements_path): "Topic :: Security", "Typing :: Typed", ], - python_requires=">=3.7", + python_requires=">=3.8", license="Proprietary", license_files=["LICENSE"], cmdclass={"egg_info": egg_info_ex}, diff --git a/disassemblers/ofrak_capstone/CHANGELOG.md b/disassemblers/ofrak_capstone/CHANGELOG.md index 669a71b1c..bc0b6a026 100644 --- a/disassemblers/ofrak_capstone/CHANGELOG.md +++ b/disassemblers/ofrak_capstone/CHANGELOG.md @@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ## [Unreleased](https://github.com/redballoonsecurity/ofrak/tree/master) +### Changed +- Update to captione==5.0.0.post1. + ## 1.0.0 - 2022-01-25 ### Added Initial release. Hello world! diff --git a/disassemblers/ofrak_capstone/requirements.txt b/disassemblers/ofrak_capstone/requirements.txt index ae1c92132..5903f94cb 100644 --- a/disassemblers/ofrak_capstone/requirements.txt +++ b/disassemblers/ofrak_capstone/requirements.txt @@ -1 +1 @@ -capstone==4.0.2 +capstone==5.0.0.post1 diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 3736e7c67..1264320b2 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -1,7 +1,7 @@ # Environment Setup & Installing OFRAK !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some packages (for example, ofrak-angr) require Python >=3.8. There are three main ways one can set up an environment to use OFRAK: diff --git a/docs/getting-started.md b/docs/getting-started.md index 8716d56c2..25d7ebec7 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -3,7 +3,7 @@ ## Quick Start - Unpack a firmware file and display it in the GUI !!! warning - OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with. + OFRAK is a Python library supporting Python3.7 and up. First and foremost, make sure your Python and pip installations are for Python3.7+! Python 3.8 is recommended, as this is the version we primarily test OFRAK with, and some packages (for example, ofrak-angr) require Python >=3.8. ```bash pip install ofrak diff --git a/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py b/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py index 5c0e0f012..e25858c39 100644 --- a/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py +++ b/ofrak_core/pytest_ofrak/patterns/basic_block_unpacker.py @@ -756,6 +756,14 @@ class BasicBlockUnpackerTestCase( operands="", mode=InstructionSetMode.NONE, ), + Instruction( + virtual_address=0x4004E0, + size=2, + disassembly="repz ret ", + mnemonic="repz ret", + operands="", + mode=InstructionSetMode.NONE, + ), ), ], 0x4004F0: [ diff --git a/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py b/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py index c632017d1..198e8391a 100644 --- a/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py +++ b/ofrak_core/pytest_ofrak/patterns/complex_block_unpacker.py @@ -231,7 +231,7 @@ class ComplexBlockUnpackerTestCase(UnpackAndVerifyTestCase[int, List[Union[Basic is_exit_point=True, exit_vaddr=None, ), - DataWord(virtual_address=32816, size=4, format_string="