From a5f2d4b2056e0531a58885b98e781cb700f9dcce Mon Sep 17 00:00:00 2001 From: Ryan O'Horo <10855297+ryanohoro@users.noreply.github.com> Date: Fri, 8 Mar 2024 18:48:52 -0600 Subject: [PATCH] Add optional overlay file extraction to ScanPe, including new event metadata, new ScanPe test --- configs/python/backend/backend.yaml | 2 + src/python/strelka/scanners/scan_pe.py | 40 ++-- .../tests/fixtures/test_overlay_zip.exe | Bin 0 -> 10538 bytes src/python/strelka/tests/test_scan_pe.py | 177 ++++++++++++++++++ 4 files changed, 207 insertions(+), 12 deletions(-) create mode 100644 src/python/strelka/tests/fixtures/test_overlay_zip.exe diff --git a/configs/python/backend/backend.yaml b/configs/python/backend/backend.yaml index cf474bfa..b6f9d019 100644 --- a/configs/python/backend/backend.yaml +++ b/configs/python/backend/backend.yaml @@ -436,6 +436,8 @@ scanners: - 'application/x-dosexec' - 'mz_file' priority: 5 + options: + extract_overlay: False 'ScanPgp': - positive: flavors: diff --git a/src/python/strelka/scanners/scan_pe.py b/src/python/strelka/scanners/scan_pe.py index fa8dd84b..dd411f13 100644 --- a/src/python/strelka/scanners/scan_pe.py +++ b/src/python/strelka/scanners/scan_pe.py @@ -392,8 +392,13 @@ class ScanPe(strelka.Scanner): """Collects metadata from PE files.""" def scan(self, data, file, options, expire_at): + extract_overlay = options.get("extract_overlay", False) + try: pe = pefile.PE(data=data) + if not pe: + self.flags.append("pe_load_error") + return except pefile.PEFormatError: self.flags.append("pe_format_error") return @@ -421,6 +426,17 @@ def scan(self, data, file, options, expire_at): } self.event["summary"] = {} + offset = pe.get_overlay_data_start_offset() + + if offset and len(data[offset:]) > 0: + self.event["overlay"] = {"size": len(data[offset:]), "extracted": False} + self.flags.append("overlay") + + if extract_overlay: + # Send extracted file back to Strelka + self.emit_file(data[offset:], name="pe_overlay") + self.event["overlay"].update({"extracted": True}) + if hasattr(pe, "DIRECTORY_ENTRY_DEBUG"): for d in pe.DIRECTORY_ENTRY_DEBUG: try: @@ -532,18 +548,18 @@ def scan(self, data, file, options, expire_at): self.event["address_of_entry_point"] = pe.OPTIONAL_HEADER.AddressOfEntryPoint self.event["image_base"] = pe.OPTIONAL_HEADER.ImageBase self.event["size_of_code"] = pe.OPTIONAL_HEADER.SizeOfCode - self.event["size_of_initialized_data"] = ( - pe.OPTIONAL_HEADER.SizeOfInitializedData - ) + self.event[ + "size_of_initialized_data" + ] = pe.OPTIONAL_HEADER.SizeOfInitializedData self.event["size_of_headers"] = pe.OPTIONAL_HEADER.SizeOfHeaders self.event["size_of_heap_reserve"] = pe.OPTIONAL_HEADER.SizeOfHeapReserve self.event["size_of_image"] = pe.OPTIONAL_HEADER.SizeOfImage self.event["size_of_stack_commit"] = pe.OPTIONAL_HEADER.SizeOfStackCommit self.event["size_of_stack_reserve"] = pe.OPTIONAL_HEADER.SizeOfStackReserve self.event["size_of_heap_commit"] = pe.OPTIONAL_HEADER.SizeOfHeapCommit - self.event["size_of_uninitialized_data"] = ( - pe.OPTIONAL_HEADER.SizeOfUninitializedData - ) + self.event[ + "size_of_uninitialized_data" + ] = pe.OPTIONAL_HEADER.SizeOfUninitializedData self.event["file_alignment"] = pe.OPTIONAL_HEADER.FileAlignment self.event["section_alignment"] = pe.OPTIONAL_HEADER.SectionAlignment self.event["checksum"] = pe.OPTIONAL_HEADER.CheckSum @@ -552,12 +568,12 @@ def scan(self, data, file, options, expire_at): self.event["minor_image_version"] = pe.OPTIONAL_HEADER.MinorImageVersion self.event["major_linker_version"] = pe.OPTIONAL_HEADER.MajorLinkerVersion self.event["minor_linker_version"] = pe.OPTIONAL_HEADER.MinorLinkerVersion - self.event["major_operating_system_version"] = ( - pe.OPTIONAL_HEADER.MajorOperatingSystemVersion - ) - self.event["minor_operating_system_version"] = ( - pe.OPTIONAL_HEADER.MinorOperatingSystemVersion - ) + self.event[ + "major_operating_system_version" + ] = pe.OPTIONAL_HEADER.MajorOperatingSystemVersion + self.event[ + "minor_operating_system_version" + ] = pe.OPTIONAL_HEADER.MinorOperatingSystemVersion self.event["major_subsystem_version"] = pe.OPTIONAL_HEADER.MajorSubsystemVersion self.event["minor_subsystem_version"] = pe.OPTIONAL_HEADER.MinorSubsystemVersion self.event["image_version"] = float( diff --git a/src/python/strelka/tests/fixtures/test_overlay_zip.exe b/src/python/strelka/tests/fixtures/test_overlay_zip.exe new file mode 100644 index 0000000000000000000000000000000000000000..408a3ad670db8dec3aa3321c35969ae90db4b636 GIT binary patch literal 10538 zcmeGiX;>6Twwn8pLq)_3+QF!pFf+&@pmGiff{0v-VswsX80eXvbk6`ol+|ENj3*kR zD4K|(!DGS28we_zXg-OOjRz{<=M|&k6^#kvD*L*J8HPx*yWjrV{XS~Ge)a0TdPlvg zepOR5(`K(h<_IARcse@~It(>i!Z`TnU=gq#J&rn}{kBab4`cC7BU7>z46ddrIW1P= zQn5-!X>bXF(^?g-P~oEK$+(h|5qt*+yHQ=L6QdB4Ej2@R%z#%0Yflgl*_#OwvH^;f zfeJOw;b9nCvACJuZ??($&z1QAfzB3l6!xKUBw-v0fU_RK4Q(T(o4{gW2pzNRNiuD{ zGa8{lQ^J3W;|Rr=$nZ5pz6RRo*zp378++F{pv4s&q4|88p{3B`dh6vP2m;Lg!(;Y;55%jrt!*_9XLZkW!Cz0%yj#n~jAa+l`Azo*IF18WHLT z-E;l;LcXui*C&XTwL~PqvmoYK2(g5*&Vt7?SwkySa)#}(1}1el)SjuyXb<4*%y~{p zjTJ#%2W=NtHiD!i&}IvWV^anpWXm=>Fkj@VA5VWhkL7dTuzDQ3a8LoI1k=}6$POEB zW`*)N7=rF&PG}N{S)okyJ!XX-a_~D2zKgMxaU6`~;6jd`gkAw%$iaOaWdqs{ltiIu0&FqtQ^b7;r1ZNA~dW(Lgi-;4qX1 z&=buA_%@;d`f_k0(gGgI(c{npz>_)p91coR2~cuS13(6yMHY+nkw?Zd5HOfRWAQLu z#FM5`GA&7jASENEXi_0TF$76cGXa;ONJ_;}B!Onq3JnpjP!VW~Rv`=5z`BuWH3Sk7 z60KY=mXJg@Es|2EDHuf$S~$ZHN(re;QD{uN8p*U+MkvK}PPdj6F)b%F(U9ASJj#?| zkcn21L>fUe3QE;2WnhBenF_g<7HfKCiwH(aE7U!i;I~>qauP`dDbD9G=1tv+Gz3hl zd6N-IsdcnMp4BT$sTQkr-7-m9l}4c?Sesx+q97F-6K?km@QHi^CF>Xsq2zbT@ViD0 zZ!Q8~*!17k15FbvRL0&!CP_%yAtJt1L(vAhNFkQ1C`O}@G6s@CajZ&1(3CovpmP;c z;tjSwKm={%>t`SQfF%UoVa6CVU6Kt}VW|y{K?GzS64oe=W&)%EmvL(_*KvE4CgaQa z4d=&q;mHNw#bC4z4EnLJF+xrlvg?j0BttGASb;PjAvX*SG)^3Tq7vuh zlr$h9z}HXg=bh=}6XfkT-Z#Ko5-1z*Efxm)%lw3d{{-m-NVgcW_JM2* zAB3i0Xed7+D#f^R-|3pJ$!taje;r+nERljylVV*0tZ`>n1~+nWKi2WrOOns9H9L$P zcx12-%GiC{q_Fwbv?L{ql6Oe0&u?0CMATqz-|4;3eJVTN$l#1r240s8jZ!MtW@ctE zl$Mqf88o4$n2hf5n4**kSsS+R-sMc&-UUNXQ;QyNb}9U1n*po6G|kg^~b^Mbl9N zv||BA1J0Iui^s3^dx~C{VSfqbxca@t!093(5g-_76~lWU4Z9!-*&r6FU~i|mE~AhI z)G5Ff1C;^37-^s%1ucE4wparT9|NB>&`SYyc`uqsu8$DT0Q!#~k^qKahh^gt3Hp@q zAwXIKR`g{Qyio?8nzOHi(Tcf#Y!65aCSOPd;T`;2HkNo^*?}%cOvMaV3*KBtTiU1?_Hkt)sKsFPr2Vm`}eU9?Vo34n;VqP%)EvT3Qv%NVDJ|#CR(eQkr6@OpUjcQU;3| zB|q1PhbzS@MJB;$3|k6k50v4!QCX~vfRlwr*TW}vYvSQ5*k>m5rs=}fYEmKPjyHU< zTFn#aHE3upqlr~zQvaqu<9VDM*k%Z+77mEIt~O9;;yo?+NyrjuMJ^oF|rLJ$R$XeWZ`6|DG~JIjlp$6 z58o#Vj6r}7lLUqt3Ng(1e^)2QnOnGUD#lJ#eJ!#AT*UEv&Ju9q`zvYCoy6cL^#^R~Cq#-PyKz-L#-J>G?ke)`YIp zuJG74w|enAKOI)tzg7F_^4Y>WAFh8e@JG>iWzJj3wdbEI{N>c?z}Ll1qH~+)?wx*m zcvETe%^2%3+lH03W}V4jyRWncabw^& z`TI|i72CYa^2-HtV&;tf_|@r{Ci{o^nWvxK`^>Sef6UM_^P5}UH3A2}YX^nfD&kcW z#p{1KU)_vm-N{&W`cC4|tIOW?{@d{SiBDV3?Wae+SAH(&(M3Vnsj0m5#`%)(HV(G& z2(U=@O!(y&)mnSZOPKWf^v4IcJ}*sv_i|2pdC9UZPi$W#DiRj%QwM%g*e|GLwlunZ z#8yGl<|~DJc05Q5b9{08WTqJQCdF#^=*-nQxS)aVp zxN5KKg()t{i(Kyg^_gq#upJ*cxrelW5Y+5d`N8VM@SF=R=}&bfwZ}i%b*KDEnJ8#( z@hkI)lAD?@%vuxY@J>Q&@r*^_|MFHV%_ZeO?7?JDuWa^9q9=?t&KqLyWIDp!OD6y9Gc z!3)i1GqcVbW0Vq42i^X?8my7$os1q}g8-SW2_;);%0@ggVBT6$zp z!9OHrnU9v(xj6l{YLWcz;WdNXnl1k!@hn z#{Qu%UxndqKOYEc`|;hA_EEbf_AwrJGRDyDkIRaJe2}1FQA~_;Q$^>#1!c|!2V?fn zB7eMiwmmN*zVu-0@!9>nFU(Fnoc8g;#=`zH77uKzdEQv?jn~o@k5fOoFyMjCjh>M( z(YBh1o_edf^ZDg|=ZaRRgkJb`h`;}fxQf=BRjcksEc+xn__NBjB~?uPusZDMvCmu& znEkzB_2PqhvkUvr5BiP2H)q4EpbbUqt=?{xt*KgSp0l>)(NUL(egj-8$8|)d+O3l*`O@uI z8%n*O+&vN;**I?FwAOQNgRXruYyG*Fw35cZ?D}=r-U|b&%x;`4dG$~helf22hsBQv zT(VjJmD`~xx3B^c8eae4M0Rl?-v-4 z&)ct?ue(KD$XNS+dGM}p>JOE8rj~m1XH2l)dc8hjer?FbxX3flZ+K86Hu_6XzRbav zjGw>TCHLyBilpL~EeAR`N$b5F=eZC5^1UY)-1oO$aZU6nvX*W9^3|QHDdpFy>(aID z?(#lY zap|?wCpC1=m2}wtaTXu)`z%fqVi~0(b$xlfFOT=-@xDCXm&gAv@;H0()^%j(PUC&) zyDxqBrSHD<-Iu=qFVZ*ryLDgQ?z=weyFTf=KIywY`QN)f!OTL@AKjIYg51rP0s6{t z47Y=yaM*$&H&pdfE{||*!S1)+38?Ddf_a^BPG{SRaR$Y`-LD%I^FYg6@R*+OO>Wyw z4Lx$Q-FnHP1-qW<9()iCtzRMK4F8#-A;@58Bp7Lf>-`>vOfQ6bGxZhn;7m2t693Ya zv(Xd|rfTeanqt4R^k!k_M7;$=6#r~6wmmG+@c%F4w%VKO1=IDaH!te=f{j~ZoF(`F SH%s`B+I{$KZ5;b62l^){AHv`O literal 0 HcmV?d00001 diff --git a/src/python/strelka/tests/test_scan_pe.py b/src/python/strelka/tests/test_scan_pe.py index 3b243a3b..698b0202 100644 --- a/src/python/strelka/tests/test_scan_pe.py +++ b/src/python/strelka/tests/test_scan_pe.py @@ -178,3 +178,180 @@ def test_scan_pe(mocker): TestCase.maxDiff = None TestCase().assertDictEqual(test_scan_event, scanner_event) + + +def test_scan_pe_overlay(mocker): + """ + Pass: Sample event matches output of scanner. + Failure: Unable to load file or sample event fails to match. + """ + + test_scan_event = { + "elapsed": mock.ANY, + "flags": ["no_certs_found", "overlay"], + "total": {"libraries": 0, "resources": 2, "sections": 2, "symbols": 0}, + "summary": { + "resource_md5": unordered( + [ + "f4741884351459aa7733725b88e693af", + "b7db84991f23a680df8e95af8946f9c9", + ] + ), + "resource_sha1": unordered( + [ + "5371904ee7671fb0b066d9323eda553269f344f9", + "cac699787884fb993ced8d7dc47b7c522c7bc734", + ] + ), + "resource_sha256": unordered( + [ + "539dc26a14b6277e87348594ab7d6e932d16aabb18612d77f29fe421a9f1d46a", + "d8df3d0358a91b3ef97c4d472b34a60f7cf9ee7f1a6f37058fc3d1af3a156a36", + ] + ), + "section_md5": unordered( + [ + "c3eafa2cd34f98a226e31b8ea3fea400", + "cc14da7fb94ef9b27a926fe95b86b44f", + ] + ), + "section_sha1": unordered( + [ + "3d584b265a558dc22fa6dfa9991ae7eafee5c1a4", + "00104b432a8e7246695843e4f2d7cf2582efa3e6", + ] + ), + "section_sha256": unordered( + [ + "86d9755b2ba9d8ffd765621f09844dd62d0b082fdc4aafa63b3b3f3ae25d9c77", + "bb31a5224e9f78905909655d9c80ba7d63f03910e4f22b296d6b7865e2a477c3", + ] + ), + }, + "debug": { + "type": "rsds", + "guid": b"a66307d0-9b84-b944-bf030bff2d7d1e4a", + "age": 1, + "pdb": b"C:\\Users\\tmcguff\\source\\repos\\HelloWorld\\HelloWorld\\obj\\x64\\Release\\HelloWorld.pdb", + }, + "file_info": { + "fixed": { + "flags": [], + "operating_systems": ["WINDOWS32"], + "type": {"primary": "APP", "sub": ""}, + }, + "string": [], + "var": {"language": None, "character_set": "Unicode"}, + "comments": "", + "company_name": ".", + "file_description": "HelloWorld", + "file_version": "1.0.0.0", + "internal_name": "HelloWorld.exe", + "legal_copyright": "Copyright © . 2020", + "legal_trademarks": "", + "original_filename": "HelloWorld.exe", + "product_name": "HelloWorld", + "product_version": "1.0.0.0", + "assembly_version": "1.0.0.0", + }, + "header": { + "machine": {"id": 34404, "type": "AMD64"}, + "magic": {"dos": "DOS", "image": "64_BIT"}, + "subsystem": "WINDOWS_CUI", + }, + "base_of_code": 8192, + "address_of_entry_point": 0, + "image_base": 5368709120, + "size_of_code": 2048, + "size_of_initialized_data": 1536, + "size_of_headers": 512, + "size_of_heap_reserve": 1048576, + "size_of_image": 24576, + "size_of_stack_commit": 16384, + "size_of_stack_reserve": 4194304, + "size_of_heap_commit": 8192, + "size_of_uninitialized_data": 0, + "file_alignment": 512, + "section_alignment": 8192, + "checksum": 0, + "major_image_version": 0, + "minor_image_version": 0, + "major_linker_version": 48, + "minor_linker_version": 0, + "major_operating_system_version": 4, + "minor_operating_system_version": 0, + "major_subsystem_version": 4, + "minor_subsystem_version": 0, + "image_version": 0.0, + "linker_version": 48.0, + "operating_system_version": 4.0, + "overlay": {"extracted": True, "size": 6442}, + "subsystem_version": 4.0, + "compile_time": "2104-07-18T17:22:04", + "dll_characteristics": unordered( + [ + "DYNAMIC_BASE", + "NX_COMPAT", + "NO_SEH", + "TERMINAL_SERVER_AWARE", + ] + ), + "image_characteristics": unordered(["EXECUTABLE_IMAGE", "LARGE_ADDRESS_AWARE"]), + "resources": unordered( + [ + { + "id": 1, + "language": {"sub": "NEUTRAL", "primary": "NEUTRAL"}, + "type": "VERSION", + "md5": "f4741884351459aa7733725b88e693af", + "sha1": "5371904ee7671fb0b066d9323eda553269f344f9", + "sha256": "d8df3d0358a91b3ef97c4d472b34a60f7cf9ee7f1a6f37058fc3d1af3a156a36", + }, + { + "id": 1, + "language": {"sub": "NEUTRAL", "primary": "NEUTRAL"}, + "type": "MANIFEST", + "md5": "b7db84991f23a680df8e95af8946f9c9", + "sha1": "cac699787884fb993ced8d7dc47b7c522c7bc734", + "sha256": "539dc26a14b6277e87348594ab7d6e932d16aabb18612d77f29fe421a9f1d46a", + }, + ] + ), + "sections": unordered( + [ + { + "address": {"physical": 1743, "virtual": 8192}, + "characteristics": ["CNT_CODE", "MEM_EXECUTE", "MEM_READ"], + "entropy": 4.621214196319175, + "name": ".text", + "size": 2048, + "md5": "cc14da7fb94ef9b27a926fe95b86b44f", + "sha1": "3d584b265a558dc22fa6dfa9991ae7eafee5c1a4", + "sha256": "bb31a5224e9f78905909655d9c80ba7d63f03910e4f22b296d6b7865e2a477c3", + }, + { + "address": {"physical": 1472, "virtual": 16384}, + "characteristics": ["CNT_INITIALIZED_DATA", "MEM_READ"], + "entropy": 4.09070377434219, + "name": ".rsrc", + "size": 1536, + "md5": "c3eafa2cd34f98a226e31b8ea3fea400", + "sha1": "00104b432a8e7246695843e4f2d7cf2582efa3e6", + "sha256": "86d9755b2ba9d8ffd765621f09844dd62d0b082fdc4aafa63b3b3f3ae25d9c77", + }, + ] + ), + "symbols": {"exported": [], "imported": [], "libraries": [], "table": []}, + } + + scanner_event = run_test_scan( + mocker=mocker, + scan_class=ScanUnderTest, + fixture_path=Path(__file__).parent / "fixtures/test_overlay_zip.exe", + options={ + "extract_overlay": True, + }, + ) + + TestCase.maxDiff = None + TestCase().assertDictEqual(test_scan_event, scanner_event)