Merge pull request #327 from ryanohoro/update_tests

Update Tests, Telemetry PR Bugs

src/python/strelka/strelka.py

@@ -217,6 +217,18 @@ def taste_yara(self, data: bytes) -> list:
     def match_flavors(self, data: bytes) -> dict:
         return {"mime": self.taste_mime(data), "yara": self.taste_yara(data)}
 
+    def check_scanners(self):
+        """attempt to import all scanners referenced in the backend configuration"""
+        logging.info("checking scanners")
+        if self.scanners:
+            for name in self.scanners:
+                try:
+                    und_name = inflection.underscore(name)
+                    scanner_import = f"strelka.scanners.{und_name}"
+                    importlib.import_module(scanner_import)
+                except ModuleNotFoundError:
+                    raise
+
     def work(self) -> None:
         """Process tasks from Redis coordinator"""
 
@@ -226,6 +238,8 @@ def work(self) -> None:
             logging.error("no coordinator specified")
             return
 
+        self.check_scanners()
+
         count = 0
         work_start = time.time()
         work_expire = work_start + self.limits.get("time_to_live", 900)
@@ -557,7 +571,6 @@ def match_scanner(
                 if re.search(neg_filename, file.name):
                     return {}
             if neg_source:
-                print(file.source, neg_source)
                 if file.source in neg_source:
                     return {}
             for pos_flavor in pos_flavors:
@@ -778,17 +791,6 @@ def emit_file(
 
                 self.files.append(extract_file)
 
-                if self.coordinator:
-                    for c in chunk_string(data):
-                        self.upload_to_coordinator(
-                            extract_file.pointer,
-                            c,
-                            self.expire_at,
-                        )
-                else:
-                    extract_file.data = data
-                self.files.append(extract_file)
-
             except Exception:
                 logging.exception("failed to emit file")
                 self.flags.append("failed_to_emit_file")

src/python/strelka/tests_configuration/test_scanner_assignment.py

@@ -78,6 +78,7 @@
     "test.yara": ["ScanUrl"],
     "test.zip": ["ScanZip"],
     "test_aes256_password.zip": ["ScanEncryptedZip", "ScanZip"],
+    "test_broken.heic": ["ScanExiftool", "ScanTranscode"],
     "test_broken_iend.png": [
         "ScanExiftool",
         "ScanLsb",
@@ -165,6 +166,9 @@
     "test_pii.csv": [],  # NOTE: ScanCcn not enabled
     "test_private.pgp": ["ScanPgp"],
     "test_public.pgp": ["ScanPgp"],
+    "test_qr.avif": ["ScanExiftool", "ScanTranscode"],
+    "test_qr.heic": ["ScanExiftool", "ScanTranscode"],
+    "test_qr.heif": ["ScanExiftool", "ScanTranscode"],
     "test_qr.jpg": [
         "ScanExiftool",
         "ScanJpeg",

src/python/strelka/tests_configuration/test_taste.py

@@ -9,6 +9,7 @@
 
 taste_expectations: dict = {
     "test.7z": {"mime": ["application/x-7z-compressed"], "yara": ["_7zip_file"]},
+    "test_qr.avif": {"mime": ["image/avif"], "yara": []},
     "test.b64": {"mime": ["text/plain"], "yara": []},  # FIXME: No file-specific match
     "test.bat": {
         "mime": ["text/x-msdos-batch"],
@@ -35,6 +36,8 @@
     "test.exe": {"mime": ["application/x-dosexec"], "yara": ["mz_file"]},
     "test.gif": {"mime": ["image/gif"], "yara": ["gif_file"]},
     "test.gz": {"mime": ["application/gzip"], "yara": ["gzip_file"]},
+    "test_qr.heic": {"mime": ["image/heic"], "yara": []},
+    "test_qr.heif": {"mime": ["image/heif"], "yara": []},
     "test.html": {"mime": ["text/html"], "yara": ["html_file"]},
     "test.ini": {"mime": ["text/plain"], "yara": ["ini_file"]},
     "test.iso": {"mime": ["application/x-iso9660-image"], "yara": ["iso_file"]},
@@ -56,7 +59,7 @@
     "test.pcapng": {
         "mime": ["application/octet-stream"],
         "yara": [],
-    },  # FIXME: pcapng_file broken
+    },  # FIXME: pcapng_file broken https://github.com/target/strelka/issues/284
     "test.pdf": {"mime": ["application/pdf"], "yara": ["pdf_file"]},
     "test.pem": {"mime": ["text/plain"], "yara": ["x509_pem_file"]},
     "test.plist": {"mime": ["text/xml"], "yara": ["plist_file", "xml_file"]},
@@ -95,6 +98,7 @@
     "test_classic.doc": {"mime": ["application/msword"], "yara": ["olecf_file"]},
     "test_embed_rar.jpg": {"mime": ["image/jpeg"], "yara": ["jpeg_file"]},
     "test_embed_rar.png": {"mime": ["image/png"], "yara": ["png_file"]},
+    "test_broken.heic": {"mime": ["image/heic"], "yara": []},
     "test_hyperlinks.html": {"mime": ["text/html"], "yara": ["html_file"]},
     "test_lzx.cab": {
         "mime": ["application/vnd.ms-cab-compressed"],