ScanUdf

[ryanohoro]

Describe the change

Adds a dedicated UDF scanner and taste yara for UDF-formatted disk images. ScanIso has limitations for what formats it supports and specifically does not support UDF files.

Also removes the pre-commit hook for large files, as test fixtures often exceed the 500KB limit.

Describe testing procedures

Added test, test fixture which are passing. Submitted to Fileshot UI and returned the expected events.

Sample output

{
  "strelka_response": [
    {
      "file": {
        "depth": 0,
        "flavors": {
          "mime": [
            "application/octet-stream"
          ],
          "yara": [
            "udf_file"
          ]
        },
        "name": "test_udf_1.50.img",
        "scanners": [
          "ScanEntropy",
          "ScanFooter",
          "ScanHash",
          "ScanHeader",
          "ScanUdf",
          "ScanTlsh",
          "ScanYara"
        ],
        "size": 1245184,
        "tree": {
          "node": "e6b22252-7f2c-4a37-8d02-d912aab51991",
          "root": "e6b22252-7f2c-4a37-8d02-d912aab51991"
        }
      },
      "request": {
        "attributes": {
          "filename": "test_udf_1.50.img",
          "metadata": {
            "source": "fileshot-webui",
            "user_name": "strelka"
          }
        },
        "client": "fileshot-webui",
        "id": "e6b22252-7f2c-4a37-8d02-d912aab51991",
        "time": 1683130898
      },
      "scan": {
        "entropy": {
          "elapsed": 0.001832,
          "entropy": 0.05834599565211478
        },
        "footer": {
          "backslash": "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00",
          "elapsed": 0.00004,
          "footer": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
        },
        "hash": {
          "elapsed": 0.028301,
          "md5": "e4869314d0fb7cddf6e71c985e4651c0",
          "sha1": "9c3892d361d684f861060dec44016087218650d6",
          "sha256": "2c375a85705ae44caf347c498799a10cdfe99a86de2ea11161f410e762693f7b",
          "ssdeep": "96:b7uHmeY3uQPebqA96q+WVc1FPFB4A0BZzgk9JeQJHqpVRIFwppGh34:b2Y+QPBOiGA0BdhEZIyP",
          "tlsh": "T14945A9305F5A0252D3A0317E8BD5C6BFE3B4E80033978796E9A8A71A9C6ED1497335DC"
        },
        "header": {
          "backslash": "\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00",
          "elapsed": 0.000079,
          "header": "\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000"
        },
        "tlsh": {
          "elapsed": 0.014869
        },
        "udf": {
          "elapsed": 0.014169,
          "files": [
            {
              "datetime": "2022-12-12 03:12:55",
              "filename": "lorem.txt",
              "size": "4015"
            }
          ],
          "meta": {
            "7zip_version": "22.01",
            "partitions": [
              {
                "created": "2023-04-25 23:44:43.364000",
                "path": "/tmp/tmpbk2w_7sq",
                "type": "Udf"
              }
            ]
          },
          "total": {
            "extracted": 1,
            "files": 1
          }
        },
        "yara": {
          "elapsed": 0.005987,
          "matches": [
            "test"
          ]
        }
      }
    },
    {
      "file": {
        "depth": 1,
        "flavors": {
          "mime": [
            "text/plain"
          ]
        },
        "name": "lorem.txt",
        "scanners": [
          "ScanEntropy",
          "ScanFooter",
          "ScanHash",
          "ScanHeader",
          "ScanTlsh",
          "ScanUrl",
          "ScanYara"
        ],
        "size": 4015,
        "source": "ScanUdf",
        "tree": {
          "node": "706d1261-9483-483d-a991-9465a9542556",
          "parent": "e6b22252-7f2c-4a37-8d02-d912aab51991",
          "root": "e6b22252-7f2c-4a37-8d02-d912aab51991"
        }
      },
      "request": {
        "attributes": {
          "filename": "test_udf_1.50.img",
          "metadata": {
            "source": "fileshot-webui",
            "user_name": "strelka"
          }
        },
        "client": "fileshot-webui",
        "id": "e6b22252-7f2c-4a37-8d02-d912aab51991",
        "time": 1683130898
      },
      "scan": {
        "entropy": {
          "elapsed": 0.00005,
          "entropy": 4.183768348776497
        },
        "footer": {
          "backslash": "itae. Et tortor consequat id porta nibh venenatis.",
          "elapsed": 0.000059,
          "footer": "itae. Et tortor consequat id porta nibh venenatis."
        },
        "hash": {
          "elapsed": 0.000189,
          "md5": "83c2df5aad9adf3e761315baea9b5b68",
          "sha1": "5030560d3a8f7e363d802cb9b1e1c82a65d60de7",
          "sha256": "7ac19ffb133c73599774fcd0d056313c497f87d091ac9a08aa73d083aa67e2e7",
          "ssdeep": "96:9Y3uQPebqA96q+WVc1FPFB4A0BZzgk9JeQJHqpVRIFwppGh344:9Y+QPBOiGA0BdhEZIyP4",
          "tlsh": "T1538165343EAA430247F4217ED7D5C9BFE288F41027CAA299D8A5FD56945E918D323294"
        },
        "header": {
          "backslash": "Lorem ipsum dolor sit amet, consectetur adipiscing",
          "elapsed": 0.000027,
          "header": "Lorem ipsum dolor sit amet, consectetur adipiscing"
        },
        "tlsh": {
          "elapsed": 0.000059,
          "match": {
            "family": "TestMatchA",
            "score": 6,
            "tlsh": "T1158165343EAA43024BF4217FD7D5C9BFE288F41027CAA399D8B5FD56945E518D323294"
          }
        },
        "url": {
          "elapsed": 0.000714
        },
        "yara": {
          "elapsed": 0.000061,
          "matches": [
            "test"
          ]
        }
      }
    }
  ]
}

Checklist

• My code follows the style guidelines of this project
• I have performed a self-review of and tested my code
• I have commented my code, particularly in hard-to-understand areas
• I have made corresponding changes to the documentation
• My changes generate no new warnings

[ryanohoro]

There's a problem with test_distribute.py that emerged before this PR. I was unable to resolve it before needing to finish this.

[phutelmyer]

Finally getting around to testing this - looks good. Thanks @ryanohoro.