scanners: Add data flow based scanning

For multi lanaguge support. Current supports Python
via inclusion of shoudli flows

Signed-off-by: John Andersen <[email protected]>

cve_bin_tool/scanners/dataflow.py

@@ -0,0 +1,77 @@
+"""
+See doc/DATA_FLOW_SCANNER.rst for more information
+"""
+import sys
+import asyncio
+import pathlib
+import platform
+from typing import Dict, NewType
+
+import dffml
+
+import dffml_feature_git.feature.definitions
+import dffml_feature_git.feature.operations
+
+
+DirectoryToScan = NewType("DirectoryToScan", pathlib.Path)
+ScanResults = NewType("ScanResults", dict)
+InputOfUnknownType = NewType("InputOfUnknownType", str)
+
+
+@dffml.op(
+    inputs={
+        "repo": dffml_feature_git.feature.definitions.git_repository,
+    },
+    outputs={
+        "result": DirectoryToScan,
+    },
+)
+async def repo_to_directory(repo):
+    return {"result": repo.directory}
+
+
+@dffml.op
+async def scan_directory(
+    directory: DirectoryToScan,
+) -> ScanResults:
+    pass
+
+
+@dffml.op
+async def scan_directory(
+    arg: InputOfUnknownType,
+) -> ScanResults:
+    pass
+
+
+COLLECTOR_DATAFLOW = dffml.DataFlow(
+    *dffml.opimp_in(dffml_feature_git.feature.operations),
+    *dffml.opimp_in(sys.modules[__name__]),
+)
+
+# CVEBinToolDataFlow = dffml.SystemContext(
+#     upstream=COLLECTOR_DATAFLOW,
+# )
+# scanner = CVEBinToolDataFlow.deployment()
+
+
+async def main():
+    # async for results in scanner():
+    async for _ctx, results in dffml.run(
+        COLLECTOR_DATAFLOW,
+        {
+            arg: [
+                dffml.Input(
+                    value=arg,
+                    definition=dffml_feature_git.feature.definitions.URL,
+                    # definition=InputOfUnknownType,
+                ),
+            ]
+            for arg in sys.argv[1:]
+        },
+    ):
+        print(_ctx, results)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

doc/DATA_FLOW_SCANNER.rst

@@ -0,0 +1,45 @@
+Data Flow Based Scanner
+#######################
+
+Implement multi language support in CVE Bin Tool via introduction of data flows
+to handle scanning. We'll then extend via overlays to add functionality such as
+shouldi were appropriate.
+
+References:
+
+- https://intel.github.io/dffml/shouldi.html
+- https://intel.github.io/dffml/examples/shouldi.html
+- https://intel.github.io/dffml/examples/dataflows.html
+
+.. note::
+
+    Tested against development version of DFFML
+    9ddcdfd6f8de743f87d41b74d53fde2c182861c7
+
+
+Install
+*******
+
+Install with extra
+
+.. code-block:: console
+
+     $ python -m pip install cve-bin-tool[dataflow]
+
+Examples
+********
+
+Scan files as usual
+
+.. code-block:: console
+    :test:
+
+     $ python -m cve_bin_tool.scanners.dataflow .
+
+Scan a git repo. Currently runs ``shoudli`` scanning. (In future we can add
+overlays to run the build then scan).
+
+.. code-block:: console
+    :test:
+
+     $ python -m cve_bin_tool.scanners.dataflow https://github.com/intel/cve-bin-tool

setup.py

@@ -47,6 +47,7 @@
     install_requires=requirements,
     extras_require={
         "PDF": ["reportlab"],
+        "dataflow": ["dffml", "dffml-feature-git"],
     },
     packages=find_packages(
         exclude=["locales", "presentation"],