Implement a basic __main__.py file able to process a --excavator arg.

.gitignore

@@ -6,5 +6,10 @@ __pycache__/
 # Sphinx documentation
 docs/_build/
 
+# Build files
+venv/
+
 # Temporary files
 _.*
+/output/
+/tests/_scratch/

Makefile

@@ -0,0 +1,7 @@
+default: example
+
+example:
+	./venv/bin/python3 run_example.py -d .
+
+test:
+	@./venv/bin/python3 -m unittest

autoarchaeologist/__main__.py

@@ -0,0 +1,73 @@
+import argparse
+import importlib
+import os
+import sys
+
+from autoarchaeologist import Excavation
+
+EXCAVATORS = [
+    'examples.showcase'
+]
+
+
+def action_for_args(args):
+    if args.excavator is not None:
+        return ("excavator", load_excavator_by_name(args.excavator))
+    else:
+        raise argparse.ArgumentError(None, "no valid excavation was requsted")
+
+
+def load_excavator_by_name(excavator):
+    package_name, excavation_prefix = excavator.split('.')
+    excavation_name = f"{excavation_prefix}_excavation"
+
+    exacations_package = importlib.import_module(package_name)
+    return getattr(exacations_package, excavation_name)
+
+
+def parse_arguments(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-d", "--dir", default="/tmp/_autoarchaologist")
+    parser.add_argument('--excavator', choices=EXCAVATORS)
+    parser.add_argument('filename')
+
+    return parser.parse_args(args=argv)
+
+
+def process_arguments(args):
+    if args.dir == ".":
+        args.dir = os.path.join(os.getcwd(), "output", "_autoarchaologist")
+    if args.filename is not None:
+        args.filename = os.path.abspath(args.filename)
+    else:
+        raise ValueError()
+
+    return args
+
+
+def perform_excavation(args):
+    match action_for_args(args):
+        case "excavator", AnExcavation:
+            assert issubclass(AnExcavation, Excavation)
+            ctx = AnExcavation(html_dir=args.dir)
+        case action, _:
+            raise NotImplementedError(f"action: {action}")
+
+    ff = ctx.add_file_artifact(args.filename)
+
+    ctx.start_examination()
+
+    return ctx
+
+
+if __name__ == "__main__":
+    args = process_arguments(parse_arguments())
+
+    try:
+        os.mkdir(args.dir)
+    except FileExistsError:
+        pass
+
+    ctx = perform_excavation(args)
+    ctx.produce_html()
+    print("Now point your browser at", ctx.filename_for(ctx).link)

ddhf/ddhf/decorated_context.py

@@ -110,13 +110,29 @@ def from_argv(self):
     "AUTOARCHAEOLOGIST_BITSTORE_CACHE": "ddhf_bitstore_cache",
 }
 
-def main(job, html_subdir="tmp", **kwargs):
+def parse_arguments(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', '--out', default='/tmp/_autoarchaologist')
+
+    args = parser.parse_args(args=argv)
+    if args.out == '.':
+        args.out = os.path.join(os.getcwd(), "_autoarchaologist")
+    return args
+
+def main(job, html_subdir, **kwargs):
+    args = parse_arguments()
+    kwargs["html_dir"] = args.out
+
     ''' A standard main routine to reduce boiler-plate '''
     for key in os.environ:
         i = OK_ENVS.get(key)
         if i:
             kwargs[i] = os.environ[key]
 
+    if 'html_dir' not in kwargs:
+        raise AttributeError("missing: html_dir")
+
+
     kwargs['html_dir'] = os.path.join(kwargs['html_dir'], html_subdir)
     kwargs.setdefault('download_links', True)
     kwargs.setdefault('download_limit', 1 << 20)

examples/__init__.py

@@ -0,0 +1,3 @@
+from .examples_showcase import ShowcaseExcacation
+
+showcase_excavation = ShowcaseExcacation

examples/examples_showcase.py

@@ -0,0 +1,15 @@
+from autoarchaeologist.base.excavation import Excavation
+from autoarchaeologist.generic.bigtext import BigText
+from autoarchaeologist.generic.samesame import SameSame
+from autoarchaeologist.data_general.absbin import AbsBin
+from autoarchaeologist.data_general.papertapechecksum import DGC_PaperTapeCheckSum
+
+
+class ShowcaseExcacation(Excavation):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.add_examiner(BigText)
+        self.add_examiner(AbsBin)
+        self.add_examiner(DGC_PaperTapeCheckSum)
+        self.add_examiner(SameSame)

run_example.py

@@ -1,32 +1,17 @@
-
 import os
+import sys
+from types import SimpleNamespace
 
-import autoarchaeologist
-
-from autoarchaeologist.generic.bigdigits import BigDigits
-from autoarchaeologist.generic.samesame import SameSame
-from autoarchaeologist.data_general.absbin import AbsBin
-from autoarchaeologist.data_general.papertapechecksum import DGC_PaperTapeCheckSum
-
+from autoarchaeologist.__main__ import parse_arguments, process_arguments, perform_excavation
+from examples import ShowcaseExcacation
 
 if __name__ == "__main__":
+    argv = sys.argv[1:]
+    # force the example as the filename
+    argv.append("examples/30001393.bin")
+    args = process_arguments(parse_arguments(argv=argv))
 
-    ctx = autoarchaeologist.Excavation()
-
-    ctx.add_examiner(BigDigits)
-    ctx.add_examiner(AbsBin)
-    ctx.add_examiner(DGC_PaperTapeCheckSum)
-    ctx.add_examiner(SameSame)
-
-    ff = ctx.add_file_artifact("examples/30001393.bin")
-
-    ctx.start_examination()
-
-    try:
-        os.mkdir("/tmp/_autoarchaologist")
-    except FileExistsError:
-        pass
-
-    ctx.produce_html(html_dir="/tmp/_autoarchaologist")
+    ctx = perform_excavation(args, ("excavator", ShowcaseExcacation))
+    ctx.produce_html()
 
     print("Now point your browser at", ctx.filename_for(ctx).link)

tests/test_example.py

@@ -0,0 +1,51 @@
+import os
+import shutil
+from types import SimpleNamespace
+import unittest
+
+TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+SCRATCH_DIR = os.path.join(TESTS_DIR, "_scratch")
+
+from autoarchaeologist.__main__ import perform_excavation
+
+
+class Test_Example_BasicHtml(unittest.TestCase):
+    """
+    Ensure run_example produces expected HTML files for the example input.
+    """
+
+    DIR_TREE = None
+
+    @classmethod
+    def setUpClass(cls):
+        args = SimpleNamespace(
+            dir=SCRATCH_DIR,
+            filename="examples/30001393.bin",
+            excavator="examples.showcase",
+        )
+        shutil.rmtree(args.dir, ignore_errors=True)
+        os.makedirs(args.dir, exist_ok=True)
+        ctx = perform_excavation(args)
+        ctx.produce_html()
+        cls.DIR_TREE = list(os.walk(args.dir))
+
+    def toplevel(self):
+        return self.__class__.DIR_TREE[0]
+
+    def toplevel_dirnames(self):
+        _, dirs, __ = self.toplevel()
+        dirs.sort()
+        return dirs
+
+    def toplevel_filenames(self):
+        _, __, filenames = self.toplevel()
+        return filenames
+
+    def test_produces_top_level_index(self):
+        toplevel_filenames = self.toplevel_filenames()
+        self.assertTrue("index.html" in toplevel_filenames)
+        self.assertTrue("index.css" in toplevel_filenames)
+
+    def test_produces_digest_directories(self):
+        toplevel_dirnames = self.toplevel_dirnames()
+        self.assertEqual(toplevel_dirnames, ['08', '79', 'fa'])

tests/test_main.py

@@ -0,0 +1,100 @@
+import importlib
+import os
+import shutil
+import sys
+from types import SimpleNamespace
+import unittest
+
+TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+SCRATCH_DIR = os.path.join(TESTS_DIR, "_scratch")
+ROOT_DIR = os.path.normpath(os.path.join(TESTS_DIR, ".."))
+
+sys.path.append(TESTS_DIR)
+
+from examples import ShowcaseExcacation
+from autoarchaeologist.__main__ import process_arguments, perform_excavation, \
+    action_for_args
+from autoarchaeologist.base.artifact import ArtifactBase, ArtifactStream
+
+
+class Test_Main_arguments(unittest.TestCase):
+    """
+    Ensure run_example excavates the expected artifacts for the example input.
+    """
+
+    def test_processing_excavator_argument_loads_excavation(self):
+        args = process_arguments(SimpleNamespace(
+            dir=SCRATCH_DIR,
+            filename=os.path.join(ROOT_DIR, 'examples/30001393.bin'),
+            excavator='examples.showcase',
+        ))
+
+        action_name, action_arg = action_for_args(args)
+        self.assertIs(action_name, "excavator")
+        self.assertIs(action_arg, ShowcaseExcacation)
+
+class Test_Main_processing(unittest.TestCase):
+    """
+    Ensure run_example excavates the expected artifacts for the example input.
+    """
+
+    ARGS = None
+
+    @classmethod
+    def setUpClass(cls):
+        args = process_arguments(SimpleNamespace(
+            dir=SCRATCH_DIR,
+            filename=os.path.join(ROOT_DIR, 'examples/30001393.bin'),
+            excavator='examples.showcase',
+        ))
+        shutil.rmtree(args.dir, ignore_errors=True)
+        os.makedirs(args.dir, exist_ok=True)
+        # record the unchanging bits against the test case
+        cls.ARGS = args
+
+    def assertArtifactIsChild(self, artifact, parent):
+        assert issubclass(artifact.__class__, ArtifactBase)
+        self.assertEqual(list(artifact.parents), [parent])
+
+    def test_excavated_three_total_artifacts(self):
+        excavation = perform_excavation(self.ARGS)
+
+        arfifact_hash_keys = list(excavation.hashes.keys())
+        self.assertEqual(len(arfifact_hash_keys), 3)
+
+    def test_excavated_one_top_level_artifact(self):
+        excavation = perform_excavation(self.ARGS)
+
+        excavatoin_child_count = len(excavation.children)
+        self.assertEqual(excavatoin_child_count, 1)
+
+    def test_produces_top_level_artifact(self):
+        excavation = perform_excavation(self.ARGS)
+
+        artifact = excavation.children[0]
+        self.assertIsInstance(artifact, ArtifactStream)
+        self.assertEqual(artifact.digest, '083a3d5e3098aec38ee5d9bc9f9880d3026e120ff8f058782d49ee3ccafd2a6c')
+        self.assertTrue(artifact.digest in excavation.hashes)
+
+    def test_produces_top_level_artifact_whose_parent_is_excavation(self):
+        excavation = perform_excavation(self.ARGS)
+
+        artifact = excavation.children[0]
+        self.assertArtifactIsChild(artifact, excavation)
+
+    def test_produces_two_children_of_the_top_level(self):
+        excavation = perform_excavation(self.ARGS)
+
+        artifact = excavation.children[0]
+        artifact_children = sorted(artifact.children, key=lambda a: a.digest)
+        self.assertEqual(len(artifact_children), 2)
+        self.assertTrue(artifact_children[0].digest in excavation.hashes)
+        self.assertTrue(artifact_children[0].digest.startswith('79'))
+        self.assertArtifactIsChild(artifact_children[0], artifact)
+        self.assertTrue(artifact_children[1].digest in excavation.hashes)
+        self.assertTrue(artifact_children[1].digest.startswith('fa'))
+        self.assertArtifactIsChild(artifact_children[1], artifact)
+
+
+if __name__ == '__main__':
+    unittest.main()