Add GetarStorage and storage tests

flowws/Workflow.py

@@ -1,7 +1,8 @@
 import argparse
 import pkg_resources
 
-from . import DirectoryStorage
+from .DirectoryStorage import DirectoryStorage
+from .GetarStorage import GetarStorage
 
 class Workflow:
     def __init__(self, stages, storage):
@@ -21,7 +22,7 @@ def run_from_command(cls, args=None, module_names='flowws_modules', scope={}):
 
         parser = argparse.ArgumentParser(
             description='Run a workflow')
-        parser.add_argument('--directory', help='Storage directory to use')
+        parser.add_argument('--storage', help='Storage location to use')
         parser.add_argument('-d', '--define', nargs=2, action='append', default=[],
             help='Define a workflow-specific value')
         parser.add_argument('workflow', nargs=argparse.REMAINDER,
@@ -38,8 +39,14 @@ def run_from_command(cls, args=None, module_names='flowws_modules', scope={}):
 
             scope[name] = val
 
-        if args.directory:
-            storage = DirectoryStorage(args.directory)
+        if args.storage:
+            location = args.storage
+
+            if any(location.endswith(suffix)
+                   for suffix in ('.zip', '.tar', '.sqlite')):
+                storage = GetarStorage(location)
+            else:
+                storage = DirectoryStorage(location)
         else:
             storage = DirectoryStorage()
 

flowws/__init__.py

@@ -1,6 +1,8 @@
 from .version import __version__
 
 from .Argument import Argument
-from .DirectoryStorage import DirectoryStorage
 from .Stage import Stage
 from .Workflow import Workflow
+
+from .DirectoryStorage import DirectoryStorage
+from .GetarStorage import GetarStorage

test/__init__.py

@@ -0,0 +1 @@
+# let python use relative imports within this directory

test/internal.py

@@ -0,0 +1,29 @@
+class StorageTestBase:
+    def test_basic_creation(self):
+        with self.storage.open('test_file', 'w') as f:
+            f.write('1')
+
+        with self.storage.open('test_file', 'a') as f:
+            f.write('2')
+
+        with self.storage.open('test_file', 'r') as f:
+            self.assertEqual(f.read(), '12')
+
+    def test_binary(self):
+        with self.storage.open('test_file', 'wb') as f:
+            f.write(b'1')
+
+        with self.storage.open('test_file', 'ab') as f:
+            f.write(b'2')
+
+        with self.storage.open('test_file', 'rb') as f:
+            self.assertEqual(f.read(), b'12')
+
+    def test_file_not_found(self):
+        with self.assertRaises(FileNotFoundError):
+            with self.storage.open('new_file', 'r') as f:
+                pass
+
+        # appending to a file that doesn't exist should just create it
+        with self.storage.open('new_file', 'a') as f:
+            pass

test/test_directory_storage.py

@@ -0,0 +1,17 @@
+
+import tempfile
+import os
+import unittest
+
+import flowws
+
+from .internal import StorageTestBase
+
+class TestDirectoryStorage(unittest.TestCase, StorageTestBase):
+    def setUp(self):
+        self.tempdir = tempfile.TemporaryDirectory()
+        dirname = os.path.join(self.tempdir.name, 'test')
+        self.storage = flowws.DirectoryStorage(dirname)
+
+    def tearDown(self):
+        self.tempdir.cleanup()

test/test_getar_storage.py

@@ -0,0 +1,35 @@
+
+import tempfile
+import os
+import unittest
+
+import flowws
+
+from .internal import StorageTestBase
+
+class TestTarStorage(unittest.TestCase, StorageTestBase):
+    def setUp(self):
+        self.tempdir = tempfile.TemporaryDirectory()
+        filename = os.path.join(self.tempdir.name, 'test.tar')
+        self.storage = flowws.GetarStorage(filename)
+
+    def tearDown(self):
+        self.tempdir.cleanup()
+
+class TestZipStorage(unittest.TestCase, StorageTestBase):
+    def setUp(self):
+        self.tempdir = tempfile.TemporaryDirectory()
+        filename = os.path.join(self.tempdir.name, 'test.zip')
+        self.storage = flowws.GetarStorage(filename)
+
+    def tearDown(self):
+        self.tempdir.cleanup()
+
+class TestSqliteStorage(unittest.TestCase, StorageTestBase):
+    def setUp(self):
+        self.tempdir = tempfile.TemporaryDirectory()
+        filename = os.path.join(self.tempdir.name, 'test.sqlite')
+        self.storage = flowws.GetarStorage(filename)
+
+    def tearDown(self):
+        self.tempdir.cleanup()