[WIP] Remote repositories for versioning

datumaro/cli/__main__.py

@@ -56,22 +56,47 @@ def make_parser():
 
     parser.add_argument('--version', action='version', version=VERSION)
     _LogManager._define_loglevel_option(parser)
+    parser.add_argument('--detached', action='store_true',
+        help=argparse.SUPPRESS)
+        # help="Work in VCS-detached mode. VCS operations will be unavailable.")
 
     known_contexts = [
-        ('project', contexts.project, "Actions with project (deprecated)"),
+        ('project', contexts.project, "Actions with project"),
+        ('repo', contexts.repository, "Actions with VCS repositories"),
+        ('remote', contexts.remote, "Actions with data remotes"),
         ('source', contexts.source, "Actions with data sources"),
         ('model', contexts.model, "Actions with models"),
     ]
     known_commands = [
-        ('create', commands.create, "Create project"),
+        ("Project modification:", None, ''),
+        ('create', commands.create, "Create empty project"),
         ('import', commands.import_, "Create project from existing dataset"),
         ('add', commands.add, "Add data source to project"),
         ('remove', commands.remove, "Remove data source from project"),
+
+        ("", None, ''),
+        ("Project versioning:", None, ''),
+        ('check_updates', commands.check_updates, "Check remote repository for updates"),
+        ('fetch', commands.fetch, "Fetch updates from remote repository"),
+        ('pull', commands.pull, "Pull updates from remote repository"),
+        ('push', commands.push, "Push updates to remote repository"),
+        ('checkout', commands.checkout, "Switch to another branch or revision"),
+        ('commit', commands.commit, "Commit changes in tracked files"),
+        ('status', commands.status, "Show status information"),
+        ('refs', commands.refs, "List branches and revisions"),
+        ('tag', commands.tag, "Give name to revision"),
+        ('track', commands.track, "Start tracking a local file or directory"),
+        ('update', commands.update, "Change data source revision"),
+
+        ("", None, ''),
+        ("Dataset and project operations:", None, ''),
         ('export', commands.export, "Export project in some format"),
-        ('filter', commands.filter, "Filter project"),
-        ('transform', commands.transform, "Transform project"),
+        ('filter', commands.filter, "Filter project items"),
+        ('transform', commands.transform, "Modify project items"),
+        ('apply', commands.apply, "Apply a few transforms to project"),
+        ('build', commands.build, "Build project"),
         ('merge', commands.merge, "Merge projects"),
-        ('convert', commands.convert, "Convert dataset into another format"),
+        ('convert', commands.convert, "Convert dataset between formats"),
         ('diff', commands.diff, "Compare projects with intersection"),
         ('ediff', commands.ediff, "Compare projects for equality"),
         ('stats', commands.stats, "Compute project statistics"),
@@ -104,7 +129,8 @@ def make_parser():
     subcommands = parser.add_subparsers(title=subcommands_desc,
         description="", help=argparse.SUPPRESS)
     for command_name, command, _ in known_contexts + known_commands:
-        add_subparser(subcommands, command_name, command.build_parser)
+        if command is not None:
+            add_subparser(subcommands, command_name, command.build_parser)
 
     return parser
 
@@ -119,8 +145,15 @@ def main(args=None):
         parser.print_help()
         return 1
 
+    if args.detached:
+        from datumaro.components.project import ProjectVcs
+        ProjectVcs.G_DETACHED = True
+
     try:
-        return args.command(args)
+        retcode = args.command(args)
+        if retcode is None:
+            retcode = 0
+        return retcode
     except CliException as e:
         log.error(e)
         return 1

datumaro/cli/commands/__init__.py

@@ -7,7 +7,8 @@
 from . import (
     create, add, remove, import_,
     explain,
-    export, merge, convert, transform, filter,
+    export, merge, convert, apply, transform, filter, build, update,
     diff, ediff, stats,
+    commit, fetch, pull, push, track, checkout, refs, status, check_updates, tag,
     info, validate
 )

datumaro/cli/commands/apply.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_apply_parser as build_parser

datumaro/cli/commands/build.py

@@ -0,0 +1,7 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+# pylint: disable=unused-import
+
+from ..contexts.project import build_build_parser as build_parser

datumaro/cli/commands/check_updates.py

@@ -0,0 +1,26 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+
+from ..util.project import load_project
+
+
+def build_parser(parser_ctor=argparse.ArgumentParser):
+    parser = parser_ctor()
+
+    parser.add_argument('targets', nargs='*',
+        help="Names of sources and models")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    parser.set_defaults(command=check_updates_command)
+
+    return parser
+
+def check_updates_command(args):
+    project = load_project(args.project_dir)
+
+    project.vcs.check_updates(targets=args.targets)
+
+    return 0

datumaro/cli/commands/checkout.py

@@ -0,0 +1,39 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+
+from ..util.project import load_project
+
+
+def build_parser(parser_ctor=argparse.ArgumentParser):
+    parser = parser_ctor()
+
+    parser.add_argument('_positionals', nargs=argparse.REMAINDER,
+        help=argparse.SUPPRESS) # args can't be resolved automatically
+    parser.add_argument('rev', nargs='?',
+        help="Commit or tag (default: current)")
+    parser.add_argument('targets', nargs='*',
+        help="Names of sources, models, tracked files and dirs (default: all)")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    parser.set_defaults(command=checkout_command)
+
+    return parser
+
+def checkout_command(args):
+    try:
+        pos = args._positionals.index('--')
+        has_sep = True
+    except ValueError:
+        pos = 1
+        has_sep = False
+    args.rev = args._positionals[:pos] or []
+    args.targets = args._positionals[pos + has_sep:]
+
+    project = load_project(args.project_dir)
+
+    project.vcs.checkout(rev=args.rev, targets=args.targets)
+
+    return 0

datumaro/cli/commands/commit.py

@@ -0,0 +1,27 @@
+# Copyright (C) 2020 Intel Corporation
+#
+# SPDX-License-Identifier: MIT
+
+import argparse
+
+from ..util.project import load_project
+
+
+def build_parser(parser_ctor=argparse.ArgumentParser):
+    parser = parser_ctor()
+
+    parser.add_argument('paths', nargs='*',
+        help="Files to include in the commit (default: all tracked)")
+    parser.add_argument('-m', '--message', required=True, help="Commit message")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the project to operate on (default: current dir)")
+    parser.set_defaults(command=commit_command)
+
+    return parser
+
+def commit_command(args):
+    project = load_project(args.project_dir)
+
+    project.vcs.commit(args.paths, args.message)
+
+    return 0

datumaro/cli/commands/create.py

@@ -4,4 +4,64 @@
 
 # pylint: disable=unused-import
 
-from ..contexts.project import build_create_parser as build_parser
+import argparse
+import logging as log
+import os
+import os.path as osp
+import shutil
+
+from datumaro.components.project import \
+    PROJECT_DEFAULT_CONFIG as DEFAULT_CONFIG
+from datumaro.components.project import Project
+
+from ..util import CliException, MultilineFormatter
+
+
+def build_parser(parser_ctor=argparse.ArgumentParser):
+    parser = parser_ctor(help="Create empty project",
+        description="""
+            Create a new empty project.|n
+            |n
+            Examples:|n
+            - Create a project in the current directory:|n
+            |s|screate -n myproject|n
+            |n
+            - Create a project in other directory:|n
+            |s|screate -o path/I/like/
+        """,
+        formatter_class=MultilineFormatter)
+
+    parser.add_argument('-o', '--output-dir', default='.', dest='dst_dir',
+        help="Save directory for the new project (default: current dir")
+    parser.add_argument('-n', '--name', default=None,
+        help="Name of the new project (default: same as project dir)")
+    parser.add_argument('--overwrite', action='store_true',
+        help="Overwrite existing files in the save directory")
+    parser.set_defaults(command=create_command)
+
+    return parser
+
+def create_command(args):
+    project_dir = osp.abspath(args.dst_dir)
+
+    project_env_dir = osp.join(project_dir, DEFAULT_CONFIG.env_dir)
+    if osp.isdir(project_env_dir) and os.listdir(project_env_dir):
+        if args.overwrite:
+            shutil.rmtree(project_env_dir, ignore_errors=True)
+        else:
+            raise CliException("Directory '%s' already exists "
+                "(pass --overwrite to overwrite)" % project_env_dir)
+
+    project_name = args.name
+    if project_name is None:
+        project_name = osp.basename(project_dir)
+
+    log.info("Creating project at '%s'" % project_dir)
+
+    Project.generate(project_dir, {
+        'project_name': project_name,
+    })
+
+    log.info("Project has been created at '%s'" % project_dir)
+
+    return 0

datumaro/cli/commands/diff.py

@@ -2,6 +2,87 @@
 #
 # SPDX-License-Identifier: MIT
 
-# pylint: disable=unused-import
+import argparse
+import logging as log
+import os
+import os.path as osp
+import shutil
 
-from ..contexts.project import build_diff_parser as build_parser
+from datumaro.components.operations import DistanceComparator
+from datumaro.util import error_rollback
+
+from ..util import CliException, MultilineFormatter
+from ..util.project import generate_next_file_name, load_project
+from ..contexts.project.diff import DatasetDiffVisualizer
+
+
+def build_parser(parser_ctor=argparse.ArgumentParser):
+    parser = parser_ctor(help="Compare projects",
+        description="""
+        Compares two projects, match annotations by distance.|n
+        |n
+        Examples:|n
+        - Compare two projects, match boxes if IoU > 0.7,|n
+        |s|s|s|sprint results to Tensorboard:
+        |s|sdiff path/to/other/project -o diff/ -v tensorboard --iou-thresh 0.7
+        """,
+        formatter_class=MultilineFormatter)
+
+    parser.add_argument('other_project_dir',
+        help="Directory of the second project to be compared")
+    parser.add_argument('-o', '--output-dir', dest='dst_dir', default=None,
+        help="Directory to save comparison results (default: do not save)")
+    parser.add_argument('-v', '--visualizer',
+        default=DatasetDiffVisualizer.DEFAULT_FORMAT.name,
+        choices=[f.name for f in DatasetDiffVisualizer.OutputFormat],
+        help="Output format (default: %(default)s)")
+    parser.add_argument('--iou-thresh', default=0.5, type=float,
+        help="IoU match threshold for detections (default: %(default)s)")
+    parser.add_argument('--conf-thresh', default=0.5, type=float,
+        help="Confidence threshold for detections (default: %(default)s)")
+    parser.add_argument('--overwrite', action='store_true',
+        help="Overwrite existing files in the save directory")
+    parser.add_argument('-p', '--project', dest='project_dir', default='.',
+        help="Directory of the first project to be compared (default: current dir)")
+    parser.set_defaults(command=diff_command)
+
+    return parser
+
+@error_rollback('on_error', implicit=True)
+def diff_command(args):
+    first_project = load_project(args.project_dir)
+
+    try:
+        second_project = load_project(args.other_project_dir)
+    except FileNotFoundError:
+        if first_project.vcs.is_ref(args.other_project_dir):
+            raise NotImplementedError("It seems that you're trying to compare "
+                "different revisions of the project. "
+                "Comparisons between project revisions are not implemented yet.")
+        raise
+
+    comparator = DistanceComparator(iou_threshold=args.iou_thresh)
+
+    dst_dir = args.dst_dir
+    if dst_dir:
+        if not args.overwrite and osp.isdir(dst_dir) and os.listdir(dst_dir):
+            raise CliException("Directory '%s' already exists "
+                "(pass --overwrite to overwrite)" % dst_dir)
+    else:
+        dst_dir = generate_next_file_name('%s-%s-diff' % (
+            first_project.config.project_name,
+            second_project.config.project_name)
+        )
+    dst_dir = osp.abspath(dst_dir)
+    log.info("Saving diff to '%s'" % dst_dir)
+
+    if not osp.exists(dst_dir):
+        on_error.do(shutil.rmtree, dst_dir, ignore_errors=True)
+
+    visualizer = DatasetDiffVisualizer(save_dir=dst_dir,
+        comparator=comparator, output_format=args.visualizer)
+    visualizer.save_dataset_diff(
+        first_project.make_dataset(),
+        second_project.make_dataset())
+
+    return 0