Merge pull request #333 from datalad/bf-update-uuid-set

include root-dataset and sub-dataset in UUID-set
datalad · Feb 24, 2023 · 9c5a61d · 9c5a61d
2 parents e5d409c + 99d9fe3
commit 9c5a61d
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 30 deletions.
diff --git a/datalad_metalad/add.py b/datalad_metalad/add.py
@@ -472,16 +472,17 @@ def add_finite_set(metadata_objects: List[JSONType],
                 tvl_us_cache=tvl_us_cache,
                 mrr_cache=mrr_cache)
 
-            error_result = check_dataset_ids(
-                metadata_store,
-                UUID(dataset_id),
-                add_parameter)
-
-            if error_result:
-                if not allow_id_mismatch:
-                    yield error_result
-                    continue
-                lgr.warning(error_result["message"])
+            if not un_versioned_path:
+                error_result = check_dataset_ids(
+                    metadata_store,
+                    UUID(dataset_id),
+                    add_parameter)
+
+                if error_result:
+                    if not allow_id_mismatch:
+                        yield error_result
+                        continue
+                    lgr.warning(error_result["message"])
 
             # If the key "path" is present in the metadata
             # dictionary, we assume that the metadata-dictionary describes
@@ -673,6 +674,8 @@ def _get_top_nodes(realm: Path,
         # path element in the version list (which confusingly is also called
         # "path".
         assert ap.dataset_path in (top_level_dataset_tree_path, None)
+
+        # We leave the creation of the respective nodes to auto_create
         return get_top_nodes_and_metadata_root_record(
             mapper_family=default_mapper_family,
             realm=str(realm),
@@ -684,10 +687,10 @@ def _get_top_nodes(realm: Path,
             sub_dataset_version=None,
             auto_create=True)
 
-    # This is an aggregated add. The inter-dataset path (aka. dataset-tree-path)
+    # This is an aggregated add. The inter-dataset path (aka: dataset-tree-path)
     # must not be "", and the un-versioned path must be "".
-    assert ap.dataset_path != MetadataPath("")
-    assert ap.unversioned_path == MetadataPath("")
+    assert ap.dataset_path != top_level_dataset_tree_path
+    assert ap.unversioned_path == top_level_dataset_tree_path
 
     # We get the dataset tree for the root version. From this we have to load
     # or create a metadata root record for the sub-dataset id and sub-dataset

diff --git a/datalad_metalad/tests/test_add.py b/datalad_metalad/tests/test_add.py
@@ -9,10 +9,8 @@
 # ## ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
 """Test metadata adding"""
 import json
-import os
 import tempfile
 import time
-from pathlib import Path
 from typing import (
     List,
     Union,
@@ -79,13 +77,6 @@
 }
 
 
-additional_keys_unknown_template = {
-    "root_dataset_id": "<unknown>",
-    "root_dataset_version": "aaaaaaa0000000000000000222222222",
-    "dataset_path": "sub_0/sub_0.0/dataset_0.0.0"
-}
-
-
 def _assert_raise_mke_with_keys(exception_keys: List[str],
                                 *args,
                                 **kwargs):
@@ -403,7 +394,9 @@ def test_override_key_allowed(file_name=None):
 def _get_top_nodes(git_repo,
                    dataset_id,
                    dataset_version,
-                   dataset_tree_path=""):
+                   dataset_tree_path="",
+                   sub_dataset_id=None,
+                   sub_dataset_version=None):
 
     # Ensure that metadata was created
     tree_version_list, uuid_set, mrr = \
@@ -414,8 +407,8 @@ def _get_top_nodes(git_repo,
             primary_data_version=dataset_version,
             prefix_path=MetadataPath(""),
             dataset_tree_path=MetadataPath(dataset_tree_path),
-            sub_dataset_id=None,
-            sub_dataset_version=None)
+            sub_dataset_id=sub_dataset_id,
+            sub_dataset_version=sub_dataset_version)
 
     assert_is_not_none(tree_version_list)
     assert_is_not_none(uuid_set)
@@ -534,6 +527,8 @@ def test_subdataset_add_dataset_end_to_end(file_name=None):
         assert_result_count(res, 0, type='file')
 
         # Verify dataset level metadata was added
+        dataset_id = UUID(metadata_template["dataset_id"])
+        dataset_version = metadata_template["dataset_version"]
         root_dataset_id = UUID(additional_keys_template["root_dataset_id"])
         root_dataset_version = additional_keys_template["root_dataset_version"]
         dataset_tree_path = MetadataPath(
@@ -543,7 +538,9 @@ def test_subdataset_add_dataset_end_to_end(file_name=None):
             git_repo,
             root_dataset_id,
             root_dataset_version,
-            additional_keys_template["dataset_path"])
+            additional_keys_template["dataset_path"],
+            dataset_id,
+            dataset_version)
 
         _, _, dataset_tree = tree_version_list.get_dataset_tree(
             root_dataset_version,
@@ -585,6 +582,8 @@ def test_subdataset_add_file_end_to_end(file_name=None):
         assert_result_count(res, 0, type='dataset')
 
         # Verify dataset level metadata was added
+        dataset_id = UUID(metadata_template["dataset_id"])
+        dataset_version = metadata_template["dataset_version"]
         root_dataset_id = UUID(additional_keys_template["root_dataset_id"])
         root_dataset_version = additional_keys_template["root_dataset_version"]
         dataset_tree_path = MetadataPath(
@@ -594,7 +593,9 @@ def test_subdataset_add_file_end_to_end(file_name=None):
             git_repo,
             root_dataset_id,
             root_dataset_version,
-            additional_keys_template["dataset_path"])
+            additional_keys_template["dataset_path"],
+            dataset_id,
+            dataset_version)
 
         _, _, dataset_tree = tree_version_list.get_dataset_tree(
             root_dataset_version,
@@ -644,7 +645,7 @@ def test_current_dir_add_end_to_end(file_name=None):
 
         expected = {
             **metadata_template,
-            **additional_keys_unknown_template,
+            **additional_keys_template,
             "type": "dataset",
             "dataset_id": str(another_id),
         }
@@ -703,7 +704,7 @@ def test_add_file_dump_end_to_end(file_name=None):
 
         expected = {
             **metadata_template,
-            **additional_keys_unknown_template,
+            **additional_keys_template,
             "type": "file",
             "path": test_path,
             "dataset_id": str(another_id)

diff --git a/setup.cfg b/setup.cfg
@@ -16,7 +16,7 @@ python_requires = >= 3.7
 install_requires =
     six
     datalad >= 0.18
-    datalad-metadata-model >=0.3.6
+    datalad-metadata-model >=0.3.10
     pytest
     pyyaml
 test_requires =