From 5371cd21c34ee80542eee3334d8151729c487bf1 Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Wed, 30 Oct 2024 12:00:49 +0100 Subject: [PATCH 01/13] Add new feature to set metadata --- ibridges/meta.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/ibridges/meta.py b/ibridges/meta.py index 2873cd32..c7db3ee0 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -115,6 +115,41 @@ def __str__(self) -> str: return "\n".join(f" - {{name: {meta.name}, value: {meta.value}, units: {meta.units}}}" for meta in meta_list) + def __getitem__(self, key: str) -> list[tuple]: + """Access the metadata like a dictionary of tuples. + + Parameters + ---------- + key + The key to get all metadata for. + + Raises + ------ + KeyError + If the key does not exist. + + """ + items = [(m.name, m.value, m.units) for m in self if m.name == key] + if len(items) == 0: + raise KeyError(f"Meta data item with name '{key}' not found.") + + def __setitem__(self, key: str, set_value: Union[str, tuple]): + """Set the value and units of a metadata key. + + Parameters + ---------- + key + The key for which to set the value and units. + set_value + Which value the metadata item is set to. + + """ + if isinstance(set_value, str) or set_value is None: + self.item.metadata.set(key, set_value, None) + else: + self.item.metadata.set(key, set_value[0], set_value[1]) + + def add(self, key: str, value: str, units: Optional[str] = None): """Add metadata to an item. From b3c30fb6e5409ee5234ad50c3a6ad8e05816ea9c Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Thu, 7 Nov 2024 14:45:17 +0100 Subject: [PATCH 02/13] Add metadataitem --- ibridges/meta.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/ibridges/meta.py b/ibridges/meta.py index c7db3ee0..5069666a 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -115,6 +115,19 @@ def __str__(self) -> str: return "\n".join(f" - {{name: {meta.name}, value: {meta.value}, units: {meta.units}}}" for meta in meta_list) + def find_all(self, key: Union[str, None, ...], value: Union[str, None, ...], + units: Union[str, None, ...]): + all_items = [] + for item in self: + if key is not ... and key != item.name: + continue + if value is not ... and value != item.value: + continue + if units is not ... and units != item.units: + continue + all_items.append(MetaDataItem(self, item)) + return all_items + def __getitem__(self, key: str) -> list[tuple]: """Access the metadata like a dictionary of tuples. @@ -129,6 +142,20 @@ def __getitem__(self, key: str) -> list[tuple]: If the key does not exist. """ + original_key = key + if isinstance(key, str): + key = (key, ..., ...) + elif len(key) == 2: + key = (*key, ...) + if len(key) > 3: + raise ValueError("Too many arguments for '[]', use key, value, units.") + all_items = self.find_all(*key) + if len(all_items) == 0: + raise KeyError(f"Cannot find metadata item with '{original_key}'") + if len(all_items) > 1: + raise ValueError(f"Found multiple items with key '{original_key}', specify value and " + "units as well, for example: meta[key, value, units].") + return all_items[0] items = [(m.name, m.value, m.units) for m in self if m.name == key] if len(items) == 0: raise KeyError(f"Meta data item with name '{key}' not found.") @@ -362,3 +389,51 @@ def from_dict(self, meta_dict: dict): self.add(*meta_tuple) except ValueError: pass + +class MetaDataItem(): + def __init__(self, ibridges_meta, prc_meta): + self._ibridges_meta = ibridges_meta + self._prc_meta = prc_meta + + @property + def key(self) -> str: + return self._prc_meta.name + + @key.setter + def key(self, new_key: str): + if new_key == self._prc_meta.name: + return + new_item_values = [new_key, self._prc_meta.value, self._prc_meta.units] + self._rename(new_item_values) + + @property + def value(self) -> str: + return self._prc_meta.value + + @value.setter + def value(self, new_value): + if new_value == self._prc_meta.name: + return + new_item_values = [self._prc_meta.name, new_value, self._prc_meta.units] + self._rename(new_item_values) + + @property + def units(self, new_units): + return self._prc_meta.units + + @units.setter + def units(self, new_units): + if new_units == self._prc_meta.units: + return + new_item_values = [self._prc_meta.name, self._prc_meta.value, new_units] + self._rename(new_item_values) + + def _rename(self, new_item_key): + try: + _new_item = self._ibridges_meta[*new_item_key] + except KeyError: + self._ibridges_meta.add(*new_item_key) + self._ibridges_meta.item.metadata.remove(self._prc_meta) + self._prc_meta = self._ibridges_meta[*new_item_key]._prc_meta + else: + raise ValueError(f"Cannot change key/value/units to '{new_item_key}' metadata item already exists.") From ca6d94c3ba8f571364b5bfdbe2ffa41b3851c595 Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Fri, 8 Nov 2024 14:33:10 +0100 Subject: [PATCH 03/13] Update metadata --- ibridges/meta.py | 259 +++++++++++++++++++++++++++++------------------ 1 file changed, 161 insertions(+), 98 deletions(-) diff --git a/ibridges/meta.py b/ibridges/meta.py index 5069666a..e8c20c7c 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -48,7 +48,7 @@ class MetaData: def __init__( self, item: Union[irods.data_object.iRODSDataObject, irods.collection.iRODSCollection], - blacklist: str = r"^org_*", + blacklist: Optional[str] = r"^org_*", ): """Initialize the metadata object.""" self.item = item @@ -56,14 +56,11 @@ def __init__( def __iter__(self) -> Iterator: """Iterate over all metadata key/value/units triplets.""" - if self.blacklist is None: - yield from self.item.metadata.items() - return for meta in self.item.metadata.items(): - if self.blacklist and re.match(self.blacklist, meta.name) is None: - yield meta + if not self.blacklist or re.match(self.blacklist, meta.name) is None: + yield MetaDataItem(self, meta) else: - warnings.warn(f"Ignoring metadata entry with value {meta.name}, because it matches " + warnings.warn(f"Ignoring metadata entry with key {meta.name}, because it matches " f"the blacklist {self.blacklist}.") def __len__(self) -> int: @@ -87,20 +84,12 @@ def __contains__(self, val: Union[str, Sequence]) -> bool: True """ - if isinstance(val, str): - val = [val] - all_attrs = ["name", "value", "units"][: len(val)] - for meta in self: - n_same = 0 - for i_attr, attr in enumerate(all_attrs): - if getattr(meta, attr) == val[i_attr] or val[i_attr] is None: - n_same += 1 - else: - break - if n_same == len(val): - return True + search_pattern = _pad_search_pattern(val) + if len(self.find_all(*search_pattern)) > 0: + return True return False + def __repr__(self) -> str: """Create a sorted representation of the metadata.""" return f"MetaData<{self.item.path}>" @@ -108,27 +97,21 @@ def __repr__(self) -> str: def __str__(self) -> str: """Return a string showing all metadata entries.""" # Sort the list of items name -> value -> units, where None is the lowest - meta_list = list(self) - meta_list = sorted(meta_list, key=lambda m: (m.units is None, m.units)) - meta_list = sorted(meta_list, key=lambda m: (m.value is None, m.value)) - meta_list = sorted(meta_list, key=lambda m: (m.name is None, m.name)) - return "\n".join(f" - {{name: {meta.name}, value: {meta.value}, units: {meta.units}}}" - for meta in meta_list) - - def find_all(self, key: Union[str, None, ...], value: Union[str, None, ...], - units: Union[str, None, ...]): + meta_list = sorted(list(self)) + return "\n".join(f" - {meta}" for meta in meta_list) + + def find_all(self, key = ..., value = ..., units = ...): + """Find all metadata entries. + + Wildcards can be used by leaving the key/value/units at default. + """ all_items = [] - for item in self: - if key is not ... and key != item.name: - continue - if value is not ... and value != item.value: - continue - if units is not ... and units != item.units: - continue - all_items.append(MetaDataItem(self, item)) + for meta_item in self: + if meta_item.matches(key, value, units): + all_items.append(meta_item) return all_items - def __getitem__(self, key: str) -> list[tuple]: + def __getitem__(self, key: Union[str, Sequence[str]]) -> MetaDataItem: """Access the metadata like a dictionary of tuples. Parameters @@ -142,40 +125,14 @@ def __getitem__(self, key: str) -> list[tuple]: If the key does not exist. """ - original_key = key - if isinstance(key, str): - key = (key, ..., ...) - elif len(key) == 2: - key = (*key, ...) - if len(key) > 3: - raise ValueError("Too many arguments for '[]', use key, value, units.") - all_items = self.find_all(*key) + search_pattern = _pad_search_pattern(key) + all_items = self.find_all(*search_pattern) if len(all_items) == 0: - raise KeyError(f"Cannot find metadata item with '{original_key}'") + raise KeyError(f"Cannot find metadata item with '{key}'") if len(all_items) > 1: - raise ValueError(f"Found multiple items with key '{original_key}', specify value and " + raise ValueError(f"Found multiple items with key '{key}', specify value and " "units as well, for example: meta[key, value, units].") return all_items[0] - items = [(m.name, m.value, m.units) for m in self if m.name == key] - if len(items) == 0: - raise KeyError(f"Meta data item with name '{key}' not found.") - - def __setitem__(self, key: str, set_value: Union[str, tuple]): - """Set the value and units of a metadata key. - - Parameters - ---------- - key - The key for which to set the value and units. - set_value - Which value the metadata item is set to. - - """ - if isinstance(set_value, str) or set_value is None: - self.item.metadata.set(key, set_value, None) - else: - self.item.metadata.set(key, set_value[0], set_value[1]) - def add(self, key: str, value: str, units: Optional[str] = None): """Add metadata to an item. @@ -281,24 +238,12 @@ def delete(self, key: str, value: Union[None, str] = ..., # type: ignore >>> meta.delete("mass") """ - try: - if value is ... or units is ...: - all_metas = self.item.metadata.get_all(key) - for meta in all_metas: - if value is ... or value == meta.value and units is ... or units == meta.units: - self.item.metadata.remove(meta) - else: - self.item.metadata.remove(key, value, units) - except irods.exception.CAT_SUCCESS_BUT_WITH_NO_INFO as error: - raise KeyError( - f"Cannot delete metadata with key '{key}', value '{value}'" - f" and units '{units}' since it does not exist." - ) from error - except irods.exception.CAT_NO_ACCESS_PERMISSION as error: - raise ValueError( - f"Cannot delete metadata due to insufficient permission " - f"for path '{self.item.path}'." - ) from error + all_meta_items = self.find_all(key, value, units) + if len(all_meta_items) == 0: + raise KeyError(f"Cannot delete items with key={key}, value={value} and units={units}, " + "since no metadata entries exist with those values.") + for meta_item in all_meta_items: + meta_item.remove() def clear(self): """Delete all metadata entries belonging to the item. @@ -356,9 +301,9 @@ def to_dict(self, keys: Optional[list] = None) -> dict: if isinstance(self.item, irods.data_object.iRODSDataObject): meta_dict["checksum"] = self.item.checksum if keys is None: - meta_dict["metadata"] = [(m.name, m.value, m.units) for m in self] + meta_dict["metadata"] = [tuple(m) for m in self] else: - meta_dict["metadata"] = [(m.name, m.value, m.units) for m in self if m.name in keys] + meta_dict["metadata"] = [tuple(m) for m in self if m.key in keys] return meta_dict def from_dict(self, meta_dict: dict): @@ -391,12 +336,32 @@ def from_dict(self, meta_dict: dict): pass class MetaDataItem(): - def __init__(self, ibridges_meta, prc_meta): + """Interface for metadata entries. + + This is a substitute of the python-irodsclient iRODSMeta object. + It implements setting the key/value/units, allows for sorting and can + remove itself. + + This class is generally created by the MetaData class, not directly + created by the user. + + Parameters + ---------- + ibridges_meta: + A MetaData object that the MetaDataItem is part of. + prc_meta: + A PRC iRODSMeta object that points to the entry. + + """ + + def __init__(self, ibridges_meta: MetaData, prc_meta: irods.iRODSMeta): + """Initialize the MetaDataItem object.""" self._ibridges_meta = ibridges_meta - self._prc_meta = prc_meta + self._prc_meta: irods.iRODSMeta = prc_meta @property def key(self) -> str: + """Return the key of the metadata item.""" return self._prc_meta.name @key.setter @@ -407,33 +372,131 @@ def key(self, new_key: str): self._rename(new_item_values) @property - def value(self) -> str: + def value(self) -> Optional[str]: + """Return the value of the metadata item.""" return self._prc_meta.value @value.setter - def value(self, new_value): + def value(self, new_value: Optional[str]): if new_value == self._prc_meta.name: return new_item_values = [self._prc_meta.name, new_value, self._prc_meta.units] self._rename(new_item_values) @property - def units(self, new_units): + def units(self) -> Optional[str]: + """Return the units of the metadata item.""" return self._prc_meta.units @units.setter - def units(self, new_units): + def units(self, new_units: Optional[str]): if new_units == self._prc_meta.units: return new_item_values = [self._prc_meta.name, self._prc_meta.value, new_units] self._rename(new_item_values) - def _rename(self, new_item_key): + def __repr__(self) -> str: + """Representation of the MetaDataItem.""" + return f"" + + def __str__(self) -> str: + """User readable representation of MetaDataItem.""" + return f"(key: {self.key}, value: {self.value}, units: {self.units})" + + def __iter__(self) -> Iterator[Optional[str]]: + """Allow iteration over key, value, units.""" + yield self.key + yield self.value + yield self.units + + def _rename(self, new_item_key: Sequence[str]): try: - _new_item = self._ibridges_meta[*new_item_key] + _new_item = self._ibridges_meta[new_item_key] except KeyError: self._ibridges_meta.add(*new_item_key) - self._ibridges_meta.item.metadata.remove(self._prc_meta) - self._prc_meta = self._ibridges_meta[*new_item_key]._prc_meta + try: + self._ibridges_meta.item.metadata.remove(self._prc_meta) + # If we get an error, roll back the added metadata + except irods.exception.CAT_NO_ACCESS_PERMISSION as error: + self._ibridges_meta.delete(*new_item_key) + raise ValueError( + f"Cannot rename metadata due to insufficient permission " + f"for path '{self.item.path}'." + ) from error + self._prc_meta = self._ibridges_meta[new_item_key]._prc_meta else: - raise ValueError(f"Cannot change key/value/units to '{new_item_key}' metadata item already exists.") + raise ValueError(f"Cannot change key/value/units to '{new_item_key}' metadata item " + "already exists.") + + def __getattribute__(self, attr: str): + """Add name attribute and check if the metadata item is already removed.""" + if attr == "name": + return self.__getattribute__("key") + if attr == "_prc_meta" and super().__getattribute__(attr) is None: + raise KeyError("Cannot remove metadata item: it has already been removed.") + return super().__getattribute__(attr) + + def remove(self): + """Remove the metadata item.""" + try: + self._ibridges_meta.item.metadata.remove(self._prc_meta) + except irods.exception.CAT_SUCCESS_BUT_WITH_NO_INFO as error: + raise KeyError( + f"Cannot delete metadata with key '{self.key}', value '{self.value}'" + f" and units '{self.units}' since it does not exist." + ) from error + except irods.exception.CAT_NO_ACCESS_PERMISSION as error: + raise ValueError( + f"Cannot delete metadata due to insufficient permission " + f"for path '{self.item.path}'." + ) from error + self._prc_meta = None + + def __lt__(self, other: MetaDataItem) -> bool: + """Compare two metadata items for sorting mainly.""" + if not isinstance(other, MetaDataItem): + raise TypeError(f"Comparison between MetaDataItem and {type(other)} " + "not supported.") + comp_key = _comp_str_none(self.key, other.key) + if comp_key is not None: + return comp_key + comp_value = _comp_str_none(self.value, other.value) + if comp_value is not None: + return comp_value + comp_units = _comp_str_none(self.units, other.units) + if comp_units is not True: + return False + return True + + def matches(self, key, value, units): + """See whether the metadata item matches the key,value,units pattern.""" + if key is not ... and key != self.key: + print(key, self.key, key is self.key) + return False + if value is not ... and value != self.value: + return False + if units is not ... and units != self.units: + return False + return True + +def _comp_str_none(obj: Optional[str], other: Optional[str]) -> Optional[bool]: + if obj is None and other is not None: + return True + if obj is not None and other is None: + return False + if str(obj) == str(other): + return None + return str(obj) < str(other) + +def _pad_search_pattern(search_pattern) -> tuple: + if isinstance(search_pattern, str): + padded_pattern = (search_pattern, ..., ...) + elif len(search_pattern) == 1: + padded_pattern = (*search_pattern, ..., ...) + elif len(search_pattern) == 2: + padded_pattern = (*search_pattern, ...) + elif len(search_pattern) > 3: + raise ValueError("Too many arguments for '[]', use key, value, units.") + else: + padded_pattern = tuple(search_pattern) + return padded_pattern From 8a3c4f165bfa09c6a418ded8ca6f1fec4c2c2e66 Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Fri, 8 Nov 2024 14:35:47 +0100 Subject: [PATCH 04/13] Fix pylint --- ibridges/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibridges/meta.py b/ibridges/meta.py index e8c20c7c..e11dada8 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -423,7 +423,7 @@ def _rename(self, new_item_key: Sequence[str]): f"Cannot rename metadata due to insufficient permission " f"for path '{self.item.path}'." ) from error - self._prc_meta = self._ibridges_meta[new_item_key]._prc_meta + self._prc_meta = self._ibridges_meta[new_item_key]._prc_meta # pylint: disable=protrected-access else: raise ValueError(f"Cannot change key/value/units to '{new_item_key}' metadata item " "already exists.") From 63d3069da28e3471e5a7d0a2caefb584633df526 Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Fri, 8 Nov 2024 14:36:03 +0100 Subject: [PATCH 05/13] Fix spelling mistake --- ibridges/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibridges/meta.py b/ibridges/meta.py index e11dada8..1a06f9a2 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -423,7 +423,7 @@ def _rename(self, new_item_key: Sequence[str]): f"Cannot rename metadata due to insufficient permission " f"for path '{self.item.path}'." ) from error - self._prc_meta = self._ibridges_meta[new_item_key]._prc_meta # pylint: disable=protrected-access + self._prc_meta = self._ibridges_meta[new_item_key]._prc_meta # pylint: disable=protected-access else: raise ValueError(f"Cannot change key/value/units to '{new_item_key}' metadata item " "already exists.") From 70d51536e188cb8d467e9b882a0b48203a1466ca Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Fri, 8 Nov 2024 15:58:04 +0100 Subject: [PATCH 06/13] Finish tests --- docker/irods_client/tests/test_meta.py | 95 +++++++++++++++++++++++++- ibridges/meta.py | 9 ++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/docker/irods_client/tests/test_meta.py b/docker/irods_client/tests/test_meta.py index 0fdfc4ac..50aaa278 100644 --- a/docker/irods_client/tests/test_meta.py +++ b/docker/irods_client/tests/test_meta.py @@ -5,7 +5,7 @@ from pytest import mark from ibridges.data_operations import Operations -from ibridges.meta import MetaData +from ibridges.meta import MetaData, MetaDataItem from ibridges.path import IrodsPath @@ -66,6 +66,7 @@ def test_meta(item_name, request): assert "x" in meta assert ("y", "z") not in meta assert ("y", "x") in meta + meta.clear() @mark.parametrize("item_name", ["collection", "dataobject"]) def test_metadata_todict(item_name, request): @@ -108,3 +109,95 @@ def test_metadata_export(item_name, request, session, tmpdir): with open(tmp_file, "r", encoding="utf-8"): new_meta_dict = json.load(tmp_file) assert isinstance(new_meta_dict, dict) + +@mark.parametrize("item_name", ["collection", "dataobject"]) +def test_metadata_getitem(item_name, request): + item = request.getfixturevalue(item_name) + meta = MetaData(item) + meta.clear() + + assert len(meta) == 0 + meta.add("some_key", "some_value", "some_units") + assert isinstance(meta["some_key"], MetaDataItem) + meta.add("some_key", "some_value", None) + meta.add("some_key", "other_value", "some_units") + meta.add("other_key", "third_value", "other_units") + with pytest.raises(ValueError): + meta["some_key"] + with pytest.raises(ValueError): + meta["some_key", "some_value"] + assert isinstance(meta["some_key", "some_value", "some_units"], MetaDataItem) + assert tuple(meta["other_key"]) == ("other_key", "third_value", "other_units") + with pytest.raises(KeyError): + meta["unknown"] + with pytest.raises(KeyError): + meta["some_key", "unknown"] + with pytest.raises(KeyError): + meta["some_key", "some_value", "unknown"] + meta.clear() + + +@mark.parametrize("item_name", ["collection", "dataobject"]) +def test_metadata_rename(item_name, request, session): + item = request.getfixturevalue(item_name) + meta = MetaData(item) + meta.clear() + + + meta.add("some_key", "some_value", "some_units") + meta["some_key"].key = "new_key" + assert ("new_key", "some_value", "some_units") in meta + assert len(meta) == 1 + + meta["new_key"].value = "new_value" + assert ("new_key", "new_value", "some_units") in meta + assert len(meta) == 1 + + meta["new_key"].units = "new_units" + assert ("new_key", "new_value", "new_units") in meta + assert len(meta) == 1 + + meta.add("new_key", "new_value", "other_units") + with pytest.raises(ValueError): + meta["new_key", "new_value", "other_units"].units = "new_units" + assert len(meta) == 2 + meta["new_key", "new_value", "other_units"].remove() + + meta.add("new_key", "other_value", "new_units") + with pytest.raises(ValueError): + meta["new_key", "other_value", "new_units"].value = "new_value" + assert len(meta) == 2 + meta["new_key", "other_value", "new_units"].remove() + + meta.add("other_key", "new_value", "new_units") + with pytest.raises(ValueError): + meta["other_key", "new_value", "new_units"].key = "new_key" + assert len(meta) == 2 + + with pytest.raises(ValueError): + meta["other_key"].key = "org_something" + assert len(meta) == 2 + assert "other_key" in meta + + meta.clear() + + +@mark.parametrize("item_name", ["collection", "dataobject"]) +def test_metadata_findall(item_name, request, session): + item = request.getfixturevalue(item_name) + meta = MetaData(item) + meta.clear() + + + meta.add("some_key", "some_value", "some_units") + meta.add("some_key", "some_value", None) + meta.add("some_key", "other_value", "some_units") + meta.add("other_key", "third_value", "other_units") + + assert len(meta.find_all()) == 4 + assert len(meta.find_all(key="some_key")) == 3 + assert isinstance(meta.find_all(key="some_key")[0], MetaDataItem) + assert len(meta.find_all(key="?")) == 0 + assert len(meta.find_all(value="some_value")) == 2 + assert len(meta.find_all(units="some_units")) == 2 + diff --git a/ibridges/meta.py b/ibridges/meta.py index 1a06f9a2..c702a097 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -124,6 +124,14 @@ def __getitem__(self, key: Union[str, Sequence[str]]) -> MetaDataItem: KeyError If the key does not exist. + + Examples + -------- + >>> meta["some_key"] + ("some_key", "some_value", "some_units") + >>> meta["some_key", "some_value"] + >>> meta["some_key", "some_value", "some_units"] + """ search_pattern = _pad_search_pattern(key) all_items = self.find_all(*search_pattern) @@ -471,7 +479,6 @@ def __lt__(self, other: MetaDataItem) -> bool: def matches(self, key, value, units): """See whether the metadata item matches the key,value,units pattern.""" if key is not ... and key != self.key: - print(key, self.key, key is self.key) return False if value is not ... and value != self.value: return False From 0588b16b25e3aaf22c32a64370cc1ef3054aac11 Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Mon, 11 Nov 2024 14:32:58 +0100 Subject: [PATCH 07/13] Fix issue with setting values --- ibridges/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibridges/meta.py b/ibridges/meta.py index c702a097..3adf0d37 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -386,7 +386,7 @@ def value(self) -> Optional[str]: @value.setter def value(self, new_value: Optional[str]): - if new_value == self._prc_meta.name: + if new_value == self._prc_meta.value: return new_item_values = [self._prc_meta.name, new_value, self._prc_meta.units] self._rename(new_item_values) From b3cb28ce4b676b38a81c0051c3dfb17648f70d6a Mon Sep 17 00:00:00 2001 From: chstaiger Date: Mon, 11 Nov 2024 14:53:00 +0100 Subject: [PATCH 08/13] fix regex for py3.12 --- ibridges/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibridges/meta.py b/ibridges/meta.py index 3adf0d37..380d7079 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -48,7 +48,7 @@ class MetaData: def __init__( self, item: Union[irods.data_object.iRODSDataObject, irods.collection.iRODSCollection], - blacklist: Optional[str] = r"^org_*", + blacklist: Optional[str] = r"^org_[\s\S]+", ): """Initialize the metadata object.""" self.item = item From 02f3b077f5305fcc7626181226503de4c906b145 Mon Sep 17 00:00:00 2001 From: qubixes <44498096+qubixes@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:59:09 +0100 Subject: [PATCH 09/13] Update ibridges/meta.py Co-authored-by: chStaiger --- ibridges/meta.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ibridges/meta.py b/ibridges/meta.py index 380d7079..55ebe8b2 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -101,7 +101,7 @@ def __str__(self) -> str: return "\n".join(f" - {meta}" for meta in meta_list) def find_all(self, key = ..., value = ..., units = ...): - """Find all metadata entries. + """Find all metadata entries belonging to the data object/collection. Wildcards can be used by leaving the key/value/units at default. """ From 1007bf550070173ddad10c9049fb33f828dbca84 Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Thu, 14 Nov 2024 15:48:46 +0100 Subject: [PATCH 10/13] Add more information about metadata --- docs/source/metadata.rst | 66 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/docs/source/metadata.rst b/docs/source/metadata.rst index 14a2b3ce..6186c72d 100644 --- a/docs/source/metadata.rst +++ b/docs/source/metadata.rst @@ -6,8 +6,8 @@ Metadata iRODS offers metadata as key, value, units triplets. The type of the keys, values and units is always a string. Below we show how to create a :doc:`Metadata ` object from a data object or collection. -The Metadata object --------------------- +The MetaData class +------------------ .. code-block:: python @@ -17,8 +17,23 @@ The Metadata object session = interactive_auth() meta = IrodsPath(session, "~", "collection_or_dataobject").meta + # Show all metadata entries with print. + print(meta) + With the object :code:`meta` we can now access and manipulate the metadata of the data object. +The MetaDataItem class +---------------------- + +As explained above, the metadata of a collection or dataobject can have multiple entries. You can iterate over +these entries as follows: + +.. code-block:: python + + for item in meta: + print(item.key, item.value, item.units) + + Add metadata ------------ To add metadata, you always need to provide a key and a value, the units are optional and can be left out. @@ -26,8 +41,6 @@ To add metadata, you always need to provide a key and a value, the units are opt .. code-block:: python meta.add('NewKey', 'NewValue', 'NewUnit') - print(meta) - .. note:: You can have several metadata entries with the same key but different values and units, @@ -46,6 +59,51 @@ same key first. This mirrors the implementation of the `iCommands Date: Thu, 14 Nov 2024 16:37:08 +0100 Subject: [PATCH 11/13] Improve error messaging --- docker/irods_client/tests/test_meta.py | 43 ++++++++++++++++++++++++++ ibridges/meta.py | 13 ++++++++ 2 files changed, 56 insertions(+) diff --git a/docker/irods_client/tests/test_meta.py b/docker/irods_client/tests/test_meta.py index 50aaa278..ec0994b9 100644 --- a/docker/irods_client/tests/test_meta.py +++ b/docker/irods_client/tests/test_meta.py @@ -201,3 +201,46 @@ def test_metadata_findall(item_name, request, session): assert len(meta.find_all(value="some_value")) == 2 assert len(meta.find_all(units="some_units")) == 2 +@mark.parametrize("item_name", ["collection", "dataobject"]) +def test_metadata_findall(item_name, request, session): + item = request.getfixturevalue(item_name) + meta = MetaData(item) + meta.clear() + + + meta.add("some_key", "some_value", "some_units") + meta.add("some_key", "some_value", None) + meta.add("some_key", "other_value", "some_units") + meta.add("other_key", "third_value", "other_units") + + assert len(meta.find_all()) == 4 + assert len(meta.find_all(key="some_key")) == 3 + assert isinstance(meta.find_all(key="some_key")[0], MetaDataItem) + assert len(meta.find_all(key="?")) == 0 + assert len(meta.find_all(value="some_value")) == 2 + assert len(meta.find_all(units="some_units")) == 2 + +@mark.parametrize("item_name", ["collection", "dataobject"]) +def test_metadata_findall(item_name, request, session): + item = request.getfixturevalue(item_name) + meta = MetaData(item) + meta.clear() + + with pytest.raises(ValueError): + meta.add("", "some_value") + with pytest.raises(TypeError): + meta.add(None, "some_value") + with pytest.raises(TypeError): + meta.add(10, "some_value") + + with pytest.raises(ValueError): + meta.add("key", "") + with pytest.raises(TypeError): + meta.add("key", None) + with pytest.raises(TypeError): + meta.add("key", 10) + + with pytest.raises(TypeError): + meta.add("key", "value", 10) + with pytest.raises(TypeError): + meta.add("key", "value", None) diff --git a/ibridges/meta.py b/ibridges/meta.py index 55ebe8b2..76a1fac3 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -182,6 +182,18 @@ def add(self, key: str, value: str, units: Optional[str] = None): self.item.metadata.add(key, value, units) except irods.exception.CAT_NO_ACCESS_PERMISSION as error: raise PermissionError("UPDATE META: no permissions") from error + except irods.message.Bad_AVU_Field as error: + if key == "": + raise ValueError("Key cannot be of size zero.") from error + if value == "": + raise ValueError("Value cannot be of size zero.") from error + if not isinstance(value, (str, bytes)): + raise TypeError(f"Value should have type str or bytes-like, " + f"not {type(value)}.") from error + if not isinstance(units, (str, bytes)): + raise TypeError(f"Value should have type str or bytes-like, " + f"not {type(value)}.") from error + raise error def set(self, key: str, value: str, units: Optional[str] = None): """Set the metadata entry. @@ -478,6 +490,7 @@ def __lt__(self, other: MetaDataItem) -> bool: def matches(self, key, value, units): """See whether the metadata item matches the key,value,units pattern.""" + units = None if units == "" else units if key is not ... and key != self.key: return False if value is not ... and value != self.value: From 52ba276eb078232318f51062d0c21e52c227a0bd Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Thu, 14 Nov 2024 16:38:14 +0100 Subject: [PATCH 12/13] Fix test name --- docker/irods_client/tests/test_meta.py | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/docker/irods_client/tests/test_meta.py b/docker/irods_client/tests/test_meta.py index ec0994b9..ef34a24c 100644 --- a/docker/irods_client/tests/test_meta.py +++ b/docker/irods_client/tests/test_meta.py @@ -201,27 +201,9 @@ def test_metadata_findall(item_name, request, session): assert len(meta.find_all(value="some_value")) == 2 assert len(meta.find_all(units="some_units")) == 2 -@mark.parametrize("item_name", ["collection", "dataobject"]) -def test_metadata_findall(item_name, request, session): - item = request.getfixturevalue(item_name) - meta = MetaData(item) - meta.clear() - - - meta.add("some_key", "some_value", "some_units") - meta.add("some_key", "some_value", None) - meta.add("some_key", "other_value", "some_units") - meta.add("other_key", "third_value", "other_units") - - assert len(meta.find_all()) == 4 - assert len(meta.find_all(key="some_key")) == 3 - assert isinstance(meta.find_all(key="some_key")[0], MetaDataItem) - assert len(meta.find_all(key="?")) == 0 - assert len(meta.find_all(value="some_value")) == 2 - assert len(meta.find_all(units="some_units")) == 2 @mark.parametrize("item_name", ["collection", "dataobject"]) -def test_metadata_findall(item_name, request, session): +def test_metadata_errors(item_name, request, session): item = request.getfixturevalue(item_name) meta = MetaData(item) meta.clear() From ed9465f6dffb2c8a579213cb9432357cc60d0aee Mon Sep 17 00:00:00 2001 From: Raoul Schram Date: Thu, 14 Nov 2024 16:43:56 +0100 Subject: [PATCH 13/13] Fix test and error message --- docker/irods_client/tests/test_meta.py | 3 +-- ibridges/meta.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/irods_client/tests/test_meta.py b/docker/irods_client/tests/test_meta.py index ef34a24c..baaf439b 100644 --- a/docker/irods_client/tests/test_meta.py +++ b/docker/irods_client/tests/test_meta.py @@ -224,5 +224,4 @@ def test_metadata_errors(item_name, request, session): with pytest.raises(TypeError): meta.add("key", "value", 10) - with pytest.raises(TypeError): - meta.add("key", "value", None) + diff --git a/ibridges/meta.py b/ibridges/meta.py index 76a1fac3..349ed5f6 100644 --- a/ibridges/meta.py +++ b/ibridges/meta.py @@ -191,7 +191,7 @@ def add(self, key: str, value: str, units: Optional[str] = None): raise TypeError(f"Value should have type str or bytes-like, " f"not {type(value)}.") from error if not isinstance(units, (str, bytes)): - raise TypeError(f"Value should have type str or bytes-like, " + raise TypeError(f"Units should have type str or bytes-like, " f"not {type(value)}.") from error raise error