Merge pull request #291 from iBridges-for-iRODS/develop

v1.3.0

docker/irods_client/tests/test_data_ops.py

@@ -135,7 +135,7 @@ def test_meta_archive(session, testdata, tmpdir):
     sync(session, testdata, ipath)
     assert len(list(ipath.meta)) == 0
     meta_list = [
-        (ipath, ("root", "true", None)),
+        (ipath, ("root", "true", "")),
         (ipath / "more_data", ("more_data", "false", "kg")),
         (ipath / "more_data" / "polarbear.txt", ("is_polar", "true", "bool")),
     ]

docker/irods_client/tests/test_meta.py

@@ -5,7 +5,7 @@
 from pytest import mark
 
 from ibridges.data_operations import Operations
-from ibridges.meta import MetaData
+from ibridges.meta import MetaData, MetaDataItem
 from ibridges.path import IrodsPath
 
 
@@ -23,7 +23,7 @@ def test_meta(item_name, request):
     assert len(meta) == 1
     assert list(meta)[0].name == "x"
     assert list(meta)[0].value == "y"
-    assert list(meta)[0].units is None
+    assert list(meta)[0].units == ""
     assert "x" in meta
     assert ("x", "y") in meta
     assert "y" not in meta
@@ -66,6 +66,7 @@ def test_meta(item_name, request):
     assert "x" in meta
     assert ("y", "z") not in meta
     assert ("y", "x") in meta
+    meta.clear()
 
 @mark.parametrize("item_name", ["collection", "dataobject"])
 def test_metadata_todict(item_name, request):
@@ -108,3 +109,136 @@ def test_metadata_export(item_name, request, session, tmpdir):
     with open(tmp_file, "r", encoding="utf-8"):
         new_meta_dict = json.load(tmp_file)
     assert isinstance(new_meta_dict, dict)
+
+@mark.parametrize("item_name", ["collection", "dataobject"])
+def test_metadata_getitem(item_name, request):
+    item = request.getfixturevalue(item_name)
+    meta = MetaData(item)
+    meta.clear()
+
+    assert len(meta) == 0
+    meta.add("some_key", "some_value", "some_units")
+    assert isinstance(meta["some_key"], MetaDataItem)
+    meta.add("some_key", "some_value", None)
+    meta.add("some_key", "other_value", "some_units")
+    meta.add("other_key", "third_value", "other_units")
+    with pytest.raises(ValueError):
+        meta["some_key"]
+    with pytest.raises(ValueError):
+        meta["some_key", "some_value"]
+    assert isinstance(meta["some_key", "some_value", "some_units"], MetaDataItem)
+    assert tuple(meta["other_key"]) == ("other_key", "third_value", "other_units")
+    with pytest.raises(KeyError):
+        meta["unknown"]
+    with pytest.raises(KeyError):
+        meta["some_key", "unknown"]
+    with pytest.raises(KeyError):
+        meta["some_key", "some_value", "unknown"]
+    meta.clear()
+
+
+@mark.parametrize("item_name", ["collection", "dataobject"])
+def test_metadata_setitem(item_name, request):
+    item = request.getfixturevalue(item_name)
+    meta = MetaData(item)
+    meta.clear()
+
+    meta.add("some_key", "some_value", "some_units")
+    meta["some_key"] = ("some_key", "new_value", "new_units")
+    meta["some_key"] = ("some_key", "new_value")
+
+    with pytest.raises(TypeError):
+        meta["some_key"] = "new_value"
+
+    with pytest.raises(ValueError):
+        meta["some_key"] = ("some_key", "new_value")
+
+
+@mark.parametrize("item_name", ["collection", "dataobject"])
+def test_metadata_rename(item_name, request, session):
+    item = request.getfixturevalue(item_name)
+    meta = MetaData(item)
+    meta.clear()
+
+
+    meta.add("some_key", "some_value", "some_units")
+    meta["some_key"].key = "new_key"
+    assert ("new_key", "some_value", "some_units") in meta
+    assert len(meta) == 1
+
+    meta["new_key"].value = "new_value"
+    assert ("new_key", "new_value", "some_units") in meta
+    assert len(meta) == 1
+
+    meta["new_key"].units = "new_units"
+    assert ("new_key", "new_value", "new_units") in meta
+    assert len(meta) == 1
+
+    meta.add("new_key", "new_value", "other_units")
+    with pytest.raises(ValueError):
+        meta["new_key", "new_value", "other_units"].units = "new_units"
+    assert len(meta) == 2
+    meta["new_key", "new_value", "other_units"].remove()
+
+    meta.add("new_key", "other_value", "new_units")
+    with pytest.raises(ValueError):
+        meta["new_key", "other_value", "new_units"].value = "new_value"
+    assert len(meta) == 2
+    meta["new_key", "other_value", "new_units"].remove()
+
+    meta.add("other_key", "new_value", "new_units")
+    with pytest.raises(ValueError):
+        meta["other_key", "new_value", "new_units"].key = "new_key"
+    assert len(meta) == 2
+
+    with pytest.raises(ValueError):
+        meta["other_key"].key = "org_something"
+    assert len(meta) == 2
+    assert "other_key" in meta
+
+    meta.clear()
+
+
+@mark.parametrize("item_name", ["collection", "dataobject"])
+def test_metadata_findall(item_name, request, session):
+    item = request.getfixturevalue(item_name)
+    meta = MetaData(item)
+    meta.clear()
+
+
+    meta.add("some_key", "some_value", "some_units")
+    meta.add("some_key", "some_value", None)
+    meta.add("some_key", "other_value", "some_units")
+    meta.add("other_key", "third_value", "other_units")
+
+    assert len(meta.find_all()) == 4
+    assert len(meta.find_all(key="some_key")) == 3
+    assert isinstance(meta.find_all(key="some_key")[0], MetaDataItem)
+    assert len(meta.find_all(key="?")) == 0
+    assert len(meta.find_all(value="some_value")) == 2
+    assert len(meta.find_all(units="some_units")) == 2
+
+
+@mark.parametrize("item_name", ["collection", "dataobject"])
+def test_metadata_errors(item_name, request, session):
+    item = request.getfixturevalue(item_name)
+    meta = MetaData(item)
+    meta.clear()
+
+    with pytest.raises(ValueError):
+        meta.add("", "some_value")
+    with pytest.raises(TypeError):
+        meta.add(None, "some_value")
+    with pytest.raises(TypeError):
+        meta.add(10, "some_value")
+
+    with pytest.raises(ValueError):
+        meta.add("key", "")
+    with pytest.raises(TypeError):
+        meta.add("key", None)
+    with pytest.raises(TypeError):
+        meta.add("key", 10)
+
+    with pytest.raises(TypeError):
+        meta.add("key", "value", 10)
+

docs/source/irods_search.rst

@@ -2,7 +2,7 @@ iRODS Search
 ============
 
 `iBridges` offers an easy way to search for data. You can pass a combination of path, metadata,
-item type and checksum. The output will be a list of :class:`ibridges.path.IrodsPath`, which contain information where to find the item on the iRODS server.
+item type and checksum. The output will be a list of :class:`ibridges.path.CachedIrodsPath`, which contain information where to find the item on the iRODS server.
 
 .. note::
 
@@ -29,6 +29,11 @@ To find all subcollections and dataobjects in a collection use the `%` as wildca
 
     search_data(session, path_pattern="subcoll/%")
   	
+.. note::
+
+    The output of a search is a :class:`ibridges.path.CachedIrodsPath`. It contains the information about the data object or collection at the time of the search.
+    This information is not refetched from the server, i.e. the size of the path will always remain the size at the time of the search. 
+
 
 Search data by metadata
 -----------------------
@@ -66,7 +71,7 @@ A query with metadata will look like:
 	# and one metadata entry that has value=="value", but they do not have to be
 	# for the same entry as in the above.
 	search_data(session, metadata=[MetaSearch(key="key"), MetaSearch(value="value")])
-	
+
 Use the `%` as a wild card again to match any combination of characters.
 
 

docs/source/metadata.rst

@@ -6,8 +6,8 @@ Metadata
 iRODS offers metadata as key, value, units triplets. The type of the keys, values and units is always a string.
 Below we show how to create a :doc:`Metadata <api/generated/ibridges.meta.MetaData>` object from a data object or collection.
 
-The Metadata object
---------------------
+The MetaData class
+------------------
 
 .. code-block:: python
 
@@ -17,17 +17,30 @@ The Metadata object
     session = interactive_auth()
     meta = IrodsPath(session, "~", "collection_or_dataobject").meta
 
+    # Show all metadata entries with print.
+    print(meta)
+
 With the object :code:`meta` we can now access and manipulate the metadata of the data object.
 
+The MetaDataItem class
+----------------------
+
+As explained above, the metadata of a collection or dataobject can have multiple entries. You can iterate over
+these entries as follows:
+
+.. code-block:: python
+
+    for item in meta:
+        print(item.key, item.value, item.units)
+
+
 Add metadata
 ------------
 To add metadata, you always need to provide a key and a value, the units are optional and can be left out.
 
 .. code-block:: python
 
     meta.add('NewKey', 'NewValue', 'NewUnit')
-    print(meta)
-
 	
 .. note::
     You can have several metadata entries with the same key but different values and units,
@@ -46,6 +59,51 @@ same key first. This mirrors the implementation of the `iCommands <https://rdm-d
     meta.set('ExistingKey', 'Value', 'Unit')
 
 
+Find metadata items
+-------------------
+
+If you want to find all items with a certain key/value/units, you can use the ``find_all`` method
+which returns a list of items:
+
+.. code-block:: python
+
+
+    # Find all metadata items with key "some_key".
+    items = meta.find_all(key="some_key")
+
+    # Find all metadata items with value "some_value".
+    items = meta.find_all(value="some_value")
+
+    # Find all metadata items with some units "some_units".
+    items = meta.find_all(units="some_units")
+
+    # Find all metadata items with key == "some_key" and value == "some_value"
+    items = meta.find_all(key="some_key", value="some_value")
+
+If you are searching for one specific metadata item, then you can also use the following notation,
+which will either give back one metadata item, raise a ``KeyError`` if no item matches the criteria, or
+a ``ValueError`` if more than one value matches the criteria:
+
+.. code-block:: python
+
+    item = meta["some_key"]
+    item = meta["some_key", "some_value", "some_units"]
+
+Modify metadata items
+---------------------
+
+You can also rename the ``key``, ``value`` and ``units`` of a metadata item, by setting it to a new value:
+
+.. code-block:: python
+
+    item = metadata["some_key"]
+    item.key = "new_key"
+    item.value = "new_value"
+    item.units = "new_units"
+
+If you are trying to rename the metadata item so that it would overwrite an existing metadata item,
+ibridges will throw an error.
+
 Delete metadata
 ---------------
 Below are examples on how to delete metadata entries:

ibridges/data_operations.py

@@ -35,6 +35,7 @@ def upload(
     options: Optional[dict] = None,
     dry_run: bool = False,
     metadata: Union[None, str, Path] = None,
+    **kwargs
 ) -> Operations:
     """Upload a local directory or file to iRODS.
 
@@ -62,6 +63,8 @@ def upload(
         Whether to do a dry run before uploading the files/folders.
     metadata:
         If not None, it should point to a file that contains the metadata for the upload.
+    kwargs:
+        Extra arguments for executing the upload operation, e.g. progress_bar = False.
 
     Returns
     -------
@@ -121,7 +124,7 @@ def upload(
     if metadata is not None:
         ops.add_meta_upload(idest_path, metadata)
     if not dry_run:
-        ops.execute(session, ignore_err=ignore_err)
+        ops.execute(session, ignore_err=ignore_err, **kwargs)
     return ops
 
 
@@ -136,6 +139,7 @@ def download(
     options: Optional[dict] = None,
     dry_run: bool = False,
     metadata: Union[None, str, Path] = None,
+    **kwargs
 ) -> Operations:
     """Download a collection or data object to the local filesystem.
 
@@ -165,6 +169,8 @@ def download(
     metadata:
         If not None, the path to store the metadata to in JSON format.
         It is recommended to use the .json suffix.
+    kwargs:
+        Extra arguments for executing the download operation, e.g. progress_bar = False.
 
     Returns
     -------
@@ -229,7 +235,7 @@ def download(
     ops.resc_name = resc_name
     ops.options = options
     if not dry_run:
-        ops.execute(session, ignore_err=ignore_err)
+        ops.execute(session, ignore_err=ignore_err, **kwargs)
     return ops
 
 
@@ -272,6 +278,7 @@ def sync(
     resc_name: str = "",
     options: Optional[dict] = None,
     metadata: Union[None, str, Path] = None,
+    **kwargs
 ) -> Operations:
     """Synchronize data between local and remote copies.
 
@@ -313,6 +320,8 @@ def sync(
         More options for the download/upload
     metadata:
         If not None, the location to get the metadata from or store it to.
+    kwargs:
+        Extra arguments for executing the sync operation, e.g. progress_bar = False.
 
 
     Returns
@@ -352,7 +361,7 @@ def sync(
     ops.resc_name = resc_name
     ops.options = options
     if not dry_run:
-        ops.execute(session, ignore_err=ignore_err)
+        ops.execute(session, ignore_err=ignore_err, **kwargs)
 
     return ops