Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add basic metadata options to CLI #293

Merged
merged 12 commits into from
Dec 2, 2024
Merged
22 changes: 22 additions & 0 deletions docker/irods_client/tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@

from ibridges import IrodsPath
from ibridges.data_operations import create_collection
from ibridges.meta import MetaData


@pytest.fixture(scope="module")
Expand Down Expand Up @@ -135,6 +136,8 @@ def test_list_cli(config, pass_opts, irods_env_file, collection):
subprocess.run(["ibridges", "init", irods_env_file], **pass_opts)
subprocess.run(["ibridges", "list"], **pass_opts)
subprocess.run(["ibridges", "list", f"irods:{collection.path}"], **pass_opts)
subprocess.run(["ibridges", "list", "-s"], **pass_opts)
subprocess.run(["ibridges", "list", "-l"], **pass_opts)


@mark.parametrize(
Expand Down Expand Up @@ -174,3 +177,22 @@ def test_search_cli(session, config, pass_opts, irods_env_file, testdata, search
else:
assert len([x for x in stripped_str.split("\n") if not x.startswith("Your iRODS")]) == nlines
ipath_coll.remove()

@mark.parametrize("item_name", ["collection", "dataobject"])
def test_meta_cli(item_name, request, pass_opts):
item = request.getfixturevalue(item_name)
meta = MetaData(item)
meta.clear()
cli_path = f"irods:{item.path}"

ret = subprocess.run(["ibridges", "meta-list", cli_path], capture_output=True, **pass_opts)
assert len(ret.stdout.strip("\n").split("\n")) == 1

subprocess.run(["ibridges", "meta-add", cli_path, "key", "value", "units"], **pass_opts)
meta.refresh()
assert ("key", "value", "units") in meta

subprocess.run(["ibridges", "meta-del", cli_path, "--key", "key"], **pass_opts)
meta.refresh()
meta = MetaData(item)
assert ("key", "value", "units") not in meta
65 changes: 63 additions & 2 deletions docs/source/cli.rst
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ useful to cache the password so that the next commands will no longer ask for yo
Listing remote files
--------------------

To list the dataobjects and collections that are available on the iRODS server, you can use the :code:`ibridges list` command:
To list the data objects and collections that are available on the iRODS server, you can use the :code:`ibridges list` command:

.. code:: shell

Expand All @@ -94,9 +94,19 @@ If you want to list a collection relative to your `irods_home`, you can use `~`

ibridges list "irods:~/collection_in_home"


It is generally best to avoid spaces in collection and data object names. If you really need them, you must enclose the path with `"`. That also holds true for local paths.

If you want to have a list that is easier to parse with other command line tools, you can use:

.. code:: shell

ibridges list --short

You can also see the checksums and sizes of data objects with the long format:

.. code:: shell

ibridges list --long

.. note::
Note that all data objects and collections on the iRODS server are always preceded with "irods:". This is done to distinguish local and remote files.
Expand Down Expand Up @@ -274,3 +284,54 @@ or
.. code:: shell

ibridges search --metadata "key" --item_type data_object


Metadata commands
-----------------

Listing metadata
^^^^^^^^^^^^^^^^

Listing metadata entries for a single collection or data object can be done with the :code:`meta-list`
subcommand:

.. code:: shell

ibridges meta-list "irods:some_collection"

Adding new metadata
^^^^^^^^^^^^^^^^^^^

To add new metadata for a single collection or data object, you can use the :code:`meta-add` subcommand:

.. code:: shell

ibridges meta-add "irods:some_collection" some_key some_value, some_units

The :code:`some_units` argument can be left out, in which case the units will be set to the empty string.

Deleting metadata
^^^^^^^^^^^^^^^^^

Metadata can also again be deleted with the CLI using the :code:`meta-del` subcommand:

.. code:: shell

ibridges meta-del "irods:some_collection" --key some_key --value some_value --units some_units

All of the :code:`--key`, :code:`--value` and :code:`--units` are optional. They serve to constrain
which metadata items will be deleted. For example, if you only set the key:

.. code:: shell

ibridges meta-del "irods:some_collection" --key some_key

then **all** metadata items with that key will be deleted. You can delete all metadata for a single
collection or data object with:

.. code:: shell

ibridges meta-del "irods:some_collection"

You will be asked to confirm this operation.

185 changes: 165 additions & 20 deletions ibridges/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,12 @@
List the content of a collections, if no path is given, the home collection will be listed.
tree:
List a collection and subcollections in a hierarchical way.
meta-list:
List the metadata of a collection or data object.
meta-add:
Add a new metadata entry to a collection or data object.
meta-del:
Delete metadata entries for a collection or data object.
mkcoll:
Create the collection and all its parent collections.
setup:
Expand All @@ -68,6 +74,9 @@
ibridges list irods:~/collection
ibridges mkcoll irods://~/bli/bla/blubb
ibridges tree irods:~/collection
ibridges meta-list irods:~/collection
ibridges meta-add irods:~/collection key value units
ibridges meta-del irods:~/collection key value units
ibridges search --path-pattern "%.txt"
ibridges search --metadata "key" "value" "units"
ibridges search --metadata "key" --metadata "key2" "value2"
Expand All @@ -84,7 +93,7 @@



def main() -> None:
def main() -> None: #pylint: disable=too-many-branches
"""CLI pointing to different entrypoints."""
# ensure .irods folder
irods_loc = Path.home() / ".irods"
Expand All @@ -106,7 +115,7 @@ def main() -> None:
ibridges_sync()
elif subcommand == "init":
ibridges_init()
elif subcommand == "list":
elif subcommand in ["list", "ls"]:
ibridges_list()
elif subcommand == "mkcoll":
ibridges_mkcoll()
Expand All @@ -116,6 +125,12 @@ def main() -> None:
ibridges_setup()
elif subcommand == "search":
ibridges_search()
elif subcommand in ["meta", "meta-list"]:
ibridges_meta_list()
elif subcommand == "meta-add":
ibridges_meta_add()
elif subcommand in ["meta-del", "meta-rm"]:
ibridges_meta_del()
else:
print(f"Invalid subcommand ({subcommand}). For help see ibridges --help")
sys.exit(1)
Expand Down Expand Up @@ -230,23 +245,6 @@ def ibridges_init():
print("ibridges init was succesful.")


def _list_coll(session: Session, remote_path: IrodsPath):
if remote_path.collection_exists():
print(str(remote_path) + ":")
coll = get_collection(session, remote_path)
print("\n".join([" " + sub.path for sub in coll.data_objects]))
print(
"\n".join(
[
" C- " + sub.path
for sub in coll.subcollections
if not str(remote_path) == sub.path
]
)
)
else:
raise ValueError(f"Irods path '{remote_path}' is not a collection.")


def ibridges_setup():
"""Use templates to create an iRODS environment json."""
Expand Down Expand Up @@ -311,6 +309,29 @@ def ibridges_setup():
handle.write(json_str)


def _list_coll(session: Session, remote_path: IrodsPath, metadata: bool = False):
if remote_path.collection_exists():
print(str(remote_path) + ":")
if metadata:
print(remote_path.meta)
print()
coll = get_collection(session, remote_path)
for data_obj in coll.data_objects:
print(" " + data_obj.path)
if metadata and len((remote_path / data_obj.name).meta) > 0:
print((remote_path / data_obj.name).meta)
print()
for sub_coll in coll.subcollections:
if str(remote_path) == sub_coll.path:
continue
print(" C- " + sub_coll.path)
if metadata and len((remote_path / sub_coll.name).meta) > 0:
print((remote_path / sub_coll.name).meta)
print()
else:
raise ValueError(f"Irods path '{remote_path}' is not a collection.")


def ibridges_list():
"""List a collection on iRODS."""
parser = argparse.ArgumentParser(
Expand All @@ -323,10 +344,134 @@ def ibridges_list():
default=None,
nargs="?",
)
parser.add_argument(
"-m", "--metadata",
help="Show metadata for each iRODS location.",
action="store_true",
)
parser.add_argument(
"-s", "--short",
help="Display available data objects/collections in short form.",
action="store_true"
)
parser.add_argument(
"-l", "--long",
help="Display available data objects/collections in long form.",
action="store_true",
)

args, _ = parser.parse_known_args()
with _cli_auth(ienv_path=_get_ienv_path()) as session:
ipath = _parse_remote(args.remote_path, session)
if args.long:
for cur_path in ipath.walk(depth=1):
if str(cur_path) == str(ipath):
continue
if cur_path.collection_exists():
print(f"C- {cur_path.name}")
else:
print(f"{cur_path.checksum: <50} {cur_path.size: <12} {cur_path.name}")
elif args.short:
print(" ".join([x.name for x in ipath.walk(depth=1) if str(x) != str(ipath)]))
else:
_list_coll(session, ipath, args.metadata)


def ibridges_meta_list():
"""List metadata of a a collection on iRODS."""
parser = argparse.ArgumentParser(
prog="ibridges meta-list", description="List a collection on iRODS."
)
parser.add_argument(
"remote_path",
help="Path to remote iRODS location starting with 'irods:'",
type=str,
default=None,
nargs="?",
)

args, _ = parser.parse_known_args()
with _cli_auth(ienv_path=_get_ienv_path()) as session:
_list_coll(session, _parse_remote(args.remote_path, session))
ipath = _parse_remote(args.remote_path, session)
print(str(ipath) + ":\n")
print(ipath.meta)

def ibridges_meta_add():
"""Add new metadata to an iRODS item."""
parser = argparse.ArgumentParser(
prog="ibridges meta-add", description="Add metadata entry."
)
parser.add_argument(
"remote_path",
help="Path to add a new metadata item to.",
type=str,
)
parser.add_argument(
"key",
help="Key for the new metadata item.",
type=str,
)
parser.add_argument(
"value",
help="Value for the new metadata item.",
type=str
)
parser.add_argument(
"units",
help="Units for the new metadata item.",
type=str,
default="",
nargs="?"
)
args = parser.parse_args()
with _cli_auth(ienv_path=_get_ienv_path()) as session:
ipath = _parse_remote(args.remote_path, session)
ipath.meta.add(args.key, args.value, args.units)

def ibridges_meta_del():
"""Delete metadata from an iRODS item."""
parser = argparse.ArgumentParser(
prog="ibridges meta-del", description="Delete metadata entries.",
)
parser.add_argument(
"remote_path",
help="Path to delete metadata entries from.",
)
parser.add_argument(
"--key",
help="Key for which to delete the entries.",
type=str,
default=...,
)
parser.add_argument(
"--value",
help="Value for which to delete the entries.",
type=str,
default=...,
)
parser.add_argument(
"--units",
help="Units for which to delete the entries.",
type=str,
default=...,
)
parser.add_argument(
"--ignore-blacklist",
help="Ignore the metadata blacklist.",
action="store_true"
)
args = parser.parse_args()
with _cli_auth(ienv_path=_get_ienv_path()) as session:
ipath = _parse_remote(args.remote_path, session)
meta = ipath.meta
if args.ignore_blacklist:
meta.blacklist = None
if args.key is ... and args.value is ... and args.units is ...:
answer = input("This command will delete all metadata for path {ipath},"
" are you sure? [y/n]")
if answer.lower() != "y":
return
meta.delete(args.key, args.value, args.units)


def _create_coll(session: Session, remote_path: IrodsPath):
Expand Down
Loading