Merge pull request #669 from christian-monch/patch-create-sibling-ria

Patch RIA and ORA code to work on multiple client platforms

.codespell-exclude

@@ -0,0 +1 @@
+            froms=ds.repo.get_revisions()[1],

.github/workflows/codespell.yml

@@ -17,3 +17,5 @@ jobs:
         uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v1
+        with:
+          exclude_file: .codespell-exclude

changelog.d/20240514_212532_michael.hanke_patch_create_sibling_ria.md

@@ -0,0 +1,10 @@
+### 💫 Enhancements and new features
+
+- A large 3k-line patch set replaces almost the entire RIA implementation,
+  including the ORA special remote, and the `create-sibling-ria` command.
+  The new implementation brings uniform support for Windows clients, progress
+  reporting for uploads and downloads via SSH, and a faster and more
+  robust behavior for SSH-based operations (based on the new remote
+  shell feature).
+  Fixes https://github.com/datalad/datalad-next/issues/654 via
+  https://github.com/datalad/datalad-next/pull/669 (by @christian-monch)

datalad_next/patches/add_method_url2transport_path.py

@@ -0,0 +1,81 @@
+"""Add the method :meth:`url2transport_path` to RIA IO-abstraction classes
+
+This patch adds the method :meth:`url2transport_path` to the IO-abstraction
+classes: :class:`datalad.distributed.ora_remote.LocalIO`, and to the class
+:class:`datalad.distributed.ora_remote.HTTPRemoteIO`.
+
+This method is required by the patches that add Windows-client support to
+RIA-code. It converts internally used abstract paths to concrete paths that are
+platform- andIO-abstraction specific and on which IO-operations cam be
+performed.
+"""
+from __future__ import annotations
+
+import logging
+from re import compile
+from pathlib import (
+    Path,
+    PurePosixPath,
+)
+
+from datalad_next.consts import on_windows
+from . import apply_patch
+
+
+# The methods are patched into the ora_remote/ria_remote. Use the same logger.
+lgr = logging.getLogger('datalad.customremotes.ria_remote')
+
+
+drive_letter_matcher = compile('^/[A-Z]:')
+
+
+def str2windows_path(url_path: PurePosixPath):
+    path_str = str(url_path)
+    match = drive_letter_matcher.match(path_str)
+    if match:
+        if path_str[3] == '/':
+            return Path(*([f'{path_str[1]}:', '/'] + path_str[4:].split('/')))
+        else:
+            lgr.warning(f'Non-absolute Windows-path detected: {path_str}')
+            return Path(*([f'{path_str[1]}:'] + path_str[3:].split('/')))
+    else:
+        return Path(path_str)
+
+
+def local_io_url2transport_path(
+        self,
+        url_path: PurePosixPath
+) -> Path | PurePosixPath:
+    assert url_path.__class__ is PurePosixPath
+    if on_windows:
+        return str2windows_path(url_path)
+    else:
+        return Path(url_path)
+
+
+def http_remote_io_url2transport_path(
+        self,
+        url_path: PurePosixPath
+) -> Path | PurePosixPath:
+    assert url_path.__class__ is PurePosixPath
+    return url_path
+
+
+# Add a `url2transport_path`-method to `ora_remote.LocalIO`
+apply_patch(
+    'datalad.distributed.ora_remote',
+    'LocalIO',
+    'url2transport_path',
+    local_io_url2transport_path,
+    expect_attr_present=False,
+)
+
+
+# Add a `url2transport_path`-method to `ora_remote.HTTPRemoteIO`
+apply_patch(
+    'datalad.distributed.ora_remote',
+    'HTTPRemoteIO',
+    'url2transport_path',
+    http_remote_io_url2transport_path,
+    expect_attr_present=False,
+)

datalad_next/patches/customremotes_main.py

@@ -10,6 +10,8 @@
 that handler in a context manager to ensure releasing any resources. This
 replaces the custom `stop()` method, which is undocumented and only used by the
 `datalad-archive` special remote.
+
+This patch also adds code that allows to patch a class that is already loaded
 """
 
 from contextlib import closing
@@ -117,6 +119,11 @@ def patched_underscore_main(args: list, cls: Type[SpecialRemote]):
     """
     assert cls is not None
     from annexremote import Master
+
+    # Reload the class, to allow `cls` itself to be patched.
+    new_module = __import__(cls.__module__, fromlist=[cls.__name__])
+    cls = getattr(new_module, cls.__name__)
+
     master = Master()
     # this context manager use relies on patching in a close() below
     with closing(cls(master)) as remote:

datalad_next/patches/enabled.py

@@ -17,6 +17,5 @@
     # the following two patches have been taken verbatim from datalad-ria
     ssh_exec,
     sshconnector,
-    # this replaces SSHRemoteIO entirely
-    replace_sshremoteio,
+    patch_ria_ora,
 )