Provide replacement for SSHRemoteIO based on shell

changelog.d/20240425_090037_michael.hanke_ssh_shell.md

@@ -0,0 +1,15 @@
+### 💫 Enhancements and new features
+
+- A rewrite of `SSHRemoteIO`, the RIA SSH-operations implementation from
+  datalad-core is provided as a patch. It is based on the new `shell`
+  feature, and provides more robust operations. It's IO performance is
+  at the same level as `scp`-based down/uploads. In contrast to the
+  original implementation, it support fine-grained progress reporting
+  for uploads and downloads.
+  Via https://github.com/datalad/datalad-next/pull/655 (by @mih)
+- The `SpecialRemote` base class in datalad-core is patched to support
+  a standard `close()` method for implementing resource release and cleanup
+  operations. The main special remote entrypoint has been altered to
+  run implementations within a `closing()` context manager to guarantee
+  execution of such handlers.
+  Via https://github.com/datalad/datalad-next/pull/655 (by @mih)

datalad_next/patches/customremotes_main.py

@@ -1,12 +1,18 @@
-"""Connect ``log_progress``-style progress reporting to git-annex
+"""Connect ``log_progress``-style progress reporting to git-annex, add `close()`
 
 This patch introduces a dedicated progress log handler as a proxy between
 standard datalad progress logging and a git-annex special remote as
 an approach to report (data transfer) progress to a git-annex parent process.
 
 This functionality is only (to be) used in dedicated special remote processes.
+
+This patch also adds a standard `close()` handler to special remotes, and calls
+that handler in a context manager to ensure releasing any resources. This
+replaces the custom `stop()` method, which is undocumented and only used by the
+`datalad-archive` special remote.
 """
 
+from contextlib import closing
 import logging
 from typing import (
     Dict,
@@ -112,23 +118,38 @@ def patched_underscore_main(args: list, cls: Type[SpecialRemote]):
     assert cls is not None
     from annexremote import Master
     master = Master()
-    remote = cls(master)
-    master.LinkRemote(remote)
+    # this context manager use relies on patching in a close() below
+    with closing(cls(master)) as remote:
+        master.LinkRemote(remote)
+
+        # we add an additional handler to the logger to deal with
+        # progress reports
+        dlroot_lgr = logging.getLogger('datalad')
+        phandler = AnnexProgressLogHandler(remote)
+        phandler.addFilter(only_progress_logrecords)
+        dlroot_lgr.addHandler(phandler)
+
+        # run the remote
+        master.Listen()
+        # cleanup special case datalad-core `archive` remote
+        # nobody should do this, use `close()`
+        if hasattr(remote, 'stop'):
+            remote.stop()
 
-    # we add an additional handler to the logger to deal with
-    # progress reports
-    dlroot_lgr = logging.getLogger('datalad')
-    phandler = AnnexProgressLogHandler(remote)
-    phandler.addFilter(only_progress_logrecords)
-    dlroot_lgr.addHandler(phandler)
 
-    # run the remote
-    master.Listen()
-    # cleanup
-    if hasattr(remote, 'stop'):
-        remote.stop()
+# a default cleanup handler for CoreBaseSpecialRemote
+# this enables us to use a standard `closing()` context manager with
+# special remotes
+def specialremote_defaultclose_noop(self):
+    pass
 
 
+apply_patch(
+    'datalad.customremotes', 'SpecialRemote', 'close',
+    specialremote_defaultclose_noop,
+    msg='Retrofit `SpecialRemote` with a `close()` handler',
+    expect_attr_present=False,
+)
 apply_patch(
     'datalad.customremotes.main', None, '_main',
     patched_underscore_main,

datalad_next/patches/enabled.py

@@ -14,9 +14,9 @@
     create_sibling_gitlab,
     run,
     update,
-    ora_remote,
-    # the following patches have been taken verbatim from datalad-ria
+    # the following two patches have been taken verbatim from datalad-ria
     ssh_exec,
-    sshremoteio,
     sshconnector,
+    # this replaces SSHRemoteIO entirely
+    replace_sshremoteio,
 )