ls-url/get-url/import-url: introduce --fs-config

Fixes iterative#9886

dvc/commands/get_url.py

@@ -3,7 +3,7 @@
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -19,6 +19,7 @@ def run(self):
                 out=self.args.out,
                 jobs=self.args.jobs,
                 force=self.args.force,
+                config=self.args.fs_config,
             )
             return 0
         except DvcException:
@@ -58,4 +59,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Override local file or folder if exists.",
     )
+    get_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     get_parser.set_defaults(func=CmdGetUrl)

dvc/commands/imp_url.py

@@ -3,7 +3,7 @@
 
 from dvc.cli import completion
 from dvc.cli.command import CmdBase
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 from dvc.exceptions import DvcException
 
 logger = logging.getLogger(__name__)
@@ -22,6 +22,7 @@ def run(self):
                 jobs=self.args.jobs,
                 force=self.args.force,
                 version_aware=self.args.version_aware,
+                config=self.args.fs_config,
             )
         except DvcException:
             logger.exception(
@@ -114,4 +115,11 @@ def add_parser(subparsers, parent_parser):
         default=False,
         help="Import using cloud versioning. Implied if the URL contains a version ID.",
     )
+    import_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     import_parser.set_defaults(func=CmdImportUrl)

dvc/commands/ls_url.py

@@ -2,7 +2,7 @@
 import logging
 
 from dvc.cli.command import CmdBaseNoRepo
-from dvc.cli.utils import append_doc_link
+from dvc.cli.utils import DictAction, append_doc_link
 
 from .ls import show_entries
 
@@ -13,7 +13,11 @@ class CmdListUrl(CmdBaseNoRepo):
     def run(self):
         from dvc.repo import Repo
 
-        entries = Repo.ls_url(self.args.url, recursive=self.args.recursive)
+        entries = Repo.ls_url(
+            self.args.url,
+            recursive=self.args.recursive,
+            config=self.args.fs_config,
+        )
         if entries:
             show_entries(entries, with_color=True, with_size=self.args.size)
         return 0
@@ -43,4 +47,11 @@ def add_parser(subparsers, parent_parser):
         action="store_true",
         help="Show sizes.",
     )
+    lsurl_parser.add_argument(
+        "--fs-config",
+        type=str,
+        nargs="*",
+        action=DictAction,
+        help="Config options for the target url.",
+    )
     lsurl_parser.set_defaults(func=CmdListUrl)

dvc/dependency/__init__.py

@@ -15,6 +15,7 @@
     **ARTIFACT_SCHEMA,
     **RepoDependency.REPO_SCHEMA,
     Output.PARAM_FILES: [DIR_FILES_SCHEMA],
+    Output.PARAM_FS_CONFIG: dict,
 }
 
 
@@ -36,7 +37,10 @@ def loadd_from(stage, d_list):
         p = d.pop(Output.PARAM_PATH, None)
         files = d.pop(Output.PARAM_FILES, None)
         hash_name = d.pop(Output.PARAM_HASH, None)
-        ret.append(_get(stage, p, d, files=files, hash_name=hash_name))
+        fs_config = d.pop(Output.PARAM_FS_CONFIG, None)
+        ret.append(
+            _get(stage, p, d, files=files, hash_name=hash_name, fs_config=fs_config)
+        )
     return ret
 
 

dvc/output.py

@@ -97,6 +97,7 @@ def loadd_from(stage, d_list):
         files = d.pop(Output.PARAM_FILES, None)
         push = d.pop(Output.PARAM_PUSH, True)
         hash_name = d.pop(Output.PARAM_HASH, None)
+        fs_config = d.pop(Output.PARAM_FS_CONFIG, None)
         ret.append(
             _get(
                 stage,
@@ -111,6 +112,7 @@ def loadd_from(stage, d_list):
                 files=files,
                 push=push,
                 hash_name=hash_name,
+                fs_config=fs_config,
             )
         )
     return ret
@@ -313,6 +315,7 @@ class Output:
     PARAM_PUSH = "push"
     PARAM_CLOUD = "cloud"
     PARAM_HASH = "hash"
+    PARAM_FS_CONFIG = "fs_config"
 
     DoesNotExistError: Type[DvcException] = OutputDoesNotExistError
     IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError
@@ -351,6 +354,7 @@ def __init__(  # noqa: PLR0913
         if meta.version_id or files:
             fs_kwargs["version_aware"] = True
 
+        self.def_fs_config = fs_config
         if fs_config is not None:
             fs_kwargs.update(**fs_config)
 
@@ -872,6 +876,9 @@ def dumpd(self, **kwargs):  # noqa: C901, PLR0912
 
         ret[self.PARAM_PATH] = path
 
+        if self.def_fs_config:
+            ret[self.PARAM_FS_CONFIG] = self.def_fs_config
+
         if not self.IS_DEPENDENCY:
             ret.update(self.annot.to_dict())
             if not self.use_cache:
@@ -1537,4 +1544,5 @@ def _merge_dir_version_meta(self, other: "Output"):
     Output.PARAM_REMOTE: str,
     Output.PARAM_PUSH: bool,
     Output.PARAM_FILES: [DIR_FILES_SCHEMA],
+    Output.PARAM_FS_CONFIG: dict,
 }

dvc/repo/imp_url.py

@@ -26,7 +26,7 @@ def imp_url(  # noqa: C901, PLR0913
     to_remote=False,
     jobs=None,
     force=False,
-    fs_config=None,
+    config=None,
     version_aware: bool = False,
 ):
     out = resolve_output(url, out, force=force)
@@ -50,9 +50,9 @@ def imp_url(  # noqa: C901, PLR0913
         url = relpath(url, wdir)
 
     if version_aware:
-        if fs_config is None:
-            fs_config = {}
-        fs_config["version_aware"] = True
+        if config is None:
+            config = {}
+        config["version_aware"] = True
 
     stage = self.stage.create(
         single_stage=True,
@@ -62,7 +62,7 @@ def imp_url(  # noqa: C901, PLR0913
         deps=[url],
         outs=[out],
         erepo=erepo,
-        fs_config=fs_config,
+        fs_config=config,
     )
 
     try:

tests/func/test_import_url.py

@@ -247,7 +247,7 @@ def test_import_url_fs_config(tmp_dir, dvc, workspace, mocker):
     url = "remote://workspace/foo"
     get_fs_config = mocker.spy(dvc_fs, "get_fs_config")
     dep_init = mocker.spy(Dependency, "__init__")
-    dvc.imp_url(url, fs_config={"jobs": 42})
+    dvc.imp_url(url, config={"jobs": 42})
 
     dep_init_kwargs = dep_init.call_args[1]
     assert dep_init_kwargs.get("fs_config") == {"jobs": 42}

tests/unit/command/test_get_url.py

@@ -11,4 +11,4 @@ def test_get_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", out="out", jobs=5, force=False)
+    m.assert_called_once_with("src", out="out", jobs=5, force=False, config=None)

tests/unit/command/test_imp_url.py

@@ -34,6 +34,7 @@ def test_import_url(mocker, dvc):
         jobs=4,
         force=False,
         version_aware=False,
+        config=None,
     )
 
 
@@ -83,6 +84,7 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected, dvc):
         jobs=None,
         force=False,
         version_aware=False,
+        config=None,
         **expected,
     )
 
@@ -115,6 +117,7 @@ def test_import_url_to_remote(mocker, dvc):
         jobs=None,
         force=False,
         version_aware=False,
+        config=None,
     )
 
 

tests/unit/command/test_ls_url.py

@@ -10,7 +10,7 @@ def test_ls_url(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", recursive=False)
+    m.assert_called_once_with("src", recursive=False, config=None)
 
 
 def test_recursive(mocker):
@@ -21,4 +21,4 @@ def test_recursive(mocker):
 
     assert cmd.run() == 0
 
-    m.assert_called_once_with("src", recursive=True)
+    m.assert_called_once_with("src", recursive=True, config=None)