diff --git a/dvc/commands/get_url.py b/dvc/commands/get_url.py index e96deb50e14..707bb777615 100644 --- a/dvc/commands/get_url.py +++ b/dvc/commands/get_url.py @@ -3,7 +3,7 @@ from dvc.cli import completion from dvc.cli.command import CmdBaseNoRepo -from dvc.cli.utils import append_doc_link +from dvc.cli.utils import DictAction, append_doc_link from dvc.exceptions import DvcException logger = logging.getLogger(__name__) @@ -19,6 +19,7 @@ def run(self): out=self.args.out, jobs=self.args.jobs, force=self.args.force, + config=self.args.fs_config, ) return 0 except DvcException: @@ -58,4 +59,11 @@ def add_parser(subparsers, parent_parser): default=False, help="Override local file or folder if exists.", ) + get_parser.add_argument( + "--fs-config", + type=str, + nargs="*", + action=DictAction, + help="Config options for the target url.", + ) get_parser.set_defaults(func=CmdGetUrl) diff --git a/dvc/commands/imp_url.py b/dvc/commands/imp_url.py index 21d826c57b3..d03d0b3d323 100644 --- a/dvc/commands/imp_url.py +++ b/dvc/commands/imp_url.py @@ -3,7 +3,7 @@ from dvc.cli import completion from dvc.cli.command import CmdBase -from dvc.cli.utils import append_doc_link +from dvc.cli.utils import DictAction, append_doc_link from dvc.exceptions import DvcException logger = logging.getLogger(__name__) @@ -22,6 +22,7 @@ def run(self): jobs=self.args.jobs, force=self.args.force, version_aware=self.args.version_aware, + config=self.args.fs_config, ) except DvcException: logger.exception( @@ -114,4 +115,11 @@ def add_parser(subparsers, parent_parser): default=False, help="Import using cloud versioning. Implied if the URL contains a version ID.", ) + import_parser.add_argument( + "--fs-config", + type=str, + nargs="*", + action=DictAction, + help="Config options for the target url.", + ) import_parser.set_defaults(func=CmdImportUrl) diff --git a/dvc/commands/ls_url.py b/dvc/commands/ls_url.py index 80034f54918..94a11d369b8 100644 --- a/dvc/commands/ls_url.py +++ b/dvc/commands/ls_url.py @@ -2,7 +2,7 @@ import logging from dvc.cli.command import CmdBaseNoRepo -from dvc.cli.utils import append_doc_link +from dvc.cli.utils import DictAction, append_doc_link from .ls import show_entries @@ -13,7 +13,11 @@ class CmdListUrl(CmdBaseNoRepo): def run(self): from dvc.repo import Repo - entries = Repo.ls_url(self.args.url, recursive=self.args.recursive) + entries = Repo.ls_url( + self.args.url, + recursive=self.args.recursive, + config=self.args.fs_config, + ) if entries: show_entries(entries, with_color=True, with_size=self.args.size) return 0 @@ -43,4 +47,11 @@ def add_parser(subparsers, parent_parser): action="store_true", help="Show sizes.", ) + lsurl_parser.add_argument( + "--fs-config", + type=str, + nargs="*", + action=DictAction, + help="Config options for the target url.", + ) lsurl_parser.set_defaults(func=CmdListUrl) diff --git a/dvc/dependency/__init__.py b/dvc/dependency/__init__.py index bf2c64204cc..2904da8a8a5 100644 --- a/dvc/dependency/__init__.py +++ b/dvc/dependency/__init__.py @@ -15,6 +15,7 @@ **ARTIFACT_SCHEMA, **RepoDependency.REPO_SCHEMA, Output.PARAM_FILES: [DIR_FILES_SCHEMA], + Output.PARAM_FS_CONFIG: dict, } @@ -36,7 +37,10 @@ def loadd_from(stage, d_list): p = d.pop(Output.PARAM_PATH, None) files = d.pop(Output.PARAM_FILES, None) hash_name = d.pop(Output.PARAM_HASH, None) - ret.append(_get(stage, p, d, files=files, hash_name=hash_name)) + fs_config = d.pop(Output.PARAM_FS_CONFIG, None) + ret.append( + _get(stage, p, d, files=files, hash_name=hash_name, fs_config=fs_config) + ) return ret diff --git a/dvc/output.py b/dvc/output.py index ae8b303fb26..5cf213300a2 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -97,6 +97,7 @@ def loadd_from(stage, d_list): files = d.pop(Output.PARAM_FILES, None) push = d.pop(Output.PARAM_PUSH, True) hash_name = d.pop(Output.PARAM_HASH, None) + fs_config = d.pop(Output.PARAM_FS_CONFIG, None) ret.append( _get( stage, @@ -111,6 +112,7 @@ def loadd_from(stage, d_list): files=files, push=push, hash_name=hash_name, + fs_config=fs_config, ) ) return ret @@ -313,6 +315,7 @@ class Output: PARAM_PUSH = "push" PARAM_CLOUD = "cloud" PARAM_HASH = "hash" + PARAM_FS_CONFIG = "fs_config" DoesNotExistError: Type[DvcException] = OutputDoesNotExistError IsNotFileOrDirError: Type[DvcException] = OutputIsNotFileOrDirError @@ -351,6 +354,7 @@ def __init__( # noqa: PLR0913 if meta.version_id or files: fs_kwargs["version_aware"] = True + self.def_fs_config = fs_config if fs_config is not None: fs_kwargs.update(**fs_config) @@ -872,6 +876,9 @@ def dumpd(self, **kwargs): # noqa: C901, PLR0912 ret[self.PARAM_PATH] = path + if self.def_fs_config: + ret[self.PARAM_FS_CONFIG] = self.def_fs_config + if not self.IS_DEPENDENCY: ret.update(self.annot.to_dict()) if not self.use_cache: @@ -1537,4 +1544,5 @@ def _merge_dir_version_meta(self, other: "Output"): Output.PARAM_REMOTE: str, Output.PARAM_PUSH: bool, Output.PARAM_FILES: [DIR_FILES_SCHEMA], + Output.PARAM_FS_CONFIG: dict, } diff --git a/dvc/repo/imp_url.py b/dvc/repo/imp_url.py index 4deb8ad9c59..b82f79581ce 100644 --- a/dvc/repo/imp_url.py +++ b/dvc/repo/imp_url.py @@ -26,7 +26,7 @@ def imp_url( # noqa: C901, PLR0913 to_remote=False, jobs=None, force=False, - fs_config=None, + config=None, version_aware: bool = False, ): out = resolve_output(url, out, force=force) @@ -50,9 +50,9 @@ def imp_url( # noqa: C901, PLR0913 url = relpath(url, wdir) if version_aware: - if fs_config is None: - fs_config = {} - fs_config["version_aware"] = True + if config is None: + config = {} + config["version_aware"] = True stage = self.stage.create( single_stage=True, @@ -62,7 +62,7 @@ def imp_url( # noqa: C901, PLR0913 deps=[url], outs=[out], erepo=erepo, - fs_config=fs_config, + fs_config=config, ) try: diff --git a/tests/unit/command/test_get_url.py b/tests/unit/command/test_get_url.py index a2b27495bcc..3598c2f5c26 100644 --- a/tests/unit/command/test_get_url.py +++ b/tests/unit/command/test_get_url.py @@ -11,4 +11,4 @@ def test_get_url(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", out="out", jobs=5, force=False) + m.assert_called_once_with("src", out="out", jobs=5, force=False, config=None) diff --git a/tests/unit/command/test_imp_url.py b/tests/unit/command/test_imp_url.py index 8944cd4c26a..b023be5529a 100644 --- a/tests/unit/command/test_imp_url.py +++ b/tests/unit/command/test_imp_url.py @@ -34,6 +34,7 @@ def test_import_url(mocker, dvc): jobs=4, force=False, version_aware=False, + config=None, ) @@ -83,6 +84,7 @@ def test_import_url_no_exec_download_flags(mocker, flag, expected, dvc): jobs=None, force=False, version_aware=False, + config=None, **expected, ) @@ -115,6 +117,7 @@ def test_import_url_to_remote(mocker, dvc): jobs=None, force=False, version_aware=False, + config=None, ) diff --git a/tests/unit/command/test_ls_url.py b/tests/unit/command/test_ls_url.py index 84f341856fb..10fef6d560d 100644 --- a/tests/unit/command/test_ls_url.py +++ b/tests/unit/command/test_ls_url.py @@ -10,7 +10,7 @@ def test_ls_url(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", recursive=False) + m.assert_called_once_with("src", recursive=False, config=None) def test_recursive(mocker): @@ -21,4 +21,4 @@ def test_recursive(mocker): assert cmd.run() == 0 - m.assert_called_once_with("src", recursive=True) + m.assert_called_once_with("src", recursive=True, config=None)