From 82134e772da3b77351ed5bc403ac5a607ad794e3 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sun, 11 Jun 2023 10:53:05 +0800 Subject: [PATCH 1/3] feat: add option to expand vars when exporting Signed-off-by: Frost Ming --- src/pdm/cli/commands/export.py | 1 + src/pdm/cli/completions/pdm.bash | 2 +- src/pdm/cli/completions/pdm.fish | 1 + src/pdm/cli/completions/pdm.ps1 | 2 +- src/pdm/cli/completions/pdm.zsh | 1 + src/pdm/formats/requirements.py | 5 ++++- src/pdm/models/backends.py | 18 +++++++++++++----- 7 files changed, 22 insertions(+), 8 deletions(-) diff --git a/src/pdm/cli/commands/export.py b/src/pdm/cli/commands/export.py index b67c5db590..2f1702fc72 100644 --- a/src/pdm/cli/commands/export.py +++ b/src/pdm/cli/commands/export.py @@ -45,6 +45,7 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: action="store_true", help="Read the list of packages from pyproject.toml", ) + parser.add_argument("--expandvars", action="store_true", help="Expand environment variables in requirements") def handle(self, project: Project, options: argparse.Namespace) -> None: if options.pyproject: diff --git a/src/pdm/cli/completions/pdm.bash b/src/pdm/cli/completions/pdm.bash index c384aae6bd..4ae325e676 100644 --- a/src/pdm/cli/completions/pdm.bash +++ b/src/pdm/cli/completions/pdm.bash @@ -49,7 +49,7 @@ _pdm_a919b69078acdf0a_complete() ;; (export) - opts="--dev --format --global --group --help --lockfile --no-default --output --production --project --pyproject --verbose --without-hashes" + opts="--dev --expandvars --format --global --group --help --lockfile --no-default --output --production --project --pyproject --verbose --without-hashes" ;; (fix) diff --git a/src/pdm/cli/completions/pdm.fish b/src/pdm/cli/completions/pdm.fish index f15ad8effa..973250b5a7 100644 --- a/src/pdm/cli/completions/pdm.fish +++ b/src/pdm/cli/completions/pdm.fish @@ -102,6 +102,7 @@ complete -c pdm -A -n '__fish_seen_subcommand_from config' -l verbose -d 'Use `- # export complete -c pdm -A -n '__fish_seen_subcommand_from export' -l dev -d 'Select dev dependencies' +complete -c pdm -A -n '__fish_seen_subcommand_from export' -l expandvars -d 'Expand environment variables in pyproject.toml' complete -c pdm -A -n '__fish_seen_subcommand_from export' -l format -d 'Specify the export file format' complete -c pdm -A -n '__fish_seen_subcommand_from export' -l global -d 'Use the global project, supply the project root with `-p` option' complete -c pdm -A -n '__fish_seen_subcommand_from export' -l group -d 'Select group of optional-dependencies separated by comma or dev-dependencies (with `-d`). Can be supplied multiple times, use ":all" to include all groups under the same species.' diff --git a/src/pdm/cli/completions/pdm.ps1 b/src/pdm/cli/completions/pdm.ps1 index d5fa703a24..d58f68bbb0 100644 --- a/src/pdm/cli/completions/pdm.ps1 +++ b/src/pdm/cli/completions/pdm.ps1 @@ -247,7 +247,7 @@ function TabExpansion($line, $lastWord) { } "export" { $completer.AddOpts(@( - [Option]::new(@("--dev", "--output", "--global", "--no-default", "--prod", "--production", "-g", "-d", "-o", "--without-hashes", "-L", "--lockfile")), + [Option]::new(@("--dev", "--output", "--global", "--no-default", "--expandvars", "--prod", "--production", "-g", "-d", "-o", "--without-hashes", "-L", "--lockfile")), $formatOption, $sectionOption, $projectOption diff --git a/src/pdm/cli/completions/pdm.zsh b/src/pdm/cli/completions/pdm.zsh index 3356451a1a..f08c4ed0b9 100644 --- a/src/pdm/cli/completions/pdm.zsh +++ b/src/pdm/cli/completions/pdm.zsh @@ -145,6 +145,7 @@ _pdm() { {-g,--global}'[Use the global project, supply the project root with `-p` option]' {-f+,--format+}"[Specify the export file format]:format:(pipfile poetry flit requirements setuppy)" "--without-hashes[Don't include artifact hashes]" + "--expandvars[Expand environment variables in requirements]" {-L,--lockfile}'[Specify another lockfile path, or use `PDM_LOCKFILE` env variable. Default: pdm.lock]:lockfile:_files' {-o+,--output+}"[Write output to the given file, or print to stdout if not given]:output file:_files" {-G+,--group+}'[Select group of optional-dependencies or dev-dependencies(with -d). Can be supplied multiple times, use ":all" to include all groups under the same species]:group:_pdm_groups' diff --git a/src/pdm/formats/requirements.py b/src/pdm/formats/requirements.py index c18ebb9845..6207e230c5 100644 --- a/src/pdm/formats/requirements.py +++ b/src/pdm/formats/requirements.py @@ -9,6 +9,7 @@ from pdm.formats.base import make_array from pdm.models.requirements import FileRequirement, Requirement, parse_requirement +from pdm.utils import expand_env_vars_in_auth if TYPE_CHECKING: from argparse import Namespace @@ -187,7 +188,7 @@ def export( else: assert isinstance(candidate, Requirement) req = candidate - lines.append(project.backend.expand_line(req.as_line())) + lines.append(project.backend.expand_line(req.as_line(), options.expandvars)) if options.hashes and getattr(candidate, "hashes", None): for item in sorted(set(candidate.hashes.values())): # type: ignore[attr-defined] lines.append(f" \\\n --hash={item}") @@ -195,6 +196,8 @@ def export( sources = project.pyproject.settings.get("source", []) for source in sources: url = source["url"] + if options.expandvars: + url = expand_env_vars_in_auth(url) prefix = "--index-url" if source["name"] == "pypi" else "--extra-index-url" lines.append(f"{prefix} {url}\n") if not source.get("verify_ssl", True): diff --git a/src/pdm/models/backends.py b/src/pdm/models/backends.py index e1583a9a21..a1a1f45e39 100644 --- a/src/pdm/models/backends.py +++ b/src/pdm/models/backends.py @@ -14,7 +14,7 @@ class BuildBackend(metaclass=abc.ABCMeta): def __init__(self, root: Path) -> None: self.root = root - def expand_line(self, line: str) -> str: + def expand_line(self, line: str, expand_env: bool = True) -> str: return line def relative_path_to_url(self, path: str) -> str: @@ -45,8 +45,11 @@ def build_system(cls) -> dict: class PDMBackend(BuildBackend): - def expand_line(self, req: str) -> str: - return expand_env_vars(req).replace("file:///${PROJECT_ROOT}", path_to_url(self.root.as_posix())) + def expand_line(self, req: str, expand_env: bool = True) -> str: + line = req.replace("file:///${PROJECT_ROOT}", path_to_url(self.root.as_posix())) + if expand_env: + line = expand_env_vars(line) + return line def relative_path_to_url(self, path: str) -> str: if os.path.isabs(path): @@ -86,8 +89,13 @@ def __format__(self, __format_spec: str) -> str: class EnvContext: + def __init__(self, expand: bool = True) -> None: + self.expand = expand + def __format__(self, __format_spec: str) -> str: name, sep, default = __format_spec.partition(":") + if not self.expand: + return f"${{{name}}}" if name in os.environ: return os.environ[name] if not sep: @@ -96,9 +104,9 @@ def __format__(self, __format_spec: str) -> str: class HatchBackend(BuildBackend): - def expand_line(self, line: str) -> str: + def expand_line(self, line: str, expand_env: bool = True) -> str: return line.format( - env=EnvContext(), + env=EnvContext(expand=expand_env), root=PathContext(self.root), home=PathContext(Path.home()), ) From c744726338bca19c2f1cc053604f1342f42981db Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sun, 11 Jun 2023 10:53:46 +0800 Subject: [PATCH 2/3] add news Signed-off-by: Frost Ming --- news/1997.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 news/1997.feature.md diff --git a/news/1997.feature.md b/news/1997.feature.md new file mode 100644 index 0000000000..2b50d2bf4c --- /dev/null +++ b/news/1997.feature.md @@ -0,0 +1 @@ +Add option to expand environment variables when exporting requirements. From 032fe52e8a2223b986fd1f8810146caca25ca6f1 Mon Sep 17 00:00:00 2001 From: Frost Ming Date: Sun, 11 Jun 2023 11:24:34 +0800 Subject: [PATCH 3/3] fix tests Signed-off-by: Frost Ming --- tests/test_formats.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tests/test_formats.py b/tests/test_formats.py index 85b7c6056b..166fb79035 100644 --- a/tests/test_formats.py +++ b/tests/test_formats.py @@ -29,7 +29,7 @@ def test_convert_pipfile(project): def test_convert_requirements_file(project, is_dev): golden_file = FIXTURES / "requirements.txt" assert requirements.check_fingerprint(project, golden_file) - options = Namespace(dev=is_dev, group=None) + options = Namespace(dev=is_dev, group=None, expandvars=False) result, settings = requirements.convert(project, golden_file, options) group = settings["dev-dependencies"]["dev"] if is_dev else result["dependencies"] dev_group = settings["dev-dependencies"]["dev"] @@ -50,7 +50,7 @@ def test_convert_requirements_file_without_name(project, vcs): req_file = project.root.joinpath("reqs.txt") project.root.joinpath("reqs.txt").write_text("git+https://github.com/test-root/demo.git\n") assert requirements.check_fingerprint(project, str(req_file)) - result, _ = requirements.convert(project, str(req_file), Namespace(dev=False, group=None)) + result, _ = requirements.convert(project, str(req_file), Namespace(dev=False, group=None, expandvars=None)) assert result["dependencies"] == ["demo @ git+https://github.com/test-root/demo.git"] @@ -124,7 +124,7 @@ def test_convert_flit(project): def test_import_requirements_with_group(project): golden_file = FIXTURES / "requirements.txt" assert requirements.check_fingerprint(project, golden_file) - result, settings = requirements.convert(project, golden_file, Namespace(dev=False, group="test")) + result, settings = requirements.convert(project, golden_file, Namespace(dev=False, group="test", expandvars=False)) group = result["optional-dependencies"]["test"] dev_group = settings["dev-dependencies"]["dev"] @@ -139,16 +139,24 @@ def test_keep_env_vars_in_source(project, monkeypatch): monkeypatch.setenv("USER", "foo") monkeypatch.setenv("PASSWORD", "bar") project.pyproject.settings["source"] = [{"url": "https://${USER}:${PASSWORD}@test.pypi.org/simple", "name": "pypi"}] - result = requirements.export(project, [], Namespace()) + result = requirements.export(project, [], Namespace(expandvars=False)) assert result.strip().splitlines()[-1] == "--index-url https://${USER}:${PASSWORD}@test.pypi.org/simple" +def test_expand_env_vars_in_source(project, monkeypatch): + monkeypatch.setenv("USER", "foo") + monkeypatch.setenv("PASSWORD", "bar") + project.pyproject.settings["source"] = [{"url": "https://foo:bar@test.pypi.org/simple", "name": "pypi"}] + result = requirements.export(project, [], Namespace(expandvars=True)) + assert result.strip().splitlines()[-1] == "--index-url https://foo:bar@test.pypi.org/simple" + + def test_export_replace_project_root(project): artifact = FIXTURES / "artifacts/first-2.0.2-py2.py3-none-any.whl" shutil.copy2(artifact, project.root) with cd(project.root): req = parse_requirement(f"./{artifact.name}") - result = requirements.export(project, [req], Namespace(hashes=False)) + result = requirements.export(project, [req], Namespace(hashes=False, expandvars=False)) assert "${PROJECT_ROOT}" not in result