From 19db87553e3cfc8ed32d2220ef8df3f8367033b2 Mon Sep 17 00:00:00 2001 From: Florent Jeannot <12172017+FlorentJeannot@users.noreply.github.com> Date: Mon, 7 Aug 2023 13:13:45 +0200 Subject: [PATCH 1/2] Direct references show extra requirements in .txt files (#1582) --- piptools/utils.py | 4 +++- tests/test_cli_compile.py | 7 +++++-- tests/test_utils.py | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index 402175347..fdfba40e7 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -155,7 +155,9 @@ def _build_direct_reference_best_efforts(ireq: InstallRequirement) -> str: # If we get here then we have a requirement that supports direct reference. # We need to remove the egg if it exists and keep the rest of the fragments. - direct_reference = f"{ireq.name.lower()} @ {ireq.link.url_without_fragment}" + lowered_ireq_name = canonicalize_name(ireq.name) + extras = f"[{','.join(sorted(ireq.extras))}]" if ireq.extras else "" + direct_reference = f"{lowered_ireq_name}{extras} @ {ireq.link.url_without_fragment}" fragments = [] # Check if there is any fragment to add to the URI. diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index ac75927ea..498b4a80b 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -779,7 +779,10 @@ def test_direct_reference_with_extras(runner): ) out = runner.invoke(cli, ["-n", "--rebuild", "--no-build-isolation"]) assert out.exit_code == 0 - assert "pip-tools @ git+https://github.com/jazzband/pip-tools@6.2.0" in out.stderr + assert ( + "pip-tools[coverage,testing] @ git+https://github.com/jazzband/pip-tools@6.2.0" + in out.stderr + ) assert "pytest==" in out.stderr assert "pytest-cov==" in out.stderr @@ -2955,7 +2958,7 @@ def test_compile_recursive_extras(runner, tmp_path, current_resolver): os.fspath(tmp_path / "pyproject.toml"), ], ) - expected = rf"""foo @ {tmp_path.as_uri()} + expected = rf"""foo[footest] @ {tmp_path.as_uri()} small-fake-a==0.2 small-fake-b==0.3 """ diff --git a/tests/test_utils.py b/tests/test_utils.py index c549a0b71..c89461fe0 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -79,6 +79,11 @@ def test_format_requirement(from_line): "example @ https://example.com/example.zip?egg=test", id="direct reference with egg in query", ), + pytest.param( + "example[b,c,a] @ https://example.com/example.zip", + "example[a,b,c] @ https://example.com/example.zip", + id="direct reference with optional dependency", + ), pytest.param( "file:./vendor/package.zip", "file:./vendor/package.zip", From d1e92150c229d08e6975bffdf5bca9037a55e697 Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Mon, 7 Aug 2023 17:21:11 +0200 Subject: [PATCH 2/2] Fix revealed default config in header if requirements in subfolder (#1904) Co-authored-by: Sviatoslav Sydorenko Co-authored-by: Ryan Walker --- piptools/locations.py | 4 +- piptools/scripts/compile.py | 8 ++-- piptools/scripts/sync.py | 8 ++-- piptools/utils.py | 13 ++++-- tests/conftest.py | 15 ++++--- tests/test_cli_compile.py | 12 ++++++ tests/test_cli_sync.py | 13 ++++++ tests/test_utils.py | 81 +++++++++++++++++++++++++++++++++++++ 8 files changed, 136 insertions(+), 18 deletions(-) diff --git a/piptools/locations.py b/piptools/locations.py index f31891cae..fa2413fc7 100644 --- a/piptools/locations.py +++ b/piptools/locations.py @@ -5,5 +5,5 @@ # The user_cache_dir helper comes straight from pip itself CACHE_DIR = user_cache_dir("pip-tools") -# The project defaults specific to pip-tools should be written to this filename -CONFIG_FILE_NAME = ".pip-tools.toml" +# The project defaults specific to pip-tools should be written to this filenames +DEFAULT_CONFIG_FILE_NAMES = (".pip-tools.toml", "pyproject.toml") diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 697c45074..acb0a937d 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -20,7 +20,7 @@ from .._compat import parse_requirements from ..cache import DependencyCache from ..exceptions import NoCandidateFound, PipToolsError -from ..locations import CACHE_DIR, CONFIG_FILE_NAME +from ..locations import CACHE_DIR, DEFAULT_CONFIG_FILE_NAMES from ..logging import log from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository @@ -314,8 +314,10 @@ def _determine_linesep( allow_dash=False, path_type=str, ), - help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " - "pyproject.toml.", + help=( + f"Read configuration from TOML file. By default, looks for the following " + f"files in the given order: {', '.join(DEFAULT_CONFIG_FILE_NAMES)}." + ), is_eager=True, callback=override_defaults_from_config_file, ) diff --git a/piptools/scripts/sync.py b/piptools/scripts/sync.py index e162feac6..307ce1b21 100755 --- a/piptools/scripts/sync.py +++ b/piptools/scripts/sync.py @@ -17,7 +17,7 @@ from .. import sync from .._compat import Distribution, parse_requirements from ..exceptions import PipToolsError -from ..locations import CONFIG_FILE_NAME +from ..locations import DEFAULT_CONFIG_FILE_NAMES from ..logging import log from ..repositories import PyPIRepository from ..utils import ( @@ -98,8 +98,10 @@ allow_dash=False, path_type=str, ), - help=f"Read configuration from TOML file. By default, looks for a {CONFIG_FILE_NAME} or " - "pyproject.toml.", + help=( + f"Read configuration from TOML file. By default, looks for the following " + f"files in the given order: {', '.join(DEFAULT_CONFIG_FILE_NAMES)}." + ), is_eager=True, callback=override_defaults_from_config_file, ) diff --git a/piptools/utils.py b/piptools/utils.py index fdfba40e7..70f1ee17f 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -12,6 +12,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, Iterable, Iterator, TypeVar, cast +from click.core import ParameterSource + if sys.version_info >= (3, 11): import tomllib else: @@ -31,7 +33,7 @@ from pip._vendor.pkg_resources import get_distribution from piptools._compat import PIP_VERSION -from piptools.locations import CONFIG_FILE_NAME +from piptools.locations import DEFAULT_CONFIG_FILE_NAMES from piptools.subprocess_utils import run_python_snippet if TYPE_CHECKING: @@ -369,8 +371,11 @@ def get_compile_command(click_ctx: click.Context) -> str: # Exclude config option if it's the default one if option_long_name == "--config": - default_config = select_config_file(click_ctx.params.get("src_files", ())) - if value == default_config: + parameter_source = click_ctx.get_parameter_source(option_name) + if ( + str(value) in DEFAULT_CONFIG_FILE_NAMES + or parameter_source == ParameterSource.DEFAULT + ): continue # Skip options without a value @@ -656,7 +661,7 @@ def select_config_file(src_files: tuple[str, ...]) -> Path | None: ( candidate_dir / config_file for candidate_dir in candidate_dirs - for config_file in (CONFIG_FILE_NAME, "pyproject.toml") + for config_file in DEFAULT_CONFIG_FILE_NAMES if (candidate_dir / config_file).is_file() ), None, diff --git a/tests/conftest.py b/tests/conftest.py index 62c3b22f6..4834f4bc1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -31,7 +31,7 @@ from piptools._compat import PIP_VERSION, Distribution from piptools.cache import DependencyCache from piptools.exceptions import NoCandidateFound -from piptools.locations import CONFIG_FILE_NAME +from piptools.locations import DEFAULT_CONFIG_FILE_NAMES from piptools.logging import log from piptools.repositories import PyPIRepository from piptools.repositories.base import BaseRepository @@ -452,13 +452,16 @@ def make_config_file(tmpdir_cwd): """ def _maker( - pyproject_param: str, new_default: Any, config_file_name: str = CONFIG_FILE_NAME + pyproject_param: str, + new_default: Any, + config_file_name: str = DEFAULT_CONFIG_FILE_NAMES[0], ) -> Path: - # Make a config file with this one config default override - config_path = tmpdir_cwd / pyproject_param - config_file = config_path / config_file_name - config_path.mkdir(exist_ok=True) + # Create a nested directory structure if config_file_name includes directories + config_dir = (tmpdir_cwd / config_file_name).parent + config_dir.mkdir(exist_ok=True, parents=True) + # Make a config file with this one config default override + config_file = tmpdir_cwd / config_file_name config_to_dump = {"tool": {"pip-tools": {pyproject_param: new_default}}} config_file.write_text(tomli_w.dumps(config_to_dump)) return cast(Path, config_file.relative_to(tmpdir_cwd)) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 498b4a80b..7dd301399 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -2978,6 +2978,18 @@ def test_config_option(pip_conf, runner, tmp_path, make_config_file): assert "Dry-run, so nothing updated" in out.stderr +def test_default_config_option(pip_conf, runner, make_config_file, tmpdir_cwd): + make_config_file("dry-run", True) + + req_in = tmpdir_cwd / "requirements.in" + req_in.touch() + + out = runner.invoke(cli) + + assert out.exit_code == 0 + assert "Dry-run, so nothing updated" in out.stderr + + def test_no_config_option_overrides_config_with_defaults( pip_conf, runner, tmp_path, make_config_file ): diff --git a/tests/test_cli_sync.py b/tests/test_cli_sync.py index 88457bda9..20e9689ec 100644 --- a/tests/test_cli_sync.py +++ b/tests/test_cli_sync.py @@ -374,6 +374,19 @@ def test_default_python_executable_option(run, runner): ] +@mock.patch("piptools.sync.run") +def test_default_config_option(run, runner, make_config_file, tmpdir_cwd): + make_config_file("dry-run", True) + + with open(sync.DEFAULT_REQUIREMENTS_FILE, "w") as reqs_txt: + reqs_txt.write("six==1.10.0") + + out = runner.invoke(cli) + + assert out.exit_code == 1 + assert "Would install:" in out.stdout + + @mock.patch("piptools.sync.run") def test_config_option(run, runner, make_config_file): config_file = make_config_file("dry-run", True) diff --git a/tests/test_utils.py b/tests/test_utils.py index c89461fe0..7d31af92f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -6,6 +6,7 @@ import shlex import sys from pathlib import Path +from textwrap import dedent import pip import pytest @@ -31,6 +32,7 @@ lookup_table, lookup_table_from_tuples, override_defaults_from_config_file, + select_config_file, ) @@ -415,6 +417,36 @@ def test_get_compile_command_with_config(tmpdir_cwd, config_file, expected_comma assert get_compile_command(ctx) == expected_command +@pytest.mark.parametrize("config_file", ("pyproject.toml", ".pip-tools.toml")) +@pytest.mark.parametrize( + "config_file_content", + ( + pytest.param("", id="empty config file"), + pytest.param("[tool.pip-tools]", id="empty config section"), + pytest.param("[tool.pip-tools]\ndry-run = true", id="non-empty config section"), + ), +) +def test_get_compile_command_does_not_include_default_config_if_reqs_file_in_subdir( + tmpdir_cwd, config_file, config_file_content +): + """ + Test that ``get_compile_command`` does not include default config file + if requirements file is in a subdirectory. + Regression test for issue GH-1903. + """ + default_config_file = Path(config_file) + default_config_file.write_text(config_file_content) + + (tmpdir_cwd / "subdir").mkdir() + req_file = Path("subdir/requirements.in") + req_file.touch() + req_file.write_bytes(b"") + + # Make sure that the default config file is not included + with compile_cli.make_context("pip-compile", [req_file.as_posix()]) as ctx: + assert get_compile_command(ctx) == f"pip-compile {req_file.as_posix()}" + + def test_get_compile_command_escaped_filenames(tmpdir_cwd): """ Test that get_compile_command output (re-)escapes ' -- '-escaped filenames. @@ -683,3 +715,52 @@ def test_callback_config_file_defaults_unreadable_toml(make_config_file): "config", "/dev/null/path/does/not/exist/my-config.toml", ) + + +def test_select_config_file_no_files(tmpdir_cwd): + assert select_config_file(()) is None + + +@pytest.mark.parametrize("filename", ("pyproject.toml", ".pip-tools.toml")) +def test_select_config_file_returns_config_in_cwd(make_config_file, filename): + config_file = make_config_file("dry-run", True, filename) + assert select_config_file(()) == config_file + + +def test_select_config_file_returns_empty_config_file_in_cwd(tmpdir_cwd): + config_file = Path(".pip-tools.toml") + config_file.touch() + + assert select_config_file(()) == config_file + + +def test_select_config_file_cannot_find_config_in_cwd(tmpdir_cwd, make_config_file): + make_config_file("dry-run", True, "subdir/pyproject.toml") + assert select_config_file(()) is None + + +def test_select_config_file_with_config_file_in_subdir(tmpdir_cwd, make_config_file): + config_file = make_config_file("dry-run", True, "subdir/.pip-tools.toml") + + requirement_file = Path("subdir/requirements.in") + requirement_file.touch() + + assert select_config_file((requirement_file.as_posix(),)) == config_file + + +def test_select_config_file_prefers_pip_tools_toml_over_pyproject_toml(tmpdir_cwd): + pip_tools_file = Path(".pip-tools.toml") + pip_tools_file.touch() + + pyproject_file = Path("pyproject.toml") + pyproject_file.write_text( + dedent( + """\ + [build-system] + requires = ["setuptools>=63", "setuptools_scm[toml]>=7"] + build-backend = "setuptools.build_meta" + """ + ) + ) + + assert select_config_file(()) == pip_tools_file