From 5d328a455039e68c6b168b47bb80f8e58a97494d Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 22 Sep 2024 13:36:12 -0400 Subject: [PATCH] Avoid retaining forks when `requires-python` range changes (#7624) ## Summary If the `requires-python` bound expands, the space covered by `resolution-markers` may no longer include all supported Python versions. In such cases, we need to avoid reusing the forks (but we _can_ reuse the preferred versions). Closes https://github.com/astral-sh/uv/issues/7618. --- crates/uv/src/commands/project/lock.rs | 64 +++---- crates/uv/tests/lock.rs | 228 ++++++++++++++++++++++++- 2 files changed, 261 insertions(+), 31 deletions(-) diff --git a/crates/uv/src/commands/project/lock.rs b/crates/uv/src/commands/project/lock.rs index 6506603dc9d0..5ea58d4050cd 100644 --- a/crates/uv/src/commands/project/lock.rs +++ b/crates/uv/src/commands/project/lock.rs @@ -476,14 +476,15 @@ async fn do_lock( debug!("Starting clean resolution"); // Determine whether we can reuse the existing package versions. - let reusable_lock = existing_lock.as_ref().and_then(|lock| match &lock { - ValidatedLock::Preferable(lock) => Some(lock), + let versions_lock = existing_lock.as_ref().and_then(|lock| match &lock { ValidatedLock::Satisfies(lock) => Some(lock), + ValidatedLock::Preferable(lock) => Some(lock), + ValidatedLock::Versions(lock) => Some(lock), ValidatedLock::Unusable(_) => None, }); // If an existing lockfile exists, build up a set of preferences. - let LockedRequirements { preferences, git } = reusable_lock + let LockedRequirements { preferences, git } = versions_lock .map(|lock| read_lock_requirements(lock, upgrade)) .unwrap_or_default(); @@ -493,13 +494,21 @@ async fn do_lock( state.git.insert(reference, sha); } + // Determine whether we can reuse the existing package forks. + let forks_lock = existing_lock.as_ref().and_then(|lock| match &lock { + ValidatedLock::Satisfies(lock) => Some(lock), + ValidatedLock::Preferable(lock) => Some(lock), + ValidatedLock::Versions(_) => None, + ValidatedLock::Unusable(_) => None, + }); + // When we run the same resolution from the lockfile again, we could get a different result the // second time due to the preferences causing us to skip a fork point (see the // `preferences-dependent-forking` packse scenario). To avoid this, we store the forks in the // lockfile. We read those after all the lockfile filters, to allow the forks to change when // the environment changed, e.g. the python bound check above can lead to different forking. let resolver_markers = ResolverMarkers::universal( - reusable_lock + forks_lock .map(|lock| lock.fork_markers().to_vec()) .unwrap_or_else(|| { environments @@ -582,13 +591,16 @@ async fn do_lock( #[derive(Debug)] enum ValidatedLock { - /// An existing lockfile was provided, but its contents should be ignored. - Unusable(Lock), /// An existing lockfile was provided, and it satisfies the workspace requirements. Satisfies(Lock), - /// An existing lockfile was provided, and the locked versions should be preferred if possible, - /// even though the lockfile does not satisfy the workspace requirements. + /// An existing lockfile was provided, but its contents should be ignored. + Unusable(Lock), + /// An existing lockfile was provided, and the locked versions and forks should be preferred if + /// possible, even though the lockfile does not satisfy the workspace requirements. Preferable(Lock), + /// An existing lockfile was provided, and the locked versions should be preferred if possible, + /// though the forks should be ignored. + Versions(Lock), } impl ValidatedLock { @@ -666,11 +678,17 @@ impl ValidatedLock { .map(SupportedEnvironments::as_markers) .unwrap_or_default() { - let _ = writeln!( - printer.stderr(), - "Ignoring existing lockfile due to change in supported environments" - ); - return Ok(Self::Unusable(lock)); + return Ok(Self::Versions(lock)); + } + + // If the Requires-Python bound has changed, we have to perform a clean resolution, since + // the set of `resolution-markers` may no longer cover the entire supported Python range. + if lock.requires_python().range() != requires_python.range() { + return if lock.fork_markers().is_empty() { + Ok(Self::Preferable(lock)) + } else { + Ok(Self::Versions(lock)) + }; } match upgrade { @@ -687,19 +705,6 @@ impl ValidatedLock { } } - // If the Requires-Python bound in the lockfile is weaker or equivalent to the - // Requires-Python bound in the workspace, we should have the necessary wheels to perform - // a locked resolution. - if lock.requires_python().range() != requires_python.range() { - // On the other hand, if the bound in the lockfile is stricter, meaning the - // bound has since been weakened, we have to perform a clean resolution to ensure - // we fetch the necessary wheels. - debug!("Ignoring existing lockfile due to change in `requires-python`"); - - // It's fine to prefer the existing versions, though. - return Ok(Self::Preferable(lock)); - } - // If the user provided at least one index URL (from the command line, or from a configuration // file), don't use the existing lockfile if it references any registries that are no longer // included in the current configuration. @@ -835,9 +840,10 @@ impl ValidatedLock { #[must_use] fn into_lock(self) -> Lock { match self { - ValidatedLock::Unusable(lock) => lock, - ValidatedLock::Satisfies(lock) => lock, - ValidatedLock::Preferable(lock) => lock, + Self::Unusable(lock) => lock, + Self::Satisfies(lock) => lock, + Self::Preferable(lock) => lock, + Self::Versions(lock) => lock, } } } diff --git a/crates/uv/tests/lock.rs b/crates/uv/tests/lock.rs index 7c14ab160d11..a17c062c9374 100644 --- a/crates/uv/tests/lock.rs +++ b/crates/uv/tests/lock.rs @@ -10805,7 +10805,6 @@ fn lock_constrained_environment() -> Result<()> { ----- stdout ----- ----- stderr ----- - Ignoring existing lockfile due to change in supported environments Resolved 8 packages in [TIME] error: The lockfile at `uv.lock` needs to be updated, but `--locked` was provided. To update the lockfile, run `uv lock`. "###); @@ -10816,7 +10815,6 @@ fn lock_constrained_environment() -> Result<()> { ----- stdout ----- ----- stderr ----- - Ignoring existing lockfile due to change in supported environments Resolved 8 packages in [TIME] Added colorama v0.4.6 "###); @@ -13099,3 +13097,229 @@ fn lock_invalid_project_table() -> Result<()> { Ok(()) } + +/// See: +#[test] +fn lock_change_requires_python() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.12" + dependencies = [ + "anyio <3 ; python_version == '3.12'", + "anyio >3, <4 ; python_version > '3.12'", + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 5 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.12" + resolution-markers = [ + "python_full_version < '3.13'", + "python_full_version >= '3.13'", + ] + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "anyio" + version = "2.2.0" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "python_full_version < '3.13'", + ] + dependencies = [ + { name = "idna", marker = "python_full_version < '3.13'" }, + { name = "sniffio", marker = "python_full_version < '3.13'" }, + ] + sdist = { url = "https://files.pythonhosted.org/packages/d3/e6/901a94731af20e7109415525666cb3753a2bd1edd19616c2730448dffd0d/anyio-2.2.0.tar.gz", hash = "sha256:4a41c5b3a65ed92e469d51b6fba3779301850ea2e352afcf9e36c46f21ee14a9", size = 97217 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/49/c3/b83a3c02c7d6f66932e9a72621d7f207cbfd2bd72b4c8931567ee386fb55/anyio-2.2.0-py3-none-any.whl", hash = "sha256:aa3da546ed17f097ca876c78024dea380a3b7fa80759abfdda59f12176a3dac8", size = 65320 }, + ] + + [[package]] + name = "anyio" + version = "3.7.1" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "python_full_version >= '3.13'", + ] + dependencies = [ + { name = "idna", marker = "python_full_version >= '3.13'" }, + { name = "sniffio", marker = "python_full_version >= '3.13'" }, + ] + sdist = { url = "https://files.pythonhosted.org/packages/28/99/2dfd53fd55ce9838e6ff2d4dac20ce58263798bd1a0dbe18b3a9af3fcfce/anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780", size = 142927 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/19/24/44299477fe7dcc9cb58d0a57d5a7588d6af2ff403fdd2d47a246c91a3246/anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5", size = 80896 }, + ] + + [[package]] + name = "idna" + version = "3.6" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", size = 175426 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f", size = 61567 }, + ] + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "anyio", version = "2.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.13'" }, + { name = "anyio", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + ] + + [package.metadata] + requires-dist = [ + { name = "anyio", marker = "python_full_version == '3.12.*'", specifier = "<3" }, + { name = "anyio", marker = "python_full_version >= '3.13'", specifier = ">3,<4" }, + ] + + [[package]] + name = "sniffio" + version = "1.3.1" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, + ] + "### + ); + }); + + // Lower the `requires-python`, expanding the set of supported versions. Loosen the upper-bound + // on `anyio` to ensure that we respect the already-locked version. + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.10" + dependencies = [ + "anyio <3 ; python_version == '3.12'", + "anyio >3 ; python_version > '3.12'", + ] + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 5 packages in [TIME] + "###); + + let lock = fs_err::read_to_string(context.temp_dir.join("uv.lock")).unwrap(); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.10" + resolution-markers = [ + "python_full_version < '3.12'", + "python_full_version == '3.12.*'", + "python_full_version >= '3.13'", + ] + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "anyio" + version = "2.2.0" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "python_full_version == '3.12.*'", + ] + dependencies = [ + { name = "idna", marker = "python_full_version == '3.12.*'" }, + { name = "sniffio", marker = "python_full_version == '3.12.*'" }, + ] + sdist = { url = "https://files.pythonhosted.org/packages/d3/e6/901a94731af20e7109415525666cb3753a2bd1edd19616c2730448dffd0d/anyio-2.2.0.tar.gz", hash = "sha256:4a41c5b3a65ed92e469d51b6fba3779301850ea2e352afcf9e36c46f21ee14a9", size = 97217 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/49/c3/b83a3c02c7d6f66932e9a72621d7f207cbfd2bd72b4c8931567ee386fb55/anyio-2.2.0-py3-none-any.whl", hash = "sha256:aa3da546ed17f097ca876c78024dea380a3b7fa80759abfdda59f12176a3dac8", size = 65320 }, + ] + + [[package]] + name = "anyio" + version = "3.7.1" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "python_full_version >= '3.13'", + ] + dependencies = [ + { name = "idna", marker = "python_full_version >= '3.13'" }, + { name = "sniffio", marker = "python_full_version >= '3.13'" }, + ] + sdist = { url = "https://files.pythonhosted.org/packages/28/99/2dfd53fd55ce9838e6ff2d4dac20ce58263798bd1a0dbe18b3a9af3fcfce/anyio-3.7.1.tar.gz", hash = "sha256:44a3c9aba0f5defa43261a8b3efb97891f2bd7d804e0e1f56419befa1adfc780", size = 142927 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/19/24/44299477fe7dcc9cb58d0a57d5a7588d6af2ff403fdd2d47a246c91a3246/anyio-3.7.1-py3-none-any.whl", hash = "sha256:91dee416e570e92c64041bd18b900d1d6fa78dff7048769ce5ac5ddad004fbb5", size = 80896 }, + ] + + [[package]] + name = "idna" + version = "3.6" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", size = 175426 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f", size = 61567 }, + ] + + [[package]] + name = "project" + version = "0.1.0" + source = { virtual = "." } + dependencies = [ + { name = "anyio", version = "2.2.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.12.*'" }, + { name = "anyio", version = "3.7.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.13'" }, + ] + + [package.metadata] + requires-dist = [ + { name = "anyio", marker = "python_full_version == '3.12.*'", specifier = "<3" }, + { name = "anyio", marker = "python_full_version >= '3.13'", specifier = ">3" }, + ] + + [[package]] + name = "sniffio" + version = "1.3.1" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, + ] + "### + ); + }); + + Ok(()) +}