diff --git a/docs/history/hatch.md b/docs/history/hatch.md index 0cfd412c8..834ba4704 100644 --- a/docs/history/hatch.md +++ b/docs/history/hatch.md @@ -15,13 +15,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ***Added:*** - Add ability to control the source of Python distributions +- Upgrade PyApp to 0.22.0 for binary builds ***Fixed:*** - The `fmt` command no longer hides the commands that are being executed - Add default timeout for network requests, useful when installing Python distributions - Fix syntax highlighting contrast for the `config show` command -- Upgrade PyApp to 0.22.0 for binary builds ## [1.11.1](https://github.com/pypa/hatch/releases/tag/hatch-v1.11.1) - 2024-05-23 ## {: #hatch-v1.11.1 } diff --git a/docs/how-to/python/custom.md b/docs/how-to/python/custom.md new file mode 100644 index 000000000..621eb1c90 --- /dev/null +++ b/docs/how-to/python/custom.md @@ -0,0 +1,30 @@ +# How to use custom Python distributions + +---- + +The built-in [Python management](../../tutorials/python/manage.md) capabilities offer full support for using custom distributions. + +## Configuration + +Configuring custom Python distributions is done entirely through three environment variables that must all be defined, for each desired distribution. In the following sections, the placeholder `` is the uppercased version of the distribution name with periods replaced by underscores e.g. `pypy3.10` would become `PYPY3_10`. + +### Source + +The `HATCH_PYTHON_CUSTOM_SOURCE_` variable is the URL to the distribution's archive. The value must end with the archive's real file extension, which is used to determine the extraction method. + +The following extensions are supported: + +| Extensions | Description | +| --- | --- | +| | A [tar file](https://en.wikipedia.org/wiki/Tar_(computing)) with [bzip2 compression](https://en.wikipedia.org/wiki/Bzip2) | +| | A [tar file](https://en.wikipedia.org/wiki/Tar_(computing)) with [gzip compression](https://en.wikipedia.org/wiki/Gzip) | +| | A [tar file](https://en.wikipedia.org/wiki/Tar_(computing)) with [Zstandard compression](https://en.wikipedia.org/wiki/Zstd) | +| | A [ZIP file](https://en.wikipedia.org/wiki/ZIP_(file_format)) with [DEFLATE compression](https://en.wikipedia.org/wiki/Deflate) | + +### Python path + +The `HATCH_PYTHON_CUSTOM_PATH_` variable is the path to the Python interpreter within the archive. This path is relative to the root of the archive and must be a Unix-style path, even on Windows. + +### Version + +The `HATCH_PYTHON_CUSTOM_VERSION_` variable is the version of the distribution. This value is used to determine whether updates are required and is displayed in the output of the [`python show`](../../cli/reference.md#hatch-python-show) command. diff --git a/mkdocs.yml b/mkdocs.yml index 28b8d2074..152245532 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -131,11 +131,13 @@ nav: - Dependency resolution: how-to/environment/dependency-resolution.md - Static analysis: - Customize behavior: how-to/static-analysis/behavior.md - - Plugins: - - Testing builds: how-to/plugins/testing-builds.md + - Python: + - Custom distributions: how-to/python/custom.md - Publishing: - Authentication: how-to/publish/auth.md - Repository selection: how-to/publish/repo.md + - Plugins: + - Testing builds: how-to/plugins/testing-builds.md - Tutorials: - Python: - Management: tutorials/python/manage.md diff --git a/src/hatch/config/constants.py b/src/hatch/config/constants.py index dec52f847..2ad442fe1 100644 --- a/src/hatch/config/constants.py +++ b/src/hatch/config/constants.py @@ -30,4 +30,6 @@ class PublishEnvVars: class PythonEnvVars: - SOURCE_PREFIX = 'HATCH_PYTHON_SOURCE_' + CUSTOM_SOURCE_PREFIX = 'HATCH_PYTHON_CUSTOM_SOURCE_' + CUSTOM_PATH_PREFIX = 'HATCH_PYTHON_CUSTOM_PATH_' + CUSTOM_VERSION_PREFIX = 'HATCH_PYTHON_CUSTOM_VERSION_' diff --git a/src/hatch/python/resolve.py b/src/hatch/python/resolve.py index 67ecf2980..f25002cdb 100644 --- a/src/hatch/python/resolve.py +++ b/src/hatch/python/resolve.py @@ -17,6 +17,26 @@ from hatch.utils.fs import Path +# Use an artificially high epoch to ensure that custom distributions are always considered newer +CUSTOM_DISTRIBUTION_VERSION_EPOCH = 100 + + +def custom_env_var(prefix: str, name: str) -> str: + return f'{prefix}{name.upper().replace(".", "_")}' + + +def get_custom_source(name: str) -> str | None: + return os.environ.get(custom_env_var(PythonEnvVars.CUSTOM_SOURCE_PREFIX, name)) + + +def get_custom_version(name: str) -> str | None: + return os.environ.get(custom_env_var(PythonEnvVars.CUSTOM_VERSION_PREFIX, name)) + + +def get_custom_path(name: str) -> str | None: + return os.environ.get(custom_env_var(PythonEnvVars.CUSTOM_PATH_PREFIX, name)) + + class Distribution(ABC): def __init__(self, name: str, source: str) -> None: self.__name = name @@ -28,8 +48,7 @@ def name(self) -> str: @cached_property def source(self) -> str: - env_var = f'{PythonEnvVars.SOURCE_PREFIX}{self.name.upper().replace(".", "_")}' - return os.environ.get(env_var, self.__source) + return self.__source if (custom_source := get_custom_source(self.name)) is None else custom_source @cached_property def archive_name(self) -> str: @@ -41,7 +60,7 @@ def unpack(self, archive: Path, directory: Path) -> None: with zipfile.ZipFile(archive, 'r') as zf: zf.extractall(directory) - elif self.source.endswith('.tar.gz'): + elif self.source.endswith(('.tar.gz', '.tgz')): import tarfile with tarfile.open(archive, 'r:gz') as tf: @@ -49,7 +68,7 @@ def unpack(self, archive: Path, directory: Path) -> None: tf.extractall(directory, filter='data') else: tf.extractall(directory) # noqa: S202 - elif self.source.endswith('.tar.bz2'): + elif self.source.endswith(('.tar.bz2', '.bz2')): import tarfile with tarfile.open(archive, 'r:bz2') as tf: @@ -57,7 +76,7 @@ def unpack(self, archive: Path, directory: Path) -> None: tf.extractall(directory, filter='data') else: tf.extractall(directory) # noqa: S202 - elif self.source.endswith('.tar.zst'): + elif self.source.endswith(('.tar.zst', '.tar.zstd')): import tarfile import zstandard @@ -89,6 +108,9 @@ class CPythonStandaloneDistribution(Distribution): def version(self) -> Version: from packaging.version import Version + if (custom_version := get_custom_version(self.name)) is not None: + return Version(f'{CUSTOM_DISTRIBUTION_VERSION_EPOCH}!{custom_version}') + # .../cpython-3.12.0%2B20231002-... # .../cpython-3.7.9-... _, _, remaining = self.source.partition('/cpython-') @@ -99,6 +121,9 @@ def version(self) -> Version: @cached_property def python_path(self) -> str: + if (custom_path := get_custom_path(self.name)) is not None: + return custom_path + if self.name == '3.7': if sys.platform == 'win32': return r'python\install\python.exe' @@ -116,12 +141,18 @@ class PyPyOfficialDistribution(Distribution): def version(self) -> Version: from packaging.version import Version + if (custom_version := get_custom_version(self.name)) is not None: + return Version(f'{CUSTOM_DISTRIBUTION_VERSION_EPOCH}!{custom_version}') + *_, remaining = self.source.partition('/pypy/') _, version, *_ = remaining.split('-') return Version(f'0!{version[1:]}') @cached_property def python_path(self) -> str: + if (custom_path := get_custom_path(self.name)) is not None: + return custom_path + directory = self.archive_name for extension in ('.tar.bz2', '.zip'): if directory.endswith(extension): diff --git a/tests/python/test_core.py b/tests/python/test_core.py index 06661d8fc..1b0eae919 100644 --- a/tests/python/test_core.py +++ b/tests/python/test_core.py @@ -5,7 +5,7 @@ from hatch.config.constants import PythonEnvVars from hatch.python.core import InstalledDistribution, PythonManager from hatch.python.distributions import ORDERED_DISTRIBUTIONS -from hatch.python.resolve import get_distribution +from hatch.python.resolve import custom_env_var, get_distribution from hatch.utils.structures import EnvVars @@ -15,7 +15,7 @@ def test_custom_source(platform, current_arch, name): pytest.skip('No macOS 3.7 distribution for ARM') dist = get_distribution(name) - with EnvVars({f'{PythonEnvVars.SOURCE_PREFIX}{name.upper().replace(".", "_")}': 'foo'}): + with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_SOURCE_PREFIX, name): 'foo'}): assert dist.source == 'foo' diff --git a/tests/python/test_resolve.py b/tests/python/test_resolve.py index c46566391..7af955cf7 100644 --- a/tests/python/test_resolve.py +++ b/tests/python/test_resolve.py @@ -3,8 +3,9 @@ import pytest +from hatch.config.constants import PythonEnvVars from hatch.errors import PythonDistributionResolutionError, PythonDistributionUnknownError -from hatch.python.resolve import get_distribution +from hatch.python.resolve import custom_env_var, get_distribution from hatch.utils.structures import EnvVars @@ -34,6 +35,15 @@ def test_cpython_standalone(self): assert version.epoch == 0 assert version.base_version == '3.11.3' + def test_cpython_standalone_custom(self): + name = '3.11' + dist = get_distribution(name) + with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_VERSION_PREFIX, name): '9000.42'}): + version = dist.version + + assert version.epoch == 100 + assert '.'.join(map(str, version.release)) == '9000.42' + def test_pypy(self): url = 'https://downloads.python.org/pypy/pypy3.10-v7.3.12-aarch64.tar.bz2' dist = get_distribution('pypy3.10', url) @@ -42,6 +52,29 @@ def test_pypy(self): assert version.epoch == 0 assert version.base_version == '7.3.12' + def test_pypy_custom(self): + name = 'pypy3.10' + dist = get_distribution(name) + with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_VERSION_PREFIX, name): '9000.42'}): + version = dist.version + + assert version.epoch == 100 + assert '.'.join(map(str, version.release)) == '9000.42' + + +class TestDistributionPaths: + def test_cpython_standalone_custom(self): + name = '3.11' + dist = get_distribution(name) + with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_PATH_PREFIX, name): 'foo/bar/python'}): + assert dist.python_path == 'foo/bar/python' + + def test_pypy_custom(self): + name = 'pypy3.10' + dist = get_distribution(name) + with EnvVars({custom_env_var(PythonEnvVars.CUSTOM_PATH_PREFIX, name): 'foo/bar/python'}): + assert dist.python_path == 'foo/bar/python' + @pytest.mark.parametrize( ('system', 'variant'),