diff --git a/build/mach_initialize.py b/build/mach_initialize.py index 1e66e9fb164ed..e4616d2847c37 100644 --- a/build/mach_initialize.py +++ b/build/mach_initialize.py @@ -9,6 +9,7 @@ import platform import shutil import site +import subprocess import sys if sys.version_info[0] < 3: @@ -167,6 +168,11 @@ class MetaPathFinder(object): """.strip() +def _scrub_system_site_packages(): + site_paths = set(site.getsitepackages() + [site.getusersitepackages()]) + sys.path = [path for path in sys.path if path not in site_paths] + + def _activate_python_environment(topsrcdir): # We need the "mach" module to access the logic to parse virtualenv # requirements. Since that depends on "packaging" (and, transitively, @@ -193,6 +199,65 @@ def _activate_python_environment(topsrcdir): True, os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"), ) + + if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"): + env_var = ( + "MOZ_AUTOMATION" + if os.environ.get("MOZ_AUTOMATION") + else "MACH_USE_SYSTEM_PYTHON" + ) + + has_pip = ( + subprocess.run( + [sys.executable, "-c", "import pip"], stderr=subprocess.DEVNULL + ).returncode + == 0 + ) + # There are environments in CI that aren't prepared to provide any Mach dependency + # packages. Changing this is a nontrivial endeavour, so guard against having + # non-optional Mach requirements. + assert ( + not requirements.pypi_requirements + ), "Mach pip package requirements must be optional." + if has_pip: + pip = [sys.executable, "-m", "pip"] + check_result = subprocess.run( + pip + ["check"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + if check_result.returncode: + print(check_result.stdout, file=sys.stderr) + subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr) + raise Exception( + 'According to "pip check", the current Python ' + "environment has package-compatibility issues." + ) + + package_result = requirements.validate_environment_packages(pip) + if not package_result.has_all_packages: + print( + "Skipping automatic management of Python dependencies since " + f"the '{env_var}' environment variable is set.\n" + "The following issues were found while validating your Python " + "environment:" + ) + print(package_result.report()) + sys.exit(1) + else: + # Pip isn't installed to the system Python environment, so we can't use + # it to verify compatibility with Mach. Remove the system site-packages + # from the import scope so that Mach behaves as though all of its + # (optional) dependencies are not installed. + _scrub_system_site_packages() + + elif sys.prefix == sys.base_prefix: + # We're in an environment where we normally use the Mach virtualenv, + # but we're running a "nativecmd" such as "create-mach-environment". + # Remove global site packages from sys.path to improve isolation accordingly. + _scrub_system_site_packages() + sys.path[0:0] = [ os.path.join(topsrcdir, pth.path) for pth in requirements.pth_requirements + requirements.vendored_requirements @@ -222,12 +287,6 @@ def initialize(topsrcdir): if os.path.exists(deleted_dir): shutil.rmtree(deleted_dir, ignore_errors=True) - if sys.prefix == sys.base_prefix: - # We are not in a virtualenv. Remove global site packages - # from sys.path. - site_paths = set(site.getsitepackages() + [site.getusersitepackages()]) - sys.path = [path for path in sys.path if path not in site_paths] - state_dir = _create_state_dir() _activate_python_environment(topsrcdir) diff --git a/python/mach/mach/requirements.py b/python/mach/mach/requirements.py index cf2f89abb92c6..2be2d5f1cd2d0 100644 --- a/python/mach/mach/requirements.py +++ b/python/mach/mach/requirements.py @@ -1,9 +1,10 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - +import json import os from pathlib import Path +import subprocess from packaging.requirements import Requirement @@ -13,6 +14,26 @@ """.strip() +class EnvironmentPackageValidationResult: + def __init__(self): + self._package_discrepancies = [] + self.has_all_packages = True + + def add_discrepancy(self, requirement, found): + self._package_discrepancies.append((requirement, found)) + self.has_all_packages = False + + def report(self): + lines = [] + for requirement, found in self._package_discrepancies: + if found: + error = f'Installed with unexpected version "{found}"' + else: + error = "Not installed" + lines.append(f"{requirement}: {error}") + return "\n".join(lines) + + class PthSpecifier: def __init__(self, path): self.path = path @@ -61,6 +82,35 @@ def __init__(self): self.pypi_optional_requirements = [] self.vendored_requirements = [] + def validate_environment_packages(self, pip_command): + result = EnvironmentPackageValidationResult() + if not self.pypi_requirements and not self.pypi_optional_requirements: + return result + + pip_json = subprocess.check_output( + pip_command + ["list", "--format", "json"], universal_newlines=True + ) + + installed_packages = json.loads(pip_json) + installed_packages = { + package["name"]: package["version"] for package in installed_packages + } + for pkg in self.pypi_requirements: + installed_version = installed_packages.get(pkg.requirement.name) + if not installed_version or not pkg.requirement.specifier.contains( + installed_version + ): + result.add_discrepancy(pkg.requirement, installed_version) + + for pkg in self.pypi_optional_requirements: + installed_version = installed_packages.get(pkg.requirement.name) + if installed_version and not pkg.requirement.specifier.contains( + installed_version + ): + result.add_discrepancy(pkg.requirement, installed_version) + + return result + @classmethod def from_requirements_definition( cls, diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py index f177d96f59038..96c2ccbcdb4a0 100644 --- a/python/mozbuild/mozbuild/virtualenv.py +++ b/python/mozbuild/mozbuild/virtualenv.py @@ -222,29 +222,10 @@ def up_to_date(self): if current_paths != required_paths: return False - if ( - env_requirements.pypi_requirements - or env_requirements.pypi_optional_requirements - ): - pip_json = self._run_pip( - ["list", "--format", "json"], stdout=subprocess.PIPE - ).stdout - installed_packages = json.loads(pip_json) - installed_packages = { - package["name"]: package["version"] for package in installed_packages - } - for pkg in env_requirements.pypi_requirements: - if not pkg.requirement.specifier.contains( - installed_packages.get(pkg.requirement.name, None) - ): - return False - - for pkg in env_requirements.pypi_optional_requirements: - installed_version = installed_packages.get(pkg.requirement.name, None) - if installed_version and not pkg.requirement.specifier.contains( - installed_packages.get(pkg.requirement.name, None) - ): - return False + pip = os.path.join(self.bin_path, "pip") + package_result = env_requirements.validate_environment_packages([pip]) + if not package_result.has_all_packages: + return False return True