From 99ad911a9541fcd11308a7075cb1abbb4d79ce01 Mon Sep 17 00:00:00 2001 From: Mitchell Hentges Date: Mon, 27 Sep 2021 20:27:22 +0000 Subject: [PATCH] Bug 1723031: In CI, assert Mach pypi package deps using system Python r=ahal There's an existing algorithm to check if a virtualenv's installed packages are up-to-date with its requirements. This patch extracts that logic so that, in cases where we can't automatically download needed pip packages, we can at least assert that the ones installed to the system Python are sufficient to meet our requirements. The current only case in which this system-checking logic is applied is when starting the Mach virtualenv and the `MOZ_AUTOMATION` or `MACH_USE_SYSTEM_PYTHON` environment variable is set. Differential Revision: https://phabricator.services.mozilla.com/D122890 --- build/mach_initialize.py | 71 +++++++++++++++++++++++--- python/mach/mach/requirements.py | 52 ++++++++++++++++++- python/mozbuild/mozbuild/virtualenv.py | 27 ++-------- 3 files changed, 120 insertions(+), 30 deletions(-) diff --git a/build/mach_initialize.py b/build/mach_initialize.py index 1e66e9fb164ed..e4616d2847c37 100644 --- a/build/mach_initialize.py +++ b/build/mach_initialize.py @@ -9,6 +9,7 @@ import platform import shutil import site +import subprocess import sys if sys.version_info[0] < 3: @@ -167,6 +168,11 @@ class MetaPathFinder(object): """.strip() +def _scrub_system_site_packages(): + site_paths = set(site.getsitepackages() + [site.getusersitepackages()]) + sys.path = [path for path in sys.path if path not in site_paths] + + def _activate_python_environment(topsrcdir): # We need the "mach" module to access the logic to parse virtualenv # requirements. Since that depends on "packaging" (and, transitively, @@ -193,6 +199,65 @@ def _activate_python_environment(topsrcdir): True, os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"), ) + + if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"): + env_var = ( + "MOZ_AUTOMATION" + if os.environ.get("MOZ_AUTOMATION") + else "MACH_USE_SYSTEM_PYTHON" + ) + + has_pip = ( + subprocess.run( + [sys.executable, "-c", "import pip"], stderr=subprocess.DEVNULL + ).returncode + == 0 + ) + # There are environments in CI that aren't prepared to provide any Mach dependency + # packages. Changing this is a nontrivial endeavour, so guard against having + # non-optional Mach requirements. + assert ( + not requirements.pypi_requirements + ), "Mach pip package requirements must be optional." + if has_pip: + pip = [sys.executable, "-m", "pip"] + check_result = subprocess.run( + pip + ["check"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True, + ) + if check_result.returncode: + print(check_result.stdout, file=sys.stderr) + subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr) + raise Exception( + 'According to "pip check", the current Python ' + "environment has package-compatibility issues." + ) + + package_result = requirements.validate_environment_packages(pip) + if not package_result.has_all_packages: + print( + "Skipping automatic management of Python dependencies since " + f"the '{env_var}' environment variable is set.\n" + "The following issues were found while validating your Python " + "environment:" + ) + print(package_result.report()) + sys.exit(1) + else: + # Pip isn't installed to the system Python environment, so we can't use + # it to verify compatibility with Mach. Remove the system site-packages + # from the import scope so that Mach behaves as though all of its + # (optional) dependencies are not installed. + _scrub_system_site_packages() + + elif sys.prefix == sys.base_prefix: + # We're in an environment where we normally use the Mach virtualenv, + # but we're running a "nativecmd" such as "create-mach-environment". + # Remove global site packages from sys.path to improve isolation accordingly. + _scrub_system_site_packages() + sys.path[0:0] = [ os.path.join(topsrcdir, pth.path) for pth in requirements.pth_requirements + requirements.vendored_requirements @@ -222,12 +287,6 @@ def initialize(topsrcdir): if os.path.exists(deleted_dir): shutil.rmtree(deleted_dir, ignore_errors=True) - if sys.prefix == sys.base_prefix: - # We are not in a virtualenv. Remove global site packages - # from sys.path. - site_paths = set(site.getsitepackages() + [site.getusersitepackages()]) - sys.path = [path for path in sys.path if path not in site_paths] - state_dir = _create_state_dir() _activate_python_environment(topsrcdir) diff --git a/python/mach/mach/requirements.py b/python/mach/mach/requirements.py index cf2f89abb92c6..2be2d5f1cd2d0 100644 --- a/python/mach/mach/requirements.py +++ b/python/mach/mach/requirements.py @@ -1,9 +1,10 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - +import json import os from pathlib import Path +import subprocess from packaging.requirements import Requirement @@ -13,6 +14,26 @@ """.strip() +class EnvironmentPackageValidationResult: + def __init__(self): + self._package_discrepancies = [] + self.has_all_packages = True + + def add_discrepancy(self, requirement, found): + self._package_discrepancies.append((requirement, found)) + self.has_all_packages = False + + def report(self): + lines = [] + for requirement, found in self._package_discrepancies: + if found: + error = f'Installed with unexpected version "{found}"' + else: + error = "Not installed" + lines.append(f"{requirement}: {error}") + return "\n".join(lines) + + class PthSpecifier: def __init__(self, path): self.path = path @@ -61,6 +82,35 @@ def __init__(self): self.pypi_optional_requirements = [] self.vendored_requirements = [] + def validate_environment_packages(self, pip_command): + result = EnvironmentPackageValidationResult() + if not self.pypi_requirements and not self.pypi_optional_requirements: + return result + + pip_json = subprocess.check_output( + pip_command + ["list", "--format", "json"], universal_newlines=True + ) + + installed_packages = json.loads(pip_json) + installed_packages = { + package["name"]: package["version"] for package in installed_packages + } + for pkg in self.pypi_requirements: + installed_version = installed_packages.get(pkg.requirement.name) + if not installed_version or not pkg.requirement.specifier.contains( + installed_version + ): + result.add_discrepancy(pkg.requirement, installed_version) + + for pkg in self.pypi_optional_requirements: + installed_version = installed_packages.get(pkg.requirement.name) + if installed_version and not pkg.requirement.specifier.contains( + installed_version + ): + result.add_discrepancy(pkg.requirement, installed_version) + + return result + @classmethod def from_requirements_definition( cls, diff --git a/python/mozbuild/mozbuild/virtualenv.py b/python/mozbuild/mozbuild/virtualenv.py index f177d96f59038..96c2ccbcdb4a0 100644 --- a/python/mozbuild/mozbuild/virtualenv.py +++ b/python/mozbuild/mozbuild/virtualenv.py @@ -222,29 +222,10 @@ def up_to_date(self): if current_paths != required_paths: return False - if ( - env_requirements.pypi_requirements - or env_requirements.pypi_optional_requirements - ): - pip_json = self._run_pip( - ["list", "--format", "json"], stdout=subprocess.PIPE - ).stdout - installed_packages = json.loads(pip_json) - installed_packages = { - package["name"]: package["version"] for package in installed_packages - } - for pkg in env_requirements.pypi_requirements: - if not pkg.requirement.specifier.contains( - installed_packages.get(pkg.requirement.name, None) - ): - return False - - for pkg in env_requirements.pypi_optional_requirements: - installed_version = installed_packages.get(pkg.requirement.name, None) - if installed_version and not pkg.requirement.specifier.contains( - installed_packages.get(pkg.requirement.name, None) - ): - return False + pip = os.path.join(self.bin_path, "pip") + package_result = env_requirements.validate_environment_packages([pip]) + if not package_result.has_all_packages: + return False return True