Bug 1723031: In CI, assert Mach pypi package deps using system Python…

… r=ahal

There's an existing algorithm to check if a virtualenv's installed
packages are up-to-date with its requirements. This patch
extracts that logic so that, in cases where we can't automatically
download needed pip packages, we can at least assert that the ones
installed to the system Python are sufficient to meet our requirements.

The current only case in which this system-checking logic is applied
is when starting the Mach virtualenv and the `MOZ_AUTOMATION` or
`MACH_USE_SYSTEM_PYTHON` environment variable is set.

Differential Revision: https://phabricator.services.mozilla.com/D122890

build/mach_initialize.py

@@ -9,6 +9,7 @@
 import platform
 import shutil
 import site
+import subprocess
 import sys
 
 if sys.version_info[0] < 3:
@@ -167,6 +168,11 @@ class MetaPathFinder(object):
 """.strip()
 
 
+def _scrub_system_site_packages():
+    site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
+    sys.path = [path for path in sys.path if path not in site_paths]
+
+
 def _activate_python_environment(topsrcdir):
     # We need the "mach" module to access the logic to parse virtualenv
     # requirements. Since that depends on "packaging" (and, transitively,
@@ -193,6 +199,65 @@ def _activate_python_environment(topsrcdir):
         True,
         os.path.join(topsrcdir, "build", "mach_virtualenv_packages.txt"),
     )
+
+    if os.environ.get("MACH_USE_SYSTEM_PYTHON") or os.environ.get("MOZ_AUTOMATION"):
+        env_var = (
+            "MOZ_AUTOMATION"
+            if os.environ.get("MOZ_AUTOMATION")
+            else "MACH_USE_SYSTEM_PYTHON"
+        )
+
+        has_pip = (
+            subprocess.run(
+                [sys.executable, "-c", "import pip"], stderr=subprocess.DEVNULL
+            ).returncode
+            == 0
+        )
+        # There are environments in CI that aren't prepared to provide any Mach dependency
+        # packages. Changing this is a nontrivial endeavour, so guard against having
+        # non-optional Mach requirements.
+        assert (
+            not requirements.pypi_requirements
+        ), "Mach pip package requirements must be optional."
+        if has_pip:
+            pip = [sys.executable, "-m", "pip"]
+            check_result = subprocess.run(
+                pip + ["check"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                universal_newlines=True,
+            )
+            if check_result.returncode:
+                print(check_result.stdout, file=sys.stderr)
+                subprocess.check_call(pip + ["list", "-v"], stdout=sys.stderr)
+                raise Exception(
+                    'According to "pip check", the current Python '
+                    "environment has package-compatibility issues."
+                )
+
+            package_result = requirements.validate_environment_packages(pip)
+            if not package_result.has_all_packages:
+                print(
+                    "Skipping automatic management of Python dependencies since "
+                    f"the '{env_var}' environment variable is set.\n"
+                    "The following issues were found while validating your Python "
+                    "environment:"
+                )
+                print(package_result.report())
+                sys.exit(1)
+        else:
+            # Pip isn't installed to the system Python environment, so we can't use
+            # it to verify compatibility with Mach. Remove the system site-packages
+            # from the import scope so that Mach behaves as though all of its
+            # (optional) dependencies are not installed.
+            _scrub_system_site_packages()
+
+    elif sys.prefix == sys.base_prefix:
+        # We're in an environment where we normally use the Mach virtualenv,
+        # but we're running a "nativecmd" such as "create-mach-environment".
+        # Remove global site packages from sys.path to improve isolation accordingly.
+        _scrub_system_site_packages()
+
     sys.path[0:0] = [
         os.path.join(topsrcdir, pth.path)
         for pth in requirements.pth_requirements + requirements.vendored_requirements
@@ -222,12 +287,6 @@ def initialize(topsrcdir):
     if os.path.exists(deleted_dir):
         shutil.rmtree(deleted_dir, ignore_errors=True)
 
-    if sys.prefix == sys.base_prefix:
-        # We are not in a virtualenv. Remove global site packages
-        # from sys.path.
-        site_paths = set(site.getsitepackages() + [site.getusersitepackages()])
-        sys.path = [path for path in sys.path if path not in site_paths]
-
     state_dir = _create_state_dir()
     _activate_python_environment(topsrcdir)
 

python/mach/mach/requirements.py

@@ -1,9 +1,10 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
+import json
 import os
 from pathlib import Path
+import subprocess
 
 from packaging.requirements import Requirement
 
@@ -13,6 +14,26 @@
 """.strip()
 
 
+class EnvironmentPackageValidationResult:
+    def __init__(self):
+        self._package_discrepancies = []
+        self.has_all_packages = True
+
+    def add_discrepancy(self, requirement, found):
+        self._package_discrepancies.append((requirement, found))
+        self.has_all_packages = False
+
+    def report(self):
+        lines = []
+        for requirement, found in self._package_discrepancies:
+            if found:
+                error = f'Installed with unexpected version "{found}"'
+            else:
+                error = "Not installed"
+            lines.append(f"{requirement}: {error}")
+        return "\n".join(lines)
+
+
 class PthSpecifier:
     def __init__(self, path):
         self.path = path
@@ -61,6 +82,35 @@ def __init__(self):
         self.pypi_optional_requirements = []
         self.vendored_requirements = []
 
+    def validate_environment_packages(self, pip_command):
+        result = EnvironmentPackageValidationResult()
+        if not self.pypi_requirements and not self.pypi_optional_requirements:
+            return result
+
+        pip_json = subprocess.check_output(
+            pip_command + ["list", "--format", "json"], universal_newlines=True
+        )
+
+        installed_packages = json.loads(pip_json)
+        installed_packages = {
+            package["name"]: package["version"] for package in installed_packages
+        }
+        for pkg in self.pypi_requirements:
+            installed_version = installed_packages.get(pkg.requirement.name)
+            if not installed_version or not pkg.requirement.specifier.contains(
+                installed_version
+            ):
+                result.add_discrepancy(pkg.requirement, installed_version)
+
+        for pkg in self.pypi_optional_requirements:
+            installed_version = installed_packages.get(pkg.requirement.name)
+            if installed_version and not pkg.requirement.specifier.contains(
+                installed_version
+            ):
+                result.add_discrepancy(pkg.requirement, installed_version)
+
+        return result
+
     @classmethod
     def from_requirements_definition(
         cls,

python/mozbuild/mozbuild/virtualenv.py

@@ -222,29 +222,10 @@ def up_to_date(self):
             if current_paths != required_paths:
                 return False
 
-        if (
-            env_requirements.pypi_requirements
-            or env_requirements.pypi_optional_requirements
-        ):
-            pip_json = self._run_pip(
-                ["list", "--format", "json"], stdout=subprocess.PIPE
-            ).stdout
-            installed_packages = json.loads(pip_json)
-            installed_packages = {
-                package["name"]: package["version"] for package in installed_packages
-            }
-            for pkg in env_requirements.pypi_requirements:
-                if not pkg.requirement.specifier.contains(
-                    installed_packages.get(pkg.requirement.name, None)
-                ):
-                    return False
-
-            for pkg in env_requirements.pypi_optional_requirements:
-                installed_version = installed_packages.get(pkg.requirement.name, None)
-                if installed_version and not pkg.requirement.specifier.contains(
-                    installed_packages.get(pkg.requirement.name, None)
-                ):
-                    return False
+        pip = os.path.join(self.bin_path, "pip")
+        package_result = env_requirements.validate_environment_packages([pip])
+        if not package_result.has_all_packages:
+            return False
 
         return True