Plumb reproducible build env vars more thoroughly.

CHANGES.md

@@ -1,5 +1,20 @@
 # Release Notes
 
+## 2.20.3
+
+This release fixes both PEX building and lock creation via
+`pex3 lock {create,sync}` to be reproducible in more cases. Previously,
+if a requirement only available in source form (an sdist, a local
+project or a VCS requirement) had a build that was not reproducible due
+to either file timestamps (where the `SOURCE_DATE_EPOCH` standard was
+respected) or random iteration order (e.g.: the `setup.py` used sets in
+certain in-opportune ways), Pex's outputs would mirror the problematic
+requirement's non-reproducibility. Now Pex plumbs a fixed
+`SOURCE_DATE_EPOCH` and `PYTHONHASHSEED` to all places sources are
+built.
+
+* Plumb reproducible build env vars more thoroughly. (#2554)
+
 ## 2.20.2
 
 This release fixes an old bug handling certain sdist zips under

pex/bin/pex.py

@@ -1254,7 +1254,9 @@ def main(args=None):
     try:
         with global_environment(options) as env:
             try:
-                resolver_configuration = resolver_options.configure(options)
+                resolver_configuration = resolver_options.configure(
+                    options, use_system_time=options.use_system_time
+                )
             except resolver_options.InvalidConfigurationError as e:
                 die(str(e))
 

pex/build_system/pep_517.py

@@ -75,6 +75,7 @@ def _default_build_system(
                     resolved=resolved_dists,
                     build_backend=DEFAULT_BUILD_BACKEND,
                     backend_path=(),
+                    use_system_time=resolver.use_system_time(),
                     **extra_env
                 )
             )

pex/build_system/pep_518.py

@@ -7,6 +7,7 @@
 import subprocess
 
 from pex.build_system import DEFAULT_BUILD_BACKEND
+from pex.common import REPRODUCIBLE_BUILDS_ENV
 from pex.dist_metadata import Distribution
 from pex.interpreter import PythonInterpreter
 from pex.pex import PEX
@@ -80,13 +81,16 @@ def create(
         build_backend,  # type: str
         backend_path,  # type: Tuple[str, ...]
         extra_requirements=None,  # type: Optional[Iterable[str]]
+        use_system_time=False,  # type: bool
         **extra_env  # type: str
     ):
         # type: (...) -> Union[BuildSystem, Error]
         pex_builder = PEXBuilder()
         pex_builder.info.venv = True
         pex_builder.info.venv_site_packages_copies = True
         pex_builder.info.venv_bin_path = BinPath.PREPEND
+        # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect.
+        pex_builder.info.venv_hermetic_scripts = False
         for req in requires:
             pex_builder.add_requirement(req)
         for dist in resolved:
@@ -144,6 +148,8 @@ def create(
                 env.update(extra_env)
             if backend_path:
                 env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(backend_path))
+            if not use_system_time:
+                env.update(REPRODUCIBLE_BUILDS_ENV)
             return cls(
                 venv_pex=venv_pex, build_backend=build_backend, requires=tuple(requires), env=env
             )
@@ -190,4 +196,5 @@ def load_build_system(
             build_backend=build_system_table.build_backend,
             backend_path=build_system_table.backend_path,
             extra_requirements=extra_requirements,
+            use_system_time=resolver.use_system_time(),
         )

pex/cache/dirs.py

@@ -136,7 +136,7 @@ def iter_transitive_dependents(self):
 
     PIP = Value(
         "pip",
-        version=0,
+        version=1,
         name="Pip Versions",
         description="Isolated Pip caches and Pip PEXes Pex uses to resolve distributions.",
     )

pex/cli/commands/lock.py

@@ -799,7 +799,10 @@ def _resolve_targets(
         # type: (...) -> Union[Targets, Error]
 
         target_config = target_configuration or target_options.configure(
-            self.options, pip_configuration=resolver_options.create_pip_configuration(self.options)
+            self.options,
+            pip_configuration=resolver_options.create_pip_configuration(
+                self.options, use_system_time=False
+            ),
         )
         if style is not LockStyle.UNIVERSAL:
             return target_config.resolve_targets()
@@ -867,7 +870,9 @@ def _gather_requirements(
     def _create(self):
         # type: () -> Result
 
-        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        pip_configuration = resolver_options.create_pip_configuration(
+            self.options, use_system_time=False
+        )
         target_configuration = target_options.configure(
             self.options, pip_configuration=pip_configuration
         )
@@ -963,7 +968,9 @@ def _export(self, requirement_configuration=RequirementConfiguration()):
             )
 
         lockfile_path, lock_file = self._load_lockfile()
-        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        pip_configuration = resolver_options.create_pip_configuration(
+            self.options, use_system_time=False
+        )
         targets = target_options.configure(
             self.options, pip_configuration=pip_configuration
         ).resolve_targets()
@@ -1091,7 +1098,9 @@ def _create_lock_update_request(
     ):
         # type: (...) -> Union[LockUpdateRequest, Error]
 
-        pip_configuration = resolver_options.create_pip_configuration(self.options)
+        pip_configuration = resolver_options.create_pip_configuration(
+            self.options, use_system_time=False
+        )
         lock_updater = LockUpdater.create(
             lock_file=lock_file,
             repos_configuration=pip_configuration.repos_configuration,
@@ -1506,7 +1515,8 @@ def _sync(self):
         # type: () -> Result
 
         resolver_configuration = cast(
-            LockRepositoryConfiguration, resolver_options.configure(self.options)
+            LockRepositoryConfiguration,
+            resolver_options.configure(self.options, use_system_time=False),
         )
         production_assert(isinstance(resolver_configuration, LockRepositoryConfiguration))
         pip_configuration = resolver_configuration.pip_configuration

pex/common.py

@@ -54,6 +54,15 @@
 _UNIX_EPOCH = datetime(year=1970, month=1, day=1, hour=0, minute=0, second=0, tzinfo=None)
 DETERMINISTIC_DATETIME_TIMESTAMP = (DETERMINISTIC_DATETIME - _UNIX_EPOCH).total_seconds()
 
+# N.B.: The `SOURCE_DATE_EPOCH` env var is semi-standard magic for controlling
+# build tools. Wheel, for example, has supported this since 2016.
+# See:
+# + https://reproducible-builds.org/docs/source-date-epoch/
+# + https://github.com/pypa/wheel/blob/1b879e53fed1f179897ed47e55a68bc51df188db/wheel/archive.py#L36-L39
+REPRODUCIBLE_BUILDS_ENV = dict(
+    PYTHONHASHSEED="0", SOURCE_DATE_EPOCH=str(int(DETERMINISTIC_DATETIME_TIMESTAMP))
+)
+
 
 def is_pyc_dir(dir_path):
     # type: (Text) -> bool

pex/pip/installation.py

@@ -11,7 +11,7 @@
 from pex import pex_warnings, third_party
 from pex.atomic_directory import atomic_directory
 from pex.cache.dirs import CacheDir
-from pex.common import pluralize, safe_mkdtemp
+from pex.common import REPRODUCIBLE_BUILDS_ENV, pluralize, safe_mkdtemp
 from pex.dist_metadata import Requirement
 from pex.interpreter import PythonInterpreter
 from pex.orderedset import OrderedSet
@@ -42,6 +42,7 @@ def _pip_installation(
     iter_distribution_locations,  # type: Callable[[], Iterator[str]]
     fingerprint,  # type: str
     interpreter=None,  # type: Optional[PythonInterpreter]
+    use_system_time=False,  # type: bool
 ):
     # type: (...) -> Pip
     pip_root = CacheDir.PIP.path(str(version))
@@ -54,6 +55,8 @@ def _pip_installation(
 
             isolated_pip_builder = PEXBuilder(path=chroot.work_dir)
             isolated_pip_builder.info.venv = True
+            # Allow REPRODUCIBLE_BUILDS_ENV PYTHONHASHSEED env var to take effect if needed.
+            isolated_pip_builder.info.venv_hermetic_scripts = False
             for dist_location in iter_distribution_locations():
                 isolated_pip_builder.add_dist_location(dist=dist_location)
             with named_temporary_file(prefix="", suffix=".py", mode="w") as fp:
@@ -78,7 +81,11 @@ def _pip_installation(
             isolated_pip_builder.freeze()
     pip_cache = os.path.join(pip_root, "pip_cache")
     pip_pex = ensure_venv(PEX(pip_pex_path, interpreter=pip_interpreter))
-    pip_venv = PipVenv(venv_dir=pip_pex.venv_dir, execute_args=tuple(pip_pex.execute_args()))
+    pip_venv = PipVenv(
+        venv_dir=pip_pex.venv_dir,
+        execute_env=REPRODUCIBLE_BUILDS_ENV if not use_system_time else {},
+        execute_args=tuple(pip_pex.execute_args()),
+    )
     return Pip(pip=pip_venv, version=version, pip_cache=pip_cache)
 
 
@@ -98,6 +105,7 @@ def _vendored_installation(
     interpreter=None,  # type: Optional[PythonInterpreter]
     resolver=None,  # type: Optional[Resolver]
     extra_requirements=(),  # type: Tuple[Requirement, ...]
+    use_system_time=False,  # type: bool
 ):
     # type: (...) -> Pip
 
@@ -111,6 +119,7 @@ def expose_vendored():
             iter_distribution_locations=expose_vendored,
             interpreter=interpreter,
             fingerprint=_fingerprint(extra_requirements),
+            use_system_time=use_system_time,
         )
 
     if not resolver:
@@ -171,6 +180,7 @@ def iter_distribution_locations():
         iter_distribution_locations=iter_distribution_locations,
         interpreter=interpreter,
         fingerprint=_fingerprint(extra_requirements),
+        use_system_time=use_system_time,
     )
 
 
@@ -204,6 +214,7 @@ def _resolved_installation(
     resolver=None,  # type: Optional[Resolver]
     interpreter=None,  # type: Optional[PythonInterpreter]
     extra_requirements=(),  # type: Tuple[Requirement, ...]
+    use_system_time=False,  # type: bool
 ):
     # type: (...) -> Pip
     targets = Targets.from_target(LocalInterpreter.create(interpreter))
@@ -222,6 +233,7 @@ def _resolved_installation(
             iter_distribution_locations=_bootstrap_pip(version, interpreter=interpreter),
             interpreter=interpreter,
             fingerprint=_fingerprint(extra_requirements),
+            use_system_time=use_system_time,
         )
 
     requirements_by_project_name = OrderedDict(
@@ -269,6 +281,7 @@ def resolve_distribution_locations():
         iter_distribution_locations=resolve_distribution_locations,
         interpreter=interpreter,
         fingerprint=_fingerprint(extra_requirements),
+        use_system_time=use_system_time,
     )
 
 
@@ -277,6 +290,7 @@ class PipInstallation(object):
     interpreter = attr.ib()  # type: PythonInterpreter
     version = attr.ib()  # type: PipVersionValue
     extra_requirements = attr.ib()  # type: Tuple[Requirement, ...]
+    use_system_time = attr.ib()  # type: bool
 
     def check_python_applies(self):
         # type: () -> None
@@ -396,6 +410,7 @@ def get_pip(
         interpreter=interpreter or PythonInterpreter.get(),
         version=calculated_version,
         extra_requirements=extra_requirements,
+        use_system_time=resolver.use_system_time() if resolver else False,
     )
     pip = _PIP.get(installation)
     if pip is None:
@@ -405,13 +420,15 @@ def get_pip(
                 interpreter=interpreter,
                 resolver=resolver,
                 extra_requirements=installation.extra_requirements,
+                use_system_time=installation.use_system_time,
             )
         else:
             pip = _resolved_installation(
                 version=installation.version,
                 resolver=resolver,
                 interpreter=interpreter,
                 extra_requirements=installation.extra_requirements,
+                use_system_time=installation.use_system_time,
             )
         _PIP[installation] = pip
     return pip

pex/pip/tool.py

@@ -272,6 +272,7 @@ def analyze(self, line):
 @attr.s(frozen=True)
 class PipVenv(object):
     venv_dir = attr.ib()  # type: str
+    execute_env = attr.ib()  # type: Mapping[str, str]
     _execute_args = attr.ib()  # type: Tuple[str, ...]
 
     def execute_args(self, *args):
@@ -431,6 +432,7 @@ def _spawn_pip_isolated(
                 popen_kwargs["stdout"] = sys.stderr.fileno()
             popen_kwargs.update(stderr=subprocess.PIPE)
 
+            env.update(self._pip.execute_env)
             args = self._pip.execute_args(*command)
 
             rendered_env = " ".join(

pex/resolve/configured_resolver.py

@@ -45,8 +45,13 @@ def default(cls):
     pip_configuration = attr.ib()  # type: PipConfiguration
 
     def is_default_repos(self):
+        # type: () -> bool
         return self.pip_configuration.repos_configuration == _DEFAULT_REPOS
 
+    def use_system_time(self):
+        # type: () -> bool
+        return self.pip_configuration.build_configuration.use_system_time
+
     def resolve_lock(
         self,
         lock,  # type: Lockfile

pex/resolve/lockfile/json_codec.py

@@ -231,6 +231,8 @@ def parse_version_specifier(
         for index, constraint in enumerate(get("constraints", list))
     ]
 
+    use_system_time = get("use_system_time", bool, optional=True)
+
     excluded = [
         parse_requirement(req, path=".excluded[{index}]".format(index=index))
         for index, req in enumerate(get("excluded", list, optional=True) or ())
@@ -352,6 +354,10 @@ def assemble_tag(
             prefer_older_binary=get("prefer_older_binary", bool),
             use_pep517=get("use_pep517", bool, optional=True),
             build_isolation=get("build_isolation", bool),
+            # N.B.: Although locks are now always generated under SOURCE_DATE_EPOCH=fixed and
+            # PYTHONHASHSEED=0 (aka: `use_system_time=False`), that did not use to be the case. In
+            # those old locks there was no "use_system_time" field.
+            use_system_time=use_system_time if use_system_time is not None else True,
         ),
         transitive=get("transitive", bool),
         excluded=excluded,
@@ -399,6 +405,7 @@ def as_json_data(
         "prefer_older_binary": lockfile.prefer_older_binary,
         "use_pep517": lockfile.use_pep517,
         "build_isolation": lockfile.build_isolation,
+        "use_system_time": lockfile.use_system_time,
         "transitive": lockfile.transitive,
         "excluded": [str(exclude) for exclude in lockfile.excluded],
         "overridden": [str(override) for override in lockfile.overridden],

pex/resolve/lockfile/model.py

@@ -106,6 +106,7 @@ def extract_requirement(req):
             prefer_older_binary=build_configuration.prefer_older_binary,
             use_pep517=build_configuration.use_pep517,
             build_isolation=build_configuration.build_isolation,
+            use_system_time=build_configuration.use_system_time,
             transitive=transitive,
             excluded=SortedTuple(excluded),
             overridden=SortedTuple(overridden),
@@ -130,6 +131,7 @@ def extract_requirement(req):
     prefer_older_binary = attr.ib()  # type: bool
     use_pep517 = attr.ib()  # type: Optional[bool]
     build_isolation = attr.ib()  # type: bool
+    use_system_time = attr.ib()  # type: bool
     transitive = attr.ib()  # type: bool
     excluded = attr.ib()  # type: SortedTuple[Requirement]
     overridden = attr.ib()  # type: SortedTuple[Requirement]
@@ -147,6 +149,7 @@ def build_configuration(self):
             prefer_older_binary=self.prefer_older_binary,
             use_pep517=self.use_pep517,
             build_isolation=self.build_isolation,
+            use_system_time=self.use_system_time,
         )
 
     def dependency_configuration(self):

pex/resolve/resolver_configuration.py

@@ -98,6 +98,7 @@ def create(
         prefer_older_binary=False,  # type: bool
         use_pep517=None,  # type: Optional[bool]
         build_isolation=True,  # type: bool
+        use_system_time=False,  # type: bool
     ):
         # type: (...) -> BuildConfiguration
         return cls(
@@ -108,6 +109,7 @@ def create(
             prefer_older_binary=prefer_older_binary,
             use_pep517=use_pep517,
             build_isolation=build_isolation,
+            use_system_time=use_system_time,
         )
 
     allow_builds = attr.ib(default=True)  # type: bool
@@ -117,6 +119,7 @@ def create(
     prefer_older_binary = attr.ib(default=False)  # type: bool
     use_pep517 = attr.ib(default=None)  # type: Optional[bool]
     build_isolation = attr.ib(default=True)  # type: bool
+    use_system_time = attr.ib(default=False)  # type: bool
 
     def __attrs_post_init__(self):
         # type: () -> None