Avoid failure in Python package installation and sanity check when $PIP_REQUIRE_VIRTUALENV is set

easybuild/easyblocks/generic/pythonbundle.py

@@ -32,11 +32,10 @@
 
 from easybuild.easyblocks.generic.bundle import Bundle
 from easybuild.easyblocks.generic.pythonpackage import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
-from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd
+from easybuild.easyblocks.generic.pythonpackage import PythonPackage, get_pylibdirs, pick_python_cmd, set_py_env_vars
 from easybuild.tools.build_log import EasyBuildError
 from easybuild.tools.filetools import which
 from easybuild.tools.modules import get_software_root
-import easybuild.tools.environment as env
 
 
 class PythonBundle(Bundle):
@@ -135,8 +134,7 @@ def prepare_step(self, *args, **kwargs):
 
     def extensions_step(self, *args, **kwargs):
         """Install extensions (usually PythonPackages)"""
-        # don't add user site directory to sys.path (equivalent to python -s)
-        env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
+        set_py_env_vars(self.log)
         super(PythonBundle, self).extensions_step(*args, **kwargs)
 
     def test_step(self):
@@ -171,16 +169,16 @@ def make_module_extra(self, *args, **kwargs):
         return txt
 
     def load_module(self, *args, **kwargs):
+        """(Re)set environment variables after loading module file.
+
+        Required here to ensure the variables are also defined for stand-alone installations,
+        because the environment is reset to the initial environment right before loading the module.
         """
-        Make sure that $PYTHONNOUSERSITE is defined after loading module file for this software."""
 
         super(PythonBundle, self).load_module(*args, **kwargs)
-
-        # Don't add user site directory to sys.path (equivalent to python -s),
-        # to avoid that any Python packages installed in $HOME/.local/lib affect the sanity check.
         # Required here to ensure that it is defined for sanity check commands of the bundle
         # because the environment is reset to the initial environment right before loading the module
-        env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
+        set_py_env_vars(self.log)
 
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for bundle of Python package."""

easybuild/easyblocks/generic/pythonpackage.py

@@ -42,7 +42,7 @@
 
 import easybuild.tools.environment as env
 from easybuild.base import fancylogger
-from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES
+from easybuild.easyblocks.python import EBPYTHONPREFIXES, EXTS_FILTER_PYTHON_PACKAGES, set_py_env_vars
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.framework.easyconfig.default import DEFAULT_CONFIG
 from easybuild.framework.easyconfig.templates import TEMPLATE_CONSTANTS
@@ -189,8 +189,8 @@ def det_pylibdir(plat_specific=False, python_cmd=None):
     if LooseVersion(det_python_version(python_cmd)) >= LooseVersion('3.12'):
         # Python 3.12 removed distutils but has a core sysconfig module which is similar
         pathname = 'platlib' if plat_specific else 'purelib'
-        vars = {'platbase': prefix, 'base': prefix}
-        pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars)
+        vars_param = {'platbase': prefix, 'base': prefix}
+        pycode = 'import sysconfig; print(sysconfig.get_path("%s", vars=%s))' % (pathname, vars_param)
     else:
         args = 'plat_specific=%s, prefix="%s"' % (plat_specific, prefix)
         pycode = "import distutils.sysconfig; print(distutils.sysconfig.get_python_lib(%s))" % args
@@ -438,15 +438,7 @@ def __init__(self, *args, **kwargs):
         self.use_setup_py = False
         self.determine_install_command()
 
-        # avoid that pip (ab)uses $HOME/.cache/pip
-        # cfr. https://pip.pypa.io/en/stable/reference/pip_install/#caching
-        env.setvar('XDG_CACHE_HOME', os.path.join(self.builddir, 'xdg-cache-home'))
-        self.log.info("Using %s as pip cache directory", os.environ['XDG_CACHE_HOME'])
-        # Users or sites may require using a virtualenv for user installations
-        # We need to disable this to be able to install into the modules
-        env.setvar('PIP_REQUIRE_VIRTUALENV', 'false')
-        # Don't let pip connect to PYPI to check for a new version
-        env.setvar('PIP_DISABLE_PIP_VERSION_CHECK', 'true')
+        set_py_env_vars(self.log)
 
     def determine_install_command(self):
         """
@@ -734,9 +726,7 @@ def prepare_step(self, *args, **kwargs):
     def configure_step(self):
         """Configure Python package build/install."""
 
-        # don't add user site directory to sys.path (equivalent to python -s)
-        # see https://www.python.org/dev/peps/pep-0370/
-        env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
+        set_py_env_vars(self.log)
 
         if self.python_cmd is None:
             self.prepare_python()
@@ -969,16 +959,14 @@ def run(self, *args, **kwargs):
                         step_method(self)()
 
     def load_module(self, *args, **kwargs):
+        """(Re)set environment variables after loading module file for this software.
+
+        Required here to ensure the variables are also defined for stand-alone installations,
+        because the environment is reset to the initial environment right before loading the module.
         """
-        Make sure that $PYTHONNOUSERSITE is defined after loading module file for this software."""
 
         super(PythonPackage, self).load_module(*args, **kwargs)
-
-        # don't add user site directory to sys.path (equivalent to python -s),
-        # to avoid that any Python packages installed in $HOME/.local/lib affect the sanity check;
-        # required here to ensure that it is defined for stand-alone installations,
-        # because the environment is reset to the initial environment right before loading the module
-        env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
+        set_py_env_vars(self.log)
 
     def sanity_check_step(self, *args, **kwargs):
         """
@@ -995,13 +983,9 @@ def sanity_check_step(self, *args, **kwargs):
             extra_modules = kwargs.get('extra_modules', None)
             self.fake_mod_data = self.sanity_check_load_module(extension=extension, extra_modules=extra_modules)
 
-        # don't add user site directory to sys.path (equivalent to python -s)
-        # see https://www.python.org/dev/peps/pep-0370/;
-        # must be set here to ensure that it is defined when running sanity check for extensions,
-        # since load_module is not called for every extension,
-        # to avoid that any Python packages installed in $HOME/.local/lib affect the sanity check;
+        # Must be called here since load_module is not called for every extension,
         # see also https://github.com/easybuilders/easybuild-easyblocks/issues/1877
-        env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
+        set_py_env_vars(self.log)
 
         if self.cfg.get('download_dep_fail', False):
             self.log.info("Detection of downloaded dependencies enabled, checking output of installation command...")

easybuild/easyblocks/p/python.py

@@ -61,6 +61,17 @@
 
 EBPYTHONPREFIXES = 'EBPYTHONPREFIXES'
 
+# Environment variables and values to avoid common issues during Python package installations and usage in EasyBuild
+PY_ENV_VARS = {
+    # don't add user site directory to sys.path (equivalent to python -s), see https://www.python.org/dev/peps/pep-0370
+    'PYTHONNOUSERSITE': '1',
+    # Users or sites may require using a virtualenv for user installations
+    # We need to disable this to be able to install into modules
+    'PIP_REQUIRE_VIRTUALENV': 'false',
+    # Don't let pip connect to PYPI to check for a new version
+    'PIP_DISABLE_PIP_VERSION_CHECK': 'true',
+}
+
 # We want the following import order:
 # 1. Packages installed into VirtualEnv
 # 2. Packages installed into $EBPYTHONPREFIXES (e.g. our modules)
@@ -109,6 +120,22 @@
 """ % {'EBPYTHONPREFIXES': EBPYTHONPREFIXES}
 
 
+def set_py_env_vars(log, verbose=False):
+    """Set environment variables required/useful for installing or using Python packages"""
+
+    py_vars = PY_ENV_VARS.copy()
+    # avoid that pip (ab)uses $HOME/.cache/pip
+    # cfr. https://pip.pypa.io/en/stable/reference/pip_install/#caching
+    py_vars['XDG_CACHE_HOME'] = os.path.join(tempfile.gettempdir(), 'xdg-cache-home')
+    # Only set (all) environment variables if any has a different value to
+    # avoid (non)changes (and log messages) for each package in a bundle
+    set_required = any(os.environ.get(name, None) != value for name, value in py_vars.items())
+    if set_required:
+        for name, value in py_vars.items():
+            env.setvar(name, value, verbose=verbose)
+        log.info("Using %s as pip cache directory", os.environ['XDG_CACHE_HOME'])
+
+
 class EB_Python(ConfigureMake):
     """Support for building/installing Python
     - default configure/build_step/make install works fine
@@ -161,9 +188,9 @@ def __init__(self, *args, **kwargs):
         }
 
         exts_default_options = self.cfg.get_ref('exts_default_options')
-        for key in ext_defaults:
+        for key, value in ext_defaults.items():
             if key not in exts_default_options:
-                exts_default_options[key] = ext_defaults[key]
+                exts_default_options[key] = value
         self.log.debug("exts_default_options: %s", self.cfg['exts_default_options'])
 
         self.install_pip = self.cfg['install_pip']
@@ -296,8 +323,7 @@ def prepare_for_extensions(self):
         self.cfg['exts_defaultclass'] = "PythonPackage"
         self.cfg['exts_filter'] = EXTS_FILTER_PYTHON_PACKAGES
 
-        # don't add user site directory to sys.path (equivalent to python -s)
-        env.setvar('PYTHONNOUSERSITE', '1')
+        set_py_env_vars(self.log)
 
         # don't pass down any build/install options that may have been specified
         # 'make' options do not make sense for when building/installing Python libraries (usually via 'python setup.py')
@@ -434,9 +460,8 @@ def configure_step(self):
                 env.setvar('TCLTK_CFLAGS', '-I%s/include -I%s/include' % (tcl, tk))
                 env.setvar('TCLTK_LIBS', tcltk_libs)
 
-        # don't add user site directory to sys.path (equivalent to python -s)
         # This matters e.g. when python installs the bundled pip & setuptools (for >= 3.4)
-        env.setvar('PYTHONNOUSERSITE', '1')
+        set_py_env_vars(self.log)
 
         super(EB_Python, self).configure_step()
 

easybuild/easyblocks/t/tensorflow.py

@@ -41,7 +41,7 @@
 import easybuild.tools.environment as env
 import easybuild.tools.toolchain as toolchain
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage, det_python_version
-from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES
+from easybuild.easyblocks.python import EXTS_FILTER_PYTHON_PACKAGES, PY_ENV_VARS
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools import run, LooseVersion
 from easybuild.tools.build_log import EasyBuildError, print_warning
@@ -894,7 +894,7 @@ def build_step(self):
             action_env['EBPYTHONPREFIXES'] = INHERIT
 
         # Ignore user environment for Python
-        action_env['PYTHONNOUSERSITE'] = '1'
+        action_env.update(PY_ENV_VARS)
 
         # TF 2 (final) sets this in configure
         if LooseVersion(self.version) < LooseVersion('2.0'):

easybuild/easyblocks/t/tensorflow_compression.py

@@ -32,6 +32,7 @@
 
 import easybuild.tools.environment as env
 from easybuild.easyblocks.generic.pythonpackage import PythonPackage
+from easybuild.easyblocks.python import PY_ENV_VARS
 from easybuild.tools import LooseVersion
 from easybuild.tools.modules import get_software_version
 from easybuild.tools.run import run_cmd
@@ -104,7 +105,7 @@ def build_step(self):
             action_env['EBPYTHONPREFIXES'] = INHERIT
 
         # Ignore user environment for Python
-        action_env['PYTHONNOUSERSITE'] = '1'
+        action_env.update(PY_ENV_VARS)
 
         # Use the same configuration (i.e. environment) for compiling and using host tools
         # This means that our action_envs are (almost) always passed