bpo-28833: Fix cross-compilation of third-party extension modules

Doc/whatsnew/3.9.rst

@@ -367,3 +367,21 @@ CPython bytecode changes
   :keyword:`assert` statement. Previously, the assert statement would not work
   correctly if the :exc:`AssertionError` exception was being shadowed.
   (Contributed by Zackery Spytz in :issue:`34880`.)
+
+Build Changes
+-------------
+
+* Cross compilation of third-party extension modules is done with the
+  ``PYTHON_PROJECT_BASE`` environment variable set to the path of the
+  directory where Python has been cross-compiled. The private environment
+  variables ``_PYTHON_HOST_PLATFORM``, ``_PYTHON_SYSCONFIGDATA_NAME`` and
+  ``_PYTHON_PROJECT_BASE`` are not used anymore.
+
+  For example the following command builds a wheel file to be transfered and
+  installed with pip on the target platform, provided the native python
+  interpreter and the cross-compiled one both have the wheel package
+  installed:
+
+  .. code-block:: shell
+
+        $ PYTHON_PROJECT_BASE=/path/to/builddir python setup.py bdist_wheel

Lib/distutils/command/build.py

@@ -6,6 +6,7 @@
 from distutils.core import Command
 from distutils.errors import DistutilsOptionError
 from distutils.util import get_platform
+from distutils.sysconfig import cross_compiling, get_config_var
 
 
 def show_compilers():
@@ -86,7 +87,8 @@ def finalize_options(self):
         # Make it so Python 2.x and Python 2.x with --with-pydebug don't
         # share the same build directories. Doing so confuses the build
         # process for C modules
-        if hasattr(sys, 'gettotalrefcount'):
+        if (cross_compiling and 'd' in get_config_var('ABIFLAGS') or
+                (not cross_compiling and hasattr(sys, "gettotalrefcount"))):
             plat_specifier += '-pydebug'
 
         # 'build_purelib' and 'build_platlib' just default to 'lib' and

Lib/distutils/command/build_ext.py

@@ -237,7 +237,11 @@ def finalize_options(self):
                 self.library_dirs.append(sysconfig.get_config_var('LIBDIR'))
             else:
                 # building python standard extensions
-                self.library_dirs.append('.')
+                if 'PYTHON_PROJECT_BASE' in os.environ:
+                    self.library_dirs.append(os.path.realpath(
+                                os.environ['PYTHON_PROJECT_BASE']))
+                else:
+                    self.library_dirs.append('.')
 
         # The argument parsing will result in self.define being a string, but
         # it has to be a list of 2-tuples.  All the preprocessor symbols
@@ -732,7 +736,7 @@ def get_libraries(self, ext):
                     link_libpython = True
                 elif sys.platform == 'cygwin':
                     link_libpython = True
-                elif '_PYTHON_HOST_PLATFORM' in os.environ:
+                elif 'PYTHON_PROJECT_BASE' in os.environ:
                     # We are cross-compiling for one of the relevant platforms
                     if get_config_var('ANDROID_API_LEVEL') != 0:
                         link_libpython = True

Lib/distutils/command/install.py

@@ -8,7 +8,7 @@
 from distutils import log
 from distutils.core import Command
 from distutils.debug import DEBUG
-from distutils.sysconfig import get_config_vars
+from distutils.sysconfig import get_config_vars, cross_compiling
 from distutils.errors import DistutilsPlatformError
 from distutils.file_util import write_file
 from distutils.util import convert_path, subst_vars, change_root
@@ -281,12 +281,10 @@ def finalize_options(self):
         # about needing recursive variable expansion (shudder).
 
         py_version = sys.version.split()[0]
-        (prefix, exec_prefix) = get_config_vars('prefix', 'exec_prefix')
-        try:
-            abiflags = sys.abiflags
-        except AttributeError:
-            # sys.abiflags may not be defined on all platforms.
-            abiflags = ''
+        prefix, exec_prefix, abiflags = get_config_vars('prefix',
+                                                'exec_prefix', 'ABIFLAGS')
+        # sys.abiflags may not be defined on all platforms.
+        abiflags = '' if abiflags is None else abiflags
         self.config_vars = {'dist_name': self.distribution.get_name(),
                             'dist_version': self.distribution.get_version(),
                             'dist_fullname': self.distribution.get_fullname(),
@@ -418,8 +416,13 @@ def finalize_unix(self):
                     raise DistutilsOptionError(
                           "must not supply exec-prefix without prefix")
 
-                self.prefix = os.path.normpath(sys.prefix)
-                self.exec_prefix = os.path.normpath(sys.exec_prefix)
+                if cross_compiling:
+                    prefix, exec_prefix = get_config_vars('prefix',
+                                                          'exec_prefix')
+                else:
+                    prefix, exec_prefix = sys.prefix, sys.exec_prefix
+                self.prefix = os.path.normpath(prefix)
+                self.exec_prefix = os.path.normpath(exec_prefix)
 
             else:
                 if self.exec_prefix is None:

Lib/distutils/sysconfig.py

@@ -14,8 +14,8 @@
 import re
 import sys
 
+from sysconfig import cross_compiling, get_project_base, get_build_time_vars
 from .errors import DistutilsPlatformError
-from .util import get_platform, get_host_platform
 
 # These are needed in a couple of spots, so just compute them once.
 PREFIX = os.path.normpath(sys.prefix)
@@ -26,8 +26,13 @@
 # Path to the base directory of the project. On Windows the binary may
 # live in project/PCbuild/win32 or project/PCbuild/amd64.
 # set for cross builds
-if "_PYTHON_PROJECT_BASE" in os.environ:
-    project_base = os.path.abspath(os.environ["_PYTHON_PROJECT_BASE"])
+if cross_compiling:
+    project_base = get_project_base()
+    build_time_vars = get_build_time_vars()
+    PREFIX = build_time_vars['prefix']
+    BASE_PREFIX = PREFIX
+    EXEC_PREFIX = build_time_vars['exec_prefix']
+    BASE_EXEC_PREFIX = EXEC_PREFIX
 else:
     if sys.executable:
         project_base = os.path.dirname(os.path.abspath(sys.executable))
@@ -46,7 +51,7 @@ def _is_python_source_dir(d):
             return True
     return False
 
-_sys_home = getattr(sys, '_home', None)
+_sys_home = getattr(sys, '_home', None) if not cross_compiling else None
 
 if os.name == 'nt':
     def _fix_pcbuild(d):
@@ -64,6 +69,9 @@ def _python_build():
 
 python_build = _python_build()
 
+if cross_compiling and not python_build:
+    raise RuntimeError('PYTHON_PROJECT_BASE is not a build directory')
+
 
 # Calculate the build qualifier flags if they are defined.  Adding the flags
 # to the include and lib directories only makes sense for an installation, not
@@ -256,8 +264,10 @@ def get_makefile_filename():
         return os.path.join(_sys_home or project_base, "Makefile")
     lib_dir = get_python_lib(plat_specific=0, standard_lib=1)
     config_file = 'config-{}{}'.format(get_python_version(), build_flags)
-    if hasattr(sys.implementation, '_multiarch'):
-        config_file += '-%s' % sys.implementation._multiarch
+    multiarch = (get_config_var('MULTIARCH') if cross_compiling else
+                 getattr(sys.implementation, '_multiarch', ''))
+    if multiarch:
+        config_file += '-%s' % multiarch
     return os.path.join(lib_dir, config_file, 'Makefile')
 
 
@@ -431,15 +441,7 @@ def expand_makefile_vars(s, vars):
 
 def _init_posix():
     """Initialize the module as appropriate for POSIX systems."""
-    # _sysconfigdata is generated at build time, see the sysconfig module
-    name = os.environ.get('_PYTHON_SYSCONFIGDATA_NAME',
-        '_sysconfigdata_{abi}_{platform}_{multiarch}'.format(
-        abi=sys.abiflags,
-        platform=sys.platform,
-        multiarch=getattr(sys.implementation, '_multiarch', ''),
-    ))
-    _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0)
-    build_time_vars = _temp.build_time_vars
+    build_time_vars = get_build_time_vars()
     global _config_vars
     _config_vars = {}
     _config_vars.update(build_time_vars)

Lib/distutils/util.py

@@ -14,6 +14,7 @@
 from distutils.spawn import spawn
 from distutils import log
 from distutils.errors import DistutilsByteCompileError
+from distutils.sysconfig import cross_compiling, get_config_var
 
 def get_host_platform():
     """Return a string that identifies the current platform.  This is used mainly to
@@ -45,8 +46,8 @@ def get_host_platform():
         return sys.platform
 
     # Set for cross builds explicitly
-    if "_PYTHON_HOST_PLATFORM" in os.environ:
-        return os.environ["_PYTHON_HOST_PLATFORM"]
+    if cross_compiling:
+        return get_config_var('PYTHON_HOST_PLATFORM')
 
     if os.name != "posix" or not hasattr(os, 'uname'):
         # XXX what about the architecture? NT is Intel or Alpha,

Lib/sysconfig.py

@@ -114,17 +114,59 @@ def _safe_realpath(path):
     _PROJECT_BASE.lower().endswith(('\\pcbuild\\win32', '\\pcbuild\\amd64'))):
     _PROJECT_BASE = _safe_realpath(os.path.join(_PROJECT_BASE, pardir, pardir))
 
+cross_compiling = False
 # set for cross builds
-if "_PYTHON_PROJECT_BASE" in os.environ:
-    _PROJECT_BASE = _safe_realpath(os.environ["_PYTHON_PROJECT_BASE"])
+if "PYTHON_PROJECT_BASE" in os.environ:
+    cross_compiling = True
+    _PROJECT_BASE = _safe_realpath(os.environ["PYTHON_PROJECT_BASE"])
+
+    # Not used when cross compiling.
+    _PREFIX = None
+    _BASE_PREFIX = None
+    _EXEC_PREFIX = None
+    _BASE_EXEC_PREFIX = None
+
+def get_project_base():
+    return _PROJECT_BASE
 
 def _is_python_source_dir(d):
     for fn in ("Setup", "Setup.local"):
         if os.path.isfile(os.path.join(d, "Modules", fn)):
             return True
     return False
 
-_sys_home = getattr(sys, '_home', None)
+_PYTHON_CONFIG = None
+def _get_python_config(name):
+    """Return the value of a python configuration variable."""
+
+    assert cross_compiling
+    global _PYTHON_CONFIG
+    if _PYTHON_CONFIG is None:
+        # The import would fail when building a native interpreter since,
+        # at the time generate-posix-vars is run, the _posixsubprocess
+        # module has not been built yet. This does not happen when
+        # generate-posix-vars is run during cross-compilation as the
+        # native interpreter being used is a full-fledged interpreter.
+        import subprocess
+
+        python_config = os.path.join(_PROJECT_BASE, 'python-config')
+        vars_ = ['version', 'abiflags', 'machdep', 'multiarch']
+        args = ['/bin/sh', python_config]
+        args.extend('--' + v for v in vars_)
+        output = subprocess.check_output(args, universal_newlines=True)
+        _PYTHON_CONFIG = dict(zip(vars_, output.split('\n')))
+        assert len(_PYTHON_CONFIG) == len(vars_)
+
+        # The specification of the sysconfigdata file name may differ
+        # across versions, so forbid Python versions mismatch.
+        sys_version = '%d.%d' % sys.version_info[:2]
+        if _PYTHON_CONFIG['version'] != sys_version:
+            raise RuntimeError('the running python version (%s) does '
+            'not match the cross-compiled version (%s)' %
+            (sys_version, _PYTHON_CONFIG['version']))
+    return _PYTHON_CONFIG.get(name)
+
+_sys_home = getattr(sys, '_home', None) if not cross_compiling else None
 
 if os.name == 'nt':
     def _fix_pcbuild(d):
@@ -146,6 +188,8 @@ def is_python_build(check_home=False):
     for scheme in ('posix_prefix', 'posix_home'):
         _INSTALL_SCHEMES[scheme]['include'] = '{srcdir}/Include'
         _INSTALL_SCHEMES[scheme]['platinclude'] = '{projectbase}/.'
+elif cross_compiling:
+    raise RuntimeError('PYTHON_PROJECT_BASE is not a build directory')
 
 
 def _subst_vars(s, local_vars):
@@ -344,13 +388,19 @@ def get_makefile_filename():
 
 
 def _get_sysconfigdata_name():
-    return os.environ.get('_PYTHON_SYSCONFIGDATA_NAME',
-        '_sysconfigdata_{abi}_{platform}_{multiarch}'.format(
-        abi=sys.abiflags,
-        platform=sys.platform,
-        multiarch=getattr(sys.implementation, '_multiarch', ''),
-    ))
+    if cross_compiling:
+        abiflags = _get_python_config('abiflags')
+        platform = _get_python_config('machdep')
+        multiarch = _get_python_config('multiarch')
+    else:
+        abiflags = sys.abiflags
+        platform = sys.platform
+        multiarch = getattr(sys.implementation, '_multiarch', '')
 
+    sysconf_name = '_sysconfigdata_%s_%s' % (abiflags, platform)
+    if multiarch:
+        sysconf_name = '%s_%s' % (sysconf_name, multiarch)
+    return sysconf_name
 
 def _generate_posix_vars():
     """Generate the Python module containing build-time variables."""
@@ -392,15 +442,17 @@ def _generate_posix_vars():
     # _sysconfigdata module manually and populate it with the build vars.
     # This is more than sufficient for ensuring the subsequent call to
     # get_platform() succeeds.
+    # The same situation exists when cross compiling.
     name = _get_sysconfigdata_name()
-    if 'darwin' in sys.platform:
+    if 'darwin' in sys.platform or cross_compiling:
         import types
         module = types.ModuleType(name)
         module.build_time_vars = vars
         sys.modules[name] = module
 
     pybuilddir = 'build/lib.%s-%s' % (get_platform(), _PY_VERSION_SHORT)
-    if hasattr(sys, "gettotalrefcount"):
+    if (cross_compiling and 'd' in vars['ABIFLAGS'] or
+            (not cross_compiling and hasattr(sys, "gettotalrefcount"))):
         pybuilddir += '-pydebug'
     os.makedirs(pybuilddir, exist_ok=True)
     destfile = os.path.join(pybuilddir, name + '.py')
@@ -415,12 +467,39 @@ def _generate_posix_vars():
     with open('pybuilddir.txt', 'w', encoding='utf8') as f:
         f.write(pybuilddir)
 
+def _get_module_attr(module, attribute):
+    _temp = __import__(module, globals(), locals(), [attribute], 0)
+    return getattr(_temp, attribute)
+
+_BUILD_TIME_VARS = None
+def get_build_time_vars():
+    global _BUILD_TIME_VARS
+    if _BUILD_TIME_VARS is None:
+        # _sysconfigdata is generated at build time, see _generate_posix_vars()
+        sysconfigdata = _get_sysconfigdata_name()
+        if cross_compiling:
+            if _is_python_source_dir(_PROJECT_BASE):
+                bdir = os.path.join(_PROJECT_BASE, 'pybuilddir.txt')
+                with open(bdir, encoding='ascii') as f:
+                    libdir = f.read().strip()
+                libdir = os.path.join(_PROJECT_BASE, libdir)
+            else:
+                libdir = os.path.join(_PROJECT_BASE,
+                           'lib', 'python%s' % _get_python_config('version'))
+            sys.path.insert(0, libdir)
+            try:
+                _BUILD_TIME_VARS =  _get_module_attr(sysconfigdata,
+                                                     'build_time_vars')
+            finally:
+                sys.path.pop(0)
+        else:
+            _BUILD_TIME_VARS =  _get_module_attr(sysconfigdata,
+                                                 'build_time_vars')
+    return _BUILD_TIME_VARS
+
 def _init_posix(vars):
     """Initialize the module as appropriate for POSIX systems."""
-    # _sysconfigdata is generated at build time, see _generate_posix_vars()
-    name = _get_sysconfigdata_name()
-    _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0)
-    build_time_vars = _temp.build_time_vars
+    build_time_vars = get_build_time_vars()
     vars.update(build_time_vars)
 
 def _init_non_posix(vars):
@@ -623,6 +702,9 @@ def get_platform():
     For other non-POSIX platforms, currently just returns 'sys.platform'.
 
     """
+    if cross_compiling:
+        return get_config_var('PYTHON_HOST_PLATFORM')
+
     if os.name == 'nt':
         if 'amd64' in sys.version.lower():
             return 'win-amd64'
@@ -636,10 +718,6 @@ def get_platform():
         # XXX what about the architecture? NT is Intel or Alpha
         return sys.platform
 
-    # Set for cross builds explicitly
-    if "_PYTHON_HOST_PLATFORM" in os.environ:
-        return os.environ["_PYTHON_HOST_PLATFORM"]
-
     # Try to distinguish various flavours of Unix
     osname, host, release, version, machine = os.uname()
 

Lib/test/pythoninfo.py

@@ -286,9 +286,7 @@ def format_groups(groups):
         "VIRTUAL_ENV",
         "WAYLAND_DISPLAY",
         "WINDIR",
-        "_PYTHON_HOST_PLATFORM",
-        "_PYTHON_PROJECT_BASE",
-        "_PYTHON_SYSCONFIGDATA_NAME",
+        "PYTHON_PROJECT_BASE",
         "__PYVENV_LAUNCHER__",
     ))
     for name, value in os.environ.items():