Build static libraries of HPy devel sources for testing.

.github/workflows/ci.yml

@@ -44,7 +44,9 @@ jobs:
         run: python -m pip install --upgrade pip wheel 'setuptools>=60.2'
 
       - name: Build
-        run: python -m pip install .
+        run: |
+          make
+          python -m pip install .
 
       - if: ${{ matrix.compiler }}
         # Only set the compiler for the tests, not for the build
@@ -79,7 +81,9 @@ jobs:
         run: python -m pip install --upgrade pip wheel
 
       - name: Build
-        run: python -m pip install .
+        run: |
+          make
+          python -m pip install .
 
       - name: Run tests
         run: |
@@ -401,7 +405,9 @@ jobs:
           python -m pip install pytest cffi
 
       - name: Build and install HPy
-        run: python -m pip install .
+        run: |
+          make
+          python -m pip install .
 
       - name: Run microbenchmarks
         run: |

.github/workflows/valgrind-tests.yml

@@ -21,7 +21,9 @@ jobs:
         run: python -m pip install --upgrade pip wheel
 
       - name: Build
-        run: python -m pip install .
+        run: |
+          make
+          python -m pip install .
 
       - name: Run tests
         env:

.gitignore

@@ -14,9 +14,11 @@ __pycache__/
 *.py[cod]
 *$py.class
 
-# C extensions
+# C extensions and static libs
 *.so
 *.o
+*.a
+*.lib
 vc140.pdb
 c_test/test_debug_handles
 

Makefile

@@ -3,7 +3,7 @@ all: hpy.universal
 
 .PHONY: hpy.universal
 hpy.universal:
-	python3 setup.py build_ext -if
+	python3 setup.py build_clib -f build_ext -if
 
 .PHONY: dist-info
 dist-info:

hpy/devel/__init__.py

@@ -52,6 +52,7 @@ def __init__(self, base_dir=_DEFAULT_BASE_DIR):
         self.base_dir = Path(base_dir)
         self.include_dir = self.base_dir.joinpath('include')
         self.src_dir = self.base_dir.joinpath('src', 'runtime')
+        self._available_static_libs = None
 
     def get_extra_include_dirs(self):
         """ Extra include directories needed by extensions in both CPython and
@@ -74,6 +75,37 @@ def get_extra_sources(self):
             self.src_dir.joinpath('helpers.c'),
         ]))
 
+    def _scan_static_lib_dir(self):
+        """ Scan the static library directory and build a dict for all
+            available static libraries. The library directory contains
+            subdirectories for each ABI and the ABI folders then contain
+            the static libraries.
+        """
+        available_libs = {}
+        lib_dir = self.base_dir.joinpath('lib')
+        if lib_dir.exists():
+            for abi_dir in lib_dir.iterdir():
+                if abi_dir.is_dir():
+                    abi = abi_dir.name
+                    # All files in '.../lib/<abi>/' are considered to be static
+                    # libraries.
+                    available_libs[abi] = \
+                        [str(x) for x in abi_dir.iterdir() if x.is_file()]
+        return available_libs
+
+    def get_static_libs(self, hpy_abi):
+        """ The list of necessary static libraries an HPy extension needs to
+            link to or 'None' (if not available). The HPy ext needs to link to
+            all static libraries in the list otherwise some function may stay
+            unresolved. For example, there is library 'hpyextra' which contains
+            compiled HPy helper functions like 'HPyArg_Parse' and such.
+            Libraries are always specific to an ABI.
+        """
+        if not self._available_static_libs:
+            # lazily initialize the dict of available (=shipped) static libs
+            self._available_static_libs = self._scan_static_lib_dir()
+        return self._available_static_libs.get(hpy_abi, None)
+
     def get_ctx_sources(self):
         """ Extra sources needed only in the CPython ABI mode.
         """
@@ -146,8 +178,10 @@ def handle_hpy_ext_modules(dist, attr, hpy_ext_modules):
 
     # add a global option --hpy-abi to setup.py
     dist.__class__.hpy_abi = DEFAULT_HPY_ABI
+    dist.__class__.hpy_use_static_libs = False
     dist.__class__.global_options += [
-        ('hpy-abi=', None, 'Specify the HPy ABI mode (default: %s)' % DEFAULT_HPY_ABI)
+        ('hpy-abi=', None, 'Specify the HPy ABI mode (default: %s)' % DEFAULT_HPY_ABI),
+        ('hpy-no-static-libs', None, 'Compile context and extra sources with extension (default: False)')
     ]
     hpydevel = HPyDevel()
     hpydevel.fix_distribution(dist)
@@ -288,11 +322,31 @@ def _finalize_hpy_ext(self, ext):
         ext.name = HPyExtensionName(ext.name)
         ext.hpy_abi = self.distribution.hpy_abi
         ext.include_dirs += self.hpydevel.get_extra_include_dirs()
-        ext.sources += self.hpydevel.get_extra_sources()
+        # If static libs should be used, then add all available libs (for
+        # the given ABI) to the extra objects. The libs will then just be added
+        # in the linking phase but nothing will be compiled in addition.
+        static_libs = self.distribution.hpy_use_static_libs
+        if static_libs:
+            static_libs = self.hpydevel.get_static_libs(ext.hpy_abi)
+            if static_libs is None or len(static_libs) != 1:
+                raise DistutilsError('Expected exactly one static library for '
+                                     'ABI "%s" but got: %r' %
+                                     (ext.hpy_abi, static_libs))
+
+        if static_libs:
+            ext.extra_objects += static_libs
+        else:
+            # If we should not use (pre-compiled) static libs or if they are
+            # not available, we just add the sources of the helpers to the
+            # extension. They are then compiler with the extension.
+            ext.sources += self.hpydevel.get_extra_sources()
         ext.define_macros.append(('HPY', None))
         if ext.hpy_abi == 'cpython':
+            # If the user disabled using static libs, we need to add the
+            # context sources in this case.
+            if not static_libs:
+                ext.sources += self.hpydevel.get_ctx_sources()
             ext.define_macros.append(('HPY_ABI_CPYTHON', None))
-            ext.sources += self.hpydevel.get_ctx_sources()
             ext._hpy_needs_stub = False
         elif ext.hpy_abi == 'hybrid':
             ext.define_macros.append(('HPY_ABI_HYBRID', None))

setup.py

@@ -1,8 +1,7 @@
-import textwrap
 import sys
-import os
 import os.path
 from setuptools import setup, Extension
+from setuptools.command.build_clib import build_clib
 import platform
 
 # this package is supposed to be installed ONLY on CPython. Try to bail out
@@ -60,6 +59,7 @@
 else:
     EXTRA_COMPILE_ARGS += ['-Werror']
 
+
 def get_scm_config():
     """
     We use this function as a hook to generate version.h before building.
@@ -93,26 +93,116 @@ def get_scm_config():
 
     return {}  # use the default config
 
+HPY_EXTRA_SOURCES = [
+    'hpy/devel/src/runtime/argparse.c',
+    'hpy/devel/src/runtime/buildvalue.c',
+    'hpy/devel/src/runtime/helpers.c',
+]
+
+HPY_CTX_SOURCES = [
+    'hpy/devel/src/runtime/ctx_bytes.c',
+    'hpy/devel/src/runtime/ctx_call.c',
+    'hpy/devel/src/runtime/ctx_capsule.c',
+    'hpy/devel/src/runtime/ctx_err.c',
+    'hpy/devel/src/runtime/ctx_module.c',
+    'hpy/devel/src/runtime/ctx_object.c',
+    'hpy/devel/src/runtime/ctx_type.c',
+    'hpy/devel/src/runtime/ctx_tracker.c',
+    'hpy/devel/src/runtime/ctx_listbuilder.c',
+    'hpy/devel/src/runtime/ctx_tuple.c',
+    'hpy/devel/src/runtime/ctx_tuplebuilder.c',
+]
+
+HPY_INCLUDE_DIRS = [
+    'hpy/devel/include',
+    'hpy/universal/src',
+    'hpy/debug/src/include',
+    'hpy/trace/src/include',
+]
+
+HPY_EXTRA_UNIVERSAL_LIB_NAME = "hpy-extra-universal"
+HPY_EXTRA_HYBRID_LIB_NAME = "hpy-extra-hybrid"
+HPY_CTX_LIB_NAME = "hpy-ctx-cpython"
+
+HPY_BUILD_CLIB_ABI_ATTR = "hpy_abi"
+
+class build_clib_hpy(build_clib):
+    """ Special build_clib command for building HPy's static libraries defined
+        by 'STATIC_LIBS' below. The behavior differs in following points:
+        (1) Option 'force' is set such that static libs will always be renewed.
+        (2) Method 'get_library_names' always returns 'None'. This is because
+            we only use this command to build static libraries for testing.
+            That means, we only use them in-place. We don't need them for
+            linking here.
+        (3) This command consumes a custom build info key
+            HPY_BUILD_CLIB_ABI_ATTR that is used to create separate build
+            temp directories for each ABI. This is necessary to avoid
+            incorrect sharing of (temporary) build artifacts.
+        (4) This command will use the include directories from command
+            'build_ext'.
+    """
+    def finalize_options(self):
+        super().finalize_options()
+        # we overwrite the include dirs and use the ones from 'build_ext'
+        build_ext_includes = self.get_finalized_command('build_ext').include_dirs or []
+        self.include_dirs = HPY_INCLUDE_DIRS + build_ext_includes
+        self.force = 1
+
+    def get_library_names(self):
+        # We only build static libraries for testing. We just use them
+        # in-place. We don't want that our extensions (i.e. 'hpy.universal'
+        # etc) link to these libs.
+        return None
+
+    def build_libraries(self, libraries):
+        # we just inherit the 'inplace' option from 'build_ext'
+        inplace = self.get_finalized_command('build_ext').inplace
+        if inplace:
+            # the inplace option requires to find the package directory
+            # using the build_py command for that
+            build_py = self.get_finalized_command('build_py')
+            lib_dir = os.path.abspath(build_py.get_package_dir('hpy.devel'))
+        else:
+            lib_dir = self.build_clib
+
+        import pathlib
+        for lib in libraries:
+            lib_name, build_info = lib
+            abi = build_info.get(HPY_BUILD_CLIB_ABI_ATTR)
+            # Call super's build_libraries with just one library in the list
+            # such that we can temporarily change the 'build_temp'.
+            orig_build_temp = self.build_temp
+            orig_build_clib = self.build_clib
+            self.build_temp = os.path.join(orig_build_temp, 'lib', abi)
+            self.build_clib = os.path.join(lib_dir, 'lib', abi)
+            # ensure that 'build_clib' directory exists
+            pathlib.Path(self.build_clib).mkdir(parents=True, exist_ok=True)
+            try:
+                super().build_libraries([lib])
+            finally:
+                self.build_temp = orig_build_temp
+                self.build_clib = orig_build_clib
+
+
+STATIC_LIBS = [(HPY_EXTRA_UNIVERSAL_LIB_NAME,
+                {'sources': HPY_EXTRA_SOURCES,
+                 HPY_BUILD_CLIB_ABI_ATTR: 'universal',
+                 'macros': [('HPY_ABI_UNIVERSAL', None)]}),
+               (HPY_EXTRA_HYBRID_LIB_NAME,
+                {'sources': HPY_EXTRA_SOURCES,
+                 HPY_BUILD_CLIB_ABI_ATTR: 'hybrid',
+                 'macros': [('HPY_ABI_HYBRID', None)]}),
+               (HPY_CTX_LIB_NAME,
+                {'sources': HPY_EXTRA_SOURCES + HPY_CTX_SOURCES,
+                 HPY_BUILD_CLIB_ABI_ATTR: 'cpython',
+                 'macros': [('HPY_ABI_CPYTHON', None)]})]
+
 EXT_MODULES = [
     Extension('hpy.universal',
               ['hpy/universal/src/hpymodule.c',
                'hpy/universal/src/ctx.c',
                'hpy/universal/src/ctx_meth.c',
                'hpy/universal/src/ctx_misc.c',
-               'hpy/devel/src/runtime/argparse.c',
-               'hpy/devel/src/runtime/buildvalue.c',
-               'hpy/devel/src/runtime/helpers.c',
-               'hpy/devel/src/runtime/ctx_bytes.c',
-               'hpy/devel/src/runtime/ctx_call.c',
-               'hpy/devel/src/runtime/ctx_capsule.c',
-               'hpy/devel/src/runtime/ctx_err.c',
-               'hpy/devel/src/runtime/ctx_module.c',
-               'hpy/devel/src/runtime/ctx_object.c',
-               'hpy/devel/src/runtime/ctx_type.c',
-               'hpy/devel/src/runtime/ctx_tracker.c',
-               'hpy/devel/src/runtime/ctx_listbuilder.c',
-               'hpy/devel/src/runtime/ctx_tuple.c',
-               'hpy/devel/src/runtime/ctx_tuplebuilder.c',
                'hpy/debug/src/debug_ctx.c',
                'hpy/debug/src/debug_ctx_cpython.c',
                'hpy/debug/src/debug_handles.c',
@@ -124,14 +214,10 @@ def get_scm_config():
                'hpy/trace/src/trace_ctx.c',
                'hpy/trace/src/_tracemod.c',
                'hpy/trace/src/autogen_trace_wrappers.c',
-               'hpy/trace/src/autogen_trace_func_table.c',
-              ],
-              include_dirs=[
-                  'hpy/devel/include',
-                  'hpy/universal/src',
-                  'hpy/debug/src/include',
-                  'hpy/trace/src/include',
-              ],
+               'hpy/trace/src/autogen_trace_func_table.c']
+              + HPY_EXTRA_SOURCES
+              + HPY_CTX_SOURCES,
+              include_dirs=HPY_INCLUDE_DIRS,
               extra_compile_args=[
                   # so we need to enable the HYBRID ABI in order to implement
                   # the legacy features
@@ -142,7 +228,6 @@ def get_scm_config():
               )
     ]
 
-
 DEV_REQUIREMENTS = [
     "pytest",
     "pytest-xdist",
@@ -158,17 +243,19 @@ def get_scm_config():
     description='A better C API for Python',
     long_description=LONG_DESCRIPTION,
     long_description_content_type='text/markdown',
-    packages = ['hpy.devel', 'hpy.debug', 'hpy.trace'],
+    packages=['hpy.devel', 'hpy.debug', 'hpy.trace'],
     include_package_data=True,
     extras_require={
         "dev": DEV_REQUIREMENTS,
     },
+    libraries=STATIC_LIBS,
     ext_modules=EXT_MODULES,
     entry_points={
         "distutils.setup_keywords": [
             "hpy_ext_modules = hpy.devel:handle_hpy_ext_modules",
         ],
     },
+    cmdclass={"build_clib": build_clib_hpy},
     use_scm_version=get_scm_config,
     setup_requires=['setuptools_scm'],
     install_requires=['setuptools>=64.0'],

test/support.py

@@ -540,7 +540,13 @@ def _build(tmpdir, ext, hpy_devel, hpy_abi, compiler_verbose=0, debug=None):
 
     # this is the equivalent of passing --hpy-abi from setup.py's command line
     dist.hpy_abi = hpy_abi
+    # For testing, we want to use static libs to avoid repeated compilation
+    # of the same sources which slows down testing.
+    dist.hpy_use_static_libs = True
     dist.hpy_ext_modules = [ext]
+    # We need to explicitly specify which Python modules we expect because some
+    # test cases create several distributions in the same temp directory.
+    dist.py_modules = [ext.name]
     hpy_devel.fix_distribution(dist)
 
     old_level = distutils.log.set_threshold(0) or 0