Python: Parse glean.h header at build time

Makefile

@@ -92,7 +92,7 @@ shellcheck: ## Run shellcheck against important shell scripts
 
 pythonlint: python-setup ## Run flake8 and black to lint Python code
 	$(GLEAN_PYENV)/bin/python3 -m flake8 glean-core/python/glean glean-core/python/tests
-	$(GLEAN_PYENV)/bin/python3 -m black --check --exclude .venv\* glean-core/python
+	$(GLEAN_PYENV)/bin/python3 -m black --check --exclude \(.venv\*\)\|\(.eggs\) glean-core/python
 	$(GLEAN_PYENV)/bin/python3 -m mypy glean-core/python/glean
 
 .PHONY: lint clippy ktlint swiftlint yamllint

glean-core/python/ffi_build.py

@@ -0,0 +1,35 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+A helper script to build the CFFI wrappers at build time.
+Run as part of the setup.py script.
+
+This is the "out-of-line", "ABI mode" option, described in the CFFI docs here:
+
+    https://cffi.readthedocs.io/en/latest/cdef.html
+"""
+
+
+import cffi
+
+
+def _load_header(path: str) -> str:
+    """
+    Load a C header file and convert it to something parseable by cffi.
+    """
+    with open(path, encoding="utf-8") as fd:
+        data = fd.read()
+    return "\n".join(
+        line for line in data.splitlines() if not line.startswith("#include")
+    )
+
+
+ffibuilder = cffi.FFI()
+ffibuilder.set_source("glean._glean_ffi", None)
+ffibuilder.cdef(_load_header("../ffi/glean.h"))
+
+
+if __name__ == "__main__":
+    ffibuilder.compile()

glean-core/python/glean/_ffi.py

@@ -3,13 +3,10 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from pathlib import Path
-import pkgutil
 import sys
 from typing import Any, List, Optional
 import weakref
 
-from cffi import FFI  # type: ignore
-
 
 def get_shared_object_filename() -> str:  # pragma: no cover
     """
@@ -27,27 +24,14 @@ def get_shared_object_filename() -> str:  # pragma: no cover
 _global_weakkeydict = weakref.WeakKeyDictionary()  # type: Any
 
 
-def _load_header(path: str) -> str:
-    """
-    Load a C header file and convert it to something parseable by cffi.
-    """
-    data = pkgutil.get_data(__name__, path)
-    if data is None:  # pragma: no cover
-        raise RuntimeError("Couldn't load 'glean.h'")
-    data_str = data.decode("utf-8")
-    return "\n".join(
-        line for line in data_str.splitlines() if not line.startswith("#include")
-    )
-
-
 # Don't load the Glean shared object / dll if we're in a (ping upload worker)
 # subprocess.
 #   (a) it's not likely to work anyway, because it won't be the same Glean
 #       singleton.
 #   (b) skipping it significantly improves startup time of the subprocess.
 if not getattr(__builtins__, "IN_GLEAN_SUBPROCESS", False):
-    ffi = FFI()
-    ffi.cdef(_load_header("glean.h"))
+    from ._glean_ffi import ffi  # type: ignore
+
     lib = ffi.dlopen(str(Path(__file__).parent / get_shared_object_filename()))
     lib.glean_enable_logging()
 else:

glean-core/python/requirements_dev.txt

@@ -1,4 +1,5 @@
 auditwheel==2.1.1
+cffi==1.13.1
 coverage==4.5.2
 flake8==3.7.8
 jsonschema==3.2.0

glean-core/python/setup.py

@@ -47,12 +47,12 @@
     version = parsed_toml["package"]["version"]
 
 requirements = [
-    "cffi==1.13.1",
+    "cffi>=1.0.0",
     "glean_parser==1.19.0",
     "iso8601==0.1.12",
 ]
 
-setup_requirements = []
+setup_requirements = ["cffi>=1.0.0"]
 
 if mingw_arch == "i686":
     shared_object_build_dir = "../../target/i686-pc-windows-gnu/debug/"
@@ -72,7 +72,6 @@
     raise ValueError("The platform {} is not supported.".format(sys.platform))
 
 
-shutil.copyfile("../ffi/glean.h", "glean/glean.h")
 shutil.copyfile("../metrics.yaml", "glean/metrics.yaml")
 shutil.copyfile("../pings.yaml", "glean/pings.yaml")
 shutil.copyfile(shared_object_build_dir + shared_object, "glean/" + shared_object)
@@ -136,9 +135,10 @@ def finalize_options(self):
     version=version,
     packages=find_packages(include=["glean", "glean.*"]),
     setup_requires=setup_requirements,
+    cffi_modules=["ffi_build.py:ffibuilder"],
     url="https://github.com/mozilla/glean",
     zip_safe=False,
-    package_data={"glean": ["glean.h", shared_object, "metrics.yaml", "pings.yaml"]},
+    package_data={"glean": [shared_object, "metrics.yaml", "pings.yaml"]},
     distclass=BinaryDistribution,
     cmdclass={"install": InstallPlatlib, "bdist_wheel": bdist_wheel},
 )