From 41ba45728c87a3e168ae0b77535431c9adea2122 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Wed, 29 Apr 2020 16:17:43 -0400 Subject: [PATCH] Python: Parse glean.h header at build time --- Makefile | 2 +- glean-core/python/ffi_build.py | 35 ++++++++++++++++++++++++++ glean-core/python/glean/_ffi.py | 20 ++------------- glean-core/python/requirements_dev.txt | 1 + glean-core/python/setup.py | 8 +++--- 5 files changed, 43 insertions(+), 23 deletions(-) create mode 100644 glean-core/python/ffi_build.py diff --git a/Makefile b/Makefile index 9fec63fd68..6258f3a99f 100644 --- a/Makefile +++ b/Makefile @@ -92,7 +92,7 @@ shellcheck: ## Run shellcheck against important shell scripts pythonlint: python-setup ## Run flake8 and black to lint Python code $(GLEAN_PYENV)/bin/python3 -m flake8 glean-core/python/glean glean-core/python/tests - $(GLEAN_PYENV)/bin/python3 -m black --check --exclude .venv\* glean-core/python + $(GLEAN_PYENV)/bin/python3 -m black --check --exclude \(.venv\*\)\|\(.eggs\) glean-core/python $(GLEAN_PYENV)/bin/python3 -m mypy glean-core/python/glean .PHONY: lint clippy ktlint swiftlint yamllint diff --git a/glean-core/python/ffi_build.py b/glean-core/python/ffi_build.py new file mode 100644 index 0000000000..94e64df54b --- /dev/null +++ b/glean-core/python/ffi_build.py @@ -0,0 +1,35 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +A helper script to build the CFFI wrappers at build time. +Run as part of the setup.py script. + +This is the "out-of-line", "ABI mode" option, described in the CFFI docs here: + + https://cffi.readthedocs.io/en/latest/cdef.html +""" + + +import cffi + + +def _load_header(path: str) -> str: + """ + Load a C header file and convert it to something parseable by cffi. + """ + with open(path, encoding="utf-8") as fd: + data = fd.read() + return "\n".join( + line for line in data.splitlines() if not line.startswith("#include") + ) + + +ffibuilder = cffi.FFI() +ffibuilder.set_source("glean._glean_ffi", None) +ffibuilder.cdef(_load_header("../ffi/glean.h")) + + +if __name__ == "__main__": + ffibuilder.compile() diff --git a/glean-core/python/glean/_ffi.py b/glean-core/python/glean/_ffi.py index 4f76f7074a..71e68453fb 100644 --- a/glean-core/python/glean/_ffi.py +++ b/glean-core/python/glean/_ffi.py @@ -3,13 +3,10 @@ # file, You can obtain one at http://mozilla.org/MPL/2.0/. from pathlib import Path -import pkgutil import sys from typing import Any, List, Optional import weakref -from cffi import FFI # type: ignore - def get_shared_object_filename() -> str: # pragma: no cover """ @@ -27,27 +24,14 @@ def get_shared_object_filename() -> str: # pragma: no cover _global_weakkeydict = weakref.WeakKeyDictionary() # type: Any -def _load_header(path: str) -> str: - """ - Load a C header file and convert it to something parseable by cffi. - """ - data = pkgutil.get_data(__name__, path) - if data is None: # pragma: no cover - raise RuntimeError("Couldn't load 'glean.h'") - data_str = data.decode("utf-8") - return "\n".join( - line for line in data_str.splitlines() if not line.startswith("#include") - ) - - # Don't load the Glean shared object / dll if we're in a (ping upload worker) # subprocess. # (a) it's not likely to work anyway, because it won't be the same Glean # singleton. # (b) skipping it significantly improves startup time of the subprocess. if not getattr(__builtins__, "IN_GLEAN_SUBPROCESS", False): - ffi = FFI() - ffi.cdef(_load_header("glean.h")) + from ._glean_ffi import ffi # type: ignore + lib = ffi.dlopen(str(Path(__file__).parent / get_shared_object_filename())) lib.glean_enable_logging() else: diff --git a/glean-core/python/requirements_dev.txt b/glean-core/python/requirements_dev.txt index 15d862fce1..4d4a3e99db 100644 --- a/glean-core/python/requirements_dev.txt +++ b/glean-core/python/requirements_dev.txt @@ -1,4 +1,5 @@ auditwheel==2.1.1 +cffi==1.13.1 coverage==4.5.2 flake8==3.7.8 jsonschema==3.2.0 diff --git a/glean-core/python/setup.py b/glean-core/python/setup.py index c4152794c3..4e181e0c59 100644 --- a/glean-core/python/setup.py +++ b/glean-core/python/setup.py @@ -47,12 +47,12 @@ version = parsed_toml["package"]["version"] requirements = [ - "cffi==1.13.1", + "cffi>=1.0.0", "glean_parser==1.19.0", "iso8601==0.1.12", ] -setup_requirements = [] +setup_requirements = ["cffi>=1.0.0"] if mingw_arch == "i686": shared_object_build_dir = "../../target/i686-pc-windows-gnu/debug/" @@ -72,7 +72,6 @@ raise ValueError("The platform {} is not supported.".format(sys.platform)) -shutil.copyfile("../ffi/glean.h", "glean/glean.h") shutil.copyfile("../metrics.yaml", "glean/metrics.yaml") shutil.copyfile("../pings.yaml", "glean/pings.yaml") shutil.copyfile(shared_object_build_dir + shared_object, "glean/" + shared_object) @@ -136,9 +135,10 @@ def finalize_options(self): version=version, packages=find_packages(include=["glean", "glean.*"]), setup_requires=setup_requirements, + cffi_modules=["ffi_build.py:ffibuilder"], url="https://github.com/mozilla/glean", zip_safe=False, - package_data={"glean": ["glean.h", shared_object, "metrics.yaml", "pings.yaml"]}, + package_data={"glean": [shared_object, "metrics.yaml", "pings.yaml"]}, distclass=BinaryDistribution, cmdclass={"install": InstallPlatlib, "bdist_wheel": bdist_wheel}, )