Merge pull request #41 from HackerFoo/static_antlr4_runtime

Statically build and link the ANTLR4 runtime.

.github/workflows/presubmit.yml

@@ -35,6 +35,8 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
+        antlr_runtime_type: [static, shared]
+        python_version: [3.5, 3.6, 3.7, 3.8, 3.9]
         include:
           - python-version: 3.5
             TOXENV: py35
@@ -61,4 +63,4 @@ jobs:
           python -m pip install --upgrade pip
           pip install -r requirements.txt
       - name: Tox
-        run: tox -e ${{ matrix.TOXENV }}
+        run: ANTLR4_RUNTIME_TYPE=${{ matrix.antlr_runtime_type }} tox -e ${{ matrix.TOXENV }}

.gitmodules

@@ -1,3 +1,6 @@
 [submodule "third_party/googletest"]
 	path = third_party/googletest
 	url = https://github.com/google/googletest.git
+[submodule "third_party/antlr4"]
+	path = third_party/antlr4
+	url = https://github.com/antlr/antlr4.git

MANIFEST.in

@@ -27,7 +27,7 @@ prune docs/env
 
 include src/*
 include src/antlr/*
-recursive-include third_party *
+graft third_party
 global-exclude .git
 exclude .gitmodules
 exclude .clang-format

Makefile

@@ -44,6 +44,10 @@ format-cpp:
 build install clean develop:
 	$(IN_ENV) python setup.py $@
 
+.PHONY: build-shared
+build-shared:
+	$(IN_ENV) python setup.py build --antlr-runtime=shared
+
 .PHONY: check
 check:
 	$(IN_ENV) python setup.py check -m -s

README.md

@@ -28,6 +28,20 @@ Test with:
 
     python setup.py test
 
+The ANTLR runtime can either be linked statically or as a shared library. Use the
+`--antlr-runtime=[static|shared]` flag to select between the two modes e.g.:
+
+    python setup.py install --antlr-runtime=shared
+
+Or, using `pip`:
+
+    pip install . --install-option="--antlr-runtime=shared" --no-use-pep517
+
+The runtime will be built and statically linked by default. This flag is available in the build_ext, build, develop, and install commands.
+
+The --no-use-pep517 flag is needed because there is currently no way to pass flags with PEP517.  
+Relevant issue: https://github.com/pypa/pip/issues/5771
+
 ## FPGA Assembly (FASM)
 
 FPGA Assembly is a file format designed by the

setup.py

@@ -18,6 +18,9 @@
 from distutils.version import LooseVersion
 from setuptools import Extension
 from setuptools.command.build_ext import build_ext
+from distutils.command.build import build
+from setuptools.command.develop import develop
+from setuptools.command.install import install
 from Cython.Build import cythonize
 import traceback
 
@@ -34,7 +37,36 @@ def __init__(self, name, sourcedir='', prefix=''):
         self.prefix = prefix
 
 
-class CMakeBuild(build_ext):
+# Used to share options between two classes.
+class SharedOptions():
+    ANTLR_RUNTIMES = ['static', 'shared']
+    options = [
+        (
+            'antlr-runtime=', None,
+            "Whether to use a 'static' or 'shared' ANTLR runtime.")
+    ]
+
+    def __init__(self):
+        self.antlr_runtime = 'static'
+
+    def initialize(self, other):
+        other.antlr_runtime = None
+
+    def load(self, other):
+        if other.antlr_runtime is not None:
+            self.antlr_runtime = other.antlr_runtime
+            assert self.antlr_runtime in SharedOptions.ANTLR_RUNTIMES, \
+                'Invalid antlr_runtime {}, expected one of {}'.format(
+                    self.antlr_runtime, SharedOptions.ANTLR_RUNTIMES)
+
+
+# Global to allow sharing options.
+shared_options = SharedOptions()
+
+
+class AntlrCMakeBuild(build_ext):
+    user_options = SharedOptions.options
+
     def copy_extensions_to_source(self):
         original_extensions = list(self.extensions)
         self.extensions = [
@@ -45,6 +77,7 @@ def copy_extensions_to_source(self):
         self.extensions = original_extensions
 
     def run(self):
+        shared_options.load(self)
         try:
             super().run()
 
@@ -79,7 +112,8 @@ def build_extension(self, ext):
             cmake_args = [
                 '-DCMAKE_INSTALL_PREFIX=' + extdir,
                 '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
-                '-DPYTHON_EXECUTABLE=' + sys.executable
+                '-DPYTHON_EXECUTABLE=' + sys.executable,
+                '-DANTLR_RUNTIME_TYPE=' + shared_options.antlr_runtime
             ]
 
             cfg = 'Debug' if self.debug else 'Release'
@@ -92,6 +126,13 @@ def build_extension(self, ext):
                 env.get('CXXFLAGS', ''), self.distribution.get_version())
             if not os.path.exists(self.build_temp):
                 os.makedirs(self.build_temp)
+
+            # When linking the ANTLR runtime statically, -fPIC is still
+            # necessary because libparse_fasm will be a shared library.
+            if shared_options.antlr_runtime == 'static':
+                for flag in ["CFLAGS", "CXXFLAGS"]:
+                    os.environ[flag] = os.environ.get(flag, "") + " -fPIC"
+
             subprocess.check_call(
                 ['cmake', ext.sourcedir] + cmake_args,
                 cwd=self.build_temp,
@@ -105,6 +146,62 @@ def build_extension(self, ext):
         else:
             super().build_extension(ext)
 
+    def initialize_options(self):
+        super().initialize_options()
+        shared_options.initialize(self)
+
+    def finalize_options(self):
+        super().finalize_options()
+        shared_options.load(self)
+
+
+class BuildCommand(build):
+    user_options = build.user_options + SharedOptions.options
+
+    def initialize_options(self):
+        super().initialize_options()
+        shared_options.initialize(self)
+
+    def finalize_options(self):
+        super().finalize_options()
+        shared_options.load(self)
+
+    def run(self):
+        shared_options.load(self)
+        super().run()
+
+
+class InstallCommand(install):
+    user_options = install.user_options + SharedOptions.options
+
+    def initialize_options(self):
+        super().initialize_options()
+        shared_options.initialize(self)
+
+    def finalize_options(self):
+        super().finalize_options()
+        shared_options.load(self)
+
+    def run(self):
+        shared_options.load(self)
+        super().run()
+
+
+class DevelopCommand(develop):
+    user_options = develop.user_options + SharedOptions.options
+
+    def initialize_options(self):
+        super().initialize_options()
+        shared_options.initialize(self)
+
+    def finalize_options(self):
+        super().finalize_options()
+        shared_options.load(self)
+
+    def run(self):
+        shared_options.load(self)
+        super().run()
+
 
 setuptools.setup(
     name="fasm",
@@ -130,6 +227,9 @@ def build_extension(self, ext):
         CMakeExtension('parse_fasm', sourcedir='src', prefix='fasm/parser')
     ] + cythonize("fasm/parser/antlr_to_tuple.pyx"),
     cmdclass={
-        'build_ext': CMakeBuild,
+        'build_ext': AntlrCMakeBuild,
+        'build': BuildCommand,
+        'develop': DevelopCommand,
+        'install': InstallCommand,
     },
 )

src/CMakeLists.txt

@@ -15,18 +15,34 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/antlr4/
 
 set(CMAKE_CXX_STANDARD 11)
 
+if(ANTLR_RUNTIME_TYPE STREQUAL "static")
+  # Required if linking to static library
+  add_definitions(-DANTLR4CPP_STATIC)
+
+  # Build the static library.
+  include(ExternalAntlr4Cpp)
+  set(ANTLR4_RUNTIME antlr4_static)
+else()
+  # Look to see if the shared library is already available.
+  find_library(ANTLR4_RUNTIME NAMES antlr4-runtime REQUIRED)
+
+  # If not, build it.
+  if(NOT ANTLR4_RUNTIME)
+    include(ExternalAntlr4Cpp)
+    set(ANTLR4_RUNTIME antlr4_shared)
+  endif()
+endif()
+
 # add antrl4cpp artifacts to project environment
 include_directories(${ANTLR4_INCLUDE_DIRS})
 
 # set variable pointing to the antlr tool that supports C++
 # this is not required if the jar file can be found under PATH environment
-set(ANTLR_EXECUTABLE ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/antlr4/antlr-4.9-complete.jar)
+set(ANTLR_EXECUTABLE ${CMAKE_CURRENT_SOURCE_DIR}/../third_party/antlr4_lib/antlr-4.9-complete.jar)
 
 # add macros to generate ANTLR Cpp code from grammar
 find_package(ANTLR REQUIRED)
 
-find_library(ANTLR4_RUNTIME NAMES antlr4-runtime REQUIRED)
-
 # Unit testing library
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/googletest EXCLUDE_FROM_ALL googletest)