Building manylinux wheels

.github/workflows/test_and_deploy.yml

@@ -110,14 +110,76 @@ jobs:
         if: ${{ always() }}
         uses: actions/upload-artifact@v3
         with:
-          name: python-package-distributions
+          name: python-sdist
           path: dist/
+  build_wheels:
+    name: Build wheels on Ubuntu
+    runs-on: ubuntu-20.04 # Can be also run for macOS
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+      - name: Set up QEMU # Required for building manylinux aarch64 wheels on x86_64
+        if: runner.os == 'Linux'
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+      - name: Build wheels
+        if: github.event_name != 'release' && github.event.action != 'released'
+        uses: pypa/[email protected] # The main configuration is in pyproject.toml
+        env:
+          CIBW_BUILD: "cp311-manylinux*" # Build only python 3.11 wheels for testing
+          # Increase verbosity to see what's going on in the build in case of failure
+          CIBW_BUILD_VERBOSITY: 3
+          CIBW_REPAIR_WHEEL_COMMAND_LINUX: >
+            auditwheel show {wheel} && auditwheel repair -w {dest_dir} {wheel}
+      - name: Build release wheels
+        if: github.event_name == 'release' && github.event.action == 'released'
+        uses: pypa/[email protected] # The main configuration is in pyproject.toml
+        env:
+          # Set NLE_RELEASE_BUILD to 1 to build release wheels
+          CIBW_ENVIRONMENT: "NLE_RELEASE_BUILD=1"
+      - name: Save wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: python-wheels
+          path: ./wheelhouse/*.whl
+  test_manylinux_3_11:
+    name: Test manylinux wheel on Ubuntu w/ Py3.11
+    needs: build_wheels
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Setup Python 3.11 env
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Ensure latest pip & wheel
+        run: python -m pip install -q --upgrade pip wheel
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - name: Get wheels artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: python-wheels
+          path: dist
+      - name: Install from wheel # Install wheel mathcing the Python version and architecture
+        run: | 
+          WHEELNAME=$(ls dist/*311*manylinux*x86_64*.whl) 
+          MODE="[all]"
+          pip install "$WHEELNAME$MODE"
+      - name: Run tests outside repo dir
+        run: |
+          REPONAME=$(basename $PWD)
+          pushd ..
+          python -m pytest --import-mode=append -svx $REPONAME/nle/tests
+          popd
 
   # TODO move this to separate workflow whenever github decides to provide basic
   #      functionalities like workflow dependencies :|
   deploy_sdist:
     name: Deploy sdist to pypi
-    needs: test_sdist_3_11
+    needs: [test_sdist_3_11, test_manylinux_3_11]  
     if: github.event_name == 'release' && github.event.action == 'released'
     runs-on: ubuntu-latest
     environment:
@@ -132,19 +194,24 @@ jobs:
           echo "v$(cat version.txt)"
           echo "${{ github.event.release.tag_name }}"
           [[ "${{ github.event.release.tag_name }}" == "v$(cat version.txt)" ]]
-      - name: Get dist artifacts from test_sdist
+      - name: Get sdist artifact # Get sdist artifact from the test_sdist job
         uses: actions/download-artifact@v3
         with:
-          name: python-package-distributions
+          name: python-sdist
           path: dist
-      - name: Install from sdist
+      - name: Get wheels artifacts # Get wheels artifacts from the build_wheels job
+        uses: actions/download-artifact@v3
+        with:
+          name: python-wheels
+          path: dist
+      - name: Report dist contents
         run: |
           pwd
           ls -R
           ls -al .
           ls -R dist/
           ls -al dist/
-    # NOTE: we assume that dist/ contains a built sdist (and only that).
+    # NOTE: we assume that dist/ contains a built sdist and wheels (and only that).
     #       Yes, we could be more defensively, but What Could Go Wrong?™
       - name: Publish package to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1

.gitignore

@@ -206,3 +206,10 @@ nle/version.py
 nle_data/
 nle/fbs/
 nle/nethackdir
+
+# CMake
+CMakeFiles
+CMakeCache.txt
+
+# cibuildwheel
+wheelhouse/

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "third_party/pybind11"]
 	path = third_party/pybind11
 	url = https://github.com/pybind/pybind11.git
+[submodule "third_party/bzip2"]
+	path = third_party/bzip2
+	url = git://sourceware.org/git/bzip2.git

CMakeLists.txt

@@ -103,6 +103,18 @@ file(
   "win/tty/*.c"
   "win/rl/winrl.cc")
 
+# static version of bzip2 library
+add_library(
+  bz2_static STATIC
+  "third_party/bzip2/blocksort.c"
+  "third_party/bzip2/bzlib.c"
+  "third_party/bzip2/compress.c"
+  "third_party/bzip2/crctable.c"
+  "third_party/bzip2/decompress.c"
+  "third_party/bzip2/huffman.c"
+  "third_party/bzip2/randtable.c")
+target_link_libraries(bz2_static)
+
 # terminal emulator library
 add_library(tmt STATIC "third_party/libtmt/tmt.c")
 set_target_properties(tmt PROPERTIES C_STANDARD 11)
@@ -114,13 +126,14 @@ set_target_properties(nethack PROPERTIES CXX_STANDARD 14 SUFFIX ".so")
 target_include_directories(
   nethack
   PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include ${NLE_INC_GEN} /usr/local/include
-         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libtmt)
-target_link_directories(nethack PUBLIC /usr/local/lib)
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libtmt
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/bzip2)
+# target_link_directories(nethack PUBLIC /usr/local/lib)
 
 # Careful with -DMONITOR_HEAP: Ironically, it fails to fclose FILE* heaplog.
 # target_compile_definitions(nethack PUBLIC "$<$<CONFIG:DEBUG>:MONITOR_HEAP>")
 
-target_link_libraries(nethack PUBLIC m fcontext bz2 tmt)
+target_link_libraries(nethack PUBLIC m fcontext bz2_static tmt)
 
 # dlopen wrapper library
 add_library(nethackdl STATIC "sys/unix/nledl.c")
@@ -154,9 +167,11 @@ add_library(
   converter STATIC ${CMAKE_CURRENT_SOURCE_DIR}/third_party/converter/converter.c
                    ${CMAKE_CURRENT_SOURCE_DIR}/third_party/converter/stripgfx.c)
 target_include_directories(
-  converter PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libtmt
-                   ${CMAKE_CURRENT_SOURCE_DIR}/third_party/converter)
-target_link_libraries(converter PUBLIC bz2 tmt)
+  converter
+  PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/third_party/libtmt
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/converter
+         ${CMAKE_CURRENT_SOURCE_DIR}/third_party/bzip2)
+target_link_libraries(converter PUBLIC bz2_static tmt)
 if(CMAKE_BUILD_TYPE MATCHES Debug)
   target_compile_options(converter PRIVATE -Wall -Wextra -pedantic -Werror)
 endif()

pyproject.toml

@@ -43,3 +43,26 @@ select = [
 allow-dict-calls-with-keyword-arguments = true
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["F401"]  # unused-import.
+
+[tool.cibuildwheel]
+# We need to build wheels for the following Python versions:
+build = "cp{38,39,310,311}-*"
+
+[tool.cibuildwheel.linux]
+# Only manylinux is supported (no musllinux)
+build = "cp{38,39,310,311}-manylinux*"
+
+# We need to clean up the build directory, all .so files, and CMakeCache.txt
+# and install the dependencies using yum, as manylinux2014 image is CentOS 7-based
+before-all = "rm -rf {project}/build {project}/*.so {project}/CMakeCache.txt && yum install -y autoconf bison cmake flex git libtool"
+
+# Only x86_64 and aarch64 (ARM) architectures are supported
+archs = "x86_64 aarch64"
+
+# Use manylinux2014 image for both x86_64 and aarch64
+manylinux-x86_64-image = "manylinux2014"
+manylinux-aarch64-image = "manylinux2014"
+
+[tool.cibuildwheel.macos]
+# For macOS wheels, we only need to install cmake
+before-all = "rm -rf {project}/build {project}/*.so {project}/CMakeCache.txt && brew install cmake"