Add cibuildwheel support

.github/workflows/build.yml

@@ -12,6 +12,10 @@ on:
   push:
   pull_request:
   workflow_dispatch:
+  release:
+    types:
+      - prereleased
+      - released
   schedule:
     # 5AM every Monday, to catch breaks due to changes in dependencies
     - cron: "0 5 * * 1"
@@ -119,7 +123,7 @@ jobs:
           name: ${{ matrix.os }}-PerformanceTestResults.txt
           path: ./out/SoarPerformanceTests/PerformanceTestResults.txt
 
-# Using powershell means we need to explicitly stop on failure
+  # Using powershell means we need to explicitly stop on failure
   Windows:
     name: build-windows
     runs-on: [windows-latest]
@@ -260,3 +264,135 @@ jobs:
         with:
           name: Windows-PerformanceTestResults.txt
           path: ./out/SoarPerformanceTests/PerformanceTestResults.txt
+
+  python_wheels:
+    name: wheel-*nix
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        os: [
+            ubuntu-latest,
+            # latest available X86_64 target
+            macos-12,
+            # latest is ARM
+            macos-latest,
+
+            windows-latest,
+          ]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Show tags
+        run: git describe --long --dirty --always --tags
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      # - name: Setup tcl (ubuntu)
+      #   if: matrix.os == 'ubuntu-latest'
+      #   run: sudo apt-get update && sudo apt-get install tcl-dev
+
+      # - name: Setup tcl (macos-latest)
+      #   if: matrix.os == 'macos-latest'
+      #   run: brew install tcl-tk
+
+      - name: Setup SWIG (macos-latest)
+        if: matrix.os == 'macos-latest'
+        run: brew install swig
+
+      - name: Build wheels
+        uses: pypa/[email protected]
+        with:
+          package-dir: Core/ClientSMLSWIG/Python/
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
+
+  python_push_dev:
+    name: Publish to test.pypi.org
+    runs-on: ubuntu-latest
+    # - Upload only every monday to prevent spamming the index with too many build artifacts.
+    # - Allow uploading on manual workflow dispatch, to enable quick dev releases.
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    # Alternative, to upload on every commit to development;
+    # if: github.event_name != 'schedule' && github.event_name != 'release'
+    needs:
+      # We depend on the python wheel builds because we need their artifacts
+      - python_wheels
+
+      # We depend on the builds themselves to gatekeep the upload if build + testing fails,
+      # usually then also the python wheel build should fail, but the normal builds do more thorough checking.
+      - Posix
+      - Windows
+
+    environment:
+      name: test-pypi
+      url: https://test.pypi.org/p/soar-sml/
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-wheels-*
+          path: wheelhouse/
+          merge-multiple: true
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          packages-dir: wheelhouse
+          skip-existing: true
+          verbose: true
+
+  python_push_release:
+    name: Publish to pypi.org
+    runs-on: ubuntu-latest
+    if: github.event_name == 'release'
+    needs:
+      # We depend on the python wheel builds because we need their artifacts
+      - python_wheels
+
+      # We depend on the builds themselves to gatekeep the upload if build + testing fails,
+      # usually then also the python wheel build should fail, but the normal builds do more thorough checking.
+      - Posix
+      - Windows
+
+    environment:
+      name: pypi
+      url: https://pypi.org/p/soar-sml/
+    permissions:
+      id-token: write
+
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-wheels-*
+          path: wheelhouse/
+          merge-multiple: true
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: wheelhouse
+          skip-existing: true
+          verbose: true

.gitignore

@@ -503,6 +503,12 @@ local.properties
 # TeXlipse plugin
 .texlipse
 
+# cibuildwheel output
+wheelhouse/
+
+# pip/enscons build output
+soar_sml/
+
 ### Soar ###
 out/
 build/

Core/ClientSMLSWIG/Python/DEVELOPING.md

@@ -0,0 +1,71 @@
+# Developing `soar-sml`
+
+Before installing development releases, it is always recommended to have an updated `pip`:
+
+```
+$ pip install --upgrade pip
+```
+
+## Instant: Installing from the latest commit
+
+To install `soar-sml` *directly* from the latest `development` branch commit, without cloning it into a project directory,
+run the following command:
+
+```
+$ pip install "git+https://github.com/SoarGroup/Soar#subdirectory=Core/ClientSMLSWIG/Python"
+```
+
+## Local: Installing from a project directory
+
+To make `soar-sml` available for import in python, while actively developing the source code for it,
+the following command is more suitable.
+
+Assuming `Soar` has been cloned under `soar/`:
+```bash
+$ pip install -e soar/Core/ClientSMLSWIG/Python
+```
+
+The `-e` stands for "editable", this means pip will have python redirect the "actual" location of the `soar_sml`
+package to be inside this project directory. Specifically, in this case, to the project root directory (`soar/`).
+
+This command will install a python package directory under the project root (`soar/soar_sml/`), where it copies
+build artifacts into, into a format that python expects.
+
+During development, other than running the `pip` command again,
+these files can be updated from builds with the `sml_python_dev` target:
+
+```bash
+$ scons sml_python_dev
+```
+
+Running the above command, and then re-running your python scripts,
+will automatically incorporate any compiled changes.
+
+## Remote: Installing a development version from the package index
+
+Weekly development versions are uploaded to [`test.pypi.org/p/soar-sml`](https://test.pypi.org/p/soar-sml),
+the latest of which can be installed with the following command:
+
+```bash
+$ pip install --pre -i https://test.pypi.org/simple/ soar-sml
+```
+
+## Packaging
+
+`build.yml` will automatically build wheels for `soar-sml` on every commit, utilizing
+[cibuildwheel](https://cibuildwheel.pypa.io/) to ease the process, and build for many python versions at once. It will
+do a quick test on every build, importing `soar_sml` and running hello world, to ensure that the wheel passes basic
+checks, and "works".
+
+`build.yml` also contains job definitions to upload to PyPI, the Python Package Index.
+
+On a GitHub release, a workflow is triggered to build the final version of wheels, and mark them
+**with the version given in the corresponding git tag**.
+(So if the release is named "Version 9.6.2", and the tag is "v9.6.2", then the auto-version script will see "v9.6.2")
+The string `releases/` is also removed from the git tag before parsing.
+
+Development versions are built and uploaded to [`test.pypi.org/p/soar-sml`](https://test.pypi.org/p/soar-sml) weekly.
+These will also upload on manual triggers.
+
+If needed, uploading to pypi can be done manually using the
+[twine CLI tool](https://twine.readthedocs.io/en/stable/#using-twine).

Core/ClientSMLSWIG/Python/README.md

@@ -0,0 +1,77 @@
+# `soar-sml`
+
+This directory contains the SWIG bindings to Soar's SML interface.
+
+This project is also published on PyPI as [soar-sml](https://pypi.org/project/soar-sml/),
+for ease of distribution and installation.
+
+Versions on PyPI will follow Soar's versioning methodology.
+
+`soar-sml` effectively bundles Soar with its distribution; you can download it, import it into Python,
+and have a fully-working Soar kernel up and running.
+
+## Importing
+
+The soar-sml package can be imported like so:
+
+```Python
+import soar_sml
+```
+
+## `Python_sml_ClientInterface` compatibility
+
+The raw build artifacts of this SWIG interface exposed these bindings under a `Python_sml_ClientInterface` namespace
+in the past, we've switched to using `soar_sml` to be more in line with python's packaging ideology. However,
+we did not want to break compatibility with all scripts by doing so, and have also published a compatibility shim.
+
+Under `compat/`, there exists a small project `soar-compat` that re-exports the classes and functions of under a
+`Python_sml_ClientInterface` namespace, effectively making every existing project compatible with the new `soar-sml`
+package.
+
+This means that code like thus will still continue to work:
+
+```Python
+import sys
+sys.path.append('/home/user/SoarSuite/bin')
+import Python_sml_ClientInterface as sml
+
+k = sml.Kernel.CreateKernelInNewThread()
+a = k.CreateAgent('soar')
+print(a.ExecuteCommandLine('echo hello world'))
+```
+
+In this case, the `sys.path.append` line is redundant (it accomplishes nothing; python already properly imports
+the package), and can be removed.
+
+This compatibility package is available on PyPI, and can be installed directly like so:
+
+```bash
+$ pip install soar-compat --no-deps
+```
+(the `--no-deps` flag is added to prevent `soar-sml` being pulled from pypi, as that is a dependency of
+`soar-compat`)
+
+However, for ease of installation, it can be installed as an ["extra"](https://stackoverflow.com/a/52475030/8700553)
+like so:
+
+```bash
+$ pip install "soar-sml[compat]"
+```
+
+Running this above command will make every Soar SML Python script on your system (or virtual environment) that uses
+`import Python_sml_ClientInterface` functional, portable, with no further modifications necessary.
+
+## Building locally
+
+Building and installing this package via pip, locally, is easy:
+
+```BASH
+$ pip install soar/Core/ClientSMLSWIG/Python
+```
+
+This will generate all the required build artifacts, install them to your python installation (system-wide, or
+inside a virtual environment), and prepare it for `import soar_sml` statements in your scripts/programs.
+
+## Developing and Packaging
+
+Notes for developing and packaging `soar-sml` can be found in [`DEVELOPING.md`](./DEVELOPING.md)

Core/ClientSMLSWIG/Python/SConscript

@@ -17,14 +17,27 @@ python_sml_alias = clone['SML_PYTHON_ALIAS']
 
 inc_path = sysconfig.get_path('include')
 if os.name == 'nt':
-    lib_path = os.path.join(sysconfig.get_config_vars('BINDIR')[0], 'libs')
+    lib_path = [
+        os.path.join(sysconfig.get_config_vars('BINDIR')[0], 'libs'),
+        # Finds the correct library path for Github Runner environments (cibuildwheel)
+        os.path.join(sysconfig.get_config_vars('LIBDEST')[0], '..\\libs'),
+    ]
     pylib = 'python' + sysconfig.get_config_vars('VERSION')[0]
 else:
     lib_path = sysconfig.get_config_vars('LIBDIR')[0]
     pylib = sysconfig.get_config_vars('LIBRARY')[0]
 lib_install_dir = clone['OUT_DIR']
 
-clone.Append(CPPPATH = inc_path, LIBPATH = lib_path, LIBS = pylib)
+clone.Append(CPPPATH = inc_path)
+
+# linux: manylinux containers do not ship with python's libraries:
+#  https://github.com/pypa/manylinux/issues/191#issuecomment-386489125
+# macos: python libraries (only their executables) do not appear to be present when installing python on
+#  GHA Runners. (python installations are handled by cibuildwheel)
+#
+# Omitting python's libraries is safe, as python itself injects its symbols when importing the library.
+if (not env['ENSCONS_ACTIVE']) or os.name == 'nt':
+    clone.Append(LIBPATH = lib_path, LIBS = pylib)
 
 if os.name == 'posix':
     clone.Append(CPPFLAGS = Split('-Wno-unused -fno-strict-aliasing'))
@@ -46,4 +59,13 @@ install_source = env.Install(lib_install_dir, source)
 install_lib = env.Install(lib_install_dir, shlib)
 install_test = env.Install(lib_install_dir, env.File('TestPythonSML.py'))
 
-env.Alias(python_sml_alias, install_lib + install_source + install_test)
+# We add soarlib to the python_sml explicitly, as some operating systems don't pick up on this dependency,
+# and crash the build.
+Import('soarlib')
+
+env.Alias(python_sml_alias, soarlib + install_lib + install_source + install_test)
+
+python_shlib = shlib
+python_source = source
+Export('python_shlib')
+Export('python_source')

Core/ClientSMLSWIG/Python/compat/Python_sml_ClientInterface.py

@@ -0,0 +1 @@
+from soar_sml import *

Core/ClientSMLSWIG/Python/compat/README.md

@@ -0,0 +1,9 @@
+# `soar-compat`
+
+A small shim library that provides compatibility with packaging/distribution methods for Soar's SML Bindings.
+
+The raw bindings, as built in Soar, are imported as `import Python_sml_ClientInterface`,
+while the pypi package `soar-sml` imports as `soar_sml`.
+
+While it possible to do `import soar_sml as Python_sml_ClientInterface`, this small library avoids that need,
+and allows all existing scripts to import Soar's SML Bindings in the way they're used to.

Core/ClientSMLSWIG/Python/compat/pyproject.toml

@@ -0,0 +1,18 @@
+[project]
+name = "soar-compat"
+description = "Compatibility layer with soar-sml"
+version = "1.0"
+license = { text = "BSD" }
+authors = [
+    {name = "Jonathan de Jong", email = "[email protected]"}
+]
+readme = "README.md"
+dependencies = ["soar-sml"]
+keywords = ["soar"]
+classifiers = [
+    "Programming Language :: Python :: 3",
+]
+
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"