From 2c99526baaab1c5bdac3462399437f1df3545c7d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Thu, 22 Oct 2020 16:27:46 -0400 Subject: [PATCH] feat: support setup helpers (#60) * feat: support setup helpers * feat: update to pybind11 2.6.0b1 * feat: bump to 2.6.0rc3 * chore: bump to 2.6.0 * ci: full release string * ci: dependabot * fix: conda-forge channel needed for 2.6.0 --- .appveyor.yml | 34 ++------- .github/dependabot.yml | 11 +++ .github/workflows/conda.yml | 45 +++++++++++ .github/workflows/pip.yml | 50 ++++++++++++ .github/workflows/wheels.yml | 143 ++++++++++++++++++++++++++++++++++ .gitignore | 144 +++++++++++++++++++++++++++++++++-- .travis.yml | 1 - README.md | 59 +++++++++----- conda.recipe/bld.bat | 6 -- conda.recipe/build.sh | 3 - conda.recipe/meta.yaml | 29 +++---- setup.py | 142 +++++++++------------------------- src/main.cpp | 5 +- 13 files changed, 492 insertions(+), 180 deletions(-) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/conda.yml create mode 100644 .github/workflows/pip.yml create mode 100644 .github/workflows/wheels.yml delete mode 100644 conda.recipe/bld.bat delete mode 100644 conda.recipe/build.sh diff --git a/.appveyor.yml b/.appveyor.yml index 770d9a5..cf8ccf8 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -11,37 +11,17 @@ environment: matrix: - PYTHON: 27 - PYTHON: 36 - - CONDA: 27 - - CONDA: 36 install: - cmd: '"%VS140COMNTOOLS%\..\..\VC\vcvarsall.bat" %PLATFORM%' - ps: | - if ($env:PYTHON) { - if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } - $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" - python -m pip install --disable-pip-version-check --user --upgrade pip setuptools - } elseif ($env:CONDA) { - if ($env:CONDA -eq "27") { $env:CONDA = "" } - if ($env:PLATFORM -eq "x64") { $env:CONDA = "$env:CONDA-x64" } - $env:PATH = "C:\Miniconda$env:CONDA\;C:\Miniconda$env:CONDA\Scripts\;$env:PATH" - conda config --set always_yes yes --set changeps1 no - conda config --add channels conda-forge - conda update -q conda - conda install -q conda-build - } + if ($env:PLATFORM -eq "x64") { $env:PYTHON = "$env:PYTHON-x64" } + $env:PATH = "C:\Python$env:PYTHON\;C:\Python$env:PYTHON\Scripts\;$env:PATH" + python -m pip install --disable-pip-version-check --upgrade --no-warn-script-location pip setuptools build_script: - ps: | - if ($env:PYTHON) { - python setup.py sdist - python -m pip install 'pybind11>=2.3' - cd dist - python -m pip install --verbose python_example-0.0.1.tar.gz - cd .. - } else { - echo "conda build conda.recipe" - conda build conda.recipe - echo "conda install --use-local python_example" - conda install --use-local python_example - } + python setup.py sdist + cd dist + python -m pip install --verbose python_example-0.0.1.tar.gz + cd .. test_script: - ps: python tests\test.py diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..c1eac3c --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + ignore: + # Offical actions have moving tags like v1 + # that are used, so they don't need updates here + - dependency-name: "actions/*" diff --git a/.github/workflows/conda.yml b/.github/workflows/conda.yml new file mode 100644 index 0000000..1f8948a --- /dev/null +++ b/.github/workflows/conda.yml @@ -0,0 +1,45 @@ +name: Conda + +on: + workflow_dispatch: + push: + branches: + - master + pull_request: + +jobs: + build: + strategy: + fail-fast: false + matrix: + platform: [ubuntu-latest, windows-latest, macos-latest] + python-version: ["3.6", "3.8"] + + runs-on: ${{ matrix.platform }} + + # The setup-miniconda action needs this to activate miniconda + defaults: + run: + shell: "bash -l {0}" + + steps: + - uses: actions/checkout@v2 + + - name: Get conda + uses: conda-incubator/setup-miniconda@v1.7.0 + with: + python-version: ${{ matrix.python-version }} + channels: conda-forge,defaults + channel-priority: strict + + - name: Prepare + run: conda install conda-build conda-verify + + - name: Build + run: conda build conda.recipe + + - name: Install + run: conda install -c ${CONDA_PREFIX}/conda-bld/ python_example + + - name: Test + run: python tests/test.py diff --git a/.github/workflows/pip.yml b/.github/workflows/pip.yml new file mode 100644 index 0000000..e5f5d27 --- /dev/null +++ b/.github/workflows/pip.yml @@ -0,0 +1,50 @@ +name: Pip + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + +jobs: + build: + strategy: + fail-fast: false + matrix: + platform: [windows-latest, macos-latest, ubuntu-latest] + python-version: ["2.7", "3.5", "3.8", "3.9"] + + runs-on: ${{ matrix.platform }} + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Add requirements + run: python -m pip install --upgrade wheel setuptools + + # Eventually Microsoft might have an action for setting up + # MSVC, but for now, this action works: + - name: Prepare compiler environment for Windows 🐍 2.7 + if: matrix.python-version == 2.7 && runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + # This makes two environment variables available in the following step(s) + - name: Set Windows 🐍 2.7 environment variables + if: matrix.python-version == 2.7 && runner.os == 'Windows' + shell: bash + run: | + echo "DISTUTILS_USE_SDK=1" >> $GITHUB_ENV + echo "MSSdk=1" >> $GITHUB_ENV + + - name: Build and install + run: pip install --verbose . + + - name: Test + run: python tests/test.py diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml new file mode 100644 index 0000000..94df2a6 --- /dev/null +++ b/.github/workflows/wheels.yml @@ -0,0 +1,143 @@ +name: Wheels + +on: + workflow_dispatch: + pull_request: + push: + branches: + - master + release: + types: + - published + +env: + CIBW_TEST_COMMAND: python {project}/tests/test.py + # This can be removed if pyproject.toml is used + CIBW_BEFORE_BUILD: pip install pybind11 + + +jobs: + build_sdist: + name: Build SDist + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + + - name: Install deps + run: python -m pip install "setuptools>=42" "setuptools_scm[toml]>=4.1.0" twine + + - name: Build SDist + run: python setup.py sdist + + - name: Check metadata + run: twine check dist/* + + - uses: actions/upload-artifact@v2 + with: + path: dist/*.tar.gz + + + build_wheels: + name: Wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-latest] + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==1.6.3 + + - name: Build wheel + run: python -m cibuildwheel --output-dir wheelhouse + env: + # Python 2.7 on Windows requires a workaround for C++11 support, + # built separately below + CIBW_SKIP: cp27-win* + + - name: Show files + run: ls -lh wheelhouse + shell: bash + + - name: Verify clean directory + run: git diff --exit-code + shell: bash + + - name: Upload wheels + uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl + + + # Windows 2.7 (requires workaround for MSVC 2008 replacement) + build_win27_wheels: + name: Py 2.7 wheels on Windows + runs-on: windows-latest + + steps: + - uses: actions/checkout@v1 + with: + submodules: true + + - uses: actions/setup-python@v2 + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==1.6.3 + + - uses: ilammy/msvc-dev-cmd@v1 + + - name: Build 64-bit wheel + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_BUILD: cp27-win_amd64 + DISTUTILS_USE_SDK: 1 + MSSdk: 1 + + - uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x86 + + - name: Build 32-bit wheel + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_BUILD: cp27-win32 + DISTUTILS_USE_SDK: 1 + MSSdk: 1 + + - name: Show files + run: ls -lh wheelhouse + shell: bash + + - name: Verify clean directory + run: git diff --exit-code + shell: bash + + - uses: actions/upload-artifact@v2 + with: + path: wheelhouse/*.whl + + + upload_all: + name: Upload if release + needs: [build_wheels, build_win27_wheels, build_sdist] + runs-on: ubuntu-latest + if: github.event_name == 'release' && github.event.action == 'published' + + steps: + - uses: actions/setup-python@v2 + + - uses: actions/download-artifact@v2 + with: + name: artifact + path: dist + + - uses: pypa/gh-action-pypi-publish@v1.4.1 + with: + user: __token__ + password: ${{ secrets.pypi_password }} diff --git a/.gitignore b/.gitignore index 839282b..78d45bd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,140 @@ +# Using https://github.com/github/gitignore/blob/master/Python.gitignore + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions *.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg MANIFEST -*.py[cod] -*.egg-info -.DS_Store -_build -_generate -build + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/.travis.yml b/.travis.yml index 18c5e22..00b06da 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,6 @@ install: - | if [ -n "$PYTHON" ]; then python setup.py sdist - python -m pip install 'pybind11>=2.3' python -m pip install --verbose dist/*.tar.gz elif [ -n "$CONDA" ]; then conda build conda.recipe --python $CONDA diff --git a/README.md b/README.md index dbad46e..2bdeb67 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,31 @@ python_example ============== +[![Gitter][gitter-badge]][gitter-link] + +| CI | status | +|----------------------|--------| +| Linux/macOS Travis | [![Travis-CI][travis-badge]][travis-link] | +| MSVC 2015 | [![AppVeyor][appveyor-badge]][appveyor-link] | +| conda.recipe | [![Conda Actions Status][actions-conda-badge]][actions-conda-link] | +| pip builds | [![Pip Actions Status][actions-pip-badge]][actions-pip-link] | +| [`cibuildwheel`][] | [![Wheels Actions Status][actions-wheels-badge]][actions-wheels-link] | + +[gitter-badge]: https://badges.gitter.im/pybind/Lobby.svg +[gitter-link]: https://gitter.im/pybind/Lobby +[actions-badge]: https://github.com/pybind/python_example/workflows/Tests/badge.svg +[actions-conda-link]: https://github.com/pybind/python_example/actions?query=workflow%3A%22Conda +[actions-conda-badge]: https://github.com/pybind/python_example/workflows/Conda/badge.svg +[actions-pip-link]: https://github.com/pybind/python_example/actions?query=workflow%3A%22Pip +[actions-pip-badge]: https://github.com/pybind/python_example/workflows/Pip/badge.svg +[actions-wheels-link]: https://github.com/pybind/python_example/actions?query=workflow%3AWheels +[actions-wheels-badge]: https://github.com/pybind/python_example/workflows/Wheels/badge.svg +[travis-link]: https://travis-ci.org/pybind/python_example +[travis-badge]: https://travis-ci.org/pybind/python_example.svg?branch=master&status=passed +[appveyor-link]: https://ci.appveyor.com/project/wjakob/python-example + +[appveyor-badge]: https://travis-ci.org/pybind/python_example.svg?branch=master&status=passed + An example project built with [pybind11](https://github.com/pybind/pybind11). Installation @@ -13,10 +38,10 @@ Installation **On Windows (Requires Visual Studio 2015)** - - For Python 3.5: + - For Python 3.5+: - clone this repository - `pip install ./python_example` - - For earlier versions of Python, including Python 2.7: + - For Python 2.7: Pybind11 requires a C++11 compliant compiler (i.e. Visual Studio 2015 on Windows). Running a regular `pip install` command will detect the version @@ -32,27 +57,21 @@ Installation Note that this requires the user building `python_example` to have registry edition rights on the machine, to be able to run the `vcvarsall.bat` script. +CI Examples +----------- + +There are examples for CI in `.github/workflows`. A simple way to produces +binary "wheels" for all platforms is illustrated in the "wheels.yml" file, +using [`cibuildwheel`][]. You can also see a basic recipe for building and +testing in `pip.yml`, and `conda.yml` has an example of a conda recipe build. -Windows runtime requirements +Windows Python 2.7 runtime requirements ---------------------------- On Windows, the Visual C++ 2015 redistributable packages are a runtime -requirement for this project. It can be found [here](https://www.microsoft.com/en-us/download/details.aspx?id=48145). - -If you use the Anaconda python distribution, you may require the Visual Studio -runtime as a platform-dependent runtime requirement for you package: - -```yaml -requirements: - build: - - python - - setuptools - - pybind11 - - run: - - python - - vs2015_runtime # [win] -``` +requirement for this project if you build for Python 2.7 (newer versions of +Python include this redistributable). It can be found +[here](https://www.microsoft.com/en-us/download/details.aspx?id=48145). Building the documentation @@ -81,3 +100,5 @@ Test call import python_example python_example.add(1, 2) ``` + +[`cibuildwheel`]: https://cibuildwheel.readthedocs.io diff --git a/conda.recipe/bld.bat b/conda.recipe/bld.bat deleted file mode 100644 index 1e575cb..0000000 --- a/conda.recipe/bld.bat +++ /dev/null @@ -1,6 +0,0 @@ -if "%ARCH%" == "32" (set PLATFORM=x86) else (set PLATFORM=x64) -call "%VS140COMNTOOLS%\..\..\VC\vcvarsall.bat" %PLATFORM% -set DISTUTILS_USE_SDK=1 -set MSSdk=1 -"%PYTHON%" setup.py install -if errorlevel 1 exit 1 diff --git a/conda.recipe/build.sh b/conda.recipe/build.sh deleted file mode 100644 index fc3a9af..0000000 --- a/conda.recipe/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -unset MACOSX_DEPLOYMENT_TARGET -${PYTHON} setup.py install; diff --git a/conda.recipe/meta.yaml b/conda.recipe/meta.yaml index a62d2eb..964116b 100644 --- a/conda.recipe/meta.yaml +++ b/conda.recipe/meta.yaml @@ -1,31 +1,34 @@ package: name: python_example - version: {{ environ.get('GIT_DESCRIBE_TAG', 'dev') }} - -build: - number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} - {% if environ.get('GIT_DESCRIBE_NUMBER', '0') == '0' %}string: py{{ environ.get('PY_VER').replace('.', '') }}_0 - {% else %}string: py{{ environ.get('PY_VER').replace('.', '') }}_{{ environ.get('GIT_BUILD_STR', 'GIT_STUB') }}{% endif %} - script_env: - - CC - - CXX + version: 0.0.1 source: - git_url: ../ + path: .. + +build: + number: 0 + script: python -m pip install . -vvv requirements: build: + - {{ compiler('cxx') }} + + host: - python - - setuptools - - pybind11 + - pip + - pybind11 >=2.6.0 run: - python - - vs2015_runtime # [win] + test: imports: - python_example + source_files: + - tests + commands: + - python tests/test.py about: summary: An example project built with pybind11. diff --git a/setup.py b/setup.py index 94e1d02..b494a9d 100644 --- a/setup.py +++ b/setup.py @@ -1,117 +1,51 @@ -from setuptools import setup, Extension -from setuptools.command.build_ext import build_ext -import sys -import setuptools - -__version__ = '0.0.1' - +from setuptools import setup -class get_pybind_include(object): - """Helper class to determine the pybind11 include path +# With setup_requires, this runs twice - once without setup_requires, and once +# with. The build only happens the second time. +try: + from pybind11.setup_helpers import Pybind11Extension, build_ext + from pybind11 import get_cmake_dir +except ImportError: + from setuptools import Extension as Pybind11Extension + from setuptools.command.build_ext import build_ext - The purpose of this class is to postpone importing pybind11 - until it is actually installed, so that the ``get_include()`` - method can be invoked. """ +import sys - def __str__(self): - import pybind11 - return pybind11.get_include() +__version__ = "0.0.1" +# The main interface is through Pybind11Extension. +# * You can add cxx_std=11/14/17, and then build_ext can be removed. +# * You can set include_pybind11=false to add the include directory yourself, +# say from a submodule. +# +# Note: +# Sort input source files if you glob sources to ensure bit-for-bit +# reproducible builds (https://github.com/pybind/python_example/pull/53) ext_modules = [ - Extension( - 'python_example', - # Sort input source files to ensure bit-for-bit reproducible builds - # (https://github.com/pybind/python_example/pull/53) - sorted(['src/main.cpp']), - include_dirs=[ - # Path to pybind11 headers - get_pybind_include(), - ], - language='c++' - ), + Pybind11Extension("python_example", + ["src/main.cpp"], + # Example: passing in the version to the compiled code + define_macros = [('VERSION_INFO', __version__)], + ), ] - -# cf http://bugs.python.org/issue26689 -def has_flag(compiler, flagname): - """Return a boolean indicating whether a flag name is supported on - the specified compiler. - """ - import tempfile - import os - with tempfile.NamedTemporaryFile('w', suffix='.cpp', delete=False) as f: - f.write('int main (int argc, char **argv) { return 0; }') - fname = f.name - try: - compiler.compile([fname], extra_postargs=[flagname]) - except setuptools.distutils.errors.CompileError: - return False - finally: - try: - os.remove(fname) - except OSError: - pass - return True - - -def cpp_flag(compiler): - """Return the -std=c++[11/14/17] compiler flag. - - The newer version is prefered over c++11 (when it is available). - """ - flags = ['-std=c++17', '-std=c++14', '-std=c++11'] - - for flag in flags: - if has_flag(compiler, flag): - return flag - - raise RuntimeError('Unsupported compiler -- at least C++11 support ' - 'is needed!') - - -class BuildExt(build_ext): - """A custom build extension for adding compiler-specific options.""" - c_opts = { - 'msvc': ['/EHsc'], - 'unix': [], - } - l_opts = { - 'msvc': [], - 'unix': [], - } - - if sys.platform == 'darwin': - darwin_opts = ['-stdlib=libc++', '-mmacosx-version-min=10.7'] - c_opts['unix'] += darwin_opts - l_opts['unix'] += darwin_opts - - def build_extensions(self): - ct = self.compiler.compiler_type - opts = self.c_opts.get(ct, []) - link_opts = self.l_opts.get(ct, []) - if ct == 'unix': - opts.append(cpp_flag(self.compiler)) - if has_flag(self.compiler, '-fvisibility=hidden'): - opts.append('-fvisibility=hidden') - - for ext in self.extensions: - ext.define_macros = [('VERSION_INFO', '"{}"'.format(self.distribution.get_version()))] - ext.extra_compile_args = opts - ext.extra_link_args = link_opts - build_ext.build_extensions(self) - - setup( - name='python_example', + name="python_example", version=__version__, - author='Sylvain Corlay', - author_email='sylvain.corlay@gmail.com', - url='https://github.com/pybind/python_example', - description='A test project using pybind11', - long_description='', + author="Sylvain Corlay", + author_email="sylvain.corlay@gmail.com", + url="https://github.com/pybind/python_example", + description="A test project using pybind11", + long_description="", ext_modules=ext_modules, - setup_requires=['pybind11>=2.5.0'], - cmdclass={'build_ext': BuildExt}, + # Note: You have to add pybind11 to both setup and install requires to make + # it available during the build. Using PEP 518's pyproject.toml is better! + setup_requires=["pybind11==2.6.0"], + install_requires=["pybind11==2.6.0"], + extras_require={"test": "pytest"}, + # Currently, build_ext only provides an optional "highest supported C++ + # level" feature, but in the future it may provide more features. + cmdclass={"build_ext": build_ext}, zip_safe=False, ) diff --git a/src/main.cpp b/src/main.cpp index 94d89a6..e341369 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,5 +1,8 @@ #include +#define STRINGIFY(x) #x +#define MACRO_STRINGIFY(x) STRINGIFY(x) + int add(int i, int j) { return i + j; } @@ -33,7 +36,7 @@ PYBIND11_MODULE(python_example, m) { )pbdoc"); #ifdef VERSION_INFO - m.attr("__version__") = VERSION_INFO; + m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO); #else m.attr("__version__") = "dev"; #endif