diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f8a8d40fb3..a84b9384f0 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -126,95 +126,121 @@ jobs: run: ./infra/scripts/helm/push-helm-charts.sh $VERSION_WITHOUT_PREFIX publish-python-sdk: + runs-on: ubuntu-latest + needs: [build-python-sdk, build-python-sdk-no-telemetry, build-python-sdk-macos-py310] + steps: + - uses: actions/download-artifact@v2 + with: + name: wheels + path: dist + - uses: pypa/gh-action-pypi-publish@v1.4.2 + with: + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} + + + build-python-sdk: + name: Build wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10" ] - os: [ ubuntu-latest, macOS-latest ] - compile-go: [ True ] - include: - - python-version: "3.7" - os: ubuntu-latest - compile-go: False - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - COMPILE_GO: ${{ matrix.compile-go }} + os: [ ubuntu-latest, macos-10.15 ] + steps: - uses: actions/checkout@v2 - - name: Setup Python - id: setup-python - uses: actions/setup-python@v2 + + - name: Build wheels + uses: pypa/cibuildwheel@v2.4.0 with: - python-version: ${{ matrix.python-version }} - architecture: x64 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 + package-dir: sdk/python + env: + CIBW_BUILD: "cp3*_x86_64" + CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64" + CIBW_ARCHS: "native" + CIBW_ENVIRONMENT: > + COMPILE_GO=True + CIBW_BEFORE_ALL_LINUX: | + yum install -y golang + CIBW_BEFORE_ALL_MACOS: | + curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg + sudo installer -pkg python.pkg -target / + CIBW_BEFORE_BUILD: | + make install-protoc-dependencies + make install-go-proto-dependencies + make install-go-ci-dependencies + + - uses: actions/upload-artifact@v2 with: - go-version: 1.17.7 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1" - - name: Install pip-tools - run: pip install pip-tools - - name: Install dependencies - run: make install-python-ci-dependencies PYTHON=${{ matrix.python-version }} - - name: Publish Python Package - run: | - cd sdk/python - python3 -m pip install --user --upgrade setuptools wheel twine - python3 setup.py sdist bdist_wheel - python3 -m twine upload --verbose dist/*.whl + name: wheels + path: ./wheelhouse/*.whl + - publish-python-sdk-no-telemetry: + build-python-sdk-no-telemetry: + name: Build no telemetry wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: - python-version: [ "3.7", "3.8", "3.9", "3.10" ] - os: [ ubuntu-latest, macOS-latest ] - compile-go: [ True ] - include: - - python-version: "3.7" - os: ubuntu-latest - compile-go: False + os: [ ubuntu-latest, macos-10.15 ] needs: get-version + steps: + - uses: actions/checkout@v2 + - run: | + cd sdk/python + sed -i.bak 's/DEFAULT_FEAST_USAGE_VALUE = "True"/DEFAULT_FEAST_USAGE_VALUE = "False"/g' feast/constants.py + sed -i.bak 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py + - name: Build wheels + uses: pypa/cibuildwheel@v2.4.0 + with: + package-dir: sdk/python + env: + CIBW_BUILD: "cp3*_x86_64" + CIBW_SKIP: "cp36-* *-musllinux_x86_64 cp310-macosx_x86_64" + CIBW_ARCHS: "native" + CIBW_ENVIRONMENT: > + COMPILE_GO=True SETUPTOOLS_SCM_PRETEND_VERSION="${{ needs.get-version.outputs.version_without_prefix }}" + CIBW_BEFORE_ALL_LINUX: | + yum install -y golang + CIBW_BEFORE_ALL_MACOS: | + curl -o python.pkg https://www.python.org/ftp/python/3.9.12/python-3.9.12-macosx10.9.pkg + sudo installer -pkg python.pkg -target / + CIBW_BEFORE_BUILD: | + make install-protoc-dependencies + make install-go-proto-dependencies + make install-go-ci-dependencies + + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: ./wheelhouse/*.whl + + build-python-sdk-macos-py310: + runs-on: macos-10.15 env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - COMPILE_GO: ${{ matrix.compile-go }} + COMPILE_GO: True steps: - uses: actions/checkout@v2 - name: Setup Python id: setup-python uses: actions/setup-python@v2 with: - python-version: ${{ matrix.python-version }} + python-version: "3.10" architecture: x64 - - name: Setup Go - id: setup-go - uses: actions/setup-go@v2 - with: - go-version: 1.17.7 - - name: Upgrade pip version - run: | - pip install --upgrade "pip>=21.3.1" - - name: Install pip-tools - run: pip install pip-tools - name: Install dependencies - run: make install-python-ci-dependencies PYTHON=${{ matrix.python-version }} - - name: Publish Python Package - env: - SETUPTOOLS_SCM_PRETEND_VERSION: ${{ needs.get-version.outputs.version_without_prefix }} + run: | + pip install -U pip setuptools wheel twine + make install-protoc-dependencies + make install-go-proto-dependencies + make install-go-ci-dependencies + - name: Build run: | cd sdk/python - sed -i 's/DEFAULT_FEAST_USAGE_VALUE = "True"/DEFAULT_FEAST_USAGE_VALUE = "False"/g' feast/constants.py - sed -i 's/NAME = "feast"/NAME = "feast-no-telemetry"/g' setup.py - python3 -m pip install --user --upgrade setuptools wheel twine python3 setup.py sdist bdist_wheel - python3 -m twine upload --verbose dist/*.whl + + - uses: actions/upload-artifact@v2 + with: + name: wheels + path: sdk/python/dist/* + publish-java-sdk: container: maven:3.6-jdk-11 diff --git a/Makefile b/Makefile index 4804271dff..41041d7c08 100644 --- a/Makefile +++ b/Makefile @@ -145,15 +145,15 @@ install-go-ci-dependencies: go get github.com/go-python/gopy go install golang.org/x/tools/cmd/goimports go install github.com/go-python/gopy + python -m pip install pybindgen==0.22.0 install-protoc-dependencies: - pip install grpcio-tools==1.44.0 + pip install grpcio-tools==1.44.0 mypy-protobuf==3.1.0 compile-protos-go: install-go-proto-dependencies install-protoc-dependencies cd sdk/python && python setup.py build_go_protos compile-go-lib: install-go-proto-dependencies install-go-ci-dependencies - python -m pip install pybindgen==0.22.0 cd sdk/python && python setup.py build_go_lib # Needs feast package to setup the feature store diff --git a/sdk/python/setup.py b/sdk/python/setup.py index af6eaea311..ed1a1a7f9f 100644 --- a/sdk/python/setup.py +++ b/sdk/python/setup.py @@ -11,7 +11,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import copy import glob +import json import os import pathlib import re @@ -19,21 +21,23 @@ import subprocess import sys from distutils.cmd import Command +from distutils.dir_util import copy_tree from pathlib import Path from subprocess import CalledProcessError -from setuptools import find_packages +from setuptools import find_packages, Extension try: from setuptools import setup from setuptools.command.build_py import build_py + from setuptools.command.build_ext import build_ext as _build_ext from setuptools.command.develop import develop from setuptools.command.install import install - from setuptools.dist import Distribution + except ImportError: from distutils.command.build_py import build_py + from distutils.command.build_ext import build_ext as _build_ext from distutils.core import setup - from distutils.dist import Distribution NAME = "feast" DESCRIPTION = "Python SDK for Feast" @@ -189,7 +193,7 @@ class BuildPythonProtosCommand(Command): def initialize_options(self): self.python_protoc = [ - "python", + sys.executable, "-m", "grpc_tools.protoc", ] # find_executable("protoc") @@ -292,7 +296,7 @@ class BuildGoProtosCommand(Command): def initialize_options(self): self.go_protoc = [ - "python", + sys.executable, "-m", "grpc_tools.protoc", ] # find_executable("protoc") @@ -331,45 +335,6 @@ def run(self): self._generate_go_protos(f"feast/{sub_folder}/*.proto") -class BuildGoEmbeddedCommand(Command): - description = "Builds Go embedded library" - user_options = [] - - def initialize_options(self) -> None: - self.path_val = _generate_path_with_gopath() - - self.go_env = {} - for var in ("GOCACHE", "GOPATH"): - self.go_env[var] = subprocess \ - .check_output(["go", "env", var]) \ - .decode("utf-8") \ - .strip() - - def finalize_options(self) -> None: - pass - - def _compile_embedded_lib(self): - print("Compile embedded go") - subprocess.check_call([ - "gopy", - "build", - "-output", - "feast/embedded_go/lib", - "-vm", - # Path of current python executable - sys.executable, - "-no-make", - "github.com/feast-dev/feast/go/embedded" - ], env={ - "PATH": self.path_val, - "CGO_LDFLAGS_ALLOW": ".*", - **self.go_env, - }) - - def run(self): - self._compile_embedded_lib() - - class BuildCommand(build_py): """Custom build command.""" @@ -378,7 +343,7 @@ def run(self): if os.getenv("COMPILE_GO", "false").lower() == "true": _ensure_go_and_proto_toolchain() self.run_command("build_go_protos") - self.run_command("build_go_lib") + build_py.run(self) @@ -390,15 +355,61 @@ def run(self): if os.getenv("COMPILE_GO", "false").lower() == "true": _ensure_go_and_proto_toolchain() self.run_command("build_go_protos") - self.run_command("build_go_lib") + develop.run(self) -class BinaryDistribution(Distribution): - """Distribution which forces a binary package with platform name - when go compilation is enabled""" - def has_ext_modules(self): - return os.getenv("COMPILE_GO", "false").lower() == "true" +class build_ext(_build_ext): + def finalize_options(self) -> None: + super().finalize_options() + if os.getenv("COMPILE_GO", "false").lower() == "false": + self.extensions = [e for e in self.extensions if not self._is_go_ext(e)] + + def _is_go_ext(self, ext: Extension): + return any(source.endswith('.go') or source.startswith('github') for source in ext.sources) + + def build_extension(self, ext: Extension): + if not self._is_go_ext(ext): + # the base class may mutate `self.compiler` + compiler = copy.deepcopy(self.compiler) + self.compiler, compiler = compiler, self.compiler + try: + return _build_ext.build_extension(self, ext) + finally: + self.compiler, compiler = compiler, self.compiler + + bin_path = _generate_path_with_gopath() + go_env = json.loads( + subprocess.check_output(["go", "env", "-json"]).decode("utf-8").strip() + ) + + destination = os.path.dirname(os.path.abspath(self.get_ext_fullpath(ext.name))) + subprocess.check_call([ + "gopy", + "build", + "-output", + destination, + "-vm", + sys.executable, + "-no-make", + *ext.sources + ], env={ + "PATH": bin_path, + "CGO_LDFLAGS_ALLOW": ".*", + **go_env, + }) + + def copy_extensions_to_source(self): + build_py = self.get_finalized_command('build_py') + for ext in self.extensions: + fullname = self.get_ext_fullname(ext.name) + modpath = fullname.split('.') + package = '.'.join(modpath[:-1]) + package_dir = build_py.get_package_dir(package) + src = os.path.join(self.build_lib, package_dir) + + # copy whole directory + copy_tree(src, package_dir) setup( @@ -453,9 +464,10 @@ def has_ext_modules(self): cmdclass={ "build_python_protos": BuildPythonProtosCommand, "build_go_protos": BuildGoProtosCommand, - "build_go_lib": BuildGoEmbeddedCommand, "build_py": BuildCommand, "develop": DevelopCommand, + "build_ext": build_ext, }, - distclass=BinaryDistribution, # generate wheel with platform-specific name + ext_modules=[Extension('feast.embedded_go.lib._embedded', + ["github.com/feast-dev/feast/go/embedded"])], )