diff --git a/.github/workflows/prep-release.yaml b/.github/workflows/prep-release.yaml index b38c7634..1217e142 100644 --- a/.github/workflows/prep-release.yaml +++ b/.github/workflows/prep-release.yaml @@ -71,6 +71,7 @@ jobs: - name: Build a binary wheel and a source tarball run: | pip install poetry + ./scripts/pre-build.sh poetry build - name: Publish distribution 📦 to Test PyPI diff --git a/pyproject.toml b/pyproject.toml index 7e256746..8c95d3fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,8 +21,8 @@ maintainers = [ "Niels Rogge ", ] repository = "https://github.com/ml6team/fondant" -include = ["*.txt", "*.rst", "*.md", "src/fondant/components/**/*.yaml"] -exclude = ["src/fondant/components"] # Exclude everything except for component specs +include = ["*.txt", "*.rst", "*.md", "fondant/components/**/*.yaml"] +exclude = ["fondant/components"] # Exclude everything except for component specs classifiers = [ "Development Status :: 2 - Pre-Alpha", "Intended Audience :: Developers", @@ -75,6 +75,9 @@ vertex = ["docker", "kfp", "google-cloud-aiplatform"] sagemaker = ["sagemaker", "boto3"] docker = ["docker"] +all = ["dask", "dask-cuda", "s3fs", "adlfs", "gcsfs", "docker", "kfp", "google-cloud-aiplatform", + "sagemaker", "boto3"] + [tool.poetry.group.test] optional = true diff --git a/scripts/post-build.sh b/scripts/post-build.sh new file mode 100755 index 00000000..0bd43717 --- /dev/null +++ b/scripts/post-build.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# This script reverts the `scripts/pre-build.sh` script by moving the package code back into a +# `src` directory. +# It should be run after running scripts/pre-build.sh and building the fondant package +set -e + +scripts_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) +root_path=$(dirname "$scripts_path") + +pushd "$root_path" +mkdir -p src/fondant +mv fondant/* src/fondant +rm -rf fondant +popd diff --git a/scripts/pre-build.sh b/scripts/pre-build.sh new file mode 100755 index 00000000..ebb71262 --- /dev/null +++ b/scripts/pre-build.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# This script unpacks the src folder before building to make the poetry `include` and `exclude` +# sections work correctly for both sdist and wheel +# (https://github.com/python-poetry/poetry/issues/8994) +# Building without running this script also works, but includes the full code for the components. +# This script makes changes to the local files, which should not be committed to git. Run +# scripts/post-build.sh to clean them up. +set -e + +scripts_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P ) +root_path=$(dirname "$scripts_path") + +pushd "$root_path" +mkdir fondant +mv src/fondant/* fondant +rm -rf src +popd diff --git a/tox.ini b/tox.ini index 71e9116b..e584a3b3 100644 --- a/tox.ini +++ b/tox.ini @@ -42,8 +42,10 @@ allowlist_externals= /bin/bash /usr/bin/bash commands_pre= - poetry lock - poetry install --all-extras - poetry show + bash ./scripts/pre-build.sh + poetry build + poetry run pip install dist/fondant-0.1.dev0-py3-none-any.whl[all] + poetry run pip list + bash ./scripts/post-build.sh commands= poetry run python -m pytest tests -vv --cov fondant --cov-report term-missing --ignore=tests/integration_tests