Add pre- and post-build script to work around Poetry bug

.github/workflows/prep-release.yaml

@@ -71,6 +71,7 @@ jobs:
       - name: Build a binary wheel and a source tarball
         run: |
           pip install poetry
+          ./scripts/pre-build.sh
           poetry build
 
       - name: Publish distribution 📦 to Test PyPI

pyproject.toml

@@ -21,8 +21,8 @@ maintainers = [
     "Niels Rogge <[email protected]>",
 ]
 repository = "https://github.com/ml6team/fondant"
-include = ["*.txt", "*.rst", "*.md", "src/fondant/components/**/*.yaml"]
-exclude = ["src/fondant/components"]  # Exclude everything except for component specs
+include = ["*.txt", "*.rst", "*.md", "fondant/components/**/*.yaml"]
+exclude = ["fondant/components"]  # Exclude everything except for component specs
 classifiers = [
     "Development Status :: 2 - Pre-Alpha",
     "Intended Audience :: Developers",
@@ -75,6 +75,9 @@ vertex = ["docker", "kfp", "google-cloud-aiplatform"]
 sagemaker = ["sagemaker", "boto3"]
 docker = ["docker"]
 
+all = ["dask", "dask-cuda", "s3fs", "adlfs", "gcsfs", "docker", "kfp", "google-cloud-aiplatform",
+    "sagemaker", "boto3"]
+
 [tool.poetry.group.test]
 optional = true
 

scripts/post-build.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+# This script reverts the `scripts/pre-build.sh` script by moving the package code back into a
+# `src` directory.
+# It should be run after running scripts/pre-build.sh and building the fondant package
+set -e
+
+scripts_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+root_path=$(dirname "$scripts_path")
+
+pushd "$root_path"
+mkdir -p src/fondant
+mv fondant/* src/fondant
+rm -rf fondant
+popd

scripts/pre-build.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# This script unpacks the src folder before building to make the poetry `include` and `exclude`
+# sections work correctly for both sdist and wheel
+# (https://github.com/python-poetry/poetry/issues/8994)
+# Building without running this script also works, but includes the full code for the components.
+# This script makes changes to the local files, which should not be committed to git. Run
+# scripts/post-build.sh to clean them up.
+set -e
+
+scripts_path=$( cd "$(dirname "${BASH_SOURCE[0]}")" ; pwd -P )
+root_path=$(dirname "$scripts_path")
+
+pushd "$root_path"
+mkdir fondant
+mv src/fondant/* fondant
+rm -rf src
+popd

tox.ini

@@ -42,8 +42,10 @@ allowlist_externals=
     /bin/bash
     /usr/bin/bash
 commands_pre=
-    poetry lock
-    poetry install --all-extras
-    poetry show
+    bash ./scripts/pre-build.sh
+    poetry build
+    poetry run pip install dist/fondant-0.1.dev0-py3-none-any.whl[all]
+    poetry run pip list
+    bash ./scripts/post-build.sh
 commands=
     poetry run python -m pytest tests -vv --cov fondant --cov-report term-missing --ignore=tests/integration_tests