Generate version with setuptools_scm and migrate to pyproject.toml (#…

…2028)

.github/workflows/pythonbuild.yml

@@ -53,8 +53,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-latest ]
-        python-version: [ "3.8", "3.11" ]
+        os: [ubuntu-latest]
+        python-version: ["3.8", "3.11"]
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -222,7 +222,7 @@ jobs:
           cd plugins/${{ matrix.plugin-names }}
           pip install .
           if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
-          pip install -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
+          pip install -U $GITHUB_WORKSPACE
           pip freeze
       - name: Test with coverage
         run: |

.github/workflows/pythonpublish.yml

@@ -18,21 +18,13 @@ jobs:
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install setuptools wheel twine
-      - name: Autobump version
-        id: bump
-        run: |
-          # from refs/tags/v1.2.3 get 1.2.3
-          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
-          echo "::set-output name=version::$VERSION"
-          VERSION=$VERSION make update_version
-        shell: bash
+          pip install build twine
       - name: Build and publish
         env:
           TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
           TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          python setup.py sdist bdist_wheel
+          python -m build
           twine upload dist/*
       - name: Autobump plugin version
         run: |

.gitignore

@@ -34,3 +34,6 @@ htmlcov
 docs/source/_tags/
 .hypothesis
 .npm
+
+# Version file is auto-generated by setuptools_scm
+flytekit/_version.py

Dockerfile.dev

@@ -26,7 +26,7 @@ COPY . /flytekit
 # 2. Install Flytekit and its plugins.
 # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4
 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories.
-RUN apt-get update && apt-get install build-essential vim libmagic1 -y \
+RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \
     && pip install --no-cache-dir -e /flytekit \
     && pip install --no-cache-dir -e /flytekit/plugins/flytekit-k8s-pod \
     && pip install --no-cache-dir -e /flytekit/plugins/flytekit-deck-standard \

Makefile

@@ -92,15 +92,3 @@ requirements: doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compil
 coverage:
 	coverage run -m pytest tests/flytekit/unit/core flytekit/types -m "not sandbox_test"
 	coverage report -m --include="flytekit/core/*,flytekit/types/*"
-
-PLACEHOLDER := "__version__\ =\ \"0.0.0+develop\""
-
-.PHONY: update_version
-update_version:
-	# ensure the placeholder is there. If grep doesn't find the placeholder
-	# it exits with exit code 1 and github actions aborts the build.
-	grep "$(PLACEHOLDER)" "flytekit/__init__.py"
-	sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "flytekit/__init__.py"
-
-	grep "$(PLACEHOLDER)" "setup.py"
-	sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "setup.py"

dev-requirements.in

@@ -17,6 +17,7 @@ google-cloud-bigquery
 google-cloud-bigquery-storage
 IPython
 keyrings.alt
+setuptools_scm
 
 # Only install tensorflow if not running on an arm Mac.
 tensorflow==2.8.1; python_version<'3.11' and (platform_machine!='arm64' or platform_system!='Darwin')

dev-requirements.txt

@@ -319,6 +319,7 @@ packaging==23.2
     #   google-cloud-bigquery
     #   marshmallow
     #   pytest
+    #   setuptools-scm
 pandas==1.5.3
     # via flytekit
 parso==0.8.3
@@ -458,6 +459,8 @@ scikit-learn==1.3.2
     # via -r dev-requirements.in
 scipy==1.11.3
     # via scikit-learn
+setuptools-scm==8.0.4
+    # via -r dev-requirements.in
 six==1.16.0
     # via
     #   asttokens
@@ -500,6 +503,7 @@ tomli==2.0.1
     #   coverage
     #   mypy
     #   pytest
+    # . setuptools-scm
 torch==1.12.1 ; python_version >= "3.11" or platform_system != "Windows"
     # via -r dev-requirements.in
 traitlets==5.13.0
@@ -526,6 +530,7 @@ typing-extensions==4.8.0
     #   mashumaro
     #   mypy
     #   rich-click
+    # . setuptools-scm
     #   tensorflow
     #   torch
     #   typing-inspect

flytekit/__init__.py

@@ -244,8 +244,7 @@
     StructuredDatasetTransformerEngine,
     StructuredDatasetType,
 )
-
-__version__ = "0.0.0+develop"
+from flytekit._version import __version__
 
 
 def current_context() -> ExecutionParameters:

flytekit/configuration/default_images.py

@@ -41,7 +41,7 @@ def find_image_for(
     def get_version_suffix(cls) -> str:
         from flytekit import __version__
 
-        if not __version__ or __version__ == "0.0.0+develop":
+        if not __version__ or "dev" in __version__:
             version_suffix = "latest"
         else:
             version_suffix = __version__

pyproject.toml

@@ -1,3 +1,97 @@
+[build-system]
+requires = ["setuptools", "setuptools_scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "flytekit"
+dynamic = ["version"]
+authors = [{ name = "Flyte Contributors", email = "[email protected]" }]
+description = "Flyte SDK for Python"
+license = { text = "Apache-2.0" }
+readme = { file = "README.md", content-type = "text/markdown" }
+requires-python = ">=3.8,<3.12"
+dependencies = [
+    # Please maintain an alphabetical order in the following list
+    "adlfs",
+    "click>=6.6,<9.0",
+    "cloudpickle>=2.0.0",
+    "cookiecutter>=1.7.3",
+    "croniter>=0.3.20,<4.0.0",
+    "dataclasses-json>=0.5.2,<0.5.12", # TODO: remove upper-bound after fixing change in contract
+    "diskcache>=5.2.1",
+    "docker>=4.0.0,<7.0.0",
+    "docstring-parser>=0.9.0",
+    "flyteidl>=1.10.0",
+    "fsspec>=2023.3.0,<=2023.9.2",
+    "gcsfs",
+    "googleapis-common-protos>=1.57",
+    "grpcio",
+    "grpcio-status",
+    "importlib-metadata",
+    "joblib",
+    "jsonpickle",
+    "keyring>=18.0.1",
+    "kubernetes>=12.0.1",
+    "marshmallow-enum",
+    # TODO: remove upper-bound after fixing change in contract
+    "marshmallow-jsonschema>=0.12.0",
+    "mashumaro>=3.9.1",
+    "numpy",
+    "pandas>=1.0.0,<2.0.0",
+    # TODO: Remove upper-bound after protobuf community fixes it. https://github.com/flyteorg/flyte/issues/4359
+    "protobuf<4.25.0",
+    "pyarrow>=4.0.0",
+    "python-json-logger>=2.0.0",
+    "pytimeparse>=1.1.8,<2.0.0",
+    "pyyaml!=6.0.0,!=5.4.0,!=5.4.1", # pyyaml is broken with cython 3: https://github.com/yaml/pyyaml/issues/601
+    "requests>=2.18.4,<3.0.0",
+    "rich",
+    "rich_click",
+    "s3fs>=0.6.0",
+    "statsd>=3.0.0,<4.0.0",
+    "typing_extensions",
+    "urllib3>=1.22,<2.0.0",
+]
+classifiers = [
+    "Intended Audience :: Science/Research",
+    "Intended Audience :: Developers",
+    "License :: OSI Approved :: Apache Software License",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Topic :: Scientific/Engineering",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Software Development",
+    "Topic :: Software Development :: Libraries",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+]
+
+[project.urls]
+Homepage = "https://github.com/flyteorg/flytekit"
+
+[project.scripts]
+pyflyte-execute = "flytekit.bin.entrypoint:execute_task_cmd"
+pyflyte-fast-execute = "flytekit.bin.entrypoint:fast_execute_task_cmd"
+pyflyte-map-execute = "flytekit.bin.entrypoint:map_execute_task_cmd"
+pyflyte = "flytekit.clis.sdk_in_container.pyflyte:main"
+flyte-cli = "flytekit.clis.flyte_cli.main:_flyte_cli"
+
+[tool.setuptools_scm]
+write_to = "flytekit/_version.py"
+
+[tool.setuptools]
+include-package-data = true
+script-files = [
+    "flytekit_scripts/flytekit_build_image.sh",
+    "flytekit_scripts/flytekit_venv",
+    "flytekit/bin/entrypoint.py",
+]
+
+[tool.setuptools.packages.find]
+include = ["flytekit", "flytekit_scripts"]
+exclude = ["boilerplate", "docs", "plugins", "tests*"]
+
 [tool.pytest.ini_options]
 norecursedirs = ["common", "workflows", "spark", "fsspec"]
 log_cli = true

setup.py

@@ -1,94 +1,3 @@
-from setuptools import find_packages, setup  # noqa
+from setuptools import setup
 
-extras_require = {}
-
-__version__ = "0.0.0+develop"
-
-setup(
-    name="flytekit",
-    version=__version__,
-    maintainer="Flyte Contributors",
-    maintainer_email="[email protected]",
-    packages=find_packages(
-        include=["flytekit", "flytekit_scripts"],
-        exclude=["boilerplate", "docs", "plugins", "tests*"],
-    ),
-    include_package_data=True,
-    url="https://github.com/flyteorg/flytekit",
-    description="Flyte SDK for Python",
-    long_description=open("README.md", encoding="utf-8").read(),
-    long_description_content_type="text/markdown",
-    entry_points={
-        "console_scripts": [
-            "pyflyte-execute=flytekit.bin.entrypoint:execute_task_cmd",
-            "pyflyte-fast-execute=flytekit.bin.entrypoint:fast_execute_task_cmd",
-            "pyflyte-map-execute=flytekit.bin.entrypoint:map_execute_task_cmd",
-            "pyflyte=flytekit.clis.sdk_in_container.pyflyte:main",
-            "flyte-cli=flytekit.clis.flyte_cli.main:_flyte_cli",
-        ]
-    },
-    install_requires=[
-        # Please maintain an alphabetical order in the following list
-        "adlfs",
-        "click>=6.6,<9.0",
-        "cloudpickle>=2.0.0",
-        "cookiecutter>=1.7.3",
-        "croniter>=0.3.20,<4.0.0",
-        "dataclasses-json>=0.5.2,<0.5.12",  # TODO: remove upper-bound after fixing change in contract
-        "diskcache>=5.2.1",
-        "docker>=4.0.0,<7.0.0",
-        "docstring-parser>=0.9.0",
-        "flyteidl>=1.10.0",
-        "fsspec>=2023.3.0,<=2023.9.2",
-        "gcsfs",
-        "googleapis-common-protos>=1.57",
-        "grpcio",
-        "grpcio-status",
-        "importlib-metadata",
-        "joblib",
-        "jsonpickle",
-        "keyring>=18.0.1",
-        "kubernetes>=12.0.1",
-        "marshmallow-enum",
-        # TODO: remove upper-bound after fixing change in contract
-        "marshmallow-jsonschema>=0.12.0",
-        "mashumaro>=3.9.1",
-        "numpy",
-        "pandas>=1.0.0,<2.0.0",
-        # TODO: Remove upper-bound after protobuf community fixes it. https://github.com/flyteorg/flyte/issues/4359
-        "protobuf<4.25.0",
-        "pyarrow>=4.0.0",
-        "python-json-logger>=2.0.0",
-        "pytimeparse>=1.1.8,<2.0.0",
-        "pyyaml!=6.0.0,!=5.4.0,!=5.4.1",  # pyyaml is broken with cython 3: https://github.com/yaml/pyyaml/issues/601
-        "requests>=2.18.4,<3.0.0",
-        "rich",
-        "rich_click",
-        "s3fs>=0.6.0",
-        "statsd>=3.0.0,<4.0.0",
-        "typing_extensions",
-        "urllib3>=1.22,<2.0.0",
-    ],
-    extras_require=extras_require,
-    scripts=[
-        "flytekit_scripts/flytekit_build_image.sh",
-        "flytekit_scripts/flytekit_venv",
-        "flytekit/bin/entrypoint.py",
-    ],
-    license="apache2",
-    python_requires=">=3.8,<3.12",
-    classifiers=[
-        "Intended Audience :: Science/Research",
-        "Intended Audience :: Developers",
-        "License :: OSI Approved :: Apache Software License",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Topic :: Scientific/Engineering",
-        "Topic :: Scientific/Engineering :: Artificial Intelligence",
-        "Topic :: Software Development",
-        "Topic :: Software Development :: Libraries",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-    ],
-)
+setup()