From f0d4f1357a770b66dc8344ab97b8b41d19eca9df Mon Sep 17 00:00:00 2001 From: "Thomas J. Fan" Date: Fri, 8 Dec 2023 11:30:19 -0500 Subject: [PATCH] Generate version with setuptools_scm and migrate to pyproject.toml (#2028) --- .github/workflows/pythonbuild.yml | 6 +- .github/workflows/pythonpublish.yml | 12 +-- .gitignore | 3 + Dockerfile.dev | 2 +- Makefile | 12 --- dev-requirements.in | 1 + dev-requirements.txt | 5 ++ flytekit/__init__.py | 3 +- flytekit/configuration/default_images.py | 2 +- pyproject.toml | 94 +++++++++++++++++++++++ setup.py | 95 +----------------------- 11 files changed, 113 insertions(+), 122 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 79f73f3209..64d9d8c52a 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -53,8 +53,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ ubuntu-latest ] - python-version: [ "3.8", "3.11" ] + os: [ubuntu-latest] + python-version: ["3.8", "3.11"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -222,7 +222,7 @@ jobs: cd plugins/${{ matrix.plugin-names }} pip install . if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi - pip install -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit + pip install -U $GITHUB_WORKSPACE pip freeze - name: Test with coverage run: | diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index 0bce90f097..c56147101d 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -18,21 +18,13 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Autobump version - id: bump - run: | - # from refs/tags/v1.2.3 get 1.2.3 - VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') - echo "::set-output name=version::$VERSION" - VERSION=$VERSION make update_version - shell: bash + pip install build twine - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | - python setup.py sdist bdist_wheel + python -m build twine upload dist/* - name: Autobump plugin version run: | diff --git a/.gitignore b/.gitignore index bceaf85f73..751af02c9b 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ htmlcov docs/source/_tags/ .hypothesis .npm + +# Version file is auto-generated by setuptools_scm +flytekit/_version.py diff --git a/Dockerfile.dev b/Dockerfile.dev index c1f90c5054..a2cada316c 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -26,7 +26,7 @@ COPY . /flytekit # 2. Install Flytekit and its plugins. # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories. -RUN apt-get update && apt-get install build-essential vim libmagic1 -y \ +RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \ && pip install --no-cache-dir -e /flytekit \ && pip install --no-cache-dir -e /flytekit/plugins/flytekit-k8s-pod \ && pip install --no-cache-dir -e /flytekit/plugins/flytekit-deck-standard \ diff --git a/Makefile b/Makefile index 0ae94f5604..3daf0da7da 100644 --- a/Makefile +++ b/Makefile @@ -92,15 +92,3 @@ requirements: doc-requirements.txt ${MOCK_FLYTE_REPO}/requirements.txt ## Compil coverage: coverage run -m pytest tests/flytekit/unit/core flytekit/types -m "not sandbox_test" coverage report -m --include="flytekit/core/*,flytekit/types/*" - -PLACEHOLDER := "__version__\ =\ \"0.0.0+develop\"" - -.PHONY: update_version -update_version: - # ensure the placeholder is there. If grep doesn't find the placeholder - # it exits with exit code 1 and github actions aborts the build. - grep "$(PLACEHOLDER)" "flytekit/__init__.py" - sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "flytekit/__init__.py" - - grep "$(PLACEHOLDER)" "setup.py" - sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "setup.py" diff --git a/dev-requirements.in b/dev-requirements.in index d015050aa4..a59f6e4c1d 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -17,6 +17,7 @@ google-cloud-bigquery google-cloud-bigquery-storage IPython keyrings.alt +setuptools_scm # Only install tensorflow if not running on an arm Mac. tensorflow==2.8.1; python_version<'3.11' and (platform_machine!='arm64' or platform_system!='Darwin') diff --git a/dev-requirements.txt b/dev-requirements.txt index d10b386a7f..b02a2d7ab6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -319,6 +319,7 @@ packaging==23.2 # google-cloud-bigquery # marshmallow # pytest + # setuptools-scm pandas==1.5.3 # via flytekit parso==0.8.3 @@ -458,6 +459,8 @@ scikit-learn==1.3.2 # via -r dev-requirements.in scipy==1.11.3 # via scikit-learn +setuptools-scm==8.0.4 + # via -r dev-requirements.in six==1.16.0 # via # asttokens @@ -500,6 +503,7 @@ tomli==2.0.1 # coverage # mypy # pytest + # . setuptools-scm torch==1.12.1 ; python_version >= "3.11" or platform_system != "Windows" # via -r dev-requirements.in traitlets==5.13.0 @@ -526,6 +530,7 @@ typing-extensions==4.8.0 # mashumaro # mypy # rich-click + # . setuptools-scm # tensorflow # torch # typing-inspect diff --git a/flytekit/__init__.py b/flytekit/__init__.py index 75037d3370..7909bbeeab 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -244,8 +244,7 @@ StructuredDatasetTransformerEngine, StructuredDatasetType, ) - -__version__ = "0.0.0+develop" +from flytekit._version import __version__ def current_context() -> ExecutionParameters: diff --git a/flytekit/configuration/default_images.py b/flytekit/configuration/default_images.py index 632f612d0e..06af49664f 100644 --- a/flytekit/configuration/default_images.py +++ b/flytekit/configuration/default_images.py @@ -41,7 +41,7 @@ def find_image_for( def get_version_suffix(cls) -> str: from flytekit import __version__ - if not __version__ or __version__ == "0.0.0+develop": + if not __version__ or "dev" in __version__: version_suffix = "latest" else: version_suffix = __version__ diff --git a/pyproject.toml b/pyproject.toml index e50c9b1800..d850ed53e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,97 @@ +[build-system] +requires = ["setuptools", "setuptools_scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "flytekit" +dynamic = ["version"] +authors = [{ name = "Flyte Contributors", email = "admin@flyte.org" }] +description = "Flyte SDK for Python" +license = { text = "Apache-2.0" } +readme = { file = "README.md", content-type = "text/markdown" } +requires-python = ">=3.8,<3.12" +dependencies = [ + # Please maintain an alphabetical order in the following list + "adlfs", + "click>=6.6,<9.0", + "cloudpickle>=2.0.0", + "cookiecutter>=1.7.3", + "croniter>=0.3.20,<4.0.0", + "dataclasses-json>=0.5.2,<0.5.12", # TODO: remove upper-bound after fixing change in contract + "diskcache>=5.2.1", + "docker>=4.0.0,<7.0.0", + "docstring-parser>=0.9.0", + "flyteidl>=1.10.0", + "fsspec>=2023.3.0,<=2023.9.2", + "gcsfs", + "googleapis-common-protos>=1.57", + "grpcio", + "grpcio-status", + "importlib-metadata", + "joblib", + "jsonpickle", + "keyring>=18.0.1", + "kubernetes>=12.0.1", + "marshmallow-enum", + # TODO: remove upper-bound after fixing change in contract + "marshmallow-jsonschema>=0.12.0", + "mashumaro>=3.9.1", + "numpy", + "pandas>=1.0.0,<2.0.0", + # TODO: Remove upper-bound after protobuf community fixes it. https://github.com/flyteorg/flyte/issues/4359 + "protobuf<4.25.0", + "pyarrow>=4.0.0", + "python-json-logger>=2.0.0", + "pytimeparse>=1.1.8,<2.0.0", + "pyyaml!=6.0.0,!=5.4.0,!=5.4.1", # pyyaml is broken with cython 3: https://github.com/yaml/pyyaml/issues/601 + "requests>=2.18.4,<3.0.0", + "rich", + "rich_click", + "s3fs>=0.6.0", + "statsd>=3.0.0,<4.0.0", + "typing_extensions", + "urllib3>=1.22,<2.0.0", +] +classifiers = [ + "Intended Audience :: Science/Research", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Topic :: Scientific/Engineering", + "Topic :: Scientific/Engineering :: Artificial Intelligence", + "Topic :: Software Development", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", +] + +[project.urls] +Homepage = "https://github.com/flyteorg/flytekit" + +[project.scripts] +pyflyte-execute = "flytekit.bin.entrypoint:execute_task_cmd" +pyflyte-fast-execute = "flytekit.bin.entrypoint:fast_execute_task_cmd" +pyflyte-map-execute = "flytekit.bin.entrypoint:map_execute_task_cmd" +pyflyte = "flytekit.clis.sdk_in_container.pyflyte:main" +flyte-cli = "flytekit.clis.flyte_cli.main:_flyte_cli" + +[tool.setuptools_scm] +write_to = "flytekit/_version.py" + +[tool.setuptools] +include-package-data = true +script-files = [ + "flytekit_scripts/flytekit_build_image.sh", + "flytekit_scripts/flytekit_venv", + "flytekit/bin/entrypoint.py", +] + +[tool.setuptools.packages.find] +include = ["flytekit", "flytekit_scripts"] +exclude = ["boilerplate", "docs", "plugins", "tests*"] + [tool.pytest.ini_options] norecursedirs = ["common", "workflows", "spark", "fsspec"] log_cli = true diff --git a/setup.py b/setup.py index c3624921aa..606849326a 100644 --- a/setup.py +++ b/setup.py @@ -1,94 +1,3 @@ -from setuptools import find_packages, setup # noqa +from setuptools import setup -extras_require = {} - -__version__ = "0.0.0+develop" - -setup( - name="flytekit", - version=__version__, - maintainer="Flyte Contributors", - maintainer_email="admin@flyte.org", - packages=find_packages( - include=["flytekit", "flytekit_scripts"], - exclude=["boilerplate", "docs", "plugins", "tests*"], - ), - include_package_data=True, - url="https://github.com/flyteorg/flytekit", - description="Flyte SDK for Python", - long_description=open("README.md", encoding="utf-8").read(), - long_description_content_type="text/markdown", - entry_points={ - "console_scripts": [ - "pyflyte-execute=flytekit.bin.entrypoint:execute_task_cmd", - "pyflyte-fast-execute=flytekit.bin.entrypoint:fast_execute_task_cmd", - "pyflyte-map-execute=flytekit.bin.entrypoint:map_execute_task_cmd", - "pyflyte=flytekit.clis.sdk_in_container.pyflyte:main", - "flyte-cli=flytekit.clis.flyte_cli.main:_flyte_cli", - ] - }, - install_requires=[ - # Please maintain an alphabetical order in the following list - "adlfs", - "click>=6.6,<9.0", - "cloudpickle>=2.0.0", - "cookiecutter>=1.7.3", - "croniter>=0.3.20,<4.0.0", - "dataclasses-json>=0.5.2,<0.5.12", # TODO: remove upper-bound after fixing change in contract - "diskcache>=5.2.1", - "docker>=4.0.0,<7.0.0", - "docstring-parser>=0.9.0", - "flyteidl>=1.10.0", - "fsspec>=2023.3.0,<=2023.9.2", - "gcsfs", - "googleapis-common-protos>=1.57", - "grpcio", - "grpcio-status", - "importlib-metadata", - "joblib", - "jsonpickle", - "keyring>=18.0.1", - "kubernetes>=12.0.1", - "marshmallow-enum", - # TODO: remove upper-bound after fixing change in contract - "marshmallow-jsonschema>=0.12.0", - "mashumaro>=3.9.1", - "numpy", - "pandas>=1.0.0,<2.0.0", - # TODO: Remove upper-bound after protobuf community fixes it. https://github.com/flyteorg/flyte/issues/4359 - "protobuf<4.25.0", - "pyarrow>=4.0.0", - "python-json-logger>=2.0.0", - "pytimeparse>=1.1.8,<2.0.0", - "pyyaml!=6.0.0,!=5.4.0,!=5.4.1", # pyyaml is broken with cython 3: https://github.com/yaml/pyyaml/issues/601 - "requests>=2.18.4,<3.0.0", - "rich", - "rich_click", - "s3fs>=0.6.0", - "statsd>=3.0.0,<4.0.0", - "typing_extensions", - "urllib3>=1.22,<2.0.0", - ], - extras_require=extras_require, - scripts=[ - "flytekit_scripts/flytekit_build_image.sh", - "flytekit_scripts/flytekit_venv", - "flytekit/bin/entrypoint.py", - ], - license="apache2", - python_requires=">=3.8,<3.12", - classifiers=[ - "Intended Audience :: Science/Research", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Topic :: Scientific/Engineering", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - "Topic :: Software Development", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - ], -) +setup()