diff --git a/.github/workflows/pythonpublish.yml b/.github/workflows/pythonpublish.yml index b143a53098..a26f54d381 100644 --- a/.github/workflows/pythonpublish.yml +++ b/.github/workflows/pythonpublish.yml @@ -1,4 +1,4 @@ -name: Upload Python Package +name: Publish Python Package on: release: @@ -8,19 +8,40 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v1 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - python setup.py sdist bdist_wheel - twine upload dist/* + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Set up Python + uses: actions/setup-python@v1 + with: + python-version: "3.x" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + - name: Autobump version + run: | + # from refs/tags/v1.2.3 get 1.2.3 + VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') + VERSION=$VERSION make update_version + shell: bash + - name: Build and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload dist/* + - name: Autobump plugin version + run: | + # from refs/tags/v1.2.3 get 1.2.3 + VERSION=$(echo $GITHUB_REF | sed 's#.*/v##') + VERSION=$VERSION make -C plugins update_all_versions + shell: bash + - name: Build all Plugins and publish + env: + TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: | + make -C plugins build_all_plugins + make -C plugins publish_all_plugins diff --git a/Makefile b/Makefile index a0f8aa26c7..797cb1906b 100644 --- a/Makefile +++ b/Makefile @@ -58,3 +58,15 @@ requirements: requirements.txt dev-requirements.txt requirements-spark3.txt ## C coverage: coverage run -m pytest tests/flytekit/unit/core flytekit/types plugins/tests coverage report -m --include="flytekit/core/*,flytekit/types/*,plugins/*" + +PLACEHOLDER := "__version__\ =\ \"develop\"" + +.PHONY: update_version +update_version: + # ensure the placeholder is there. If grep doesn't find the placeholder + # it exits with exit code 1 and github actions aborts the build. + grep "$(PLACEHOLDER)" "flytekit/__init__.py" + sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "flytekit/__init__.py" + + grep "$(PLACEHOLDER)" "setup.py" + sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "setup.py" diff --git a/flytekit/__init__.py b/flytekit/__init__.py index f1d2d71495..6e0fb4f63e 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -17,7 +17,7 @@ from flytekit.core.workflow import WorkflowFailurePolicy, reference_workflow, workflow from flytekit.loggers import logger -__version__ = "0.16.0b6" +__version__ = "develop" def current_context() -> ExecutionParameters: diff --git a/plugins/Makefile b/plugins/Makefile index 81359eec8f..e9135278ef 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -8,5 +8,18 @@ install-all-dev: .PHONY: build_all_plugins build_all_plugins: - run_all_plugins.sh python setup.py sdist bdist_wheel + ./run_all_plugins.sh python setup.py sdist bdist_wheel +.PHONY: publish_all_plugins +publish_all_plugins: + twine upload */dist/* + +PLACEHOLDER := "__version__\ =\ \"develop\"" +VERSION_FILE := "setup.py" + +.PHONY: update_all_versions +update_all_versions: + # ensure the placeholder is there. If grep doesn't find the placeholder + # it exits with exit code 1 and github actions aborts the build. + ./run_all_plugins.sh grep "$(PLACEHOLDER)" "$(VERSION_FILE)" + ./run_all_plugins.sh sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" $(VERSION_FILE) diff --git a/plugins/awssagemaker/setup.py b/plugins/awssagemaker/setup.py index 58ab099177..45a1be7087 100644 --- a/plugins/awssagemaker/setup.py +++ b/plugins/awssagemaker/setup.py @@ -6,10 +6,12 @@ plugin_requires = ["flytekit>=0.16.0b0,<1.0.0", "sagemaker-training>=3.6.2,<4.0.0"] +__version__ = "develop" + # TODO: move sagemaker install script here. setup( name=microlib_name, - version="0.1.1", + version=__version__, author="flyteorg", author_email="admin@flyte.org", description="AWS Plugins for flytekit", diff --git a/plugins/hive/setup.py b/plugins/hive/setup.py index 37754f918b..a5523cead0 100644 --- a/plugins/hive/setup.py +++ b/plugins/hive/setup.py @@ -6,9 +6,11 @@ plugin_requires = ["flytekit>=0.16.0b0,<1.0.0"] +__version__ = "develop" + setup( name=microlib_name, - version="0.1.1", + version=__version__, author="flyteorg", author_email="admin@flyte.org", description="This package holds Hive plugins for flytekit", diff --git a/plugins/kfpytorch/setup.py b/plugins/kfpytorch/setup.py index 359396598c..6c1674312d 100644 --- a/plugins/kfpytorch/setup.py +++ b/plugins/kfpytorch/setup.py @@ -6,9 +6,11 @@ plugin_requires = ["flytekit>=0.16.0b0,<1.0.0"] +__version__ = "develop" + setup( name=microlib_name, - version="0.1.1", + version=__version__, author="flyteorg", author_email="admin@flyte.org", description="K8s based Pytorch plugin for Flytekit", diff --git a/plugins/kftensorflow/setup.py b/plugins/kftensorflow/setup.py index 97995b74a2..74193c8e23 100644 --- a/plugins/kftensorflow/setup.py +++ b/plugins/kftensorflow/setup.py @@ -7,9 +7,11 @@ # TODO: Requirements are missing, add them back in later. plugin_requires = ["flytekit>=0.16.0b0,<1.0.0"] +__version__ = "develop" + setup( name=microlib_name, - version="0.1.1", + version=__version__, author="flyteorg", author_email="admin@flyte.org", description="K8s based Tensorflow plugin for flytekit", diff --git a/plugins/papermill/setup.py b/plugins/papermill/setup.py index f18e323570..395eb36aa9 100644 --- a/plugins/papermill/setup.py +++ b/plugins/papermill/setup.py @@ -6,9 +6,11 @@ plugin_requires = ["flytekit>=0.16.0b0,<1.0.0", "papermill>=1.2.0", "nbconvert>=6.0.7", "ipykernel>=5.0.0"] +__version__ = "develop" + setup( name=microlib_name, - version="0.1.1", + version=__version__, author="flyteorg", author_email="admin@flyte.org", description="This is the flytekit papermill plugin", diff --git a/plugins/pod/setup.py b/plugins/pod/setup.py index 3eab2e64b1..8d70ed0566 100644 --- a/plugins/pod/setup.py +++ b/plugins/pod/setup.py @@ -6,9 +6,11 @@ plugin_requires = ["flytekit>=0.16.0b0,<1.0.0", "k8s-proto>=0.0.3,<1.0.0"] +__version__ = "develop" + setup( name=microlib_name, - version="0.1.1", + version=__version__, author="flyteorg", author_email="admin@flyte.org", description="Flytekit plugin to support K8s Pod tasks", diff --git a/plugins/spark/setup.py b/plugins/spark/setup.py index e5aafe3acd..c1ad98bd50 100644 --- a/plugins/spark/setup.py +++ b/plugins/spark/setup.py @@ -6,9 +6,11 @@ plugin_requires = ["flytekit>=0.16.0b0,<1.0.0", "pyspark>=3.0.0"] +__version__ = "develop" + setup( name=microlib_name, - version="0.1.1", + version=__version__, author="flyteorg", author_email="admin@flyte.org", description="Spark 3 plugin for flytekit", diff --git a/setup.py b/setup.py index bdfe162080..1b122715f5 100644 --- a/setup.py +++ b/setup.py @@ -35,10 +35,11 @@ "all": spark3 + all_but_spark, } +__version__ = "develop" setup( name="flytekit", - version="0.16.0b6", + version=__version__, maintainer="Flyte Org", maintainer_email="admin@flyte.org", packages=find_packages(exclude=["tests*"]),