Dynamically update lib version and publish all plugins (#364)

Automatically publishes to pypi when a release is created. Removes the need to update any versions embedded in the files (init and setup.py) for the core library as well as plugins.

.github/workflows/pythonpublish.yml

@@ -1,4 +1,4 @@
-name: Upload Python Package
+name: Publish Python Package
 
 on:
   release:
@@ -8,19 +8,40 @@ jobs:
   deploy:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v1
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: "0"
+      - name: Set up Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Autobump version
+        run: |
+          # from refs/tags/v1.2.3 get 1.2.3
+          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+          VERSION=$VERSION make update_version
+        shell: bash
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*
+      - name: Autobump plugin version
+        run: |
+          # from refs/tags/v1.2.3 get 1.2.3
+          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+          VERSION=$VERSION make -C plugins update_all_versions
+        shell: bash
+      - name: Build all Plugins and publish
+        env:
+          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+        run: |
+          make -C plugins build_all_plugins
+          make -C plugins publish_all_plugins

Makefile

@@ -58,3 +58,15 @@ requirements: requirements.txt dev-requirements.txt requirements-spark3.txt ## C
 coverage:
 	coverage run -m pytest tests/flytekit/unit/core flytekit/types plugins/tests
 	coverage report -m --include="flytekit/core/*,flytekit/types/*,plugins/*"
+
+PLACEHOLDER := "__version__\ =\ \"develop\""
+
+.PHONY: update_version
+update_version:
+	# ensure the placeholder is there. If grep doesn't find the placeholder
+	# it exits with exit code 1 and github actions aborts the build. 
+	grep "$(PLACEHOLDER)" "flytekit/__init__.py"
+	sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "flytekit/__init__.py"
+
+	grep "$(PLACEHOLDER)" "setup.py"
+	sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" "setup.py"

flytekit/__init__.py

@@ -17,7 +17,7 @@
 from flytekit.core.workflow import WorkflowFailurePolicy, reference_workflow, workflow
 from flytekit.loggers import logger
 
-__version__ = "0.16.0b6"
+__version__ = "develop"
 
 
 def current_context() -> ExecutionParameters:

plugins/Makefile

@@ -8,5 +8,18 @@ install-all-dev:
 
 .PHONY: build_all_plugins
 build_all_plugins:
-	run_all_plugins.sh python setup.py sdist bdist_wheel
+	./run_all_plugins.sh python setup.py sdist bdist_wheel
 
+.PHONY: publish_all_plugins
+publish_all_plugins:
+	twine upload */dist/*
+
+PLACEHOLDER := "__version__\ =\ \"develop\""
+VERSION_FILE := "setup.py"
+
+.PHONY: update_all_versions
+update_all_versions:
+	# ensure the placeholder is there. If grep doesn't find the placeholder
+	# it exits with exit code 1 and github actions aborts the build. 
+	./run_all_plugins.sh grep "$(PLACEHOLDER)" "$(VERSION_FILE)"
+	./run_all_plugins.sh sed -i "s/$(PLACEHOLDER)/__version__ = \"${VERSION}\"/g" $(VERSION_FILE)

plugins/awssagemaker/setup.py

@@ -6,10 +6,12 @@
 
 plugin_requires = ["flytekit>=0.16.0b0,<1.0.0", "sagemaker-training>=3.6.2,<4.0.0"]
 
+__version__ = "develop"
+
 # TODO: move sagemaker install script here.
 setup(
     name=microlib_name,
-    version="0.1.1",
+    version=__version__,
     author="flyteorg",
     author_email="[email protected]",
     description="AWS Plugins for flytekit",

plugins/hive/setup.py

@@ -6,9 +6,11 @@
 
 plugin_requires = ["flytekit>=0.16.0b0,<1.0.0"]
 
+__version__ = "develop"
+
 setup(
     name=microlib_name,
-    version="0.1.1",
+    version=__version__,
     author="flyteorg",
     author_email="[email protected]",
     description="This package holds Hive plugins for flytekit",

plugins/kfpytorch/setup.py

@@ -6,9 +6,11 @@
 
 plugin_requires = ["flytekit>=0.16.0b0,<1.0.0"]
 
+__version__ = "develop"
+
 setup(
     name=microlib_name,
-    version="0.1.1",
+    version=__version__,
     author="flyteorg",
     author_email="[email protected]",
     description="K8s based Pytorch plugin for Flytekit",

plugins/kftensorflow/setup.py

@@ -7,9 +7,11 @@
 # TODO: Requirements are missing, add them back in later.
 plugin_requires = ["flytekit>=0.16.0b0,<1.0.0"]
 
+__version__ = "develop"
+
 setup(
     name=microlib_name,
-    version="0.1.1",
+    version=__version__,
     author="flyteorg",
     author_email="[email protected]",
     description="K8s based Tensorflow plugin for flytekit",

plugins/papermill/setup.py

@@ -6,9 +6,11 @@
 
 plugin_requires = ["flytekit>=0.16.0b0,<1.0.0", "papermill>=1.2.0", "nbconvert>=6.0.7", "ipykernel>=5.0.0"]
 
+__version__ = "develop"
+
 setup(
     name=microlib_name,
-    version="0.1.1",
+    version=__version__,
     author="flyteorg",
     author_email="[email protected]",
     description="This is the flytekit papermill plugin",

plugins/pod/setup.py

@@ -6,9 +6,11 @@
 
 plugin_requires = ["flytekit>=0.16.0b0,<1.0.0", "k8s-proto>=0.0.3,<1.0.0"]
 
+__version__ = "develop"
+
 setup(
     name=microlib_name,
-    version="0.1.1",
+    version=__version__,
     author="flyteorg",
     author_email="[email protected]",
     description="Flytekit plugin to support K8s Pod tasks",

plugins/spark/setup.py

@@ -6,9 +6,11 @@
 
 plugin_requires = ["flytekit>=0.16.0b0,<1.0.0", "pyspark>=3.0.0"]
 
+__version__ = "develop"
+
 setup(
     name=microlib_name,
-    version="0.1.1",
+    version=__version__,
     author="flyteorg",
     author_email="[email protected]",
     description="Spark 3 plugin for flytekit",

setup.py

@@ -35,10 +35,11 @@
     "all": spark3 + all_but_spark,
 }
 
+__version__ = "develop"
 
 setup(
     name="flytekit",
-    version="0.16.0b6",
+    version=__version__,
     maintainer="Flyte Org",
     maintainer_email="[email protected]",
     packages=find_packages(exclude=["tests*"]),