Update publish workflow for tag checking (#2632)

.github/workflows/publish.yml

@@ -1,4 +1,4 @@
-name: Build and upload to internal PyPI
+name: Build and upload to PyPI
 
 on:
   workflow_dispatch: # run on request (no need for PR)
@@ -40,22 +40,17 @@ jobs:
     name: Publish package
     needs: [build_wheels, build_sdist]
     environment: pypi
-    runs-on: [self-hosted, linux, x64, dev]
+    runs-on: ubuntu-latest
     permissions: write-all
     steps:
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
-      - name: Install dependencies
-        run: python -m pip install twine
       - name: Download artifacts
         uses: actions/download-artifact@v3
         with:
           # unpacks default artifact into dist/
           # if `name: artifact` is omitted, the action will create extra parent dir
           name: artifact
           path: dist
+      # to determine where to publish the source distribution to PyPI or TestPyPI
       - name: Check tag
         id: check-tag
         uses: actions-ecosystem/action-regex-match@v2
@@ -71,18 +66,15 @@ jobs:
           tag: ${{ github.ref }}
           overwrite: true
           file_glob: true
-      - name: Check dist contents
-        run: twine check dist/*
-      - name: Publish package dist to internal PyPI
-        run: |
-          export no_proxy=${{ secrets.PYPI_HOST }}
-          export REPOSITORY_URL=http://${{ secrets.PYPI_HOST }}:${{ secrets.PYPI_PORT }}
-          twine upload --verbose --repository-url $REPOSITORY_URL dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }}
-      - name: Clean up dist
-        if: ${{ always() }}
-        run: |
-          if OUTPUT=$(ls | grep -c dist)
-          then
-            echo "Cleaning up dist directory"
-            rm -r dist
-          fi
+      - name: Publish package distributions to PyPI
+        if: ${{ steps.check-tag.outputs.match != '' }}
+        uses: pypa/[email protected]
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}
+      - name: Publish package distributions to TestPyPI
+        if: ${{ steps.check-tag.outputs.match == '' }}
+        uses: pypa/[email protected]
+        with:
+          password: ${{ secrets.TESTPYPI_API_TOKEN }}
+          repository-url: https://test.pypi.org/legacy/
+          verbose: true

.github/workflows/publish_internal.yml

@@ -0,0 +1,92 @@
+name: Build and upload to internal PyPI
+
+on:
+  workflow_dispatch: # run on request (no need for PR)
+
+jobs:
+  build_wheels:
+    name: Build wheels
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Build wheels
+        uses: pypa/[email protected]
+      - uses: actions/upload-artifact@v3
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v3
+        with:
+          python-version: "3.10"
+      - name: Install pypa/build
+        run: python -m pip install build
+      - name: Build sdist
+        run: python -m build --sdist
+      - uses: actions/upload-artifact@v3
+        with:
+          path: dist/*.tar.gz
+
+  publish_package:
+    name: Publish package
+    needs: [build_wheels, build_sdist]
+    environment: pypi
+    runs-on: [self-hosted, linux, x64, dev]
+    permissions: write-all
+    steps:
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: python -m pip install twine
+      - name: Download artifacts
+        uses: actions/download-artifact@v3
+        with:
+          # unpacks default artifact into dist/
+          # if `name: artifact` is omitted, the action will create extra parent dir
+          name: artifact
+          path: dist
+      - name: Check tag
+        id: check-tag
+        uses: actions-ecosystem/action-regex-match@v2
+        with:
+          text: ${{ github.ref }}
+          regex: '^refs/heads/releases/[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+rc[0-9]+|rc[0-9]+)?$'
+      - name: Upload package distributions to github
+        if: ${{ steps.check-tag.outputs.match != '' }}
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: dist/*
+          tag: ${{ github.ref }}
+          overwrite: true
+          file_glob: true
+      - name: Check dist contents
+        run: twine check dist/*
+      - name: Publish package dist to internal PyPI
+        if: ${{ steps.check-tag.outputs.match != '' }}
+        run: |
+          export no_proxy=${{ secrets.PYPI_HOST }}
+          export REPOSITORY_URL=http://${{ secrets.PYPI_HOST }}:${{ secrets.PYPI_PORT }}
+          twine upload --verbose --repository-url $REPOSITORY_URL dist/* -u ${{ secrets.PYPI_USER }} -p ${{ secrets.PYPI_PASSWORD }}
+      - name: Publish package distributions to TestPyPI
+        if: ${{ steps.check-tag.outputs.match == '' }}
+        run: |
+          export REPOSITORY_URL=https://test.pypi.org/legacy/
+          twine upload --verbose --repository-url $REPOSITORY_URL dist/* -u __token__ -p ${{ secrets.TESTPYPI_API_TOKEN }}
+      - name: Clean up dist
+        if: ${{ always() }}
+        run: |
+          if OUTPUT=$(ls | grep -c dist)
+          then
+            echo "Cleaning up dist directory"
+            rm -r dist
+          fi

requirements/base.txt

@@ -1,10 +1,10 @@
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-# Base Algo Requirements.                                                          #
+# Base Algo Requirements.                                                     #
 natsort==8.1.*
 prettytable==3.9.*
 protobuf==3.20.*
 pyyaml
-datumaro==1.5.1rc3
+datumaro~=1.5.1rc4
 psutil==5.9.*
 scipy==1.10.*
 bayesian-optimization==1.4.*