Merge branch 'dev' into eelco/sync-table-fixes

OpenMined · May 22, 2024 · 0af9838 · 0af9838
2 parents 8cdf60d + 8a28ad6
commit 0af9838
Show file tree

Hide file tree

Showing 179 changed files with 942 additions and 18,186 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -44,14 +44,6 @@ first_value = 1
 
 [bumpversion:file:packages/grid/helm/syft/Chart.yaml]
 
-[bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube.yaml]
-
-[bumpversion:file:packages/grid/podman/podman-kube/podman-syft-kube-config.yaml]
-
 [bumpversion:file:packages/grid/helm/syft/values.yaml]
 
-[bumpversion:file:packages/hagrid/hagrid/manifest_template.yml]
-
-[bumpversion:file:packages/hagrid/hagrid/deps.py]
-
 [bumpversion:file:packages/syftcli/manifest.yml]
diff --git a/.bumpversion_stable.cfg b/.bumpversion_stable.cfg
@@ -13,7 +13,3 @@ serialize =
 	{major}.{minor}.{patch}
 
 [bumpversion:file:packages/syft/src/syft/stable_version.py]
-
-[bumpversion:file:packages/hagrid/hagrid/stable_version.py]
-
-[bumpversion:file:packages/hagrid/hagrid/cache.py]
diff --git a/.github/file-filters.yml b/.github/file-filters.yml
@@ -27,17 +27,6 @@ backend:
   - packages/grid/backend/**/*.sh
   - packages/grid/backend/**/*.mako
 
-hagrid:
-  - .github/workflows/pr-tests-hagrid.yml
-  - packages/hagrid/**/*.py
-  - packages/hagrid/**/*.cfg
-  - packages/hagrid/**/*.yml
-  - packages/hagrid/**/*.dockerfile
-  - packages/hagrid/**/*.toml
-  - packages/hagrid/**/*.txt
-  - packages/hagrid/**/*.ini
-  - packages/hagrid/**/*.sh
-
 syft:
   - .github/workflows/pr-tests-syft.yml
   - packages/syft/**/*.py

diff --git a/.github/workflows/cd-hagrid.yml b/.github/workflows/cd-hagrid.yml
diff --git a/.github/workflows/cd-post-release-tests.yml b/.github/workflows/cd-post-release-tests.yml
@@ -30,105 +30,6 @@ on:
         default: "REAL_PYPI"
 
 jobs:
-  notebook-test-hagrid:
-    if: github.event.inputs.release_platform == 'REAL_PYPI'
-    strategy:
-      max-parallel: 99
-      matrix:
-        os: [ubuntu-latest]
-        python-version: ["3.12"]
-
-    runs-on: ${{ matrix.os }}
-    steps:
-      - uses: actions/checkout@v4
-
-      # free 10GB of space
-      - name: Remove unnecessary files
-        if: matrix.os == 'ubuntu-latest'
-        run: |
-          sudo rm -rf /usr/share/dotnet
-          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
-          docker image prune --all --force
-          docker builder prune --all --force
-          docker system prune --all --force
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Upgrade pip
-        run: |
-          python -m pip install --upgrade --user pip
-
-      - name: Get pip cache dir
-        id: pip-cache
-        shell: bash
-        run: |
-          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
-
-      - name: pip cache
-        uses: actions/cache@v4
-        with:
-          path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-py${{ matrix.python-version }}-
-
-      - name: Install Hagrid, tox and uv
-        run: |
-          pip install -U hagrid
-          pip install --upgrade pip uv==0.1.35 tox tox-uv==1.5.1
-
-      - name: Hagrid Version
-        run: |
-          hagrid version
-
-      - name: Remove existing containers
-        continue-on-error: true
-        shell: bash
-        run: |
-          docker rm $(docker ps -aq) --force || true
-          docker volume prune -f || true
-          docker buildx use default || true
-
-      - name: Launch Domain
-        run: |
-          hagrid launch test-domain-1 to docker:8081 --tag=${{ inputs.syft_version }} --low-side
-
-      - name: Run tests
-        env:
-          NODE_PORT: "8081"
-          SYFT_VERSION: ${{ inputs.syft_version }}
-          EXCLUDE_NOTEBOOKS: "not 11-container-images-k8s.ipynb"
-        run: |
-          tox -e e2e.test.notebook
-
-      #Run log collector python script
-      - name: Run log collector
-        timeout-minutes: 5
-        if: failure()
-        shell: bash
-        run: |
-          python ./scripts/container_log_collector.py
-
-      # Get Job name and url
-      - name: Get job name and url
-        id: job_name
-        if: failure()
-        shell: bash
-        run: |
-          echo "job_name=$(echo ${{ github.job }})" >> $GITHUB_OUTPUT
-          echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
-
-      - name: Upload logs to GitHub
-        uses: actions/upload-artifact@master
-        if: failure()
-        with:
-          name: ${{ matrix.os }}-${{ steps.job_name.outputs.job_name }}-logs-${{ steps.job_name.outputs.date }}
-          path: ./logs/${{ steps.job_name.outputs.job_name}}/
-
   syft-install-check:
     strategy:
       max-parallel: 99
@@ -290,9 +191,8 @@ jobs:
             pip install syft[data_science,dev]==${{ inputs.syft_version }}
           fi
 
-      - name: Install Hagrid, tox and uv
+      - name: Install tox and uv
         run: |
-          pip install -U hagrid
           pip install --upgrade pip uv==0.1.35 tox tox-uv==1.5.1 tox-current-env
 
       - name: Run unit tests

diff --git a/.github/workflows/cd-syft.yml b/.github/workflows/cd-syft.yml
@@ -396,7 +396,6 @@ jobs:
           bump2version prenum --allow-dirty --no-commit
           ls **/VERSION | xargs -I {} python {}
           cat packages/grid/devspace.yaml | grep '0\.'
-          python packages/hagrid/scripts/update_manifest.py $(python packages/grid/VERSION)
 
       - name: Generate Release Metadata
         id: release_checks
@@ -460,7 +459,7 @@ jobs:
           author_name: ${{ secrets.OM_BOT_NAME }}
           author_email: ${{ secrets.OM_BOT_EMAIL }}
           message: "[syft]bump version"
-          add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/hagrid/hagrid/manifest_template.yml', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo', 'packages/hagrid/hagrid/deps.py', 'packages/grid/podman/podman-kube/podman-syft-kube.yaml' ,'packages/grid/podman/podman-kube/podman-syft-kube-config.yaml', 'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/', 'packages/grid/backend/grid/images/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml','packages/grid/helm/syft']"
+          add: "['.bumpversion.cfg', 'VERSION', 'packages/grid/VERSION','packages/syft/PYPI.md', 'packages/grid/devspace.yaml', 'packages/syft/src/syft/VERSION', 'packages/syft/setup.cfg', 'packages/grid/frontend/package.json', 'packages/syft/src/syft/__init__.py', 'packages/grid/helm/syft/Chart.yaml','packages/grid/helm/repo',  'packages/syftcli/manifest.yml', 'packages/syft/src/syft/protocol/protocol_version.json', 'packages/syft/src/syft/protocol/releases/', 'packages/grid/backend/grid/images/worker_cpu.dockerfile','packages/grid/helm/syft/values.yaml','packages/grid/helm/syft']"
 
       - name: Changes to commit to Syft Repo during stable release
         if: needs.merge-docker-images.outputs.release_tag == 'latest'
@@ -539,7 +538,6 @@ jobs:
           files: |
             ./packages/syftcli/manifest.yml
             ./build/syftcli-config/*
-            ./packages/hagrid/hagrid/manifest_template.yml
           tag_name: v${{ steps.release_checks.outputs.github_release_version }}
 
       # Checkout to gh-pages and update helm repo

diff --git a/.github/workflows/e2e-tests-notebook.yml b/.github/workflows/e2e-tests-notebook.yml
@@ -40,7 +40,7 @@ on:
         type: string
 
 jobs:
-  notebook-test-hagrid:
+  notebook-test-e2e:
     strategy:
       max-parallel: 99
       matrix:

diff --git a/.github/workflows/manual-delete-buildjet-cache.yml b/.github/workflows/manual-delete-buildjet-cache.yml
diff --git a/.github/workflows/nightlies.yml b/.github/workflows/nightlies.yml
@@ -14,10 +14,6 @@ jobs:
     if: github.repository == 'OpenMined/PySyft' # don't run on forks
     uses: OpenMined/PySyft/.github/workflows/pr-tests-linting.yml@dev
 
-  call-pr-tests-hagrid:
-    if: github.repository == 'OpenMined/PySyft' # don't run on forks
-    uses: OpenMined/PySyft/.github/workflows/pr-tests-hagrid.yml@dev
-
   call-pr-tests-syft:
     if: github.repository == 'OpenMined/PySyft' # don't run on forks
     uses: OpenMined/PySyft/.github/workflows/pr-tests-syft.yml@dev

diff --git a/.github/workflows/pr-tests-frontend.yml b/.github/workflows/pr-tests-frontend.yml
@@ -61,13 +61,13 @@ jobs:
         if: steps.changes.outputs.frontend == 'true'
         with:
           path: ${{ steps.pip-cache.outputs.dir }}
-          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-${{ hashFiles('packages/hagrid/setup.cfg') }}
+          key: ${{ runner.os }}-uv-py${{ matrix.python-version }}-frontend
           restore-keys: |
             ${{ runner.os }}-uv-py${{ matrix.python-version }}-
 
       - name: Docker on MacOS
         if: steps.changes.outputs.frontend == 'true' && matrix.os == 'macos-latest'
-        uses: crazy-max/ghaction-setup-docker@v3.1.0
+        uses: crazy-max/ghaction-setup-docker@v3.2.0
 
       - name: Install Tox
         if: steps.changes.outputs.frontend == 'true'