Run all integration tests using local core

Also cleanup and DRY our github actions a wee bit.
port-labs · Sep 10, 2024 · 9bc26af · 9bc26af
1 parent 32e9c00
commit 9bc26af
Show file tree

Hide file tree

Showing 3 changed files with 66 additions and 64 deletions.
diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml
@@ -0,0 +1,42 @@
+name: Detect Changes
+on:
+  workflow_call:
+    outputs:
+      matrix:
+        value: ${{ jobs.detect-changes.outputs.matrix }}
+        description: "Matrix of changed integrations per git commit changes"
+
+jobs:
+  detect-changes:
+    name: Detect changes
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Get list of changed files
+        id: changed-files
+        uses: tj-actions/[email protected]
+      - name: Set matrix
+        id: set-matrix
+        run: |
+          folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md"
+          changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" \
+              | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' \
+              | egrep -v '_infra' \
+              | cut -d'/' -f2 | sort | uniq)
+          if [ -z "$changed_folders" ]; then
+            changed_folders=""
+          fi
+
+          folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md"
+          other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l)
+          if [ "$other_changes" -ne 0 ]; then
+            # Add the root directory to the matrix if there are changes outside the integrations folder
+            changed_folders=$(echo -e "$changed_folders\n.")
+          fi
+
+          matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))')
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -6,36 +6,7 @@ on:
 
 jobs:
   detect-changes:
-    name: Detect changes
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-
-      - name: Get list of changed files
-        id: changed-files
-        uses: tj-actions/[email protected]
-      - name: Set matrix
-        id: set-matrix
-        run: |
-          folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md"
-          changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' | cut -d'/' -f2 | sort | uniq)
-          if [ -z "$changed_folders" ]; then
-            changed_folders=""
-          fi
-
-          folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md"
-          other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l)
-          if [ "$other_changes" -ne 0 ]; then
-            # Add the root directory to the matrix if there are changes outside the integrations folder
-            changed_folders=$(echo -e "$changed_folders\n.")
-          fi
-
-          matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))')
-          echo "matrix=$matrix" >> $GITHUB_OUTPUT
-
+    uses: ./.github/workflows/detect-changes-matrix.yml
   lint:
     name: ${{ matrix.folder == '.' && '🌊 Ocean Core' || format('🚢 {0}', matrix.folder) }}
     needs: detect-changes
@@ -44,12 +15,15 @@ jobs:
       matrix:
         folder: ${{ fromJson(needs.detect-changes.outputs.matrix) }}
     steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Install poetry
+        run: pipx install poetry
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-      - name: Checkout Repo
-        uses: actions/checkout@v4
+          cache: 'poetry'
       - name: Install dependencies
         working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }}
         run: |

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -6,36 +6,7 @@ on:
 
 jobs:
   detect-changes:
-    name: Detect changes
-    runs-on: ubuntu-latest
-    outputs:
-      matrix: ${{ steps.set-matrix.outputs.matrix }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v4
-
-      - name: Get list of changed files
-        id: changed-files
-        uses: tj-actions/[email protected]
-      - name: Set matrix
-        id: set-matrix
-        run: |
-          folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md"
-          changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' | cut -d'/' -f2 | sort | uniq)
-          if [ -z "$changed_folders" ]; then
-            changed_folders=""
-          fi
-
-          folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md"
-          other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l)
-          if [ "$other_changes" -ne 0 ]; then
-            # Add the root directory to the matrix if there are changes outside the integrations folder
-            changed_folders=$(echo -e "$changed_folders\n.")
-          fi
-
-          matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))')
-          echo "matrix=$matrix" >> $GITHUB_OUTPUT
-
+    uses: ./.github/workflows/detect-changes-matrix.yml
   test:
     name: ${{ matrix.folder == '.' && '🌊 Ocean Core' || format('🚢 {0}', matrix.folder) }}
     needs: detect-changes
@@ -44,12 +15,15 @@ jobs:
       matrix:
         folder: ${{ fromJson(needs.detect-changes.outputs.matrix) }}
     steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+      - name: Install poetry
+        run: pipx install poetry
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
         with:
           python-version: '3.11'
-      - name: Checkout Repo
-        uses: actions/checkout@v4
+          cache: 'poetry'
       - name: Install dependencies
         working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }}
         run: |
@@ -60,6 +34,18 @@ jobs:
           PYTEST_ADDOPTS: --junitxml=junit/test-results-${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || 'ocean/core' }}.xml
         run: |
           make test
+      - name: Install current core for all integrations
+        working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }}
+        if: ${{ matrix.folder == '.' }}
+        run: |
+          echo "Installing local core for all integrations"
+          SCRIPT_TO_RUN='make install/local-core' make execute/all
+      - name: Test all integrations with current core
+        working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }}
+        if: ${{ matrix.folder == '.' }}
+        run: |
+          echo "Testing all integrations with local core"
+          SCRIPT_TO_RUN="PYTEST_ADDOPTS=--junitxml=${PWD}/junit/test-results-core-change/\`pwd | xargs basename\`.xml make test" make execute/all
       - name: Publish Test Report
         uses: mikepenz/action-junit-report@v4
         if: ${{ always() }}