From 9bc26afcf848f169767117750f79778a2327b591 Mon Sep 17 00:00:00 2001 From: erikzaadi Date: Tue, 10 Sep 2024 14:05:09 +0300 Subject: [PATCH] Run all integration tests using local core Also cleanup and DRY our github actions a wee bit. --- .github/workflows/detect-changes-matrix.yml | 42 +++++++++++++++++ .github/workflows/lint.yml | 38 +++------------- .github/workflows/test.yml | 50 ++++++++------------- 3 files changed, 66 insertions(+), 64 deletions(-) create mode 100644 .github/workflows/detect-changes-matrix.yml diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml new file mode 100644 index 0000000000..91380f3d89 --- /dev/null +++ b/.github/workflows/detect-changes-matrix.yml @@ -0,0 +1,42 @@ +name: Detect Changes +on: + workflow_call: + outputs: + matrix: + value: ${{ jobs.detect-changes.outputs.matrix }} + description: "Matrix of changed integrations per git commit changes" + +jobs: + detect-changes: + name: Detect changes + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Get list of changed files + id: changed-files + uses: tj-actions/changed-files@v44.5.7 + - name: Set matrix + id: set-matrix + run: | + folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md" + changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" \ + | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' \ + | egrep -v '_infra' \ + | cut -d'/' -f2 | sort | uniq) + if [ -z "$changed_folders" ]; then + changed_folders="" + fi + + folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md" + other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l) + if [ "$other_changes" -ne 0 ]; then + # Add the root directory to the matrix if there are changes outside the integrations folder + changed_folders=$(echo -e "$changed_folders\n.") + fi + + matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "matrix=$matrix" >> $GITHUB_OUTPUT diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 4faa6de8a0..5de7346245 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,36 +6,7 @@ on: jobs: detect-changes: - name: Detect changes - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - - name: Get list of changed files - id: changed-files - uses: tj-actions/changed-files@v44.5.7 - - name: Set matrix - id: set-matrix - run: | - folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md" - changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' | cut -d'/' -f2 | sort | uniq) - if [ -z "$changed_folders" ]; then - changed_folders="" - fi - - folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md" - other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l) - if [ "$other_changes" -ne 0 ]; then - # Add the root directory to the matrix if there are changes outside the integrations folder - changed_folders=$(echo -e "$changed_folders\n.") - fi - - matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "matrix=$matrix" >> $GITHUB_OUTPUT - + uses: ./.github/workflows/detect-changes-matrix.yml lint: name: ${{ matrix.folder == '.' && '🌊 Ocean Core' || format('🚢 {0}', matrix.folder) }} needs: detect-changes @@ -44,12 +15,15 @@ jobs: matrix: folder: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Install poetry + run: pipx install poetry - name: Set up Python 3.11 uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Checkout Repo - uses: actions/checkout@v4 + cache: 'poetry' - name: Install dependencies working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }} run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 04fe15bafc..402f914128 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,36 +6,7 @@ on: jobs: detect-changes: - name: Detect changes - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - - name: Get list of changed files - id: changed-files - uses: tj-actions/changed-files@v44.5.7 - - name: Set matrix - id: set-matrix - run: | - folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md" - changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' | cut -d'/' -f2 | sort | uniq) - if [ -z "$changed_folders" ]; then - changed_folders="" - fi - - folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md" - other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l) - if [ "$other_changes" -ne 0 ]; then - # Add the root directory to the matrix if there are changes outside the integrations folder - changed_folders=$(echo -e "$changed_folders\n.") - fi - - matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "matrix=$matrix" >> $GITHUB_OUTPUT - + uses: ./.github/workflows/detect-changes-matrix.yml test: name: ${{ matrix.folder == '.' && '🌊 Ocean Core' || format('🚢 {0}', matrix.folder) }} needs: detect-changes @@ -44,12 +15,15 @@ jobs: matrix: folder: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: + - name: Checkout Repo + uses: actions/checkout@v4 + - name: Install poetry + run: pipx install poetry - name: Set up Python 3.11 uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Checkout Repo - uses: actions/checkout@v4 + cache: 'poetry' - name: Install dependencies working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }} run: | @@ -60,6 +34,18 @@ jobs: PYTEST_ADDOPTS: --junitxml=junit/test-results-${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || 'ocean/core' }}.xml run: | make test + - name: Install current core for all integrations + working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }} + if: ${{ matrix.folder == '.' }} + run: | + echo "Installing local core for all integrations" + SCRIPT_TO_RUN='make install/local-core' make execute/all + - name: Test all integrations with current core + working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }} + if: ${{ matrix.folder == '.' }} + run: | + echo "Testing all integrations with local core" + SCRIPT_TO_RUN="PYTEST_ADDOPTS=--junitxml=${PWD}/junit/test-results-core-change/\`pwd | xargs basename\`.xml make test" make execute/all - name: Publish Test Report uses: mikepenz/action-junit-report@v4 if: ${{ always() }}