Clickup integrations

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,9 @@
 # Description
 
 What -
+
 Why -
+
 How -
 
 ## Type of change
@@ -15,6 +17,40 @@ Please leave one option from the following and delete the rest:
 - [ ] Non-breaking change (fix of existing functionality that will not change current behavior)
 - [ ] Documentation (added/updated documentation)
 
+<h4> All tests should be run against the port production environment(using a testing org). </h4>
+
+### Core testing checklist
+
+- [ ] Integration able to create all default resources from scratch
+- [ ] Resync finishes successfully
+- [ ] Resync able to create entities
+- [ ] Resync able to update entities
+- [ ] Resync able to detect and delete entities
+- [ ] Scheduled resync able to abort existing resync and start a new one
+- [ ] Tested with at least 2 integrations from scratch
+- [ ] Tested with Kafka and Polling event listeners
+- [ ] Tested deletion of entities that don't pass the selector
+
+
+### Integration testing checklist
+
+- [ ] Integration able to create all default resources from scratch
+- [ ] Resync able to create entities
+- [ ] Resync able to update entities
+- [ ] Resync able to detect and delete entities
+- [ ] Resync finishes successfully
+- [ ] If new resource kind is added or updated in the integration, add example raw data, mapping and expected result to the `examples` folder in the integration directory.
+- [ ] If resource kind is updated, run the integration with the example data and check if the expected result is achieved
+- [ ] If new resource kind is added or updated, validate that live-events for that resource are working as expected
+- [ ] Docs PR link [here](#)
+
+### Preflight checklist
+
+- [ ] Handled rate limiting
+- [ ] Handled pagination
+- [ ] Implemented the code in async
+- [ ] Support Multi account
+
 ## Screenshots
 
 Include screenshots from your environment showing how the resources of the integration will look.

.github/workflows/apply-release.yml

@@ -41,7 +41,7 @@ jobs:
           git config --local user.email "[email protected]"
           git config --local user.name "GitHub Action"
 
-          ./scripts/bump-all.sh ^${{ steps.version.outputs.version }}
+          ./scripts/bump-all.sh ${{ steps.version.outputs.version }}
 
 
       - name: Open pull request

.github/workflows/ci.yml

@@ -47,17 +47,26 @@ jobs:
       contents: read
     needs: [prepare-matrix]
     strategy:
+      max-parallel: 10
       matrix:
         integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}}
+        platform:
+          - linux/amd64
+          - linux/arm64
     steps:
       - name: Check out code
         uses: actions/checkout@v4
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
         with:
-          platforms: linux/amd64,linux/arm64
+          platforms: ${{ matrix.platform }}
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          platforms: ${{ matrix.platform }}
+
       - name: Prepare Docker images tags
         id: prepare_tags
         run: |
@@ -73,6 +82,7 @@ jobs:
 
           dockerfile_path=integrations/_infra/Dockerfile
           if test -e $folder/../Dockerfile; then
+            echo "Choosing a custom Dockerfile for ${{ matrix.integration }}"
             dockerfile_path=$folder/../Dockerfile
           fi
           echo "dockerfile_path=$dockerfile_path" >> $GITHUB_OUTPUT
@@ -91,14 +101,31 @@ jobs:
             echo "is_dev_version=false" >> $GITHUB_OUTPUT
           fi
 
+      - name: Get used docker base image
+        id: get-docker-image
+        run: |
+          echo "base_image=$(cat ${{ steps.prepare_tags.outputs.dockerfile_path }} | head -n 1 | awk -F '=' '{print $2}' )" >> $GITHUB_OUTPUT
+
+      - name: Cache Docker images
+        uses: ScribeMD/[email protected]
+        with:
+          key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }}
+
       - name: Build
         uses: docker/build-push-action@v6
         with:
           context: .
           file: ${{ steps.prepare_tags.outputs.dockerfile_path }}
-          platforms: linux/amd64,linux/arm64
+          platforms: ${{ matrix.platform }}
           push: false
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
           tags: ${{ steps.prepare_tags.outputs.tags }}
           build-args: |
             BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }}
             INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }}
+
+      - name: Verify Built Image
+        run: |
+          docker run --platform ${{ matrix.platform }} --rm --entrypoint bash ${{ steps.prepare_tags.outputs.tags }} -c 'ocean version'

.github/workflows/core-test.yml

@@ -0,0 +1,88 @@
+name: 🌊 Ocean Core Tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  detect-changes:
+    uses: ./.github/workflows/detect-changes-matrix.yml
+  test:
+    name: 🌊 Ocean Core Tests
+    needs: detect-changes
+    runs-on: ubuntu-latest
+    if: ${{ needs.detect-changes.outputs.core == 'true' }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Install poetry
+        run: pipx install poetry
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'poetry'
+
+      - name: Install dependencies
+        run: |
+          make install
+
+      - name: Unit Test Core
+        env:
+          PYTEST_ADDOPTS: --junitxml=junit/unit-test-results-ocean/core.xml
+        run: |
+          make test
+
+      - name: Build core for smoke test
+        run: |
+          make build
+
+      - name: Run fake integration for core test
+        env:
+          PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }}
+          PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }}
+          PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }}
+          SMOKE_TEST_SUFFIX: ${{ github.run_id }}
+        run: |
+          ./scripts/run-smoke-test.sh
+
+      - name: Smoke Test Core
+        env:
+          PYTEST_ADDOPTS: --junitxml=junit/smoke-test-results-ocean/core.xml
+          PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }}
+          PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }}
+          PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }}
+          SMOKE_TEST_SUFFIX: ${{ github.run_id }}
+        run: |
+          make test/smoke
+
+      - name: Cleanup Smoke Test
+        env:
+          PYTEST_ADDOPTS: --junitxml=junit/smoke-test-results-ocean/core.xml
+          PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }}
+          PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }}
+          PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }}
+          SMOKE_TEST_SUFFIX: ${{ github.run_id }}
+        run: |
+          make test/smoke
+
+      - name: Install current core for all integrations
+        run: |
+          echo "Installing local core for all integrations"
+          SCRIPT_TO_RUN='make install/local-core' make execute/all
+
+      - name: Test all integrations with current core
+        run: |
+          echo "Testing all integrations with local core"
+          SCRIPT_TO_RUN="PYTEST_ADDOPTS=--junitxml=${PWD}/junit/test-results-core-change/\`pwd | xargs basename\`.xml make test" make execute/all
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v4
+        if: ${{ always() }}
+        with:
+          report_paths: '**/junit/**-test-results-**/*.xml'
+          include_passed: true
+          require_tests: true
+          fail_on_failure: true

.github/workflows/create-new-sonarcloud-project.yml

@@ -20,7 +20,7 @@ jobs:
           fetch-depth: 0
       - name: Get all changed integrations
         id: changed-integrations
-        uses: tj-actions/changed-files@v44
+        uses: tj-actions/changed-files@v45
         with:
           json: true
           dir_names: true
@@ -42,6 +42,7 @@ jobs:
     runs-on: ubuntu-latest
     name: Create new project for integrations
     strategy:
+      max-parallel: 5
       matrix: ${{ fromJson(needs.pre-run.outputs.changed_integrations) }}
     steps:
       - name: Create integration variable

.github/workflows/detect-changes-matrix.yml

@@ -0,0 +1,57 @@
+name: Detect Changes
+on:
+  workflow_call:
+    outputs:
+      matrix:
+        value: ${{ jobs.detect-changes.outputs.matrix }}
+        description: "Matrix of changed integrations / Ocean Core per git commit changes"
+      integrations:
+        description: "Matrix of changed integrations per git commit changes"
+        value: ${{ jobs.detect-changes.outputs.integrations }}
+      core:
+        value: ${{ jobs.detect-changes.outputs.core }}
+        description: "Determine if any core changes per git commit changes"
+
+jobs:
+  detect-changes:
+    name: Detect changes
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-all-matrix.outputs.matrix }}
+      integrations: ${{ steps.set-all-matrix.outputs.integrations }}
+      core: ${{ steps.set-all-matrix.outputs.core }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Get list of changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v45
+        with:
+          dir_names: true
+          json: true
+          dir_names_max_depth: 2
+          escape_json: false
+          files_yaml: |
+            core:
+              - '!integrations/**'
+              - '!scripts/*'
+              - '!scripts/*'
+              - '!./*.md'
+            integrations:
+              - 'integrations/**'
+              - '!integrations/**/*.md'
+              - '!integrations/_infra/*'
+
+      - name: Set integrations and all matrix
+        id: set-all-matrix
+        run: |
+          INTEGRATIONS=$(node -e 'integrations=${{ steps.changed-files.outputs.integrations_all_changed_files }};console.log(JSON.stringify(integrations.map(integration => integration.split("/")[1])))')
+          HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }}
+          echo "Core changes : ${HAS_CORE}"
+          MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))")
+          echo "Integration changes : ${INTEGRATIONS}"
+          echo "All changes : ${MATRIX}"
+          echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT
+          echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT
+          echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT