From b8f5e605bb326cfcdd34b3803c5af90559435fd1 Mon Sep 17 00:00:00 2001 From: Stefano Fiorucci Date: Wed, 13 Mar 2024 16:32:14 +0100 Subject: [PATCH] Notify nightly failures (#577) * first refactorings * separate unit tests in pgvector * small change to weaviate * fix format * wip * retry * try failur * restrict * retry * try using composite action * retry composite action * fix typo * update all workflows * retry * fix test * fix --- .github/actions/send_failure/action.yml | 28 ++++++++++++++++++++++ .github/workflows/astra.yml | 9 ++++++- .github/workflows/chroma.yml | 7 ++++++ .github/workflows/cohere.yml | 7 ++++++ .github/workflows/deepeval.yml | 7 ++++++ .github/workflows/elasticsearch.yml | 7 ++++++ .github/workflows/fastembed.yml | 7 ++++++ .github/workflows/google_ai.yml | 7 ++++++ .github/workflows/google_vertex.yml | 7 ++++++ .github/workflows/gradient.yml | 9 ++++++- .github/workflows/instructor_embedders.yml | 7 ++++++ .github/workflows/jina.yml | 7 ++++++ .github/workflows/llama_cpp.yml | 7 ++++++ .github/workflows/mistral.yml | 7 ++++++ .github/workflows/mongodb_atlas.yml | 7 ++++++ .github/workflows/nvidia.yml | 7 ++++++ .github/workflows/ollama.yml | 7 ++++++ .github/workflows/opensearch.yml | 7 ++++++ .github/workflows/optimum.yml | 7 ++++++ .github/workflows/pgvector.yml | 7 ++++++ .github/workflows/pinecone.yml | 7 ++++++ .github/workflows/qdrant.yml | 7 ++++++ .github/workflows/ragas.yml | 7 ++++++ .github/workflows/unstructured.yml | 7 ++++++ .github/workflows/uptrain.yml | 7 ++++++ .github/workflows/weaviate.yml | 7 ++++++ 26 files changed, 205 insertions(+), 2 deletions(-) create mode 100644 .github/actions/send_failure/action.yml diff --git a/.github/actions/send_failure/action.yml b/.github/actions/send_failure/action.yml new file mode 100644 index 000000000..8481b0b80 --- /dev/null +++ b/.github/actions/send_failure/action.yml @@ -0,0 +1,28 @@ +name: "Send failure event to Datadog" +inputs: + api-key: + description: "Datadog API key" + required: true + title: + description: "Custom title for the event" + required: true +runs: + using: "composite" + steps: + - uses: masci/datadog@v1 + with: + api-key: ${{ inputs.api-key }} + api-url: https://api.datadoghq.eu + events: | + - title: "${{ inputs.title }}" + text: "Job ${{ github.job }} in branch ${{ github.ref_name }}" + alert_type: "error" + source_type_name: "Github" + host: ${{ github.repository_owner }} + tags: + - "project:${{ github.repository }}" + - "job:${{ github.job }}" + - "run_id:${{ github.run_id }}" + - "workflow:${{ github.workflow }}" + - "branch:${{ github.ref_name }}" + - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \ No newline at end of file diff --git a/.github/workflows/astra.yml b/.github/workflows/astra.yml index d859626ff..e90edc2e9 100644 --- a/.github/workflows/astra.yml +++ b/.github/workflows/astra.yml @@ -61,4 +61,11 @@ jobs: env: ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_API_ENDPOINT }} ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_TOKEN }} - run: hatch run cov \ No newline at end of file + run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} \ No newline at end of file diff --git a/.github/workflows/chroma.yml b/.github/workflows/chroma.yml index fec309f6b..e6712d807 100644 --- a/.github/workflows/chroma.yml +++ b/.github/workflows/chroma.yml @@ -58,3 +58,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} \ No newline at end of file diff --git a/.github/workflows/cohere.yml b/.github/workflows/cohere.yml index fb6b00680..6f23760a0 100644 --- a/.github/workflows/cohere.yml +++ b/.github/workflows/cohere.yml @@ -55,3 +55,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} \ No newline at end of file diff --git a/.github/workflows/deepeval.yml b/.github/workflows/deepeval.yml index e2468fa8c..a9efc2f3a 100644 --- a/.github/workflows/deepeval.yml +++ b/.github/workflows/deepeval.yml @@ -58,3 +58,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/elasticsearch.yml b/.github/workflows/elasticsearch.yml index 688e5c48f..21efcbc34 100644 --- a/.github/workflows/elasticsearch.yml +++ b/.github/workflows/elasticsearch.yml @@ -56,3 +56,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/fastembed.yml b/.github/workflows/fastembed.yml index fe736029a..a2b076c1a 100644 --- a/.github/workflows/fastembed.yml +++ b/.github/workflows/fastembed.yml @@ -43,3 +43,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: failure() #github.event_name == 'schedule' && + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/google_ai.yml b/.github/workflows/google_ai.yml index 6093df4a4..9efeb8590 100644 --- a/.github/workflows/google_ai.yml +++ b/.github/workflows/google_ai.yml @@ -59,3 +59,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/google_vertex.yml b/.github/workflows/google_vertex.yml index 6f6c6d0d9..03890ed4a 100644 --- a/.github/workflows/google_vertex.yml +++ b/.github/workflows/google_vertex.yml @@ -58,3 +58,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/gradient.yml b/.github/workflows/gradient.yml index 61a04be7b..8fbaf6f18 100644 --- a/.github/workflows/gradient.yml +++ b/.github/workflows/gradient.yml @@ -57,4 +57,11 @@ jobs: run: hatch run docs - name: Run tests - run: hatch run cov \ No newline at end of file + run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} \ No newline at end of file diff --git a/.github/workflows/instructor_embedders.yml b/.github/workflows/instructor_embedders.yml index 09d04e9d3..5282c8e18 100644 --- a/.github/workflows/instructor_embedders.yml +++ b/.github/workflows/instructor_embedders.yml @@ -36,3 +36,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/jina.yml b/.github/workflows/jina.yml index 1f8e83a7d..1ab0e2a2b 100644 --- a/.github/workflows/jina.yml +++ b/.github/workflows/jina.yml @@ -58,3 +58,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/llama_cpp.yml b/.github/workflows/llama_cpp.yml index 89c7e5426..712e91fa2 100644 --- a/.github/workflows/llama_cpp.yml +++ b/.github/workflows/llama_cpp.yml @@ -58,3 +58,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/mistral.yml b/.github/workflows/mistral.yml index a02b5ad43..029bb974a 100644 --- a/.github/workflows/mistral.yml +++ b/.github/workflows/mistral.yml @@ -59,3 +59,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/mongodb_atlas.yml b/.github/workflows/mongodb_atlas.yml index af19776cd..bf48a75c2 100644 --- a/.github/workflows/mongodb_atlas.yml +++ b/.github/workflows/mongodb_atlas.yml @@ -56,3 +56,10 @@ jobs: - name: Run tests working-directory: integrations/mongodb_atlas run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/nvidia.yml b/.github/workflows/nvidia.yml index 6e7562c17..8b6ec030a 100644 --- a/.github/workflows/nvidia.yml +++ b/.github/workflows/nvidia.yml @@ -55,3 +55,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/ollama.yml b/.github/workflows/ollama.yml index 28b522890..c977ba116 100644 --- a/.github/workflows/ollama.yml +++ b/.github/workflows/ollama.yml @@ -76,3 +76,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/opensearch.yml b/.github/workflows/opensearch.yml index 72a01d090..da177b83c 100644 --- a/.github/workflows/opensearch.yml +++ b/.github/workflows/opensearch.yml @@ -57,3 +57,10 @@ jobs: - name: Run tests working-directory: integrations/opensearch run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/optimum.yml b/.github/workflows/optimum.yml index 3b0d137da..077413920 100644 --- a/.github/workflows/optimum.yml +++ b/.github/workflows/optimum.yml @@ -58,3 +58,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/pgvector.yml b/.github/workflows/pgvector.yml index badb2565b..647f520e1 100644 --- a/.github/workflows/pgvector.yml +++ b/.github/workflows/pgvector.yml @@ -62,3 +62,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/pinecone.yml b/.github/workflows/pinecone.yml index a82fb74de..49d421813 100644 --- a/.github/workflows/pinecone.yml +++ b/.github/workflows/pinecone.yml @@ -57,3 +57,10 @@ jobs: - name: Run tests working-directory: integrations/pinecone run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/qdrant.yml b/.github/workflows/qdrant.yml index 9f031031f..3c72b0f02 100644 --- a/.github/workflows/qdrant.yml +++ b/.github/workflows/qdrant.yml @@ -58,3 +58,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/ragas.yml b/.github/workflows/ragas.yml index e2ce46764..d3def92ca 100644 --- a/.github/workflows/ragas.yml +++ b/.github/workflows/ragas.yml @@ -58,3 +58,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/unstructured.yml b/.github/workflows/unstructured.yml index 83cad6dfc..b2778431c 100644 --- a/.github/workflows/unstructured.yml +++ b/.github/workflows/unstructured.yml @@ -70,3 +70,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/uptrain.yml b/.github/workflows/uptrain.yml index bacfa27fb..64453b0fd 100644 --- a/.github/workflows/uptrain.yml +++ b/.github/workflows/uptrain.yml @@ -54,3 +54,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }} diff --git a/.github/workflows/weaviate.yml b/.github/workflows/weaviate.yml index 051415336..69447b96b 100644 --- a/.github/workflows/weaviate.yml +++ b/.github/workflows/weaviate.yml @@ -55,3 +55,10 @@ jobs: - name: Run tests run: hatch run cov + + - name: Send event to Datadog for nightly failures + if: github.event_name == 'schedule' && failure() + uses: ./.github/actions/send_failure + with: + title: "core-integrations nightly failure: ${{ github.workflow }}" + api-key: ${{ secrets.CORE_DATADOG_API_KEY }}