Notify nightly failures (#577)

* first refactorings

* separate unit tests in pgvector

* small change to weaviate

* fix format

* wip

* retry

* try failur

* restrict

* retry

* try using composite action

* retry composite action

* fix typo

* update all workflows

* retry

* fix test

* fix

.github/actions/send_failure/action.yml

@@ -0,0 +1,28 @@
+name: "Send failure event to Datadog"
+inputs:
+  api-key:
+    description: "Datadog API key"
+    required: true
+  title:
+    description: "Custom title for the event"
+    required: true    
+runs:
+  using: "composite"
+  steps:
+    - uses: masci/datadog@v1
+      with:
+        api-key: ${{ inputs.api-key }}
+        api-url: https://api.datadoghq.eu
+        events: |
+          - title: "${{ inputs.title }}"
+            text: "Job ${{ github.job }} in branch ${{ github.ref_name }}"
+            alert_type: "error"
+            source_type_name: "Github"
+            host: ${{ github.repository_owner }}
+            tags:
+              - "project:${{ github.repository }}"
+              - "job:${{ github.job }}"
+              - "run_id:${{ github.run_id }}"
+              - "workflow:${{ github.workflow }}"
+              - "branch:${{ github.ref_name }}"
+              - "url:https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"

.github/workflows/astra.yml

@@ -61,4 +61,11 @@ jobs:
       env:
         ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_API_ENDPOINT }}
         ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_TOKEN }}
-      run: hatch run cov
+      run: hatch run cov
+
+    - name: Send event to Datadog for nightly failures
+      if: github.event_name == 'schedule' && failure()
+      uses: ./.github/actions/send_failure
+      with:
+        title: "core-integrations nightly failure: ${{ github.workflow }}"
+        api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/chroma.yml

@@ -58,3 +58,10 @@ jobs:
 
     - name: Run tests
       run: hatch run cov
+
+    - name: Send event to Datadog for nightly failures
+      if: github.event_name == 'schedule' && failure()
+      uses: ./.github/actions/send_failure
+      with:
+        title: "core-integrations nightly failure: ${{ github.workflow }}"
+        api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/cohere.yml

@@ -55,3 +55,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/deepeval.yml

@@ -58,3 +58,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/elasticsearch.yml

@@ -56,3 +56,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/fastembed.yml

@@ -43,3 +43,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: failure() #github.event_name == 'schedule' && 
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}          

.github/workflows/google_ai.yml

@@ -59,3 +59,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/google_vertex.yml

@@ -58,3 +58,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/gradient.yml

@@ -57,4 +57,11 @@ jobs:
       run: hatch run docs
 
     - name: Run tests
-      run: hatch run cov
+      run: hatch run cov
+
+    - name: Send event to Datadog for nightly failures
+      if: github.event_name == 'schedule' && failure()
+      uses: ./.github/actions/send_failure
+      with:
+        title: "core-integrations nightly failure: ${{ github.workflow }}"
+        api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/instructor_embedders.yml

@@ -36,3 +36,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/jina.yml

@@ -58,3 +58,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/llama_cpp.yml

@@ -58,3 +58,10 @@ jobs:
 
     - name: Run tests
       run: hatch run cov
+
+    - name: Send event to Datadog for nightly failures
+      if: github.event_name == 'schedule' && failure()
+      uses: ./.github/actions/send_failure
+      with:
+        title: "core-integrations nightly failure: ${{ github.workflow }}"
+        api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/mistral.yml

@@ -59,3 +59,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/mongodb_atlas.yml

@@ -56,3 +56,10 @@ jobs:
       - name: Run tests
         working-directory: integrations/mongodb_atlas
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/nvidia.yml

@@ -55,3 +55,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/ollama.yml

@@ -76,3 +76,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/opensearch.yml

@@ -57,3 +57,10 @@ jobs:
       - name: Run tests
         working-directory: integrations/opensearch
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/optimum.yml

@@ -58,3 +58,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/pgvector.yml

@@ -62,3 +62,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/pinecone.yml

@@ -57,3 +57,10 @@ jobs:
       - name: Run tests
         working-directory: integrations/pinecone
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/qdrant.yml

@@ -58,3 +58,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/ragas.yml

@@ -58,3 +58,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/unstructured.yml

@@ -70,3 +70,10 @@ jobs:
 
       - name: Run tests 
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/uptrain.yml

@@ -54,3 +54,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}

.github/workflows/weaviate.yml

@@ -55,3 +55,10 @@ jobs:
 
       - name: Run tests
         run: hatch run cov
+
+      - name: Send event to Datadog for nightly failures
+        if: github.event_name == 'schedule' && failure()
+        uses: ./.github/actions/send_failure
+        with:
+          title: "core-integrations nightly failure: ${{ github.workflow }}"
+          api-key: ${{ secrets.CORE_DATADOG_API_KEY }}