Merge pull request #12 from navikt/integration-tests

Integration tests

.github/workflows/tests.yaml

@@ -2,8 +2,16 @@ name: Tests
 
 on: push
 
+env:
+  GIT_SYNC_IMAGE: europe-north1-docker.pkg.dev/knada-gcp/knada-north/git-sync
+  DATAVERK_AIRFLOW_IMAGE: europe-north1-docker.pkg.dev/knada-gcp/knada-north/dataverk-airflow
+
+permissions:
+  contents: read
+  id-token: write
+
 jobs:
-  tests:
+  unit-tests:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -13,3 +21,79 @@ jobs:
       - run: pip3 install poetry
       - run: poetry install --with test
       - run: poetry run pytest
+
+  integration-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+      - id: auth
+        name: Authenticate with Google Cloud
+        uses: google-github-actions/auth@v1
+        with:
+          token_format: access_token
+          workload_identity_provider: projects/193123067890/locations/global/workloadIdentityPools/ci-knada-images/providers/ci-knada-images
+          service_account: [email protected]
+      - name: Install python dependencies
+        run: |
+          pip3 install apache-airflow
+          pip3 install .
+      - name: Start kind cluster
+        uses: helm/[email protected]
+      - name: Verify cluster up
+        run: |
+          kubectl cluster-info
+          kubectl get nodes
+      - name: Setup cluster for tests
+        run: |
+          kubectl create sa airflow
+          kubectl create secret generic github-app-secret --from-literal=test=test
+          kubectl create cm ca-bundle-pem --from-literal=test=test
+      - name: Setup Airflow
+        env:
+          AIRFLOW_CONN_SQLITE_DEFAULT: sqlite://?mode=ro
+        run: |
+          airflow db reset -y
+          airflow variables set quarto_token ${{ secrets.QUARTO_TOKEN }}
+
+          # envs
+          echo "AIRFLOW__CORE__DAGS_FOLDER=$(pwd)/tests-integration" >> $GITHUB_ENV
+
+          gitsync_tag=$(gcloud artifacts docker images list "$GIT_SYNC_IMAGE" --include-tags --sort-by=~Update_Time --limit=1 --format=json | jq -rc '.[0].tags')
+          echo "CLONE_REPO_IMAGE=ghcr.io/navikt/knada-git-sync/git-sync:$gitsync_tag" >> $GITHUB_ENV
+
+          dataverk_airflow_tag=$(gcloud artifacts docker images list "$DATAVERK_AIRFLOW_IMAGE" --include-tags --sort-by=~Update_Time --limit=1 --format=json | jq -rc '.[0].tags')
+          echo "KNADA_AIRFLOW_OPERATOR_IMAGE=ghcr.io/navikt/knada-images/dataverk-airflow:$dataverk_airflow_tag" >> $GITHUB_ENV
+      - name: Run tests
+        env:
+          AIRFLOW_CONN_SQLITE_DEFAULT: sqlite://?mode=ro
+          KNADA_TEAM_SECRET: secret
+          NAMESPACE: default
+          K8S_IMAGE_PULL_SECRETS: ghcr-creds
+          MARKEDSPLASSEN_HOST: data.ekstern.dev.nav.no
+          INTEGRATION_TEST: "true"
+        run: |
+          airflow dags reserialize
+          airflow dags test DataverkAirflow
+      - name: Verify successful dag tests
+        run: |
+          airflow dags list-runs -d DataverkAirflow
+
+          errorCount=0
+          for res in $(airflow dags list-runs -d DataverkAirflow -ojson | jq -rc '.[]')
+          do
+            status=$(echo $res | jq -r '.state')
+            if [ "$status" != "success" ]
+            then
+              errorCount=$((errorCount+1))
+              dag_id=$(echo $res | jq -r '.dag_id')
+              echo "DAG test error: DAG $dag_id got status $status"
+            fi
+          done
+
+          if [ $errorCount -gt 0 ]
+          then
+            exit 1
+          fi

dataverk_airflow/kubernetes_operator.py

@@ -84,11 +84,13 @@ def kubernetes_operator(
     env_vars = {
         "NLS_LANG": "NORWEGIAN_NORWAY.AL32UTF8",
         "TZ": os.getenv("TZ", "Europe/Oslo"),
-        "REQUESTS_CA_BUNDLE": CA_BUNDLE_PATH,
         "KNADA_TEAM_SECRET": os.environ["KNADA_TEAM_SECRET"],
         **extra_envs
     }
 
+    if not os.getenv("INTEGRATION_TEST"):
+        env_vars["REQUESTS_CA_BUNDLE"] = CA_BUNDLE_PATH
+
     namespace = os.environ["NAMESPACE"]
 
     if slack_channel:

dataverk_airflow/quarto_operator.py

@@ -58,7 +58,7 @@ def quarto_operator(
     working_dir = None
     try:
         working_dir = Path(quarto['path']).parent
-        host = "datamarkedsplassen.intern.nav.no" if quarto['env'] == "prod" else "datamarkedsplassen.intern.dev.nav.no"
+        host = "datamarkedsplassen.intern.nav.no" if quarto['env'] == "prod" else os.getenv("MARKEDSPLASSEN_HOST", "datamarkedsplassen.intern.dev.nav.no")
         url = f"https://{host}/quarto/update/{quarto['id']}"
         cmds = [
             f"quarto render {Path(quarto['path']).name} --to html --execute --output index.html -M self-contained:True", 

tests-integration/dataverk-airflow.py

@@ -0,0 +1,46 @@
+from airflow import DAG
+from datetime import datetime
+from kubernetes.client import models as k8s
+from airflow.models import Variable
+from dataverk_airflow import python_operator, notebook_operator, quarto_operator
+
+
+with DAG('DataverkAirflow', start_date=datetime(2023, 2, 15), schedule=None) as dag:
+    py_op = python_operator(
+        dag = dag,
+        name = "python-op",
+        repo = "navikt/dataverk-airflow",
+        script_path = "dags/notebooks/script.py",
+        requirements_path="dags/notebooks/requirements.txt",
+        retries=0,
+        startup_timeout_seconds=60,
+    )
+
+    nb_op = notebook_operator(
+        dag = dag,
+        name = "nb-op",
+        repo = "navikt/dataverk-airflow",
+        nb_path = "dags/notebooks/mynb.ipynb",
+        requirements_path="dags/notebooks/requirements.txt",
+        retries=0,
+        startup_timeout_seconds=60,
+    )
+
+    quarto_op = quarto_operator(
+        dag=dag,
+        name="quarto-op",
+        repo="navikt/dataverk-airflow",
+        quarto={
+            "path": "dags/notebooks/quarto.ipynb",
+            "env": "dev",
+            "id": "bf48d8a4-05ca-47a5-a360-bc24171baf62",
+            "token": Variable.get("quarto_token"),
+        },
+        requirements_path="dags/notebooks/requirements.txt",
+        retries=0,
+        startup_timeout_seconds=60,
+    )
+
+    py_op
+    nb_op
+    quarto_op

tests-integration/notebooks/myfile.json

@@ -0,0 +1,5 @@
+{
+    "key1": "value1",
+    "key2": "value2",
+    "key3": "value3"
+}

tests-integration/notebooks/mynb.ipynb

@@ -0,0 +1,48 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0ef98384-c2c5-454f-bda9-b61cace54950",
+   "metadata": {
+    "tags": []
+   },
+   "outputs": [],
+   "source": [
+    "print(\"hello\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "aab509ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"myfile.json\") as f:\n",
+    "    data = f.read()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

tests-integration/notebooks/requirements.txt

@@ -0,0 +1,5 @@
+jupyterlab
+pandas
+plotly
+numpy
+matplotlib

tests-integration/notebooks/script.py

@@ -0,0 +1,10 @@
+import requests
+import json
+import time
+
+print("python script test")
+
+with open("myfile.json") as f:
+    data = f.read()
+
+print("fil:", data)