Add DiracX integration tests to CI

.github/workflows/main.yml

@@ -35,6 +35,49 @@ jobs:
       - name: Upload coverage report
         uses: codecov/[email protected]
 
+  pytest-integration:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: environment.yml
+          init-shell: bash
+          post-cleanup: 'all'
+      - name: Set up environment
+        run: |
+          pip install pytest-github-actions-annotate-failures
+          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
+          pip install .
+      - name: Start demo
+        run: |
+          git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
+          ../diracx-charts/run_demo.sh --exit-when-done $PWD
+      - name: Debugging information
+        run: |
+          DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
+          export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
+          export PATH=${DIRACX_DEMO_DIR}:$PATH
+          kubectl get pods
+          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
+            echo "${pod_name}"
+            kubectl describe pod/"${pod_name}" || true
+            for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do
+              echo $pod_name $container_name
+              kubectl logs "${pod_name}" -c "${container_name}" || true
+            done
+          done
+          if [ ! -f "${DIRACX_DEMO_DIR}/.success" ]; then
+            cat "${DIRACX_DEMO_DIR}/.failed"
+            exit 1
+          fi
+      - name: Run pytest
+        run: |
+          pytest . --demo-dir ../diracx-charts/ --cov-report=xml:coverage.xml --junitxml=report.xml
+      - name: Upload coverage report
+        uses: codecov/[email protected]
+
   mypy:
     runs-on: ubuntu-latest
     steps:

codecov.yml

@@ -10,4 +10,8 @@ coverage:
         target: 100%
         informational: true
 
+codecov:
+  notify:
+    after_n_builds: 2
+
 comment: false

tests/conftest.py

@@ -222,18 +222,29 @@ def admin_user_client(test_client, test_auth_settings):
 
 
 @pytest.fixture(scope="session")
-def demo_kubectl_env(request):
+def demo_dir(request):
     demo_dir = request.config.getoption("--demo-dir")
     if demo_dir is None:
         pytest.skip("Requires a running instance of the DiracX demo")
-    kube_conf = demo_dir / ".demo" / "kube.conf"
+    demo_dir = (demo_dir / ".demo").resolve()
+    yield demo_dir
+
+
+@pytest.fixture(scope="session")
+def demo_kubectl(demo_dir):
+    kubectl = demo_dir / "kubectl"
+    if not kubectl.is_file():
+        raise RuntimeError("Could not find kubectl, is the demo running?")
+    yield str(kubectl)
+
+
+@pytest.fixture(scope="session")
+def demo_kubectl_env(demo_dir, demo_kubectl):
+    """TODO: docstring"""
+    kube_conf = demo_dir / "kube.conf"
     if not kube_conf.exists():
         raise RuntimeError(f"Could not find {kube_conf}, is the demo running?")
-    env = {
-        **os.environ,
-        "KUBECONFIG": str(kube_conf),
-        "PATH": os.environ["PATH"] + ":" + str(demo_dir / ".demo"),
-    }
-    pods_result = subprocess.check_output(["kubectl", "get", "pods"], env=env)
+    env = {**os.environ, "KUBECONFIG": str(kube_conf)}
+    pods_result = subprocess.check_output([demo_kubectl, "get", "pods"], env=env)
     assert pods_result
     yield env

tests/db/opensearch/conftest.py

@@ -2,7 +2,7 @@
 
 import secrets
 import socket
-import subprocess
+from subprocess import PIPE, Popen, check_output
 
 import pytest
 
@@ -43,19 +43,33 @@ def index_name(self, doc_id: int) -> str:
 
 
 @pytest.fixture(scope="session")
-def opensearch_conn_kwargs(demo_kubectl_env):
+def opensearch_conn_kwargs(demo_kubectl, demo_kubectl_env):
     """Fixture which forwards a port from the diracx-demo and returns the connection kwargs."""
     require_port_availability(OPENSEARCH_PORT)
-    command = [
-        "kubectl",
+
+    # Ensure the pod is running
+    cmd = [
+        demo_kubectl,
+        "get",
+        "pod/opensearch-cluster-master-0",
+        "-o",
+        "jsonpath={.status.phase}",
+    ]
+    pod_status = check_output(cmd, text=True, env=demo_kubectl_env)
+    if pod_status != "Running":
+        raise RuntimeError(f"OpenSearch pod is not running: {pod_status=}")
+
+    # Forward the actual port and wait until it has been forwarded before yielding
+    cmd = [
+        demo_kubectl,
         "port-forward",
         "service/opensearch-cluster-master",
         f"{OPENSEARCH_PORT}:9200",
     ]
-    with subprocess.Popen(
-        command, stdout=subprocess.PIPE, universal_newlines=True, env=demo_kubectl_env
-    ) as proc:
+    output_lines = []
+    with Popen(cmd, stdout=PIPE, stderr=PIPE, text=True, env=demo_kubectl_env) as proc:
         for line in proc.stdout:
+            output_lines.append(line)
             if line.startswith("Forwarding from"):
                 yield {
                     "hosts": f"admin:admin@localhost:{OPENSEARCH_PORT}",
@@ -64,6 +78,10 @@ def opensearch_conn_kwargs(demo_kubectl_env):
                 }
                 proc.kill()
                 break
+        else:
+            raise RuntimeError(
+                f"Could not start port forwarding with {cmd=}\n{output_lines=}"
+            )
     proc.wait()
 
 

tests/db/opensearch/test_connection.py

@@ -78,3 +78,17 @@ async def test_connection_error_bad_password(opensearch_conn_kwargs):
         }
     )
     await _ensure_db_unavailable(db)
+
+
+async def test_sanity_checks(opensearch_conn_kwargs):
+    """Check that the sanity checks are working as expected."""
+    db = DummyOSDB(opensearch_conn_kwargs)
+    # Check that the client is not available before entering the context manager
+    with pytest.raises(RuntimeError):
+        db.client.ping()
+
+    # It shouldn't be possible to enter the context manager twice
+    async with db.client_context():
+        assert db.client.ping()
+        with pytest.raises(AssertionError):
+            await db.__aenter__()