extensions.yml

name: Extensions full test

on:
  push:
    branches:
      - main
      - gubbins
  pull_request:
    branches:
      - main


defaults:
  run:
    shell: bash -el {0}

jobs:
  unittest:
    name: Unit test - ${{ matrix.package }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # In principle, the dependencies could be limited to the extension packages.
        # However, we want to use the local packages, and not those published on pypi
        include:
          - package: "./extensions/gubbins/gubbins-core"
            dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./diracx-core"
          - package: "./extensions/gubbins/gubbins-db"
            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core "
          - package: "./extensions/gubbins/gubbins-routers"
            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-db ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-db ./diracx-core ./diracx-routers"
          - package: "./extensions/gubbins/gubbins-client"
            dependencies: "./extensions/gubbins/gubbins-testing ./diracx-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-client ./diracx-core "
          - package: "./extensions/gubbins/gubbins-cli"
            dependencies: "./extensions/gubbins/gubbins-testing ./extensions/gubbins/gubbins-client ./extensions/gubbins/gubbins-core ./diracx-testing ./diracx-cli ./diracx-client ./diracx-core ./diracx-api"
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - uses: mamba-org/setup-micromamba@v2
        with:
          # TODO: Use a conda environment file used for the diracx/base container image
          environment-name: test-env
          create-args: >-
            python=3.11
            m2crypto
            python-gfal2
            mypy
            pip
          init-shell: bash
          post-cleanup: 'all'
      - name: Set up environment
        run: |
          pip install pytest-github-actions-annotate-failures
          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
          pip install ${{ matrix.dependencies }} ${{ matrix.package }}[types]
      - name: Run mypy
        run: |
          mypy ${{ matrix.package }}/src
      - name: Run pytest
        run: |
          cd ${{ matrix.package }}
          pip install .[testing]
          export DIRACX_EXTENSIONS=gubbins,diracx
          pytest --cov-report=xml:coverage.xml --junitxml=report.xml
      - name: Upload coverage report
        uses: codecov/codecov-action@v4.6.0


  build-wheels:
    name: Build wheels
    runs-on: "ubuntu-latest"
    if: github.event_name != 'push' || github.repository == 'DIRACGrid/diracx'
    defaults:
      run:
        # We need extglob for REFERENCE_BRANCH substitution
        shell: bash -l -O extglob {0}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Installing dependencies
        run: |
          python -m pip install \
              build \
              python-dateutil \
              pytz \
              readme_renderer[md] \
              requests \
              setuptools_scm
      - name: Build distributions
        run: |
          for pkg_dir in $PWD/diracx-*; do
            echo "Building $pkg_dir"
            python -m build --outdir $PWD/dist $pkg_dir
          done
          # Also build the diracx metapackage
          python -m build --outdir $PWD/dist .
          # And build the gubbins package
          for pkg_dir in $PWD/extensions/gubbins/gubbins-*; do
            # Skip the testing package
            if [[ "${pkg_dir}" =~ .*testing.* ]];
            then
              echo "Do not build ${pkg_dir}";
              continue;
            fi
            echo "Building $pkg_dir"
            python -m build --outdir $PWD/dist $pkg_dir
          done
      - name: 'Upload Artifact'
        uses: actions/upload-artifact@v4
        with:
          name: gubbins-whl
          path: dist/*.whl
          retention-days: 5

  # Build to docker image with the code in it
  build-image:
    needs: build-wheels
    timeout-minutes:  30
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Download gubbins wheels
        uses: actions/download-artifact@v4
        with:
            name: gubbins-whl
      - name: "Find wheels"
        id: find_wheel
        run: |
          # We need to copy them there to be able to access them in the RUN --mount
          cp diracx*.whl gubbins*.whl extensions/containers/services/
          for wheel_fn in *.whl; do
            pkg_name=$(basename "${wheel_fn}" | cut -d '-' -f 1)
            echo "${pkg_name}-wheel-name=$(ls "${pkg_name}"-*.whl)" >> $GITHUB_OUTPUT
          done
      - name: Build and export service
        uses: docker/build-push-action@v6
        with:
          context: extensions/containers/services
          tags: gubbins/services:dev
          outputs: type=docker,dest=/tmp/gubbins_services_image.tar
          build-args: |
            EXTRA_PACKAGES_TO_INSTALL=git+https://github.com/DIRACGrid/DIRAC.git@integration
            EXTENSION_CUSTOM_SOURCES_TO_INSTALL=/bindmount/gubbins_db*.whl,/bindmount/gubbins_routers*.whl,/bindmount/gubbins_client*.whl
      - name: Build and export client
        uses: docker/build-push-action@v6
        with:
          context: extensions/containers/client
          tags: gubbins/client:dev
          outputs: type=docker,dest=/tmp/gubbins_client_image.tar
      - name: Upload artifact
        uses: actions/upload-artifact@v4
        with:
          name: gubbins-services-img
          path: /tmp/gubbins_services_image.tar


  pytest-integration:
    needs: build-image
    runs-on: ubuntu-latest
    steps:
      - name: Download gubbins-image
        uses: actions/download-artifact@v4
        with:
          name: gubbins-services-img
          path: /tmp
      - name: Load image
        run: |
          docker load --input /tmp/gubbins_services_image.tar
          docker image ls -a
      - name: Checkout code
        uses: actions/checkout@v4
      - uses: mamba-org/setup-micromamba@v2
        with:
          environment-file: environment.yml
          init-shell: bash
          post-cleanup: 'all'
      - name: Set up environment
        run: |
          pip install pytest-github-actions-annotate-failures
          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-cli/[testing] ./extensions/gubbins/gubbins-core/[testing]
      - name: Start demo
        run: |
          git clone https://github.com/DIRACGrid/diracx-charts.git ../diracx-charts
          # We have to copy the code to another directory
          # and make it a git repository by itself because otherwise the
          # root in the pyproject to do not make sense once mounted
          # in the containers.
          cp -r ./extensions/gubbins /tmp/
          sed -i 's@../..@.@g' /tmp/gubbins/pyproject.toml
          sed -i 's@../../@@g' /tmp/gubbins/gubbins-*/pyproject.toml
          git init /tmp/gubbins/
          ../diracx-charts/run_demo.sh --enable-open-telemetry --enable-coverage --exit-when-done --set-value developer.autoReload=false --ci-values ../diracx-charts/demo/ci_values.yaml --ci-values ./extensions/gubbins_values.yaml --load-docker-image "gubbins/services:dev" $PWD /tmp/gubbins/
      - name: Debugging information
        run: |
          DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
          export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
          export PATH=${DIRACX_DEMO_DIR}:$PATH
          kubectl get pods
          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
            echo "${pod_name}"
            kubectl describe pod/"${pod_name}" || true
            for container_name in $(kubectl get pods $pod_name -o jsonpath='{.spec.initContainers[*].name} {.spec.containers[*].name}'); do
              echo $pod_name $container_name
              kubectl logs "${pod_name}" -c "${container_name}" || true
            done
          done
          if [ ! -f "${DIRACX_DEMO_DIR}/.success" ]; then
            cat "${DIRACX_DEMO_DIR}/.failed"
            exit 1
          fi
      - name: Run pytest
        run: |
          cd extensions/gubbins
          export DIRACX_EXTENSIONS=gubbins,diracx
          pytest --demo-dir=../../../diracx-charts/ --cov-report=xml:coverage-pytest.xml --junitxml=report.xml
      - name: Collect demo coverage
        run: |
          DIRACX_DEMO_DIR=$PWD/../diracx-charts/.demo
          export KUBECONFIG=${DIRACX_DEMO_DIR}/kube.conf
          export PATH=${DIRACX_DEMO_DIR}:$PATH
          # Shutdown the pods so we collect coverage data
          for pod_name in $(kubectl get pods -o json | jq -r '.items[] | .metadata.name' | grep -vE '(dex|minio|mysql|rabbitmq|opensearch)'); do
            kubectl delete pod/"${pod_name}"
          done
          set -x
          # Combine the coverage data from the demo and make an XML report
          coverage_data=$(mktemp)
          sudo chown -R $(id -u) "${DIRACX_DEMO_DIR}"/coverage-reports/
          coverage combine --keep --data-file "${coverage_data}" "${DIRACX_DEMO_DIR}"/coverage-reports/*

          # coverage can't handle having multiple src directories, so we need to make a fake one with symlinks
          fake_module=$(mktemp -d)

          mkdir -p "${fake_module}/src/diracx"
          for fn in "${PWD}"/*/src/diracx/*; do
            ln -sf "${fn}" "${fake_module}/src/diracx/$(basename "${fn}")"
          done


          mkdir -p "${fake_module}/src/gubbins"
          for fn in "${PWD}"/extensions/gubbins/*/src/gubbins/*; do
            ln -sf "${fn}" "${fake_module}/src/gubbins/$(basename "${fn}")"
          done

          sed -i "s@source =@source =\n    ${fake_module}/src@g" .coveragerc

          cat .coveragerc

          coverage xml -o coverage-demo.xml --data-file "${coverage_data}"
      - name: Upload coverage report
        uses: codecov/codecov-action@v4.6.0
        with:
          files: ./coverage-pytest.xml,./coverage-demo.xml

  client-generation:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4
      - uses: mamba-org/setup-micromamba@v2
        with:
          environment-file: environment.yml
          init-shell: bash
          post-cleanup: 'all'
      - name: Set up environment
        run: |
          micromamba install -c conda-forge nodejs pre-commit
          pip install git+https://github.com/DIRACGrid/DIRAC.git@integration
          pip install ./diracx-core/[testing] ./diracx-api/[testing] ./diracx-cli/[testing] ./diracx-client/[testing] ./diracx-routers/[testing] ./diracx-db/[testing] ./diracx-testing/[testing]  ./extensions/gubbins/gubbins-testing[testing] ./extensions/gubbins/gubbins-db[testing] ./extensions/gubbins/gubbins-routers/[testing] ./extensions/gubbins/gubbins-testing/[testing] -e ./extensions/gubbins/gubbins-client/[testing] ./extensions/gubbins/gubbins-core/[testing]
          npm install -g autorest
      - name: Run autorest
        run: |
          autorest --python --help
          $HOME/.autorest/\@autorest_python\@*/node_modules/\@autorest/python/venv/bin/python -m pip install --upgrade setuptools
          export DIRACX_EXTENSIONS=gubbins,diracx
          pytest --no-cov --regenerate-client extensions/gubbins/gubbins-client/tests/test_regenerate.py