Copy logs before Gardener deprovision (#1536)

.github/actions/integration-test-gardener/action.yaml

@@ -7,9 +7,15 @@ inputs:
   gardener_secret:
     description: "Kubeconfig secret for Gardener"
     required: true
-  script:
-    description: "Entry point script for test"
+  gardener_provider:
+    description: "Gardener provider to use"
     required: true
+  gardener_project_name:
+    required: true
+    description: "Name of the gardener project"
+  test_custom_domain:
+    required: true
+    description: "The custom domain used by tests"
   test_make_target:
     description: 'Make target for integration tests to run'
     default: 'test-custom-domain test-integration-gateway'
@@ -28,6 +34,9 @@ inputs:
 runs:
   using: "composite"
   steps:
+    - name: Generate cluster name
+      shell: bash
+      run: echo CLUSTER_NAME=ag-$(echo $RANDOM | md5sum | head -c 7) >> $GITHUB_ENV
     - uses: actions/setup-go@v5
       with:
         go-version-file: "go.mod"
@@ -41,14 +50,21 @@ runs:
       shell: bash
       env:
         GARDENER_TOKEN: ${{ inputs.gardener_secret }}
-      run: |
-        ./hack/ci/gardener_kubeconfig.sh
+      run: "${{ github.workspace }}/hack/ci/gardener_kubeconfig.sh"
+    - name: Provision cluster
+      shell: bash
+      env:
+        GARDENER_KUBECONFIG: "${{ github.workspace }}/gardener_kubeconfig.yaml"
+        CLUSTER_KUBECONFIG: "${{ github.workspace }}/${{ env.CLUSTER_NAME }}_kubeconfig.yaml"
+        GARDENER_PROVIDER: ${{ inputs.gardener_provider }}
+        GARDENER_PROJECT_NAME: ${{ inputs.gardener_project_name }}
+      run: "${{ github.workspace }}/hack/ci/provision-gardener.sh"
     - name: Prepare DNS SA secret
       shell: bash
       env:
         DNS_SA_BASE64: ${{ inputs.dns_secret_json }}
       run: |
-        echo "$DNS_SA_BASE64" | base64 --decode >> service-account.json
+        echo "$DNS_SA_BASE64" | base64 --decode >> "${{ github.workspace }}/service-account.json"
     - name: Run integration tests
       shell: bash
       env:
@@ -57,19 +73,15 @@ runs:
         CLIENT_SECRET: ${{ inputs.client_secret }}
         OIDC_CONFIG_URL: ${{ inputs.oidc_well_known_url }}
         TEST_SA_ACCESS_KEY_PATH: "${{ github.workspace }}/service-account.json"
-        GARDENER_KUBECONFIG: "gardener_kubeconfig.yaml"
-        PERSISTENT_CLUSTER_KUBECONFIG: "cluster_kubeconfig.yaml"
-      run: |
-        EXPORT_RESULT=true \
-        ${{ inputs.script }} ${{ inputs.test_make_target }}
+        CLUSTER_KUBECONFIG: "${{ github.workspace }}/${{ env.CLUSTER_NAME }}_kubeconfig.yaml"
+        TEST_CUSTOM_DOMAIN: ${{ inputs.test_custom_domain }}
+      run: EXPORT_RESULT=true "${{ github.workspace }}/hack/ci/custom-domain-gardener.sh" ${{ inputs.test_make_target }}
     - name: Gather deployment logs
       shell: bash
-      env:
-        PERSISTENT_CLUSTER_KUBECONFIG: "cluster_kubeconfig.yaml"
-      if: failure()
+      if: always()
       run: |
         mkdir logs
-        export KUBECONFIG="${PERSISTENT_CLUSTER_KUBECONFIG}"
+        export KUBECONFIG="${{ github.workspace }}/${{ env.CLUSTER_NAME }}_kubeconfig.yaml"
         (kubectl logs -n kyma-system deployments/istio-controller-manager || true) > logs/istio-controller-manager.log
         (kubectl logs -n kyma-system deployments/api-gateway-controller-manager || true) > logs/api-gateway-controller-manager.log
         (kubectl logs -n istio-system deployments/istio-ingressgateway || true) > logs/istio-ingressgateway.log
@@ -80,7 +92,7 @@ runs:
         (kubectl get -n kyma-system deployment ory-oathkeeper -oyaml || true) > logs/ory-oathkeeper-dump.yaml
     - name: Uploads logs
       uses: actions/upload-artifact@v4
-      if: failure()
+      if: always()
       with:
         name: logs-${{ github.job }}-${{ inputs.test_make_target }}
         path: logs/
@@ -89,3 +101,10 @@ runs:
       with:
         name: html-reports-${{ github.job }}-${{ inputs.test_make_target }}
         path: tests/integration/reports/
+    - name: Deprovision cluster
+      if: always()
+      shell: bash
+      env:
+        GARDENER_KUBECONFIG: "${{ github.workspace }}/gardener_kubeconfig.yaml"
+        GARDENER_PROJECT_NAME: ${{ inputs.gardener_project_name }}
+      run: "${{ github.workspace }}/hack/ci/deprovision-gardener.sh"

.github/workflows/call-integration-release.yaml

@@ -107,12 +107,14 @@ jobs:
         with:
           manager_image: ${{ inputs.image }}
           gardener_secret: ${{ secrets.GARDENER_TOKEN }}
-          script: ./hack/ci/custom-domain-gardener-gcp.sh
+          gardener_project_name: ${{ vars.GARDENER_PROJECT_NAME }}
+          gardener_provider: gcp
           client_id: ${{ secrets.CLIENT_ID }}
           client_secret: ${{ secrets.CLIENT_SECRET }}
           oidc_well_known_url: "${{ secrets.OIDC_ISSUER_URL }}/.well-known/openid-configuration"
           dns_secret_json: ${{ secrets.DNS_SECRET_JSON }}
           test_make_target: ${{ matrix.test_make_target }}
+          test_custom_domain: ${{ vars.TEST_CUSTOM_DOMAIN }}
 
   custom-domain-aws:
     name: Custom domain AWS
@@ -132,9 +134,11 @@ jobs:
         with:
           manager_image: ${{ inputs.image }}
           gardener_secret: ${{ secrets.GARDENER_TOKEN }}
-          script: ./hack/ci/custom-domain-gardener-aws.sh
+          gardener_project_name: ${{ vars.GARDENER_PROJECT_NAME }}
+          gardener_provider: aws
           client_id: ${{ secrets.CLIENT_ID }}
           client_secret: ${{ secrets.CLIENT_SECRET }}
           oidc_well_known_url: "${{ secrets.OIDC_ISSUER_URL }}/.well-known/openid-configuration"
           dns_secret_json: ${{ secrets.DNS_SECRET_JSON }}
           test_make_target: ${{ matrix.test_make_target }}
+          test_custom_domain: ${{ vars.TEST_CUSTOM_DOMAIN }}

.github/workflows/post-main-integration.yaml

@@ -123,11 +123,13 @@ jobs:
         with:
           manager_image: "europe-docker.pkg.dev/kyma-project/prod/api-gateway/main/api-gateway-manager:${{ needs.get-sha.outputs.sha }}"
           gardener_secret: ${{ secrets.GARDENER_TOKEN }}
-          script: ./hack/ci/custom-domain-gardener-gcp.sh
+          gardener_project_name: ${{ vars.GARDENER_PROJECT_NAME }}
+          gardener_provider: gcp
           client_id: ${{ secrets.CLIENT_ID }}
           client_secret: ${{ secrets.CLIENT_SECRET }}
           oidc_well_known_url: "${{ secrets.OIDC_ISSUER_URL }}/.well-known/openid-configuration"
           dns_secret_json: ${{ secrets.DNS_SECRET_JSON }}
+          test_custom_domain: ${{ vars.TEST_CUSTOM_DOMAIN }}
 
   custom-domain-integration-aws:
     name: Custom domain integration AWS
@@ -146,11 +148,13 @@ jobs:
         with:
           manager_image: "europe-docker.pkg.dev/kyma-project/prod/api-gateway/main/api-gateway-manager:${{ needs.get-sha.outputs.sha }}"
           gardener_secret: ${{ secrets.GARDENER_TOKEN }}
-          script: ./hack/ci/custom-domain-gardener-aws.sh
+          gardener_project_name: ${{ vars.GARDENER_PROJECT_NAME }}
+          gardener_provider: aws
           client_id: ${{ secrets.CLIENT_ID }}
           client_secret: ${{ secrets.CLIENT_SECRET }}
           oidc_well_known_url: "${{ secrets.OIDC_ISSUER_URL }}/.well-known/openid-configuration"
           dns_secret_json: ${{ secrets.DNS_SECRET_JSON }}
+          test_custom_domain: ${{ vars.TEST_CUSTOM_DOMAIN }}
 
   slack_failed_notification:
     name: Slack Notification

hack/ci/custom-domain-gardener.sh

@@ -1,9 +1,13 @@
 #!/usr/bin/env bash
 
-#
-##Description: This scripts installs and tests api-gateway custom domain test as well as gateway test using the CLI on a real Gardener GCP cluster.
-## exit on error, and raise error when variable is not set when used
-## IMG env variable expected (for make deploy), which points to the image in the registry
+# Description: This script runs given integration tests on a real Gardener cluster
+# It installs istio and api gateway and then runs make test targets provided via commandline arguments to that script
+# It requires the following env variables:
+# - IMG - API gateway image to be deployed (by make deploy)
+# - CLUSTER_NAME - Gardener cluster name
+# - CLUSTER_KUBECONFIG - Gardener cluster kubeconfig path
+# - TEST_SA_ACCESS_KEY_PATH - path to the GCP Service account json file
+# - TEST_CUSTOM_DOMAIN - a domain used by tests (a subdomain is created during tests execution)
 
 set -eo pipefail
 
@@ -40,91 +44,72 @@ function check_required_files() {
 }
 
 requiredVars=(
-    GARDENER_KUBECONFIG
-    GARDENER_PROJECT_NAME
-    CLIENT_ID
-    CLIENT_SECRET
-    OIDC_CONFIG_URL
+    IMG
+    CLUSTER_NAME
+    CLUSTER_KUBECONFIG
     TEST_SA_ACCESS_KEY_PATH
+    TEST_CUSTOM_DOMAIN
 )
 
 requiredFiles=(
-    GARDENER_KUBECONFIG
     TEST_SA_ACCESS_KEY_PATH
 )
 
 check_required_vars "${requiredVars[@]}"
 check_required_files "${requiredFiles[@]}"
 
-function cleanup() {
-  kubectl annotate shoot "${CLUSTER_NAME}" confirmation.gardener.cloud/deletion=true \
-      --overwrite \
-      -n "garden-${GARDENER_PROJECT_NAME}" \
-      --kubeconfig "${GARDENER_KUBECONFIG}"
-
-  kubectl delete shoot "${CLUSTER_NAME}" \
-    --wait="false" \
-    --kubeconfig "${GARDENER_KUBECONFIG}" \
-    -n "garden-${GARDENER_PROJECT_NAME}"
-}
-
-# Cleanup on exit, be it successful or on fail
-trap cleanup EXIT INT
-
-# Add pwd to path to be able to use binaries downloaded in scripts
-export PATH="${PATH}:${PWD}"
+echo "Executing custom domain tests in cluster ${CLUSTER_NAME}, kubeconfig ${CLUSTER_KUBECONFIG}"
+export KUBECONFIG="${CLUSTER_KUBECONFIG}"
 
-CLUSTER_NAME=ag-$(echo $RANDOM | md5sum | head -c 7)
-export CLUSTER_NAME
+export CLUSTER_DOMAIN=$(kubectl get configmap -n kube-system shoot-info -o jsonpath="{.data.domain}")
+echo "Cluster domain: ${CLUSTER_DOMAIN}"
 
-TMP_FOLDER=$(mktemp -d)
+export GARDENER_PROVIDER=$(kubectl get configmap -n kube-system shoot-info -o jsonpath="{.data.provider}")
+echo "Gardener provider: ${GARDENER_PROVIDER}"
 
-if [ -z "${PERSISTENT_CLUSTER_KUBECONFIG}" ]; then
-    export CLUSTER_KUBECONFIG="${PERSISTENT_CLUSTER_KUBECONFIG}"
-else
-    export CLUSTER_KUBECONFIG="${TMP_FOLDER}/${CLUSTER_NAME}_kubeconfig.yaml"
-fi
+echo "Custom domain: ${TEST_CUSTOM_DOMAIN}"
 
-./hack/ci/provision-gardener.sh
+export TEST_DOMAIN="${CLUSTER_DOMAIN}"
+export KYMA_DOMAIN="${CLUSTER_DOMAIN}" # it is required by env_vars.sh
+export IS_GARDENER=true
 
-export KUBECONFIG="${CLUSTER_KUBECONFIG}"
+# Add pwd to path to be able to use binaries downloaded in scripts
+export PATH="${PATH}:${PWD}"
 
-echo "installing istio"
+echo "Installing istio"
 make install-istio
 
-echo "deploying api-gateway"
+echo "Deploying api-gateway, image: ${IMG}"
 make deploy
 
-# KYMA_DOMAIN is required by the tests
-export TEST_DOMAIN="${CLUSTER_NAME}.${GARDENER_PROJECT_NAME}.shoot.live.k8s-hana.ondemand.com"
-export KYMA_DOMAIN="${TEST_DOMAIN}"
-export TEST_CUSTOM_DOMAIN="goat.build.kyma-project.io"
-export IS_GARDENER=true
-
-echo "waiting for the ingress gateway external address"
+echo "Waiting for the ingress gateway external address"
 [ "$GARDENER_PROVIDER" == "aws" ] && address_field="{.status.loadBalancer.ingress[0].hostname}" || address_field="{.status.loadBalancer.ingress[0].ip}"
 kubectl wait --timeout=300s --namespace istio-system services/istio-ingressgateway --for=jsonpath="${address_field}"
 ingress_external_address=$(kubectl get services --namespace istio-system istio-ingressgateway --output jsonpath="${address_field}")
 ingress_external_status_port=$(kubectl get services --namespace istio-system istio-ingressgateway --output jsonpath='{.spec.ports[?(@.name=="status-port")].targetPort}')
 
-echo "determined ingress external address: ${ingress_external_address} and external status port: ${ingress_external_status_port}"
+echo "Determined ingress external address: ${ingress_external_address} and external status port: ${ingress_external_status_port}"
 
-echo "waiting until it is possible to connect to the ingress gateway"
+echo "Waiting until it is possible to connect to the ingress gateway"
 trial=1
 # check if it is possible to establish connection to the ingress gateway (the exact http status code doesn't matter)
 until curl --silent --output /dev/null "http://${ingress_external_address}:${ingress_external_status_port}"
 do
   if (( trial >= 60 ))
   then
-     echo "exceeded number of trials while waiting for the ingress gateway, giving up..."
+     echo "Exceeded number of trials while waiting for the ingress gateway, giving up..."
      exit 4
   fi
-  echo "ingress gateway does not respond, trying again..."
+  echo "Ingress gateway does not respond, trying again..."
   sleep 10
   trial=$((trial + 1))
 done
+echo "Ingress gateway responded"
 
+echo "Executing tests"
 for make_target in "$@"
 do
+    echo "Executing make target $make_target"
     make $make_target
 done
+echo "Tests finished"

hack/ci/deprovision-gardener.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Description: This script deletes the Gardener cluster
+# It requires the following env variables:
+# - CLUSTER_NAME - name of the cluster to be deleted
+# - GARDENER_KUBECONFIG - Gardener kubeconfig path
+# - GARDENER_PROJECT_NAME - name of the Gardener project
+
+set -eo pipefail
+script_dir="$(dirname "$(readlink -f "$0")")"
+
+function check_required_vars() {
+  local requiredVarMissing=false
+  for var in "$@"; do
+    if [ -z "${!var}" ]; then
+      >&2 echo "Environment variable ${var} is required but not set"
+      requiredVarMissing=true
+    fi
+  done
+  if [ "${requiredVarMissing}" = true ] ; then
+    exit 2
+  fi
+}
+
+function check_required_files() {
+  local requiredFileMissing=false
+  for file in "$@"; do
+    path=$(eval echo "\$$file")
+    if [ ! -f "${path}" ]; then
+        >&2 echo "File '${path}' required but not found"
+        requiredFileMissing=true
+    fi
+  done
+  if [ "${requiredFileMissing}" = true ] ; then
+    exit 2
+  fi
+}
+
+requiredVars=(
+    CLUSTER_NAME
+    GARDENER_PROJECT_NAME
+)
+
+requiredFiles=(
+    GARDENER_KUBECONFIG
+)
+
+check_required_vars "${requiredVars[@]}"
+check_required_files "${requiredFiles[@]}"
+
+echo "Deprovisioning cluster: ${CLUSTER_NAME}"
+
+kubectl annotate shoot "${CLUSTER_NAME}" confirmation.gardener.cloud/deletion=true \
+    --overwrite \
+    -n "garden-${GARDENER_PROJECT_NAME}" \
+    --kubeconfig "${GARDENER_KUBECONFIG}"
+
+kubectl delete shoot "${CLUSTER_NAME}" \
+  --wait="false" \
+  --kubeconfig "${GARDENER_KUBECONFIG}" \
+  -n "garden-${GARDENER_PROJECT_NAME}"