feat: support scheduling gates in integration tests (#2149)

Signed-off-by: odubajDT <[email protected]> Signed-off-by: odubajDT <[email protected]> Co-authored-by: RealAnna <[email protected]>
keptn · Sep 27, 2023 · 263d17e · 263d17e
1 parent 480e1d2
commit 263d17e
Show file tree

Hide file tree

Showing 19 changed files with 700 additions and 14 deletions.
diff --git a/.github/actions/deploy-klt-on-cluster/action.yml b/.github/actions/deploy-klt-on-cluster/action.yml
@@ -8,7 +8,7 @@ inputs:
     default: "v0.18.0"
   k8s-version:
     required: false
-    description: "Kubernetes that should be used"
+    description: "Kubernetes version that should be used"
     # renovate: datasource=github-releases depName=kubernetes/kubernetes
     default: "v1.27.3"
   runtime_tag:
@@ -20,8 +20,12 @@ inputs:
     default: "test-cluster"
   helm-install:
     required: false
-    description: "Install KLT via helm instead of manifest if true"
+    description: "Install Keptn via helm instead of manifest if true"
     default: "true"
+  scheduling_gates:
+    required: false
+    description: "Use scheduling gates instead of scheduler"
+    default: "false"
 runs:
   using: "composite"
   steps:
@@ -57,10 +61,10 @@ runs:
         done
 
     - name: Install lifecycle-toolkit with manifests
-      if: ${{ inputs.helm-install == 'false' }}
+      if: inputs.helm-install == 'false' && inputs.scheduling_gates == 'false'
       shell: bash
       run: |
-        echo "Installing KLT using manifests"
+        echo "Installing Keptn using manifests"
         sed -i 's/imagePullPolicy: Always/imagePullPolicy: Never/g' ~/download/artifacts/lifecycle-operator-manifest-test/release.yaml
         sed -i 's/ghcr.io\/keptn\/deno-runtime:.*/localhost:5000\/keptn\/deno-runtime:${{ inputs.runtime_tag }}/g' \
           ~/download/artifacts/lifecycle-operator-manifest-test/release.yaml
@@ -83,14 +87,15 @@ runs:
         kubectl rollout status deployment lifecycle-operator -n keptn-lifecycle-toolkit-system -w
 
     - name: Install lifecycle-toolkit with helm
-      if: ${{ inputs.helm-install == 'true'  }}
+      if: inputs.helm-install == 'true'
       env:
         RELEASE_REGISTRY: "localhost:5000/keptn"
       shell: bash
       run: |
-        echo "Installing KLT using helm"
+        echo "Installing Keptn using helm"
         helm version
         helm install -n keptn-lifecycle-toolkit-system --create-namespace toolkit ./helm/chart  \
+          --set schedulingGatesEnabled=${{ inputs.scheduling_gates }} \
           --set scheduler.scheduler.imagePullPolicy=Never \
           --set scheduler.scheduler.image.tag=${{ inputs.runtime_tag }} \
           --set scheduler.scheduler.image.repository="localhost:5000/keptn/scheduler" \

diff --git a/.github/scripts/.helm-tests/default/result.yaml b/.github/scripts/.helm-tests/default/result.yaml
@@ -6753,6 +6753,7 @@ rules:
   - get
   - list
   - watch
+  - update
 - apiGroups:
   - ""
   resources:
@@ -7915,6 +7916,8 @@ spec:
           value: "0"
         - name: OPTIONS_CONTROLLER_LOG_LEVEL
           value: "0"
+        - name: SCHEDULING_GATES_ENABLED
+          value: "false"
         - name: KUBERNETES_CLUSTER_DOMAIN
           value: cluster.local
         image: ghcr.io/keptn/lifecycle-operator:v0.8.2

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -58,3 +58,44 @@ jobs:
         with:
           name: logs-integration-tests-${{ inputs.helm-install }}
           path: .github/scripts/logs
+
+  run-integration-test-scheduling-gates:
+    name: Run Integration Tests Scheduling Gates
+    if: ${{ inputs.helm-install == true }}
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Setup cluster
+        uses: ./.github/actions/deploy-klt-on-cluster
+        with:
+          runtime_tag: ${{ inputs.runtime_tag }}
+          scheduling_gates: "true"
+
+      - name: Install and expose Prometheus
+        uses: ./.github/actions/deploy-prometheus-on-cluster
+
+      - name: Download KUTTL
+        env:
+          BASE_URL: "https://github.com/kudobuilder/kuttl/releases"
+        run: |
+          curl -fL "${BASE_URL}/download/${{ env.KUTTL_VERSION }}/kubectl-kuttl_${KUTTL_VERSION#v}_linux_x86_64" -o kubectl-kuttl
+          chmod +x kubectl-kuttl
+          mv kubectl-kuttl /usr/local/bin
+
+      - name: Run Integration Tests
+        working-directory: .
+        run: make integration-test-scheduling-gates && make integration-test
+
+      - name: Create reports
+        if: always()
+        working-directory: ./.github/scripts
+        run: ./create-reports-full.sh
+
+      - name: Upload cluster logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: logs-integration-tests-scheduling-gates
+          path: .github/scripts/logs
diff --git a/Makefile b/Makefile
@@ -36,6 +36,14 @@ integration-test-local: install-prometheus
 	kubectl kuttl test --start-kind=false ./test/testanalysis/ --config=kuttl-test-local.yaml
 	kubectl kuttl test --start-kind=false ./test/testcertificate/ --config=kuttl-test-local.yaml
 
+.PHONY: integration-test-scheduling-gates #these tests should run on a real cluster!
+integration-test-scheduling-gates:	# to run a single test by name use --test eg. --test=expose-keptn-metric
+	kubectl kuttl test --start-kind=false ./test/scheduling-gates/ --config=kuttl-test.yaml
+
+.PHONY: integration-test-scheduling-gates-local #these tests should run on a real cluster!
+integration-test-scheduling-gates-local: install-prometheus
+	kubectl kuttl test --start-kind=false ./test/scheduling-gates/ --config=kuttl-test-local.yaml
+
 .PHONY: load-test
 load-test:
 	kubectl apply -f ./test/load/assets/templates/namespace.yaml

diff --git a/README.md b/README.md
@@ -78,6 +78,33 @@ helm repo update
 helm upgrade --install keptn klt/klt -n keptn-lifecycle-toolkit-system --create-namespace --wait
 ```
 
+### Installation without scheduler
+
+Keptn installed on Kubernetes cluster running Kubernetes >= 1.26
+does not need scheduler for a proper functionality.
+With introduction
+of [Pod scheduling gates](https://kubernetes.io/blog/2022/12/26/pod-scheduling-readiness-alpha/)
+Keptn can use this feature to substitute the functionality of Keptn scheduler.
+
+As this functionality is still disabled by default, it can be enabled by setting up
+the correct helm values.
+This will lead to a Keptn installation without a scheduler and
+with scheduling gates feature enabled.
+
+Use the following command sequence
+to install Keptn with scheduling gates enabled:
+
+```shell
+helm repo add klt https://charts.lifecycle.keptn.sh
+helm repo update
+helm upgrade --install keptn klt/klt -n keptn-lifecycle-toolkit-system --set schedulingGatesEnabled=true --create-namespace --wait
+```
+
+> **Note**
+Please be aware that scheduling gates functionality in Kubernetes is enabled by default
+in Kubernetes >= 1.27.
+To use it with Kubernetes 1.26, you need to enable it on your cluster.
+
 ## More information
 
 For more info about Keptn, please see our

diff --git a/helm/chart/README.md b/helm/chart/README.md
@@ -158,7 +158,8 @@ as well as the concept of application health checks
 
 ### Global
 
-| Name                      | Description                            | Value           |
-| ------------------------- | -------------------------------------- | --------------- |
-| `kubernetesClusterDomain` | overrides domain.local                 | `cluster.local` |
-| `imagePullSecrets`        | global value for image registry secret | `[]`            |
+| Name                      | Description                                                                                                                                     | Value           |
+| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | --------------- |
+| `kubernetesClusterDomain` | overrides domain.local                                                                                                                          | `cluster.local` |
+| `imagePullSecrets`        | global value for image registry secret                                                                                                          | `[]`            |
+| `schedulingGatesEnabled`  | enables the scheduling gates in lifecycle-operator. This feature is available in alpha version from K8s 1.27 or 1.26 enabling the alpha version | `false`         |
diff --git a/helm/chart/doc.yaml b/helm/chart/doc.yaml
@@ -233,6 +233,7 @@
 
 
 ## @section Global
-## Current available parameters: kubernetesClusterDomain, imagePullSecrets
+## Current available parameters: kubernetesClusterDomain, imagePullSecrets, schedulingGatesEnabled
 ## @param  kubernetesClusterDomain overrides domain.local
 ## @param  imagePullSecrets global value for image registry secret
+## @param  schedulingGatesEnabled enables the scheduling gates in lifecycle-operator. This feature is available in alpha version from K8s 1.27 or 1.26 enabling the alpha version