From 263d17e5a40432e4c298f30e150378cdbe3d4664 Mon Sep 17 00:00:00 2001 From: odubajDT <93584209+odubajDT@users.noreply.github.com> Date: Wed, 27 Sep 2023 12:33:21 +0200 Subject: [PATCH] feat: support scheduling gates in integration tests (#2149) Signed-off-by: odubajDT Signed-off-by: odubajDT <93584209+odubajDT@users.noreply.github.com> Co-authored-by: RealAnna <89971034+RealAnna@users.noreply.github.com> --- .../actions/deploy-klt-on-cluster/action.yml | 17 +- .../scripts/.helm-tests/default/result.yaml | 3 + .github/workflows/integration-test.yml | 41 ++ Makefile | 8 + README.md | 27 + helm/chart/README.md | 9 +- helm/chart/doc.yaml | 3 +- helm/chart/rendered.yaml | 487 ++++++++++++++++++ helm/chart/templates/deployment.yaml | 7 + ...-apiserver-authentication-reader-rbac.yaml | 4 +- .../chart/templates/keptn-scheduler-rbac.yaml | 4 +- .../templates/lifecycle-operator-rbac.yaml | 1 + helm/chart/templates/scheduler-config.yaml | 4 +- helm/chart/values.yaml | 1 + .../config/manager/manager.yaml | 2 + .../simple-deployment/00-assert.yaml | 20 + .../simple-deployment/00-install.yaml | 40 ++ .../simple-deployment/00-teststep.yaml | 4 + .../simple-deployment/01-assert.yaml | 32 ++ 19 files changed, 700 insertions(+), 14 deletions(-) create mode 100644 helm/chart/rendered.yaml create mode 100644 test/scheduling-gates/simple-deployment/00-assert.yaml create mode 100644 test/scheduling-gates/simple-deployment/00-install.yaml create mode 100644 test/scheduling-gates/simple-deployment/00-teststep.yaml create mode 100644 test/scheduling-gates/simple-deployment/01-assert.yaml diff --git a/.github/actions/deploy-klt-on-cluster/action.yml b/.github/actions/deploy-klt-on-cluster/action.yml index fb2f59700dc..b559d755b38 100644 --- a/.github/actions/deploy-klt-on-cluster/action.yml +++ b/.github/actions/deploy-klt-on-cluster/action.yml @@ -8,7 +8,7 @@ inputs: default: "v0.18.0" k8s-version: required: false - description: "Kubernetes that should be used" + description: "Kubernetes version that should be used" # renovate: datasource=github-releases depName=kubernetes/kubernetes default: "v1.27.3" runtime_tag: @@ -20,8 +20,12 @@ inputs: default: "test-cluster" helm-install: required: false - description: "Install KLT via helm instead of manifest if true" + description: "Install Keptn via helm instead of manifest if true" default: "true" + scheduling_gates: + required: false + description: "Use scheduling gates instead of scheduler" + default: "false" runs: using: "composite" steps: @@ -57,10 +61,10 @@ runs: done - name: Install lifecycle-toolkit with manifests - if: ${{ inputs.helm-install == 'false' }} + if: inputs.helm-install == 'false' && inputs.scheduling_gates == 'false' shell: bash run: | - echo "Installing KLT using manifests" + echo "Installing Keptn using manifests" sed -i 's/imagePullPolicy: Always/imagePullPolicy: Never/g' ~/download/artifacts/lifecycle-operator-manifest-test/release.yaml sed -i 's/ghcr.io\/keptn\/deno-runtime:.*/localhost:5000\/keptn\/deno-runtime:${{ inputs.runtime_tag }}/g' \ ~/download/artifacts/lifecycle-operator-manifest-test/release.yaml @@ -83,14 +87,15 @@ runs: kubectl rollout status deployment lifecycle-operator -n keptn-lifecycle-toolkit-system -w - name: Install lifecycle-toolkit with helm - if: ${{ inputs.helm-install == 'true' }} + if: inputs.helm-install == 'true' env: RELEASE_REGISTRY: "localhost:5000/keptn" shell: bash run: | - echo "Installing KLT using helm" + echo "Installing Keptn using helm" helm version helm install -n keptn-lifecycle-toolkit-system --create-namespace toolkit ./helm/chart \ + --set schedulingGatesEnabled=${{ inputs.scheduling_gates }} \ --set scheduler.scheduler.imagePullPolicy=Never \ --set scheduler.scheduler.image.tag=${{ inputs.runtime_tag }} \ --set scheduler.scheduler.image.repository="localhost:5000/keptn/scheduler" \ diff --git a/.github/scripts/.helm-tests/default/result.yaml b/.github/scripts/.helm-tests/default/result.yaml index 5e230adcd6d..7e2da92fa53 100644 --- a/.github/scripts/.helm-tests/default/result.yaml +++ b/.github/scripts/.helm-tests/default/result.yaml @@ -6753,6 +6753,7 @@ rules: - get - list - watch + - update - apiGroups: - "" resources: @@ -7915,6 +7916,8 @@ spec: value: "0" - name: OPTIONS_CONTROLLER_LOG_LEVEL value: "0" + - name: SCHEDULING_GATES_ENABLED + value: "false" - name: KUBERNETES_CLUSTER_DOMAIN value: cluster.local image: ghcr.io/keptn/lifecycle-operator:v0.8.2 diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 82ee892f65f..39f25b03c32 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -58,3 +58,44 @@ jobs: with: name: logs-integration-tests-${{ inputs.helm-install }} path: .github/scripts/logs + + run-integration-test-scheduling-gates: + name: Run Integration Tests Scheduling Gates + if: ${{ inputs.helm-install == true }} + runs-on: ubuntu-22.04 + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Setup cluster + uses: ./.github/actions/deploy-klt-on-cluster + with: + runtime_tag: ${{ inputs.runtime_tag }} + scheduling_gates: "true" + + - name: Install and expose Prometheus + uses: ./.github/actions/deploy-prometheus-on-cluster + + - name: Download KUTTL + env: + BASE_URL: "https://github.com/kudobuilder/kuttl/releases" + run: | + curl -fL "${BASE_URL}/download/${{ env.KUTTL_VERSION }}/kubectl-kuttl_${KUTTL_VERSION#v}_linux_x86_64" -o kubectl-kuttl + chmod +x kubectl-kuttl + mv kubectl-kuttl /usr/local/bin + + - name: Run Integration Tests + working-directory: . + run: make integration-test-scheduling-gates && make integration-test + + - name: Create reports + if: always() + working-directory: ./.github/scripts + run: ./create-reports-full.sh + + - name: Upload cluster logs + if: always() + uses: actions/upload-artifact@v3 + with: + name: logs-integration-tests-scheduling-gates + path: .github/scripts/logs diff --git a/Makefile b/Makefile index df494bda3c1..6ecdc5c998f 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,14 @@ integration-test-local: install-prometheus kubectl kuttl test --start-kind=false ./test/testanalysis/ --config=kuttl-test-local.yaml kubectl kuttl test --start-kind=false ./test/testcertificate/ --config=kuttl-test-local.yaml +.PHONY: integration-test-scheduling-gates #these tests should run on a real cluster! +integration-test-scheduling-gates: # to run a single test by name use --test eg. --test=expose-keptn-metric + kubectl kuttl test --start-kind=false ./test/scheduling-gates/ --config=kuttl-test.yaml + +.PHONY: integration-test-scheduling-gates-local #these tests should run on a real cluster! +integration-test-scheduling-gates-local: install-prometheus + kubectl kuttl test --start-kind=false ./test/scheduling-gates/ --config=kuttl-test-local.yaml + .PHONY: load-test load-test: kubectl apply -f ./test/load/assets/templates/namespace.yaml diff --git a/README.md b/README.md index cf2e5cb7bd4..d6c7260568d 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,33 @@ helm repo update helm upgrade --install keptn klt/klt -n keptn-lifecycle-toolkit-system --create-namespace --wait ``` +### Installation without scheduler + +Keptn installed on Kubernetes cluster running Kubernetes >= 1.26 +does not need scheduler for a proper functionality. +With introduction +of [Pod scheduling gates](https://kubernetes.io/blog/2022/12/26/pod-scheduling-readiness-alpha/) +Keptn can use this feature to substitute the functionality of Keptn scheduler. + +As this functionality is still disabled by default, it can be enabled by setting up +the correct helm values. +This will lead to a Keptn installation without a scheduler and +with scheduling gates feature enabled. + +Use the following command sequence +to install Keptn with scheduling gates enabled: + +```shell +helm repo add klt https://charts.lifecycle.keptn.sh +helm repo update +helm upgrade --install keptn klt/klt -n keptn-lifecycle-toolkit-system --set schedulingGatesEnabled=true --create-namespace --wait +``` + +> **Note** +Please be aware that scheduling gates functionality in Kubernetes is enabled by default +in Kubernetes >= 1.27. +To use it with Kubernetes 1.26, you need to enable it on your cluster. + ## More information For more info about Keptn, please see our diff --git a/helm/chart/README.md b/helm/chart/README.md index 847af1cd968..8075cc39e49 100644 --- a/helm/chart/README.md +++ b/helm/chart/README.md @@ -158,7 +158,8 @@ as well as the concept of application health checks ### Global -| Name | Description | Value | -| ------------------------- | -------------------------------------- | --------------- | -| `kubernetesClusterDomain` | overrides domain.local | `cluster.local` | -| `imagePullSecrets` | global value for image registry secret | `[]` | +| Name | Description | Value | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | +| `kubernetesClusterDomain` | overrides domain.local | `cluster.local` | +| `imagePullSecrets` | global value for image registry secret | `[]` | +| `schedulingGatesEnabled` | enables the scheduling gates in lifecycle-operator. This feature is available in alpha version from K8s 1.27 or 1.26 enabling the alpha version | `false` | diff --git a/helm/chart/doc.yaml b/helm/chart/doc.yaml index 9c52fa3a3b5..fde4d363ee1 100644 --- a/helm/chart/doc.yaml +++ b/helm/chart/doc.yaml @@ -233,6 +233,7 @@ ## @section Global -## Current available parameters: kubernetesClusterDomain, imagePullSecrets +## Current available parameters: kubernetesClusterDomain, imagePullSecrets, schedulingGatesEnabled ## @param kubernetesClusterDomain overrides domain.local ## @param imagePullSecrets global value for image registry secret +## @param schedulingGatesEnabled enables the scheduling gates in lifecycle-operator. This feature is available in alpha version from K8s 1.27 or 1.26 enabling the alpha version diff --git a/helm/chart/rendered.yaml b/helm/chart/rendered.yaml new file mode 100644 index 00000000000..8175ac13c6a --- /dev/null +++ b/helm/chart/rendered.yaml @@ -0,0 +1,487 @@ +certificateOperator: + manager: + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + env: + labelSelectorKey: keptn.sh/inject-cert + labelSelectorValue: "true" + image: + repository: ghcr.io/keptn/certificate-operator + tag: v1.1.0 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 25m + memory: 64Mi + requests: + cpu: 5m + memory: 16Mi + nodeSelector: {} + replicas: 1 + tolerations: [] + topologySpreadConstraints: [] +imagePullSecrets: [] +kubernetesClusterDomain: cluster.local +schedulingGatesEnabled: false +lifecycleManagerConfig: + controllerManagerConfigYaml: + health: + healthProbeBindAddress: :8081 + leaderElection: + leaderElect: true + resourceName: 6b866dd9.keptn.sh + metrics: + bindAddress: 127.0.0.1:8080 + webhook: + port: 9443 +lifecycleOperator: + manager: + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + env: + functionRunnerImage: ghcr.io/keptn/deno-runtime:v1.0.1 + keptnAppControllerLogLevel: "0" + keptnAppCreationRequestControllerLogLevel: "0" + keptnAppVersionControllerLogLevel: "0" + keptnEvaluationControllerLogLevel: "0" + keptnTaskControllerLogLevel: "0" + keptnTaskDefinitionControllerLogLevel: "0" + keptnWorkloadControllerLogLevel: "0" + keptnWorkloadInstanceControllerLogLevel: "0" + optionsControllerLogLevel: "0" + otelCollectorUrl: otel-collector:4317 + pythonRunnerImage: ghcr.io/keptn/python-runtime:v1.0.0 + image: + repository: ghcr.io/keptn/lifecycle-operator + tag: v0.8.2 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + nodeSelector: {} + replicas: 1 + tolerations: [] + topologySpreadConstraints: [] +lifecycleOperatorMetricsService: + ports: + - name: metrics + port: 2222 + protocol: TCP + targetPort: metrics + type: ClusterIP +lifecycleWebhookService: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + type: ClusterIP +metricsManagerConfig: + controllerManagerConfigYaml: + health: + healthProbeBindAddress: :8081 + leaderElection: + leaderElect: true + resourceName: 3f8532ca.keptn.sh + metrics: + bindAddress: 127.0.0.1:8080 + webhook: + port: 9443 +metricsOperator: + manager: + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + env: + exposeKeptnMetrics: "true" + enableKeptnAnalysis: "false" + metricsControllerLogLevel: "0" + analysisControllerLogLevel: "0" + image: + repository: ghcr.io/keptn/metrics-operator + tag: v0.8.2 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + nodeSelector: {} + replicas: 1 + tolerations: [] + topologySpreadConstraints: [] +metricsOperatorService: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + - name: custom-metrics + port: 443 + targetPort: custom-metrics + - name: metrics + port: 9999 + protocol: TCP + targetPort: metrics + type: ClusterIP +metricsWebhookService: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + type: ClusterIP +scheduler: + nodeSelector: {} + replicas: 1 + scheduler: + containerSecurityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + env: + otelCollectorUrl: otel-collector:4317 + image: + repository: ghcr.io/keptn/scheduler + tag: v0.8.2 + imagePullPolicy: Always + livenessProbe: + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + initialDelaySeconds: 15 + readinessProbe: + httpGet: + path: /healthz + port: 10259 + scheme: HTTPS + resources: + limits: + cpu: 300m + memory: 100Mi + requests: + cpu: 100m + memory: 20Mi + tolerations: [] + topologySpreadConstraints: [] +schedulerConfig: + schedulerConfigYaml: + leaderElection: + leaderElect: false + profiles: + - plugins: + permit: + enabled: + - name: KLCPermit + schedulerName: keptn-scheduler +# yamllint disable rule:line-length +## @section Keptn Scheduler +## @extra scheduler.scheduler.containerSecurityContext Sets security context +## @skip scheduler.scheduler.containerSecurityContext.allowPrivilegeEscalation +## @skip scheduler.scheduler.containerSecurityContext.capabilities.drop +## @skip scheduler.scheduler.containerSecurityContext.privileged +## @skip scheduler.scheduler.containerSecurityContext.readOnlyRootFilesystem +## @skip scheduler.scheduler.containerSecurityContext.runAsNonRoot +## @skip scheduler.scheduler.containerSecurityContext.runAsUser +## @skip scheduler.scheduler.containerSecurityContext.seccompProfile.type + +## @param scheduler.scheduler.env.otelCollectorUrl sets url for open telemetry collector + +## @param scheduler.scheduler.image.repository set image repository for scheduler +## @param scheduler.scheduler.image.tag set image tag for scheduler +## @param scheduler.scheduler.imagePullPolicy set image pull policy for scheduler + +## @extra scheduler.scheduler.livenessProbe customizable liveness probe for the scheduler +## @skip scheduler.scheduler.livenessProbe.httpGet.path +## @skip scheduler.scheduler.livenessProbe.httpGet.port +## @skip scheduler.scheduler.livenessProbe.httpGet.scheme +## @skip scheduler.scheduler.livenessProbe.initialDelaySeconds + +## @extra scheduler.scheduler.readinessProbe customizable readiness probe for the scheduler +## @skip scheduler.scheduler.readinessProbe.httpGet.path +## @skip scheduler.scheduler.readinessProbe.httpGet.port +## @skip scheduler.scheduler.readinessProbe.httpGet.scheme + +## @extra scheduler.scheduler.resources sets cpu and memory resurces/limits for scheduler +## @skip scheduler.scheduler.resources.limits.cpu +## @skip scheduler.scheduler.resources.limits.memory +## @skip scheduler.scheduler.resources.requests.cpu +## @skip scheduler.scheduler.resources.requests.memory + +## @param schedulerConfig.schedulerConfigYaml.leaderElection.leaderElect enables leader election for multiple replicas of the scheduler +## @param schedulerConfig.schedulerConfigYaml.profiles[0].plugins.permit.enabled[0].name enables permit plugin +## @param schedulerConfig.schedulerConfigYaml.profiles[0].schedulerName changes scheduler name + +## @param scheduler.nodeSelector adds node selectors for scheduler +## @param scheduler.replicas modifies replicas +## @param scheduler.tolerations adds tolerations for scheduler +## @param scheduler.topologySpreadConstraints add topology constraints for scheduler + +## @section Keptn Certificate Operator common + +## @param certificateOperator.replicas customize number of replicas + +## @param certificateOperator.nodeSelector specify custom node selectors for cert manager +## @param certificateOperator.tolerations customize tolerations for cert manager +## @param certificateOperator.topologySpreadConstraints add topology constraints for cert manager + +## @param lifecycleManagerConfig.controllerManagerConfigYaml.health.healthProbeBindAddress setup on what address to start the default health handler +## @param lifecycleManagerConfig.controllerManagerConfigYaml.leaderElection.leaderElect enable leader election for multiple replicas of the lifecycle operator +## @param lifecycleManagerConfig.controllerManagerConfigYaml.leaderElection.resourceName define LeaderElectionID +## @param lifecycleManagerConfig.controllerManagerConfigYaml.metrics.bindAddress MetricsBindAddress is the TCP address that the controller should bind to for serving prometheus metrics. It can be set to "0" to disable the metrics serving. +## @param lifecycleManagerConfig.controllerManagerConfigYaml.webhook.port setup port for the lifecycle operator admission webhook + + +## @section Keptn Certificate Operator controller +## @extra certificateOperator.manager.containerSecurityContext Sets security context for the cert manager +## @skip certificateOperator.manager.containerSecurityContext.allowPrivilegeEscalation +## @skip certificateOperator.manager.containerSecurityContext.capabilities.drop +## @skip certificateOperator.manager.containerSecurityContext.readOnlyRootFilesystem +## @skip certificateOperator.manager.containerSecurityContext.runAsGroup +## @skip certificateOperator.manager.containerSecurityContext.runAsUser +## @skip certificateOperator.manager.containerSecurityContext.seccompProfile.type + +## @param certificateOperator.manager.image.repository specify repo for manager image +## @param certificateOperator.manager.image.tag select tag for manager container +## @param certificateOperator.manager.imagePullPolicy select image pull policy for manager container + +## @param certificateOperator.manager.env.labelSelectorKey specify the label selector to find resources to generate certificates for +## @param certificateOperator.manager.env.labelSelectorValue specify the value for the label selector + +## @extra certificateOperator.manager.livenessProbe custom RBAC proxy liveness probe +## @skip certificateOperator.manager.livenessProbe.httpGet.path +## @skip certificateOperator.manager.livenessProbe.httpGet.port +## @skip certificateOperator.manager.livenessProbe.initialDelaySeconds +## @skip certificateOperator.manager.livenessProbe.periodSeconds + +## @extra certificateOperator.manager.readinessProbe custom manager readiness probe +## @skip certificateOperator.manager.readinessProbe.httpGet.path +## @skip certificateOperator.manager.readinessProbe.httpGet.port +## @skip certificateOperator.manager.readinessProbe.initialDelaySeconds +## @skip certificateOperator.manager.readinessProbe.periodSeconds + +## @extra certificateOperator.manager.resources custom limits and requests for manager container +## @skip certificateOperator.manager.resources.limits.cpu +## @skip certificateOperator.manager.resources.limits.memory +## @skip certificateOperator.manager.resources.requests.cpu +## @skip certificateOperator.manager.resources.requests.memory + +## @section Keptn Lifecycle Operator common + +## @param lifecycleOperator.replicas customize number of installed lifecycle operator replicas + +## @extra lifecycleOperatorMetricsService Adjust settings here to change the k8s service for scraping Prometheus metrics +## @skip lifecycleOperatorMetricsService.ports[0].name +## @skip lifecycleOperatorMetricsService.ports[0].port +## @skip lifecycleOperatorMetricsService.ports[0].protocol +## @skip lifecycleOperatorMetricsService.ports[0].targetPort +## @skip lifecycleOperatorMetricsService.type + +## @extra lifecycleWebhookService Mutating Webhook Configurations for lifecycle Operator +## @param lifecycleWebhookService.ports[0].port +## @param lifecycleWebhookService.ports[0].protocol +## @param lifecycleWebhookService.ports[0].targetPort +## @param lifecycleWebhookService.type + +## @param lifecycleOperator.nodeSelector add custom nodes selector to lifecycle operator +## @param lifecycleOperator.tolerations add custom tolerations to lifecycle operator +## @param lifecycleOperator.topologySpreadConstraints add custom topology constraints to lifecycle operator + +## @section Keptn Lifecycle Operator controller +## @extra lifecycleOperator.manager.containerSecurityContext Sets security context privileges +## @param lifecycleOperator.manager.containerSecurityContext.allowPrivilegeEscalation +## @param lifecycleOperator.manager.containerSecurityContext.capabilities.drop +## @param lifecycleOperator.manager.containerSecurityContext.privileged +## @param lifecycleOperator.manager.containerSecurityContext.runAsGroup +## @param lifecycleOperator.manager.containerSecurityContext.runAsNonRoot +## @param lifecycleOperator.manager.containerSecurityContext.runAsUser +## @param lifecycleOperator.manager.containerSecurityContext.seccompProfile.type + +## @param lifecycleOperator.manager.env.keptnAppControllerLogLevel sets the log level of Keptn App Controller +## @param lifecycleOperator.manager.env.keptnAppCreationRequestControllerLogLevel sets the log level of Keptn App Creation Request Controller +## @param lifecycleOperator.manager.env.keptnAppVersionControllerLogLevel sets the log level of Keptn AppVersion Controller +## @param lifecycleOperator.manager.env.keptnEvaluationControllerLogLevel sets the log level of Keptn Evaluation Controller +## @param lifecycleOperator.manager.env.keptnTaskControllerLogLevel sets the log level of Keptn Task Controller +## @param lifecycleOperator.manager.env.keptnTaskDefinitionControllerLogLevel sets the log level of Keptn TaskDefinition Controller +## @param lifecycleOperator.manager.env.keptnWorkloadControllerLogLevel sets the log level of Keptn Workload Controller +## @param lifecycleOperator.manager.env.keptnWorkloadInstanceControllerLogLevel sets the log level of Keptn WorkloadInstance Controller +## @param lifecycleOperator.manager.env.optionsControllerLogLevel sets the log level of Keptn Options Controller + +## @param lifecycleOperator.manager.env.otelCollectorUrl Sets the URL for the open telemetry collector +## @param lifecycleOperator.manager.env.functionRunnerImage specify image for deno task runtime +## @param lifecycleOperator.manager.env.pythonRunnerImage specify image for python task runtime + +## @param lifecycleOperator.manager.image.repository specify registry for manager image +## @param lifecycleOperator.manager.image.tag select tag for manager image +## @param lifecycleOperator.manager.imagePullPolicy specify pull policy for manager image + +## @extra lifecycleOperator.manager.livenessProbe custom livenessprobe for manager container +## @skip lifecycleOperator.manager.livenessProbe.httpGet.path +## @skip lifecycleOperator.manager.livenessProbe.httpGet.port +## @skip lifecycleOperator.manager.livenessProbe.initialDelaySeconds +## @skip lifecycleOperator.manager.livenessProbe.periodSeconds + +## @extra lifecycleOperator.manager.readinessProbe custom readinessprobe for manager container +## @skip lifecycleOperator.manager.readinessProbe.httpGet.path +## @skip lifecycleOperator.manager.readinessProbe.httpGet.port +## @skip lifecycleOperator.manager.readinessProbe.initialDelaySeconds +## @skip lifecycleOperator.manager.readinessProbe.periodSeconds + +## @extra lifecycleOperator.manager.resources specify limits and requests for manager container +## @skip lifecycleOperator.manager.resources.limits.cpu +## @skip lifecycleOperator.manager.resources.limits.memory +## @skip lifecycleOperator.manager.resources.requests.cpu +## @skip lifecycleOperator.manager.resources.requests.memory + + +## @section Keptn Metrics Operator common + +## @param metricsOperator.replicas customize number of installed metrics operator replicas + +## @extra metricsOperatorService.ports[0] webhook port (must correspond to Mutating Webhook Configurations) +## @param metricsOperatorService.ports[0].name +## @param metricsOperatorService.ports[0].port +## @param metricsOperatorService.ports[0].protocol +## @param metricsOperatorService.ports[0].targetPort +## @extra metricsOperatorService.ports[1] port to integrate with the K8s custom metrics API +## @param metricsOperatorService.ports[1].name +## @param metricsOperatorService.ports[1].port +## @param metricsOperatorService.ports[1].targetPort +## @extra metricsOperatorService.ports[2] port to integrate with metrics API (e.g. Keda) +## @param metricsOperatorService.ports[2].name +## @param metricsOperatorService.ports[2].port +## @param metricsOperatorService.ports[2].protocol +## @param metricsOperatorService.ports[2].targetPort +## @param metricsOperatorService.type + +## @param metricsManagerConfig.controllerManagerConfigYaml.health.healthProbeBindAddress setup on what address to start the default health handler +## @param metricsManagerConfig.controllerManagerConfigYaml.leaderElection.leaderElect decides whether to enable leader election with multiple replicas +## @param metricsManagerConfig.controllerManagerConfigYaml.leaderElection.resourceName defines LeaderElectionID +## @param metricsManagerConfig.controllerManagerConfigYaml.metrics.bindAddress MetricsBindAddress is the TCP address that the controller should bind to for serving prometheus metrics. It can be set to "0" to disable the metrics serving. +## @param metricsManagerConfig.controllerManagerConfigYaml.webhook.port + +## @extra Mutating Webhook Configurations for metrics Operator +## @param metricsWebhookService.ports[0].port +## @param metricsWebhookService.ports[0].protocol +## @param metricsWebhookService.ports[0].targetPort +## @param metricsWebhookService.type + +## @param metricsOperator.nodeSelector add custom nodes selector to metrics operator +## @param metricsOperator.tolerations add custom tolerations to metrics operator +## @param metricsOperator.topologySpreadConstraints add custom topology constraints to metrics operator + +## @section Keptn Metrics Operator controller +## @extra metricsOperator.manager.containerSecurityContext Sets security context privileges +## @param metricsOperator.manager.containerSecurityContext.allowPrivilegeEscalation +## @param metricsOperator.manager.containerSecurityContext.capabilities.drop +## @param metricsOperator.manager.containerSecurityContext.privileged +## @param metricsOperator.manager.containerSecurityContext.runAsGroup +## @param metricsOperator.manager.containerSecurityContext.runAsNonRoot +## @param metricsOperator.manager.containerSecurityContext.runAsUser +## @param metricsOperator.manager.containerSecurityContext.seccompProfile.type + + +## @param metricsOperator.manager.image.repository specify registry for manager image +## @param metricsOperator.manager.image.tag select tag for manager image + +## @param metricsOperator.manager.env.exposeKeptnMetrics enable metrics exporter +## @param metricsOperator.manager.env.metricsControllerLogLevel sets the log level of Metrics Controller +## @param metricsOperator.manager.env.analysisControllerLogLevel sets the log level of Analysis Controller +## @param metricsOperator.manager.env.enableKeptnAnalysis enables/disables the analysis feature + +## @extra metricsOperator.manager.livenessProbe custom livenessprobe for manager container +## @skip metricsOperator.manager.livenessProbe.httpGet.path +## @skip metricsOperator.manager.livenessProbe.httpGet.port +## @skip metricsOperator.manager.livenessProbe.initialDelaySeconds +## @skip metricsOperator.manager.livenessProbe.periodSeconds + +## @extra metricsOperator.manager.readinessProbe custom readinessprobe for manager container +## @skip metricsOperator.manager.readinessProbe.httpGet.path +## @skip metricsOperator.manager.readinessProbe.httpGet.port +## @skip metricsOperator.manager.readinessProbe.initialDelaySeconds +## @skip metricsOperator.manager.readinessProbe.periodSeconds + +## @extra metricsOperator.manager.resources specify limits and requests for manager container +## @skip metricsOperator.manager.resources.limits.cpu +## @skip metricsOperator.manager.resources.limits.memory +## @skip metricsOperator.manager.resources.requests.cpu +## @skip metricsOperator.manager.resources.requests.memory + + +## @section Global +## Current available parameters: kubernetesClusterDomain, imagePullSecrets, schedulingGatesEnabled +## @param kubernetesClusterDomain overrides domain.local +## @param imagePullSecrets global value for image registry secret +## @param schedulingGatesEnabled enables the scheduling gates in lifecycle-operator. This feature is available in alpha version from K8s 1.27 or 1.26 enabling the alpha version diff --git a/helm/chart/templates/deployment.yaml b/helm/chart/templates/deployment.yaml index bcd42bb59da..e784aa33a87 100644 --- a/helm/chart/templates/deployment.yaml +++ b/helm/chart/templates/deployment.yaml @@ -9,6 +9,7 @@ metadata: app.kubernetes.io/created-by: certificate-operator app.kubernetes.io/part-of: keptn-lifecycle-toolkit {{- include "chart.labels" . | nindent 4 }} +{{- if not .Values.schedulingGatesEnabled }} --- apiVersion: v1 kind: ServiceAccount @@ -17,6 +18,7 @@ metadata: namespace: {{ .Release.Namespace | quote }} labels: {{- include "chart.labels" . | nindent 4 }} +{{- end }} --- apiVersion: v1 kind: ServiceAccount @@ -203,6 +205,9 @@ spec: - name: OPTIONS_CONTROLLER_LOG_LEVEL value: {{ .Values.lifecycleOperator.manager.env.optionsControllerLogLevel | quote }} + - name: SCHEDULING_GATES_ENABLED + value: {{ .Values.schedulingGatesEnabled | quote + }} - name: KUBERNETES_CLUSTER_DOMAIN value: {{ .Values.kubernetesClusterDomain }} image: {{ .Values.lifecycleOperator.manager.image.repository }}:{{ .Values.lifecycleOperator.manager.image.tag @@ -406,6 +411,7 @@ spec: tolerations: {{- include "tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} {{- end }} +{{- if not .Values.schedulingGatesEnabled }} --- apiVersion: apps/v1 kind: Deployment @@ -493,3 +499,4 @@ spec: {{- if .Values.tolerations }} tolerations: {{- include "tplvalues.render" (dict "value" .Values.tolerations "context" .) | nindent 8 }} {{- end }} +{{- end }} diff --git a/helm/chart/templates/extension-apiserver-authentication-reader-rbac.yaml b/helm/chart/templates/extension-apiserver-authentication-reader-rbac.yaml index 5307cafb658..d646bb66b1b 100644 --- a/helm/chart/templates/extension-apiserver-authentication-reader-rbac.yaml +++ b/helm/chart/templates/extension-apiserver-authentication-reader-rbac.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.schedulingGatesEnabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -12,4 +13,5 @@ roleRef: subjects: - kind: ServiceAccount name: 'keptn-scheduler' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file + namespace: '{{ .Release.Namespace }}' +{{- end }} \ No newline at end of file diff --git a/helm/chart/templates/keptn-scheduler-rbac.yaml b/helm/chart/templates/keptn-scheduler-rbac.yaml index 10776b56ae1..31921664e7c 100644 --- a/helm/chart/templates/keptn-scheduler-rbac.yaml +++ b/helm/chart/templates/keptn-scheduler-rbac.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.schedulingGatesEnabled }} apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -198,4 +199,5 @@ roleRef: subjects: - kind: ServiceAccount name: 'keptn-scheduler' - namespace: '{{ .Release.Namespace }}' \ No newline at end of file + namespace: '{{ .Release.Namespace }}' +{{- end }} \ No newline at end of file diff --git a/helm/chart/templates/lifecycle-operator-rbac.yaml b/helm/chart/templates/lifecycle-operator-rbac.yaml index 51054332a1e..6a4317ad589 100644 --- a/helm/chart/templates/lifecycle-operator-rbac.yaml +++ b/helm/chart/templates/lifecycle-operator-rbac.yaml @@ -88,6 +88,7 @@ rules: - get - list - watch + - update - apiGroups: - "" resources: diff --git a/helm/chart/templates/scheduler-config.yaml b/helm/chart/templates/scheduler-config.yaml index d1bf70335fa..978b4391f8f 100644 --- a/helm/chart/templates/scheduler-config.yaml +++ b/helm/chart/templates/scheduler-config.yaml @@ -1,3 +1,4 @@ +{{- if not .Values.schedulingGatesEnabled }} apiVersion: v1 kind: ConfigMap metadata: @@ -13,4 +14,5 @@ data: leaderElect: {{ .Values.schedulerConfig.schedulerConfigYaml.leaderElection.leaderElect }} profiles: {{ toYaml .Values.schedulerConfig.schedulerConfigYaml.profiles | nindent - 6 }} \ No newline at end of file + 6 }} +{{- end }} \ No newline at end of file diff --git a/helm/chart/values.yaml b/helm/chart/values.yaml index 8b32ea51a4a..36ac2199875 100644 --- a/helm/chart/values.yaml +++ b/helm/chart/values.yaml @@ -42,6 +42,7 @@ certificateOperator: topologySpreadConstraints: [] imagePullSecrets: [] kubernetesClusterDomain: cluster.local +schedulingGatesEnabled: false lifecycleManagerConfig: controllerManagerConfigYaml: health: diff --git a/lifecycle-operator/config/manager/manager.yaml b/lifecycle-operator/config/manager/manager.yaml index 3468446b73e..5c7733f58e0 100644 --- a/lifecycle-operator/config/manager/manager.yaml +++ b/lifecycle-operator/config/manager/manager.yaml @@ -83,6 +83,8 @@ spec: value: "0" - name: OPTIONS_CONTROLLER_LOG_LEVEL value: "0" + - name: SCHEDULING_GATES_ENABLED + value: "false" securityContext: seccompProfile: type: RuntimeDefault diff --git a/test/scheduling-gates/simple-deployment/00-assert.yaml b/test/scheduling-gates/simple-deployment/00-assert.yaml new file mode 100644 index 00000000000..e22b09f2ae0 --- /dev/null +++ b/test/scheduling-gates/simple-deployment/00-assert.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: test + name: test +status: + readyReplicas: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + app: test + annotations: + keptn.sh/scheduling-gate-removed: "true" +status: + phase: Running +spec: + schedulerName: default-scheduler diff --git a/test/scheduling-gates/simple-deployment/00-install.yaml b/test/scheduling-gates/simple-deployment/00-install.yaml new file mode 100644 index 00000000000..c96eff8c869 --- /dev/null +++ b/test/scheduling-gates/simple-deployment/00-install.yaml @@ -0,0 +1,40 @@ +apiVersion: lifecycle.keptn.sh/v1alpha3 +kind: KeptnTaskDefinition +metadata: + name: pre-deployment-hello +spec: + function: + inline: + code: | + console.log("Pre-Deployment Task has been executed"); +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: test + name: test +spec: + replicas: 1 + selector: + matchLabels: + app: test + strategy: {} + template: + metadata: + labels: + app: test + annotations: + keptn.sh/workload: waiter + keptn.sh/version: "0.4" + keptn.sh/pre-deployment-tasks: pre-deployment-hello + keptn.sh/post-deployment-tasks: pre-deployment-hello + spec: + containers: + - image: busybox + name: busybox + command: ['sh', '-c', 'echo The app is running! && sleep infinity'] + initContainers: + - name: init-myservice + image: busybox:1.36.1 + command: ['sh', '-c', 'sleep 30'] diff --git a/test/scheduling-gates/simple-deployment/00-teststep.yaml b/test/scheduling-gates/simple-deployment/00-teststep.yaml new file mode 100644 index 00000000000..ad4f1d95d54 --- /dev/null +++ b/test/scheduling-gates/simple-deployment/00-teststep.yaml @@ -0,0 +1,4 @@ +apiVersion: kuttl.dev/v1 +kind: TestStep +commands: + - script: kubectl annotate ns $NAMESPACE keptn.sh/lifecycle-toolkit='enabled' diff --git a/test/scheduling-gates/simple-deployment/01-assert.yaml b/test/scheduling-gates/simple-deployment/01-assert.yaml new file mode 100644 index 00000000000..f9e2166ee07 --- /dev/null +++ b/test/scheduling-gates/simple-deployment/01-assert.yaml @@ -0,0 +1,32 @@ +apiVersion: lifecycle.keptn.sh/v1alpha3 +kind: KeptnWorkload +metadata: + name: waiter-waiter +--- +apiVersion: lifecycle.keptn.sh/v1alpha3 +kind: KeptnWorkloadInstance +metadata: + name: waiter-waiter-0.4 +status: + currentPhase: Completed + deploymentStatus: Succeeded + postDeploymentEvaluationStatus: Succeeded + postDeploymentStatus: Succeeded + postDeploymentTaskStatus: + - status: Succeeded + definitionName: pre-deployment-hello + preDeploymentEvaluationStatus: Succeeded + preDeploymentStatus: Succeeded + preDeploymentTaskStatus: + - status: Succeeded + definitionName: pre-deployment-hello +--- +apiVersion: lifecycle.keptn.sh/v1alpha3 +kind: KeptnApp +metadata: + name: waiter +--- +apiVersion: lifecycle.keptn.sh/v1alpha3 +kind: KeptnAppVersion +metadata: + name: waiter-0.4-6b86b273