From d7cb251b16fe4c23cd4f7dc6d38fb68b52663515 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Fri, 16 Feb 2024 16:56:42 +0200 Subject: [PATCH 1/4] Don't deploy visibility by default --- charts/kueue/templates/visibility/apiservice.yaml | 2 ++ charts/kueue/templates/visibility/role_binding.yaml | 2 ++ charts/kueue/templates/visibility/service.yaml | 2 ++ charts/kueue/values.yaml | 2 ++ config/alpha-enabled/kustomization.yaml | 1 + config/default/kustomization.yaml | 1 - config/visibility/kustomization.yaml | 7 +++++++ hack/update-helm.sh | 7 +++++++ test/e2e/config/kustomization.yaml | 1 + test/e2e/config_1_26/kustomization.yaml | 1 + 10 files changed, 25 insertions(+), 1 deletion(-) create mode 100644 config/visibility/kustomization.yaml diff --git a/charts/kueue/templates/visibility/apiservice.yaml b/charts/kueue/templates/visibility/apiservice.yaml index 31cbdf453a..8b89ccc12b 100644 --- a/charts/kueue/templates/visibility/apiservice.yaml +++ b/charts/kueue/templates/visibility/apiservice.yaml @@ -1,3 +1,4 @@ +{{- if .Values.enableVisibility }} apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: @@ -11,3 +12,4 @@ spec: namespace: '{{ .Release.Namespace }}' version: v1alpha1 versionPriority: 100 +{{- end}} diff --git a/charts/kueue/templates/visibility/role_binding.yaml b/charts/kueue/templates/visibility/role_binding.yaml index f67ff03225..a1e14f9c85 100644 --- a/charts/kueue/templates/visibility/role_binding.yaml +++ b/charts/kueue/templates/visibility/role_binding.yaml @@ -1,3 +1,4 @@ +{{- if .Values.enableVisibility }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -13,3 +14,4 @@ subjects: - kind: ServiceAccount name: kueue-controller-manager namespace: '{{ .Release.Namespace }}' +{{- end}} diff --git a/charts/kueue/templates/visibility/service.yaml b/charts/kueue/templates/visibility/service.yaml index fd88e4f986..d1a95bfdcf 100644 --- a/charts/kueue/templates/visibility/service.yaml +++ b/charts/kueue/templates/visibility/service.yaml @@ -1,3 +1,4 @@ +{{- if .Values.enableVisibility }} apiVersion: v1 kind: Service metadata: @@ -11,3 +12,4 @@ spec: targetPort: 8082 selector: control-plane: controller-manager +{{- end}} diff --git a/charts/kueue/values.yaml b/charts/kueue/values.yaml index 94725329f9..eb1430d65e 100644 --- a/charts/kueue/values.yaml +++ b/charts/kueue/values.yaml @@ -5,6 +5,8 @@ nameOverride: "" fullnameOverride: "" # Enable each function, like kustomize https://github.com/kubernetes-sigs/kueue/blob/main/config/default/kustomization.yaml enablePrometheus: false +# Enable visibility, set the default to true when the feature is no longer aplha +enableVisibility: false # Enable x509 automated certificate management using cert-manager (cert-manager.io) enableCertManager: false # Customize controlerManager diff --git a/config/alpha-enabled/kustomization.yaml b/config/alpha-enabled/kustomization.yaml index 21e4b163d9..ad4c4899e3 100644 --- a/config/alpha-enabled/kustomization.yaml +++ b/config/alpha-enabled/kustomization.yaml @@ -3,6 +3,7 @@ # Use default settings as a base. resources: - ../default +- ../visibility patches: # Modify feature gates to enable AllAlpha=true diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index f4afb3eade..29eebb3b76 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -17,7 +17,6 @@ resources: - ../components/rbac - ../components/manager - ../components/internalcert -- ../components/visibility # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # ../components/crd/kustomization.yaml - ../components/webhook diff --git a/config/visibility/kustomization.yaml b/config/visibility/kustomization.yaml new file mode 100644 index 0000000000..8dbcc994d9 --- /dev/null +++ b/config/visibility/kustomization.yaml @@ -0,0 +1,7 @@ +# This overlay builds the visibility component to be used in combination +# with other overlays. + +namespace: kueue-system +namePrefix: kueue- +resources: +- ../components/visibility diff --git a/hack/update-helm.sh b/hack/update-helm.sh index b726593c2f..cbf37d3ad4 100755 --- a/hack/update-helm.sh +++ b/hack/update-helm.sh @@ -269,4 +269,11 @@ for output_file in ${DEST_VISIBILITY_DIR}/*.yaml; do if [ "$(cat $output_file | $YQ '.subjects.[] | has("namespace")')" = "true" ]; then $YQ -N -i '.subjects.[].namespace = "{{ .Release.Namespace }}"' $output_file fi + + { + echo "{{- if .Values.enableVisibility }}" + cat $output_file + echo "{{- end}}" + }> ${output_file}.tmp + mv ${output_file}.tmp ${output_file} done diff --git a/test/e2e/config/kustomization.yaml b/test/e2e/config/kustomization.yaml index 56657fe34a..064f203d8b 100644 --- a/test/e2e/config/kustomization.yaml +++ b/test/e2e/config/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - ../../../config/dev +- ../../../config/visibility replicas: - name: kueue-controller-manager diff --git a/test/e2e/config_1_26/kustomization.yaml b/test/e2e/config_1_26/kustomization.yaml index 11d3f3eecb..70be6af201 100644 --- a/test/e2e/config_1_26/kustomization.yaml +++ b/test/e2e/config_1_26/kustomization.yaml @@ -3,6 +3,7 @@ kind: Kustomization resources: - ../../../config/dev +- ../../../config/visibility patches: - path: manager_e2e_patch.yaml From 4d00f13ada84c1226c03dff5b64fba1bfcf169b1 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Mon, 19 Feb 2024 11:44:37 +0200 Subject: [PATCH 2/4] Review Remarks --- charts/kueue/templates/_helpers.tpl | 18 ++++++++++++++++++ .../kueue/templates/visibility/apiservice.yaml | 2 +- .../templates/visibility/role_binding.yaml | 2 +- charts/kueue/templates/visibility/service.yaml | 2 +- charts/kueue/values.yaml | 2 -- hack/update-helm.sh | 2 +- 6 files changed, 22 insertions(+), 6 deletions(-) diff --git a/charts/kueue/templates/_helpers.tpl b/charts/kueue/templates/_helpers.tpl index d207db1faa..404473f248 100644 --- a/charts/kueue/templates/_helpers.tpl +++ b/charts/kueue/templates/_helpers.tpl @@ -74,3 +74,21 @@ FeatureGates - --feature-gates={{ $features | trimSuffix "," }} {{- end }} {{- end }} + +{{/* +IsFeatureGateEanabled - outputs true if the feature gate .Feature is enabled in the .List +Usage: + {{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}} +*/}} +{{- define "kueue.isFeatureGateEnabled" -}} +{{- $feature := .Feature }} +{{- $enabled := false }} +{{- range .List }} +{{- if (and (eq .name $feature) (eq .enabled true))}} +{{- $enabled = true }} +{{- end }} +{{- end }} +{{- if $enabled }} +{{- $enabled -}} +{{- end }} +{{- end }} diff --git a/charts/kueue/templates/visibility/apiservice.yaml b/charts/kueue/templates/visibility/apiservice.yaml index 8b89ccc12b..27b798f2dc 100644 --- a/charts/kueue/templates/visibility/apiservice.yaml +++ b/charts/kueue/templates/visibility/apiservice.yaml @@ -1,4 +1,4 @@ -{{- if .Values.enableVisibility }} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}} apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: diff --git a/charts/kueue/templates/visibility/role_binding.yaml b/charts/kueue/templates/visibility/role_binding.yaml index a1e14f9c85..87d9ec0199 100644 --- a/charts/kueue/templates/visibility/role_binding.yaml +++ b/charts/kueue/templates/visibility/role_binding.yaml @@ -1,4 +1,4 @@ -{{- if .Values.enableVisibility }} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/charts/kueue/templates/visibility/service.yaml b/charts/kueue/templates/visibility/service.yaml index d1a95bfdcf..9c028de845 100644 --- a/charts/kueue/templates/visibility/service.yaml +++ b/charts/kueue/templates/visibility/service.yaml @@ -1,4 +1,4 @@ -{{- if .Values.enableVisibility }} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}} apiVersion: v1 kind: Service metadata: diff --git a/charts/kueue/values.yaml b/charts/kueue/values.yaml index eb1430d65e..94725329f9 100644 --- a/charts/kueue/values.yaml +++ b/charts/kueue/values.yaml @@ -5,8 +5,6 @@ nameOverride: "" fullnameOverride: "" # Enable each function, like kustomize https://github.com/kubernetes-sigs/kueue/blob/main/config/default/kustomization.yaml enablePrometheus: false -# Enable visibility, set the default to true when the feature is no longer aplha -enableVisibility: false # Enable x509 automated certificate management using cert-manager (cert-manager.io) enableCertManager: false # Customize controlerManager diff --git a/hack/update-helm.sh b/hack/update-helm.sh index cbf37d3ad4..f51d753df8 100755 --- a/hack/update-helm.sh +++ b/hack/update-helm.sh @@ -271,7 +271,7 @@ for output_file in ${DEST_VISIBILITY_DIR}/*.yaml; do fi { - echo "{{- if .Values.enableVisibility }}" + echo '{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}}' cat $output_file echo "{{- end}}" }> ${output_file}.tmp From 2e6410a2abf0e85aa224663751010ac23f497d36 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Tue, 20 Feb 2024 13:13:15 +0200 Subject: [PATCH 3/4] Review Remarks --- charts/kueue/templates/_helpers.tpl | 2 +- charts/kueue/templates/visibility/apiservice.yaml | 2 +- charts/kueue/templates/visibility/role_binding.yaml | 2 +- charts/kueue/templates/visibility/service.yaml | 2 +- hack/update-helm.sh | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/charts/kueue/templates/_helpers.tpl b/charts/kueue/templates/_helpers.tpl index 404473f248..bdaab9eafa 100644 --- a/charts/kueue/templates/_helpers.tpl +++ b/charts/kueue/templates/_helpers.tpl @@ -78,7 +78,7 @@ FeatureGates {{/* IsFeatureGateEanabled - outputs true if the feature gate .Feature is enabled in the .List Usage: - {{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}} + {{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}} */}} {{- define "kueue.isFeatureGateEnabled" -}} {{- $feature := .Feature }} diff --git a/charts/kueue/templates/visibility/apiservice.yaml b/charts/kueue/templates/visibility/apiservice.yaml index 27b798f2dc..76224d1a1d 100644 --- a/charts/kueue/templates/visibility/apiservice.yaml +++ b/charts/kueue/templates/visibility/apiservice.yaml @@ -1,4 +1,4 @@ -{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}} apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: diff --git a/charts/kueue/templates/visibility/role_binding.yaml b/charts/kueue/templates/visibility/role_binding.yaml index 87d9ec0199..bc1db93cc6 100644 --- a/charts/kueue/templates/visibility/role_binding.yaml +++ b/charts/kueue/templates/visibility/role_binding.yaml @@ -1,4 +1,4 @@ -{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: diff --git a/charts/kueue/templates/visibility/service.yaml b/charts/kueue/templates/visibility/service.yaml index 9c028de845..0f3f70cc67 100644 --- a/charts/kueue/templates/visibility/service.yaml +++ b/charts/kueue/templates/visibility/service.yaml @@ -1,4 +1,4 @@ -{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}} apiVersion: v1 kind: Service metadata: diff --git a/hack/update-helm.sh b/hack/update-helm.sh index f51d753df8..787f2f4773 100755 --- a/hack/update-helm.sh +++ b/hack/update-helm.sh @@ -271,7 +271,7 @@ for output_file in ${DEST_VISIBILITY_DIR}/*.yaml; do fi { - echo '{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "Visibility")}}' + echo '{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}}' cat $output_file echo "{{- end}}" }> ${output_file}.tmp From e24adfeb6270198d625043c391546e9d0fa6ee98 Mon Sep 17 00:00:00 2001 From: Traian Schiau Date: Wed, 21 Feb 2024 17:23:21 +0200 Subject: [PATCH 4/4] Review Remarks --- charts/kueue/templates/_helpers.tpl | 8 ++++---- charts/kueue/templates/visibility/apiservice.yaml | 4 ++-- charts/kueue/templates/visibility/role_binding.yaml | 4 ++-- charts/kueue/templates/visibility/service.yaml | 4 ++-- hack/update-helm.sh | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/charts/kueue/templates/_helpers.tpl b/charts/kueue/templates/_helpers.tpl index bdaab9eafa..9b0a8a5606 100644 --- a/charts/kueue/templates/_helpers.tpl +++ b/charts/kueue/templates/_helpers.tpl @@ -76,15 +76,15 @@ FeatureGates {{- end }} {{/* -IsFeatureGateEanabled - outputs true if the feature gate .Feature is enabled in the .List +IsFeatureGateEnabled - outputs true if the feature gate .Feature is enabled in the .List Usage: - {{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}} + {{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }} */}} {{- define "kueue.isFeatureGateEnabled" -}} -{{- $feature := .Feature }} +{{- $feature := .Feature }} {{- $enabled := false }} {{- range .List }} -{{- if (and (eq .name $feature) (eq .enabled true))}} +{{- if (and (eq .name $feature) (eq .enabled true)) }} {{- $enabled = true }} {{- end }} {{- end }} diff --git a/charts/kueue/templates/visibility/apiservice.yaml b/charts/kueue/templates/visibility/apiservice.yaml index 76224d1a1d..497cea57f5 100644 --- a/charts/kueue/templates/visibility/apiservice.yaml +++ b/charts/kueue/templates/visibility/apiservice.yaml @@ -1,4 +1,4 @@ -{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }} apiVersion: apiregistration.k8s.io/v1 kind: APIService metadata: @@ -12,4 +12,4 @@ spec: namespace: '{{ .Release.Namespace }}' version: v1alpha1 versionPriority: 100 -{{- end}} +{{- end }} diff --git a/charts/kueue/templates/visibility/role_binding.yaml b/charts/kueue/templates/visibility/role_binding.yaml index bc1db93cc6..7cdcc3e376 100644 --- a/charts/kueue/templates/visibility/role_binding.yaml +++ b/charts/kueue/templates/visibility/role_binding.yaml @@ -1,4 +1,4 @@ -{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }} apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: @@ -14,4 +14,4 @@ subjects: - kind: ServiceAccount name: kueue-controller-manager namespace: '{{ .Release.Namespace }}' -{{- end}} +{{- end }} diff --git a/charts/kueue/templates/visibility/service.yaml b/charts/kueue/templates/visibility/service.yaml index 0f3f70cc67..662de0024a 100644 --- a/charts/kueue/templates/visibility/service.yaml +++ b/charts/kueue/templates/visibility/service.yaml @@ -1,4 +1,4 @@ -{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}} +{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }} apiVersion: v1 kind: Service metadata: @@ -12,4 +12,4 @@ spec: targetPort: 8082 selector: control-plane: controller-manager -{{- end}} +{{- end }} diff --git a/hack/update-helm.sh b/hack/update-helm.sh index 787f2f4773..d258fcbe21 100755 --- a/hack/update-helm.sh +++ b/hack/update-helm.sh @@ -271,9 +271,9 @@ for output_file in ${DEST_VISIBILITY_DIR}/*.yaml; do fi { - echo '{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand")}}' + echo '{{- if include "kueue.isFeatureGateEnabled" (dict "List" .Values.controllerManager.featureGates "Feature" "VisibilityOnDemand") }}' cat $output_file - echo "{{- end}}" + echo "{{- end }}" }> ${output_file}.tmp mv ${output_file}.tmp ${output_file} done