From f7ef2b01ea1e3378eca6bbbc6ac862869896baa6 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 19 Jul 2024 14:21:33 +0300 Subject: [PATCH 1/4] Remove unnecessary ClusterRole requirements for nodes and persistentvolumes --- config/rbac/role.yaml | 1 - .../cassandradatacenter_controller.go | 2 +- pkg/utils/k8s_utils.go | 166 ------------------ 3 files changed, 1 insertion(+), 168 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index ffd0d00e..b59de614 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -7,7 +7,6 @@ rules: - apiGroups: - "" resources: - - nodes - persistentvolumes verbs: - get diff --git a/internal/controllers/cassandra/cassandradatacenter_controller.go b/internal/controllers/cassandra/cassandradatacenter_controller.go index 7bd6bbcb..d6bef3e5 100644 --- a/internal/controllers/cassandra/cassandradatacenter_controller.go +++ b/internal/controllers/cassandra/cassandradatacenter_controller.go @@ -60,7 +60,7 @@ var ( // +kubebuilder:rbac:groups=apps,namespace=cass-operator,resources=deployments/finalizers,verbs=update // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=pods;endpoints;services;configmaps;secrets;persistentvolumeclaims;events,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,namespace=cass-operator,resources=namespaces,verbs=get -// +kubebuilder:rbac:groups=core,resources=persistentvolumes;nodes,verbs=get;list;watch +// +kubebuilder:rbac:groups=core,resources=persistentvolumes,verbs=get;list;watch // +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch // +kubebuilder:rbac:groups=policy,namespace=cass-operator,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete diff --git a/pkg/utils/k8s_utils.go b/pkg/utils/k8s_utils.go index 111f4b71..6c3a2cf7 100644 --- a/pkg/utils/k8s_utils.go +++ b/pkg/utils/k8s_utils.go @@ -5,9 +5,6 @@ import ( "os" "strings" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -44,129 +41,6 @@ func IntersectionStringSet(a, b StringSet) StringSet { return result } -// k8s Node helper functions -func GetNodeNameSet(nodes []*corev1.Node) StringSet { - result := StringSet{} - for _, node := range nodes { - result[node.Name] = true - } - return result -} - -func hasTaint(node *corev1.Node, taintKey, value string, effect corev1.TaintEffect) bool { - for _, taint := range node.Spec.Taints { - if taint.Key == taintKey && taint.Effect == effect { - if taint.Value == value { - return true - } - } - } - return false -} - -func FilterNodesWithFn(nodes []*corev1.Node, fn func(*corev1.Node) bool) []*corev1.Node { - result := []*corev1.Node{} - for _, node := range nodes { - if fn(node) { - result = append(result, node) - } - } - return result -} - -func FilterNodesWithTaintKeyValueEffect(nodes []*corev1.Node, taintKey, value string, effect corev1.TaintEffect) []*corev1.Node { - return FilterNodesWithFn(nodes, func(node *corev1.Node) bool { - return hasTaint(node, taintKey, value, effect) - }) -} - -// k8s Pod helper functions -func IsPodUnschedulable(pod *corev1.Pod) bool { - for _, condition := range pod.Status.Conditions { - if condition.Reason == corev1.PodReasonUnschedulable && - condition.Type == corev1.PodScheduled && - condition.Status == corev1.ConditionFalse { - return true - } - } - return false -} - -func GetPodNameSet(pods []*corev1.Pod) StringSet { - names := StringSet{} - for _, pod := range pods { - names[pod.Name] = true - } - - return names -} - -func GetPodNodeNameSet(pods []*corev1.Pod) StringSet { - names := StringSet{} - for _, pod := range pods { - names[pod.Spec.NodeName] = true - } - return names -} - -func FilterPodsWithFn(pods []*corev1.Pod, fn func(*corev1.Pod) bool) []*corev1.Pod { - result := []*corev1.Pod{} - for _, pod := range pods { - if fn(pod) { - result = append(result, pod) - } - } - return result -} - -func FilterPodsWithNodeInNameSet(pods []*corev1.Pod, nameSet StringSet) []*corev1.Pod { - return FilterPodsWithFn(pods, func(pod *corev1.Pod) bool { - return nameSet[pod.Spec.NodeName] - }) -} - -func FilterPodsWithAnnotationKey(pods []*corev1.Pod, key string) []*corev1.Pod { - return FilterPodsWithFn(pods, func(pod *corev1.Pod) bool { - annos := pod.ObjectMeta.Annotations - if annos != nil { - _, ok := annos[key] - return ok - } - return false - }) -} - -func FilterPodsWithLabel(pods []*corev1.Pod, label, value string) []*corev1.Pod { - return FilterPodsWithFn(pods, func(pod *corev1.Pod) bool { - labels := pod.Labels - if labels != nil { - labelValue, ok := labels[label] - return ok && labelValue == value - } - return false - }) -} - -// k8s PVC helpers -func FilterPVCsWithFn(pvcs []*corev1.PersistentVolumeClaim, fn func(*corev1.PersistentVolumeClaim) bool) []*corev1.PersistentVolumeClaim { - result := []*corev1.PersistentVolumeClaim{} - for _, pvc := range pvcs { - if fn(pvc) { - result = append(result, pvc) - } - } - return result -} - -func GetPVCSelectedNodeName(pvc *corev1.PersistentVolumeClaim) string { - annos := pvc.Annotations - if annos == nil { - annos = map[string]string{} - } - pvcNode := annos["volume.kubernetes.io/selected-node"] - return pvcNode -} - // // Migrated from operator-sdk, these are internal in newer versions // @@ -228,43 +102,3 @@ func GetOperatorNamespace() (string, error) { func isRunModeLocal() bool { return os.Getenv(ForceRunModeEnv) == string(LocalRunMode) } - -// GetGVKsFromAddToScheme takes in the runtime scheme and filters out all generic apimachinery meta types. -// It returns just the GVK specific to this scheme. -func GetGVKsFromAddToScheme(addToSchemeFunc func(*runtime.Scheme) error) ([]schema.GroupVersionKind, error) { - s := runtime.NewScheme() - err := addToSchemeFunc(s) - if err != nil { - return nil, err - } - schemeAllKnownTypes := s.AllKnownTypes() - ownGVKs := []schema.GroupVersionKind{} - for gvk := range schemeAllKnownTypes { - if !isKubeMetaKind(gvk.Kind) { - ownGVKs = append(ownGVKs, gvk) - } - } - - return ownGVKs, nil -} - -func isKubeMetaKind(kind string) bool { - if strings.HasSuffix(kind, "List") || - kind == "PatchOptions" || - kind == "GetOptions" || - kind == "DeleteOptions" || - kind == "ExportOptions" || - kind == "APIVersions" || - kind == "APIGroupList" || - kind == "APIResourceList" || - kind == "UpdateOptions" || - kind == "CreateOptions" || - kind == "Status" || - kind == "WatchEvent" || - kind == "ListOptions" || - kind == "APIGroup" { - return true - } - - return false -} From 5e56a726ebedb7effd406a9a9d0e4b6344bfee0f Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 19 Jul 2024 14:22:55 +0300 Subject: [PATCH 2/4] Add missing annotations as required by current certified-operators process, fixes #679 --- .../bases/cass-operator.clusterserviceversion.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config/manifests/bases/cass-operator.clusterserviceversion.yaml b/config/manifests/bases/cass-operator.clusterserviceversion.yaml index c6ca7aac..2552bfc3 100644 --- a/config/manifests/bases/cass-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/cass-operator.clusterserviceversion.yaml @@ -10,6 +10,16 @@ metadata: Simple provisioning, turn-key operations, and automated remediation of Apache Cassandra clusters repository: https://github.com/k8ssandra/cass-operator support: DataStax + features.operators.openshift.io/disconnected: "false" + features.operators.openshift.io/fips-compliant: "false" + features.operators.openshift.io/proxy-aware: "false" + features.operators.openshift.io/tls-profiles: "false" + features.operators.openshift.io/token-auth-aws: "false" + features.operators.openshift.io/token-auth-azure: "false" + features.operators.openshift.io/token-auth-gcp: "false" + features.operators.openshift.io/cnf: "false" + features.operators.openshift.io/cni: "false" + features.operators.openshift.io/csi: "false" name: cass-operator.v0.0.0 namespace: placeholder spec: From be756d343d2b88ec11faf55df3839853ac2f9f62 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 19 Jul 2024 14:23:29 +0300 Subject: [PATCH 3/4] Add process to create legacy Helm chart CRDs and autogenerate role.yaml, fixes #680 --- scripts/release-helm-chart.sh | 85 +++++++++++++++++++++++++---------- 1 file changed, 62 insertions(+), 23 deletions(-) diff --git a/scripts/release-helm-chart.sh b/scripts/release-helm-chart.sh index c16cb2f7..0fa20a53 100755 --- a/scripts/release-helm-chart.sh +++ b/scripts/release-helm-chart.sh @@ -6,55 +6,94 @@ if [[ ! $0 == scripts/* ]]; then fi # This script assumes k8ssandra is checked out at ../k8ssandra and is checked out at main -if [ "$#" -ne 1 ]; then - echo "Usage: scripts/release-helm-chart.sh version" +if [ "$#" -le 1 ]; then + echo "Usage: scripts/release-helm-chart.sh version legacy" echo "Script assumes you are in the correct branch / tag and that k8ssandra repository" - echo "has been checked out to ../k8ssandra/" + echo "has been checked out to ../k8ssandra/. If legacy is set, the script will generate" + echo "CRDs to the chart/crds directory" exit fi # Includes here to get all the updates even if we swap to an older branch . scripts/lib.sh +LEGACY=false +if [[ $2 == "legacy" ]]; then + LEGACY=true +fi + # This should work with BSD/MacOS mktemp and GNU one CRD_TMP_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'crd') VERSION=$1 CHART_HOME=../k8ssandra/charts/cass-operator -CRD_TARGET_PATH=$CRD_TMP_DIR TEMPLATE_HOME=$CHART_HOME/templates +CRD_TARGET_PATH=$TEMPLATE_HOME # Checkout tag git checkout v$VERSION # Create CRDs -kustomize build config/crd --output $CRD_TARGET_PATH +kustomize build config/crd --output $CRD_TMP_DIR # Rename generated CRDs to shorter format -for f in $CRD_TARGET_PATH/*; do +for f in $CRD_TMP_DIR/*; do TARGET_FILENAME=$(yq '.spec.names.plural' $f).yaml - mv $f $CRD_TARGET_PATH/$TARGET_FILENAME + mv $f $CRD_TMP_DIR/$TARGET_FILENAME done -# Add Helm conditionals to the end and beginning of CRDs before applying them to the templates path -echo "Updating CRDs in" $TEMPLATE_HOME -CRD_FILE_NAME=$TEMPLATE_HOME/crds.yaml -echo '{{- if .Values.manageCrds }}' > $CRD_FILE_NAME - -declare -a files -files=($CRD_TARGET_PATH/*) -for i in ${!files[@]}; do - echo "Processing " ${files[$i]} - yq -i '.metadata.annotations."helm.sh/resource-policy" = "keep"' ${files[$i]} - cat ${files[$i]} >> $CRD_FILE_NAME - if [[ $i -lt ${#files[@]}-1 ]]; then - echo "---" >> $CRD_FILE_NAME - fi -done -echo '{{- end }}' >> $CRD_FILE_NAME +if [ "$LEGACY" == true ]; then + echo "Updating CRDs for legacy CRD handling in Helm chart" + + # Update CRDs for legacy Helm chart + CRD_TARGET_PATH=$CHART_HOME/crds + cp -r $CRD_TMP_DIR/* $CRD_TARGET_PATH +else + # Add Helm conditionals to the end and beginning of CRDs before applying them to the templates path + echo "Updating CRDs in" $TEMPLATE_HOME + CRD_FILE_NAME=$CRD_TARGET_PATH/crds.yaml + echo '{{- if .Values.manageCrds }}' > $CRD_FILE_NAME + + declare -a files + files=($CRD_TMP_DIR/*) + for i in ${!files[@]}; do + echo "Processing " ${files[$i]} + yq -i '.metadata.annotations."helm.sh/resource-policy" = "keep"' ${files[$i]} + cat ${files[$i]} >> $CRD_FILE_NAME + if [[ $i -lt ${#files[@]}-1 ]]; then + echo "---" >> $CRD_FILE_NAME + fi + done + echo '{{- end }}' >> $CRD_FILE_NAME +fi rm -fr $CRD_TMP_DIR +# Update role.yaml +echo "Updating role.yaml" +ROLE_FILE_NAME=$TEMPLATE_HOME/role.yaml +cat <<'EOF' > $ROLE_FILE_NAME +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "k8ssandra-common.fullname" . }} + labels: {{ include "k8ssandra-common.labels" . | indent 4 }} +{{- if .Values.global.clusterScoped }} +EOF +yq -N eval-all '.rules = (.rules as $item ireduce ([]; . *+ $item)) | select(di == 0) | with_entries(select(.key | test("rules")))' config/rbac/role.yaml >> $ROLE_FILE_NAME +echo '{{- else }}' >> $ROLE_FILE_NAME +yq -N 'select(di == 0) | with_entries(select(.key | test("rules")))' config/rbac/role.yaml >> $ROLE_FILE_NAME +cat <<'EOF' >> $ROLE_FILE_NAME +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "k8ssandra-common.fullname" . }} + labels: {{ include "k8ssandra-common.labels" . | indent 4 }} +EOF +yq -N 'select(di == 1) | with_entries(select(.key | test("rules")))' config/rbac/role.yaml >> $ROLE_FILE_NAME +echo '{{- end }}' >> $ROLE_FILE_NAME + # Update version of the Chart.yaml automatically (to next minor one) CURRENT_VERSION=$(yq '.version' $CHART_HOME/Chart.yaml) next_minor_version From 7eabc31c1b5b5ebc5ad8a83e2a4f5ad9c78a72b2 Mon Sep 17 00:00:00 2001 From: Michael Burman Date: Fri, 19 Jul 2024 14:36:38 +0300 Subject: [PATCH 4/4] CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02d0710a..78413ba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ Changelog for Cass Operator, new PRs should update the `main / unreleased` secti * [ENHANCEMENT] [#184](https://github.com/k8ssandra/cass-operator/issues/349) Add CassandraDatacenter.Status fields as metrics also * [ENHANCEMENT] [#199](https://github.com/k8ssandra/cass-operator/issues/199) If .spec.readOnlyRootFilesystem is set, run the cassandra container with readOnlyRootFilesystem. Also, modify the default SecurityContext to mention runAsNonRoot: true * [ENHANCEMENT] [#595](https://github.com/k8ssandra/cass-operator/issues/595) Update Vector to 0.39.0 and enforce the TOML file format in the starting command +* [BUGFIX] [#681](https://github.com/k8ssandra/cass-operator/issues/681) Remove nodes rights from the operator as it is not required ## v1.21.1