fix: make deploy target

This PR address the issue sustainable-computing-io#381 by: * Enabling cert-manager that is necessary by webhooks during the deployment of Operator on k8s. * Added a separate config for k8s that contains the manifests that are specific for deploying on k8s. * `config/default` points to OpenShift specific manifests * `config/k8s` points to manifests specific for k8s * Added overlays in manager to isolate k8s and OpenShift Signed-off-by: Vibhu Prashar <[email protected]>
vprashar2929 · Jun 11, 2024 · 2ce068f · 2ce068f
1 parent da1c1fc
commit 2ce068f
Show file tree

Hide file tree

Showing 18 changed files with 251 additions and 86 deletions.
diff --git a/Makefile b/Makefile
@@ -246,15 +246,15 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 
 .PHONY: deploy
 deploy: install ## Deploy controller to the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/default | \
+	$(KUSTOMIZE) build config/k8s | \
 		sed  -e "s|<OPERATOR_IMG>|$(OPERATOR_IMG)|g" \
 		     -e "s|<KEPLER_IMG>|$(KEPLER_IMG)|g" \
 		| tee tmp/deploy.yaml | \
 		kubectl apply --server-side --force-conflicts -f -
 
 .PHONY: undeploy
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
-	$(KUSTOMIZE) build config/default | \
+	$(KUSTOMIZE) build config/k8s | \
 		kubectl delete --ignore-not-found=$(ignore-not-found) -f -
 
 ##@ Build Dependencies

diff --git a/bundle/manifests/kepler-operator.clusterserviceversion.yaml b/bundle/manifests/kepler-operator.clusterserviceversion.yaml
@@ -28,7 +28,7 @@ metadata:
     capabilities: Seamless Upgrades
     categories: Monitoring
     containerImage: quay.io/sustainable_computing_io/kepler-operator:0.13.0
-    createdAt: "2024-05-22T07:06:13Z"
+    createdAt: "2024-06-11T18:04:59Z"
     description: 'Deploys and Manages Kepler on Kubernetes '
     operators.operatorframework.io/builder: operator-sdk-v1.27.0
     operators.operatorframework.io/internal-objects: |-
@@ -259,9 +259,9 @@ spec:
               containers:
               - args:
                 - --openshift
+                - --deployment-namespace=kepler-operator
                 - --leader-elect
                 - --kepler.image=$(RELATED_IMAGE_KEPLER)
-                - --deployment-namespace=kepler-operator
                 - --zap-log-level=5
                 command:
                 - /manager
@@ -302,10 +302,6 @@ spec:
                   capabilities:
                     drop:
                     - ALL
-                volumeMounts:
-                - mountPath: /tmp/k8s-webhook-server/serving-certs
-                  name: cert
-                  readOnly: true
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: kepler-operator-controller-manager

diff --git a/bundle/manifests/kepler.system.sustainable.computing.io_keplers.yaml b/bundle/manifests/kepler.system.sustainable.computing.io_keplers.yaml
@@ -12,7 +12,7 @@ spec:
       clientConfig:
         service:
           name: kepler-operator-webhook-service
-          namespace: kepler-operator-system
+          namespace: kepler-operator
           path: /convert
       conversionReviewVersions:
       - v1

diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml
@@ -1,5 +1,5 @@
 # Adds namespace to all resources.
-namespace: kepler-operator-system
+namespace: kepler-operator
 
 # Value of this field is prepended to the
 # names of all resources, e.g. a deployment named
@@ -15,7 +15,7 @@ namePrefix: kepler-operator-
 bases:
 - ../crd
 - ../rbac
-- ../manager
+- ../manager/overlays/openshift
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 - ../webhook
@@ -30,8 +30,6 @@ patchesStrategicMerge:
 # endpoint w/o any authn/z, please comment the following line.
 # - manager_auth_proxy_patch.yaml
 
-
-
 # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 - manager_webhook_patch.yaml
@@ -40,33 +38,3 @@ patchesStrategicMerge:
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection
 #- webhookcainjection_patch.yaml
-
-# the following config is for teaching kustomize how to do var substitution
-vars:
-# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
-#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1
-#    name: serving-cert # this name should match the one in certificate.yaml
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: CERTIFICATE_NAME
-#  objref:
-#    kind: Certificate
-#    group: cert-manager.io
-#    version: v1
-#    name: serving-cert # this name should match the one in certificate.yaml
-#- name: SERVICE_NAMESPACE # namespace of the service
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
-#  fieldref:
-#    fieldpath: metadata.namespace
-#- name: SERVICE_NAME
-#  objref:
-#    kind: Service
-#    version: v1
-#    name: webhook-service
diff --git a/config/default/manager_config_patch.yaml b/config/default/manager_config_patch.yaml
diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml
@@ -12,14 +12,3 @@ spec:
         - containerPort: 9443
           name: webhook-server
           protocol: TCP
-        volumeMounts:
-        - mountPath: /tmp/k8s-webhook-server/serving-certs
-          name: cert
-          readOnly: true
-  # NOTE: this will be removed by the manager kustomization.yaml
-  # since OLM will add the volume
-  volumes:
-  - name: cert
-    secret:
-      defaultMode: 420
-      secretName: webhook-server-cert
diff --git a/config/k8s/default/cainjection_in_keplers.yaml b/config/k8s/default/cainjection_in_keplers.yaml
@@ -0,0 +1,7 @@
+# The following patch adds a directive for certmanager to inject CA into the CRD
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+  name: keplers.kepler.system.sustainable.computing.io
diff --git a/config/k8s/default/kustomization.yaml b/config/k8s/default/kustomization.yaml
@@ -0,0 +1,72 @@
+# Adds namespace to all resources.
+namespace: kepler-operator
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: kepler-operator-
+
+# Labels to add to all resources and selectors.
+#commonLabels:
+#  someName: someValue
+
+bases:
+- ../../crd
+- ../../rbac
+- ../../manager/overlays/k8s
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+- ../../webhook
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
+- ../../certmanager
+# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
+- ../../prometheus
+
+patchesStrategicMerge:
+# Protect the /metrics endpoint by putting it behind auth.
+# If you want your controller-manager to expose the /metrics
+# endpoint w/o any authn/z, please comment the following line.
+# - manager_auth_proxy_patch.yaml
+
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
+# crd/kustomization.yaml
+- manager_webhook_patch.yaml
+
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
+# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
+# 'CERTMANAGER' needs to be enabled to use ca injection
+- webhookcainjection_patch.yaml
+
+- cainjection_in_keplers.yaml
+
+# the following config is for teaching kustomize how to do var substitution
+vars:
+# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
+- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: serving-cert # this name should match the one in certificate.yaml
+  fieldref:
+    fieldpath: metadata.namespace
+- name: CERTIFICATE_NAME
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: serving-cert # this name should match the one in certificate.yaml
+- name: SERVICE_NAMESPACE # namespace of the service
+  objref:
+    kind: Service
+    version: v1
+    name: webhook-service
+  fieldref:
+    fieldpath: metadata.namespace
+- name: SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: webhook-service
diff --git a/config/k8s/default/manager_auth_proxy_patch.yaml b/config/k8s/default/manager_auth_proxy_patch.yaml
@@ -0,0 +1,56 @@
+# This patch inject a sidecar container which is a HTTP proxy for the
+# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                - key: kubernetes.io/arch
+                  operator: In
+                  values:
+                    - amd64
+                    - arm64
+                    - ppc64le
+                    - s390x
+                - key: kubernetes.io/os
+                  operator: In
+                  values:
+                    - linux
+      containers:
+      - name: kube-rbac-proxy
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - "ALL"
+        image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0
+        args:
+        - "--secure-listen-address=0.0.0.0:8443"
+        - "--upstream=http://127.0.0.1:8080/"
+        - "--logtostderr=true"
+        - "--v=0"
+        ports:
+        - containerPort: 8443
+          protocol: TCP
+          name: https
+        resources:
+          limits:
+            cpu: 500m
+            memory: 128Mi
+          requests:
+            cpu: 5m
+            memory: 64Mi
+      - name: manager
+        args:
+        - "--health-probe-bind-address=:8081"
+        - "--metrics-bind-address=127.0.0.1:8080"
+        - "--leader-elect"
+        - "--zap-log-level=3"
diff --git a/config/k8s/default/manager_webhook_patch.yaml b/config/k8s/default/manager_webhook_patch.yaml
@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+      volumes:
+        - name: cert
+          secret:
+            defaultMode: 420
+            secretName: webhook-server-cert
diff --git a/config/k8s/default/webhookcainjection_patch.yaml b/config/k8s/default/webhookcainjection_patch.yaml
@@ -0,0 +1,29 @@
+# This patch add annotation to admission webhook config and
+# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize.
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: mutatingwebhookconfiguration
+    app.kubernetes.io/instance: mutating-webhook-configuration
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/created-by: kepler-operator
+    app.kubernetes.io/part-of: kepler-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: mutating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/name: validatingwebhookconfiguration
+    app.kubernetes.io/instance: validating-webhook-configuration
+    app.kubernetes.io/component: webhook
+    app.kubernetes.io/created-by: kepler-operator
+    app.kubernetes.io/part-of: kepler-operator
+    app.kubernetes.io/managed-by: kustomize
+  name: validating-webhook-configuration
+  annotations:
+    cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
diff --git a/config/k8s/kustomization.yaml b/config/k8s/kustomization.yaml
@@ -0,0 +1,24 @@
+# These resources constitute the fully configured set of manifests
+# used to generate the 'manifests/' directory in a bundle.
+resources:
+- default
+
+# [WEBHOOK] To enable webhooks, uncomment all the sections with [WEBHOOK] prefix.
+# Do NOT uncomment sections with prefix [CERTMANAGER], as OLM does not support cert-manager.
+# These patches remove the unnecessary "cert" volume and its manager container volumeMount.
+# patchesJson6902:
+# - target:
+#     group: apps
+#     version: v1
+#     kind: Deployment
+#     name: controller-manager
+#     namespace: system
+#   patch: |-
+#     # Remove the manager container's "cert" volumeMount, since OLM will create and mount a set of certs.
+#     # Update the indices in this path if adding or removing containers/volumeMounts in the manager's Deployment.
+#     - op: remove
+#       path: /spec/template/spec/containers/0/volumeMounts/0
+#     # Remove the "cert" volume, since OLM will create and mount a set of certs.
+#     # Update the indices in this path if adding or removing volumes in the manager's Deployment.
+#     - op: remove
+#       path: /spec/template/spec/volumes/0
diff --git a/config/manager/kustomization.yaml → config/manager/base/kustomization.yaml b/config/manager/kustomization.yaml → config/manager/base/kustomization.yaml
diff --git a/config/manager/manager.yaml → config/manager/base/manager.yaml b/config/manager/manager.yaml → config/manager/base/manager.yaml
@@ -73,10 +73,8 @@ spec:
             value: '<KEPLER_IMG>'
         args:
         # TODO: move --openshift and deployment-namespace to openshift specific kustomize directory
-        - --openshift
         - --leader-elect
         - --kepler.image=$(RELATED_IMAGE_KEPLER)
-        - --deployment-namespace=kepler-operator
         - --zap-log-level=5
         image: '<OPERATOR_IMG>'
         imagePullPolicy: IfNotPresent

diff --git a/config/manager/overlays/k8s/kustomization.yaml b/config/manager/overlays/k8s/kustomization.yaml
@@ -0,0 +1,14 @@
+resources:
+- ../../base
+
+patchesJson6902:
+  - target:
+      group: apps
+      version: v1
+      kind: Deployment
+      name: controller
+      namespace: system
+    patch: |-
+      - op: add
+        path: /spec/template/spec/containers/0/args/0
+        value: --deployment-namespace=kepler