feat(controller): add generic cluster scoped resources to proxysettings

Makefile

@@ -1,6 +1,8 @@
 # Version
 GIT_HEAD_COMMIT ?= $(shell git rev-parse --short HEAD)
 VERSION         ?= $(or $(shell git describe --abbrev=0 --tags --match "v*" 2>/dev/null),$(GIT_HEAD_COMMIT))
+GO_OS 		    ?= $(shell go env GOOS)
+GO_ARCH 	    ?= $(shell go env GOARCH)
 
 # Defaults
 REGISTRY        ?= ghcr.io
@@ -39,6 +41,7 @@ dlv-build:
 	docker build . --build-arg "GCFLAGS=all=-N -l" --tag projectcapsule/capsule-proxy:dlv --target dlv
 
 
+KO_PLATFORM     ?= $(GOOS)/$(GO_ARCH)
 KOCACHE         ?= /tmp/ko-cache
 KO_TAGS         ?= "latest"
 
@@ -60,9 +63,9 @@ LD_FLAGS        := "-X main.Version=$(VERSION) \
 
 .PHONY: ko-build-capsule-proxy
 ko-build-capsule-proxy: ko
-	@echo Building Capsule Proxy $(KO_TAGS) >&2
+	echo Building Capsule Proxy $(KO_TAGS) for $(KO_PLATFORM) >&2
 	@LD_FLAGS=$(LD_FLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO=$(CAPSULE_PROXY_IMG) \
-		$(KO) build ./ --bare --tags=$(KO_TAGS) --local --push=false
+		$(KO) build ./ --bare --tags=$(KO_TAGS) --local --push=false --platform=$(KO_PLATFORM)
 
 .PHONY: ko-build-all
 ko-build-all: ko-build-capsule-proxy
@@ -132,8 +135,8 @@ e2e-exec:
 
 .PHONY: e2e-build
 e2e-build:
-	@echo "Building kubernetes env using Kind $${KIND_K8S_VERSION:-v1.22.0}..."
-	@kind create cluster --name capsule --image kindest/node:$${KIND_K8S_VERSION:-v1.22.0} --config ./e2e/kind.yaml --wait=120s \
+	@echo "Building kubernetes env using Kind $${KIND_K8S_VERSION:-v1.27.0}..."
+	@kind create cluster --name capsule --image kindest/node:$${KIND_K8S_VERSION:-v1.27.0} --config ./e2e/kind.yaml --wait=120s \
 		&& kubectl taint nodes capsule-worker2 key1=value1:NoSchedule
 	@helm repo add bitnami https://charts.bitnami.com/bitnami
 	@helm repo update
@@ -176,6 +179,7 @@ ifeq ($(CAPSULE_PROXY_MODE),http)
 		--set "image.pullPolicy=Never" \
 		--set "image.tag=$(VERSION)" \
 		--set "options.enableSSL=false" \
+		--set "options.logLevel=10" \
 		--set "service.type=NodePort" \
 		--set "service.nodePort=" \
 		--set "kind=DaemonSet" \
@@ -186,7 +190,7 @@ else
 	@echo "Running in HTTPS mode"
 	@echo "capsule proxy certificates..."
 	cd hack && $(MKCERT) -install && $(MKCERT) 127.0.0.1  \
-		&& kubectl --namespace capsule-systemdelete secret capsule-proxy \
+		&& kubectl --namespace capsule-system delete secret capsule-proxy || true \
 		&& kubectl --namespace capsule-system create secret generic capsule-proxy --from-file=tls.key=./127.0.0.1-key.pem --from-file=tls.crt=./127.0.0.1.pem --from-literal=ca=$$(cat $(ROOTCA) | base64 |tr -d '\n')
 	@echo "kubeconfig configurations..."
 	@cd hack \
@@ -210,6 +214,7 @@ else
 	@helm upgrade --install capsule-proxy ./charts/capsule-proxy -n capsule-system \
 		--set "image.pullPolicy=Never" \
 		--set "image.tag=$(VERSION)" \
+		--set "options.logLevel=10" \
 		--set "service.type=NodePort" \
 		--set "service.nodePort=" \
 		--set "kind=DaemonSet" \
@@ -227,7 +232,7 @@ rbac-fix:
 
 .PHONY: manifests
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=charts/capsule-proxy/crds
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=charts/capsule-proxy/crd
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.

api/v1beta1/clusterresoure.go

@@ -0,0 +1,30 @@
+package v1beta1
+
+import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+// +kubebuilder:validation:Enum=List;Update;Delete
+type ClusterResourceOperation string
+
+func (p ClusterResourceOperation) String() string {
+	return string(p)
+}
+
+const (
+	ClusterResourceOperationList ClusterResourceOperation = "List"
+)
+
+// +kubebuilder:object:generate=true
+type ClusterResource struct {
+	// APIGroups is the name of the APIGroup that contains the resources. If multiple API groups are specified, any action requested against any resource listed will be allowed. '*' represents all resources. Empty string represents v1 api resources.
+	APIGroups []string `json:"apiGroups"`
+
+	// Resources is a list of resources this rule applies to. '*' represents all resources.
+	Resources []string `json:"resources"`
+
+	// Operations which can be executed on the selected resources.
+	// +kubebuilder:default={List}
+	Operations []ClusterResourceOperation `json:"operations"`
+
+	// Select all cluster scoped resources with the given label selector.
+	Selector *metav1.LabelSelector `json:"selector"`
+}

api/v1beta1/proxysettings_types.go

@@ -15,6 +15,8 @@ type OwnerSpec struct {
 	Name string `json:"name"`
 	// Proxy settings for tenant owner.
 	ProxyOperations []v1beta2.ProxySettings `json:"proxySettings,omitempty"`
+	// Cluster Resources for tenant Owner.
+	ClusterResources []ClusterResource `json:"clusterResources,omitempty"`
 }
 
 // ProxySettingSpec defines the additional Capsule Proxy settings for additional users of the Tenant.

charts/capsule-proxy/Chart.yaml

@@ -34,4 +34,4 @@ annotations:
       url: https://capsule.clastix.io/
   artifacthub.io/changes: |
     - kind: added
-      description: add subjects for cert-manager certificate
+      description: crd lifecycle

charts/capsule-proxy/README.md

@@ -40,6 +40,25 @@ The Capsule-proxy Chart can be used to instantly deploy the Capsule-proxy on you
 
         $ helm uninstall capsule-proxy -n capsule-system
 
+## Upgrading the Chart
+
+Intsructions to upgrade the chart the versions, which may remove features or introduce breaking changes.
+
+### 0.7.x
+
+Introduces a new methode to manage all capsule-proxy CRDs and their lifecycle. We are no longer relying on the [native CRD hook with the Helm Chart](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations). The hook only allows to manage CRDs on install and uninstall but we can't deliver updates to the CRDs.
+When you newly install the chart we recommend to set  `crds.install` to `true`. This will manage the CRDs with the Helm Chart.
+
+If you are upgrading to this release, you can choose to set `global.crds.install` to `true` (by default `false`). However you need to add metadata to the existing CRDs so they can be correctly managed with the new flow. Run the following commands:
+
+```bash
+kubectl label crd proxysettings.capsule.clastix.io  app.kubernetes.io/managed-by=Helm
+kubectl annotate crd proxysettings.capsule.clastix.io meta.helm.sh/release-namespace=capsule-system # might be different
+kubectl annotate crd proxysettings.capsule.clastix.io meta.helm.sh/release-name=capsule-proxy # might be different
+```
+
+With the new CRD management we can release update CRDs bundled with the chart. The Chart can be uninstalled and the CRDs are still kept.
+
 ## Customize the installation
 
 There are two methods for specifying overrides of values during chart installation: `--values` and `--set`.
@@ -56,6 +75,13 @@ If you only need to make minor customizations, you can specify them on the comma
 
         $ helm install capsule-proxy projectcapsule/capsule-proxy --set "kind=DaemonSet" -n capsule-system
 
+### CustomResourceDefinition Lifecycle
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| crds.install | bool | `false` | Install the CustomResourceDefinitions (This also manages the lifecycle of the CRDs for update operations) |
+| crds.keep | bool | `true` | Keep the CustomResourceDefinitions (when the chart is deleted) |
+
 ### General Parameters
 
 | Key | Type | Default | Description |
@@ -70,6 +96,8 @@ If you only need to make minor customizations, you can specify them on the comma
 | certManager.generateCertificates | bool | `false` | Set if the cert manager will generate SSL certificates (self-signed or CA-signed) |
 | certManager.issuer.kind | string | `"Issuer"` | Set if the cert manager will generate either self-signed or CA signed SSL certificates. Its value will be either Issuer or ClusterIssuer |
 | certManager.issuer.name | string | `""` | Set the name of the ClusterIssuer if issuer kind is ClusterIssuer and if cert manager will generate CA signed SSL certificates |
+| crds.install | bool | `false` | Install the CustomResourceDefinitions (This also manages the lifecycle of the CRDs for update operations) |
+| crds.keep | bool | `true` | Keep the CustomResourceDefinitions (when the chart is deleted) |
 | daemonset.hostNetwork | bool | `false` | Use the host network namespace for capsule-proxy pod. |
 | daemonset.hostPort | bool | `false` | Binding the capsule-proxy listening port to the host port. |
 | hostNetwork | bool | `false` | When deployed as DaemonSet use |

charts/capsule-proxy/README.md.gotmpl

@@ -40,6 +40,25 @@ The Capsule-proxy Chart can be used to instantly deploy the Capsule-proxy on you
 
         $ helm uninstall capsule-proxy -n capsule-system
 
+## Upgrading the Chart
+
+Intsructions to upgrade the chart the versions, which may remove features or introduce breaking changes.
+
+### 0.7.x
+
+Introduces a new methode to manage all capsule-proxy CRDs and their lifecycle. We are no longer relying on the [native CRD hook with the Helm Chart](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations). The hook only allows to manage CRDs on install and uninstall but we can't deliver updates to the CRDs.
+When you newly install the chart we recommend to set  `crds.install` to `true`. This will manage the CRDs with the Helm Chart.
+
+If you are upgrading to this release, you can choose to set `global.crds.install` to `true` (by default `false`). However you need to add metadata to the existing CRDs so they can be correctly managed with the new flow. Run the following commands:
+
+```bash
+kubectl label crd proxysettings.capsule.clastix.io  app.kubernetes.io/managed-by=Helm
+kubectl annotate crd proxysettings.capsule.clastix.io meta.helm.sh/release-namespace=capsule-system # might be different
+kubectl annotate crd proxysettings.capsule.clastix.io meta.helm.sh/release-name=capsule-proxy # might be different
+```
+
+With the new CRD management we can release update CRDs bundled with the chart. The Chart can be uninstalled and the CRDs are still kept.
+
 ## Customize the installation
 
 There are two methods for specifying overrides of values during chart installation: `--values` and `--set`.
@@ -56,6 +75,17 @@ If you only need to make minor customizations, you can specify them on the comma
 
         $ helm install capsule-proxy projectcapsule/capsule-proxy --set "kind=DaemonSet" -n capsule-system
 
+### CustomResourceDefinition Lifecycle
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+{{- range .Values }}
+  {{- if (hasPrefix "crds" .Key)  }}
+| {{ .Key }} | {{ .Type }} | {{ if .Default }}{{ .Default }}{{ else }}{{ .AutoDefault }}{{ end }} | {{ if .Description }}{{ .Description }}{{ else }}{{ .AutoDescription }}{{ end }} |
+  {{- end }}
+{{- end }}
+
+
 ### General Parameters
 
 | Key | Type | Default | Description |

charts/capsule-proxy/ci/cert-manager-values.yaml

@@ -1,3 +1,6 @@
+crds:
+  install: true
+  keep: false
 options:
   enableSSL: true
   generateCertificates: false

charts/capsule-proxy/ci/deploy-values.yaml

@@ -1,3 +1,6 @@
+crds:
+  install: true
+  keep: false
 kind: DaemonSet
 imagePullSecrets: []
 certManager:

charts/capsule-proxy/ci/ds-values.yaml

@@ -1,3 +1,6 @@
+crds:
+  install: true
+  keep: false
 kind: DaemonSet
 daemonset:
   hostNetwork: true