diff --git a/Makefile b/Makefile index c9193514..dddb985d 100644 --- a/Makefile +++ b/Makefile @@ -120,6 +120,19 @@ ifneq ("$(wildcard $(GINKGO))","") GO_TEST="$(GINKGO)" -cover -output-dir=. endif +# Optional Red Hat Insights integration +ENABLE_INSIGHTS ?= false +ifeq ($(ENABLE_INSIGHTS), true) +KUSTOMIZE_DIR ?= config/insights +INSIGHTS_PROXY_NAMESPACE ?= quay.io/3scale +INSIGHTS_PROXY_NAME ?= apicast +INSIGHTS_PROXY_VERSION ?= insights-01 +export INSIGHTS_PROXY_IMG ?= $(INSIGHTS_PROXY_NAMESPACE)/$(INSIGHTS_PROXY_NAME):$(INSIGHTS_PROXY_VERSION) +export INSIGHTS_BACKEND ?= cert.console.redhat.com +else +KUSTOMIZE_DIR ?= config/default +endif + ##@ General .PHONY: all @@ -275,6 +288,9 @@ manifests: controller-gen ## Generate manifests e.g. CRD, RBAC, etc. $(CONTROLLER_GEN) rbac:roleName=role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases envsubst < hack/image_tag_patch.yaml.in > config/default/image_tag_patch.yaml envsubst < hack/image_pull_patch.yaml.in > config/default/image_pull_patch.yaml +ifeq ($(ENABLE_INSIGHTS), true) + envsubst < hack/insights_patch.yaml.in > config/insights/insights_patch.yaml +endif .PHONY: fmt fmt: add-license ## Run go fmt against code. @@ -435,11 +451,11 @@ predeploy: .PHONY: print_deploy_config print_deploy_config: predeploy ## Print deployment configurations for the controller. - $(KUSTOMIZE) build config/default + $(KUSTOMIZE) build $(KUSTOMIZE_DIR) .PHONY: deploy deploy: check_cert_manager manifests kustomize predeploy ## Deploy controller in the configured cluster in ~/.kube/config - $(KUSTOMIZE) build config/default | $(CLUSTER_CLIENT) apply -f - + $(KUSTOMIZE) build $(KUSTOMIZE_DIR) | $(CLUSTER_CLIENT) apply -f - ifeq ($(DISABLE_SERVICE_TLS), true) @echo "Disabling TLS for in-cluster communication between Services" @$(CLUSTER_CLIENT) -n $(DEPLOY_NAMESPACE) set env deployment/cryostat-operator-controller-manager DISABLE_SERVICE_TLS=true @@ -449,7 +465,7 @@ endif undeploy: ## Undeploy controller from the configured cluster in ~/.kube/config. - $(CLUSTER_CLIENT) delete --ignore-not-found=$(ignore-not-found) -f config/samples/operator_v1beta1_cryostat.yaml - $(CLUSTER_CLIENT) delete --ignore-not-found=$(ignore-not-found) -f config/samples/operator_v1beta1_clustercryostat.yaml - - $(KUSTOMIZE) build config/default | $(CLUSTER_CLIENT) delete --ignore-not-found=$(ignore-not-found) -f - + - $(KUSTOMIZE) build $(KUSTOMIZE_DIR) | $(CLUSTER_CLIENT) delete --ignore-not-found=$(ignore-not-found) -f - .PHONY: deploy_bundle deploy_bundle: check_cert_manager undeploy_bundle ## Deploy the controller in the bundle format with OLM. diff --git a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml index d16856e5..2ffce22e 100644 --- a/bundle/manifests/cryostat-operator.clusterserviceversion.yaml +++ b/bundle/manifests/cryostat-operator.clusterserviceversion.yaml @@ -54,7 +54,7 @@ metadata: capabilities: Seamless Upgrades categories: Monitoring, Developer Tools containerImage: quay.io/cryostat/cryostat-operator:2.4.0-dev - createdAt: "2023-10-27T14:13:11Z" + createdAt: "2023-11-07T21:50:35Z" description: JVM monitoring and profiling tool operatorframework.io/initialization-resource: |- { @@ -879,6 +879,15 @@ spec: spec: clusterPermissions: - rules: + - apiGroups: + - "" + resources: + - configmaps + - configmaps/finalizers + - secrets + - services + verbs: + - '*' - apiGroups: - "" resources: @@ -916,6 +925,13 @@ spec: - statefulsets verbs: - '*' + - apiGroups: + - apps + resources: + - deployments + - deployments/finalizers + verbs: + - '*' - apiGroups: - apps.openshift.io resources: @@ -1084,6 +1100,10 @@ spec: valueFrom: fieldRef: fieldPath: metadata.annotations['olm.targetNamespaces'] + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace image: quay.io/cryostat/cryostat-operator:2.4.0-dev imagePullPolicy: Always livenessProbe: diff --git a/config/insights/insights_patch.yaml b/config/insights/insights_patch.yaml new file mode 100644 index 00000000..7ad97173 --- /dev/null +++ b/config/insights/insights_patch.yaml @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + env: + - name: RELATED_IMAGE_INSIGHTS_PROXY + value: "quay.io/3scale/apicast:insights-01" + - name: INSIGHTS_ENABLED + value: "true" + - name: INSIGHTS_BACKEND_DOMAIN + value: "cert.console.redhat.com" diff --git a/config/insights/kustomization.yaml b/config/insights/kustomization.yaml new file mode 100644 index 00000000..cf37b360 --- /dev/null +++ b/config/insights/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- ../default + +patchesStrategicMerge: +- insights_patch.yaml diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index d231596e..277a0b7d 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -49,6 +49,10 @@ spec: env: - name: WATCH_NAMESPACE value: "" + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace resources: limits: cpu: 1000m diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0e3fb7dd..25853c46 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -5,6 +5,15 @@ metadata: creationTimestamp: null name: role rules: +- apiGroups: + - "" + resources: + - configmaps + - configmaps/finalizers + - secrets + - services + verbs: + - '*' - apiGroups: - "" resources: @@ -42,6 +51,13 @@ rules: - statefulsets verbs: - '*' +- apiGroups: + - apps + resources: + - deployments + - deployments/finalizers + verbs: + - '*' - apiGroups: - apps.openshift.io resources: diff --git a/hack/insights_patch.yaml.in b/hack/insights_patch.yaml.in new file mode 100644 index 00000000..dd30bf6e --- /dev/null +++ b/hack/insights_patch.yaml.in @@ -0,0 +1,17 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + env: + - name: RELATED_IMAGE_INSIGHTS_PROXY + value: "${INSIGHTS_PROXY_IMG}" + - name: INSIGHTS_ENABLED + value: "true" + - name: INSIGHTS_BACKEND_DOMAIN + value: "${INSIGHTS_BACKEND}" diff --git a/internal/controllers/clustercryostat_controller.go b/internal/controllers/clustercryostat_controller.go index cdb84e4a..a79da08e 100644 --- a/internal/controllers/clustercryostat_controller.go +++ b/internal/controllers/clustercryostat_controller.go @@ -38,13 +38,15 @@ type ClusterCryostatReconciler struct { *ReconcilerConfig } -func NewClusterCryostatReconciler(config *ReconcilerConfig) *ClusterCryostatReconciler { +func NewClusterCryostatReconciler(config *ReconcilerConfig) (*ClusterCryostatReconciler, error) { + delegate, err := newReconciler(config, &operatorv1beta1.ClusterCryostat{}, false) + if err != nil { + return nil, err + } return &ClusterCryostatReconciler{ ReconcilerConfig: config, - delegate: &Reconciler{ - ReconcilerConfig: config, - }, - } + delegate: delegate, + }, nil } // +kubebuilder:rbac:groups="",resources=pods;services;services/finalizers;endpoints;persistentvolumeclaims;events;configmaps;secrets;serviceaccounts,verbs=* @@ -94,7 +96,7 @@ func (r *ClusterCryostatReconciler) Reconcile(ctx context.Context, request ctrl. // SetupWithManager sets up the controller with the Manager. func (r *ClusterCryostatReconciler) SetupWithManager(mgr ctrl.Manager) error { - return r.delegate.setupWithManager(mgr, &operatorv1beta1.ClusterCryostat{}, r) + return r.delegate.setupWithManager(mgr, r) } func (r *ClusterCryostatReconciler) GetConfig() *ReconcilerConfig { diff --git a/internal/controllers/clustercryostat_controller_test.go b/internal/controllers/clustercryostat_controller_test.go index 51ef112f..4c93df77 100644 --- a/internal/controllers/clustercryostat_controller_test.go +++ b/internal/controllers/clustercryostat_controller_test.go @@ -117,6 +117,6 @@ func (t *cryostatTestInput) expectTargetNamespaces() { Expect(*cr.TargetNamespaceStatus).To(ConsistOf(t.TargetNamespaces)) } -func newClusterCryostatController(config *controllers.ReconcilerConfig) controllers.CommonReconciler { +func newClusterCryostatController(config *controllers.ReconcilerConfig) (controllers.CommonReconciler, error) { return controllers.NewClusterCryostatReconciler(config) } diff --git a/internal/controllers/common/common_utils.go b/internal/controllers/common/common_utils.go index b6c5310b..454d349f 100644 --- a/internal/controllers/common/common_utils.go +++ b/internal/controllers/common/common_utils.go @@ -23,7 +23,9 @@ import ( "strings" "time" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" logf "sigs.k8s.io/controller-runtime/pkg/log" ) @@ -37,21 +39,21 @@ type OSUtils interface { GenPasswd(length int) string } -type defaultOSUtils struct{} +type DefaultOSUtils struct{} // GetEnv returns the value of the environment variable with the provided name. If no such // variable exists, the empty string is returned. -func (o *defaultOSUtils) GetEnv(name string) string { +func (o *DefaultOSUtils) GetEnv(name string) string { return os.Getenv(name) } // GetFileContents reads and returns the entire file contents specified by the path -func (o *defaultOSUtils) GetFileContents(path string) ([]byte, error) { +func (o *DefaultOSUtils) GetFileContents(path string) ([]byte, error) { return ioutil.ReadFile(path) } // GenPasswd generates a psuedorandom password of a given length. -func (o *defaultOSUtils) GenPasswd(length int) string { +func (o *DefaultOSUtils) GenPasswd(length int) string { rand.Seed(time.Now().UnixNano()) chars := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_" b := make([]byte, length) @@ -63,13 +65,16 @@ func (o *defaultOSUtils) GenPasswd(length int) string { // ClusterUniqueName returns a name for cluster-scoped objects that is // uniquely identified by a namespace and name. -func ClusterUniqueName(kind string, name string, namespace string) string { +func ClusterUniqueName(gvk *schema.GroupVersionKind, name string, namespace string) string { // Use the SHA256 checksum of the namespaced name as a suffix nn := types.NamespacedName{Namespace: namespace, Name: name} suffix := fmt.Sprintf("%x", sha256.Sum256([]byte(nn.String()))) - return strings.ToLower(kind) + "-" + suffix + return strings.ToLower(gvk.Kind) + "-" + suffix } +// MergeLabelsAndAnnotations copies labels and annotations from a source +// to the destination ObjectMeta, overwriting any existing labels and +// annotations of the same key. func MergeLabelsAndAnnotations(dest *metav1.ObjectMeta, srcLabels, srcAnnotations map[string]string) { // Check and create labels/annotations map if absent if dest.Labels == nil { @@ -83,8 +88,23 @@ func MergeLabelsAndAnnotations(dest *metav1.ObjectMeta, srcLabels, srcAnnotation for k, v := range srcLabels { dest.Labels[k] = v } - for k, v := range srcAnnotations { dest.Annotations[k] = v } } + +// SeccompProfile returns a SeccompProfile for the restricted +// Pod Security Standard that, on OpenShift, is backwards-compatible +// with OpenShift < 4.11. +// TODO Remove once OpenShift < 4.11 support is dropped +func SeccompProfile(openshift bool) *corev1.SeccompProfile { + // For backward-compatibility with OpenShift < 4.11, + // leave the seccompProfile empty. In OpenShift >= 4.11, + // the restricted-v2 SCC will populate it for us. + if openshift { + return nil + } + return &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + } +} diff --git a/internal/controllers/common/resource_definitions/resource_definitions.go b/internal/controllers/common/resource_definitions/resource_definitions.go index ad93b5a8..d2872bd4 100644 --- a/internal/controllers/common/resource_definitions/resource_definitions.go +++ b/internal/controllers/common/resource_definitions/resource_definitions.go @@ -41,9 +41,10 @@ type ImageTags struct { } type ServiceSpecs struct { - CoreURL *url.URL - GrafanaURL *url.URL - ReportsURL *url.URL + CoreURL *url.URL + GrafanaURL *url.URL + ReportsURL *url.URL + InsightsURL *url.URL } // TLSConfig contains TLS-related information useful when creating other objects @@ -390,7 +391,7 @@ func NewPodForCR(cr *model.CryostatInstance, specs *ServiceSpecs, imageTags *Ima // Ensure PV mounts are writable FSGroup: &fsGroup, RunAsNonRoot: &nonRoot, - SeccompProfile: seccompProfile(openshift), + SeccompProfile: common.SeccompProfile(openshift), } } @@ -444,10 +445,6 @@ func NewReportContainerResource(cr *model.CryostatInstance) *corev1.ResourceRequ return resources } -// ALL capability to drop for restricted pod security. See: -// https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted -const capabilityAll corev1.Capability = "ALL" - func NewPodForReports(cr *model.CryostatInstance, imageTags *ImageTags, tls *TLSConfig, openshift bool) *corev1.PodSpec { resources := NewReportContainerResource(cr) cpus := resources.Requests.Cpu().Value() // Round to 1 if cpu request < 1000m @@ -537,7 +534,7 @@ func NewPodForReports(cr *model.CryostatInstance, imageTags *ImageTags, tls *TLS nonRoot := true podSc = &corev1.PodSecurityContext{ RunAsNonRoot: &nonRoot, - SeccompProfile: seccompProfile(openshift), + SeccompProfile: common.SeccompProfile(openshift), } } @@ -549,7 +546,7 @@ func NewPodForReports(cr *model.CryostatInstance, imageTags *ImageTags, tls *TLS containerSc = &corev1.SecurityContext{ AllowPrivilegeEscalation: &privEscalation, Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{capabilityAll}, + Drop: []corev1.Capability{constants.CapabilityAll}, }, } } @@ -729,6 +726,17 @@ func NewCoreContainer(cr *model.CryostatInstance, specs *ServiceSpecs, imageTag envs = append(envs, subprocessReportHeapEnv...) } + // Define INSIGHTS_PROXY URL if Insights integration is enabled + if specs.InsightsURL != nil { + insightsEnvs := []corev1.EnvVar{ + { + Name: "INSIGHTS_PROXY", + Value: specs.InsightsURL.String(), + }, + } + envs = append(envs, insightsEnvs...) + } + if cr.Spec.MaxWsConnections != 0 { maxWsConnections := strconv.Itoa(int(cr.Spec.MaxWsConnections)) maxWsConnectionsEnv := []corev1.EnvVar{ @@ -947,7 +955,7 @@ func NewCoreContainer(cr *model.CryostatInstance, specs *ServiceSpecs, imageTag containerSc = &corev1.SecurityContext{ AllowPrivilegeEscalation: &privEscalation, Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{capabilityAll}, + Drop: []corev1.Capability{constants.CapabilityAll}, }, } } @@ -1037,7 +1045,7 @@ func NewGrafanaContainer(cr *model.CryostatInstance, imageTag string, tls *TLSCo containerSc = &corev1.SecurityContext{ AllowPrivilegeEscalation: &privEscalation, Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{capabilityAll}, + Drop: []corev1.Capability{constants.CapabilityAll}, }, } } @@ -1097,7 +1105,7 @@ func NewJfrDatasourceContainer(cr *model.CryostatInstance, imageTag string) core containerSc = &corev1.SecurityContext{ AllowPrivilegeEscalation: &privEscalation, Capabilities: &corev1.Capabilities{ - Drop: []corev1.Capability{capabilityAll}, + Drop: []corev1.Capability{constants.CapabilityAll}, }, } } @@ -1195,18 +1203,6 @@ func newVolumeForCR(cr *model.CryostatInstance) []corev1.Volume { } } -func seccompProfile(openshift bool) *corev1.SeccompProfile { - // For backward-compatibility with OpenShift < 4.11, - // leave the seccompProfile empty. In OpenShift >= 4.11, - // the restricted-v2 SCC will populate it for us. - if openshift { - return nil - } - return &corev1.SeccompProfile{ - Type: corev1.SeccompProfileTypeRuntimeDefault, - } -} - func useEmptyDir(cr *model.CryostatInstance) bool { return cr.Spec.StorageOptions != nil && cr.Spec.StorageOptions.EmptyDir != nil && cr.Spec.StorageOptions.EmptyDir.Enabled diff --git a/internal/controllers/common/tls.go b/internal/controllers/common/tls.go index 793c6217..03723017 100644 --- a/internal/controllers/common/tls.go +++ b/internal/controllers/common/tls.go @@ -59,7 +59,7 @@ const disableServiceTLS = "DISABLE_SERVICE_TLS" func NewReconcilerTLS(config *ReconcilerTLSConfig) ReconcilerTLS { configCopy := *config if config.OSUtils == nil { - configCopy.OSUtils = &defaultOSUtils{} + configCopy.OSUtils = &DefaultOSUtils{} } return &reconcilerTLS{ ReconcilerTLSConfig: &configCopy, diff --git a/internal/controllers/constants/constants.go b/internal/controllers/constants/constants.go index 5a68859b..cc9541bf 100644 --- a/internal/controllers/constants/constants.go +++ b/internal/controllers/constants/constants.go @@ -16,6 +16,7 @@ package constants import ( certMeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" ) const ( @@ -26,9 +27,13 @@ const ( ReportsContainerPort int32 = 10000 LoopbackAddress string = "127.0.0.1" OperatorNamePrefix string = "cryostat-operator-" + OperatorDeploymentName string = "cryostat-operator-controller-manager" HttpPortName string = "http" // CAKey is the key for a CA certificate within a TLS secret CAKey = certMeta.TLSCAKey // Hostname alias for loopback address, to be used for health checks HealthCheckHostname = "cryostat-health.local" + // ALL capability to drop for restricted pod security. See: + // https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted + CapabilityAll corev1.Capability = "ALL" ) diff --git a/internal/controllers/cryostat_controller.go b/internal/controllers/cryostat_controller.go index ee07fd49..554f16e2 100644 --- a/internal/controllers/cryostat_controller.go +++ b/internal/controllers/cryostat_controller.go @@ -35,13 +35,15 @@ type CryostatReconciler struct { *ReconcilerConfig } -func NewCryostatReconciler(config *ReconcilerConfig) *CryostatReconciler { +func NewCryostatReconciler(config *ReconcilerConfig) (*CryostatReconciler, error) { + delegate, err := newReconciler(config, &operatorv1beta1.Cryostat{}, true) + if err != nil { + return nil, err + } return &CryostatReconciler{ ReconcilerConfig: config, - delegate: &Reconciler{ - ReconcilerConfig: config, - }, - } + delegate: delegate, + }, nil } // +kubebuilder:rbac:groups=operator.cryostat.io,resources=cryostats,verbs=* @@ -75,7 +77,7 @@ func (r *CryostatReconciler) Reconcile(ctx context.Context, request ctrl.Request // SetupWithManager sets up the controller with the Manager. func (r *CryostatReconciler) SetupWithManager(mgr ctrl.Manager) error { - return r.delegate.setupWithManager(mgr, &operatorv1beta1.Cryostat{}, r) + return r.delegate.setupWithManager(mgr, r) } func (r *CryostatReconciler) GetConfig() *ReconcilerConfig { diff --git a/internal/controllers/cryostat_controller_test.go b/internal/controllers/cryostat_controller_test.go index e5a910dc..2530f4ed 100644 --- a/internal/controllers/cryostat_controller_test.go +++ b/internal/controllers/cryostat_controller_test.go @@ -28,6 +28,6 @@ var _ = Describe("CryostatController", func() { c.commonTests() }) -func newCryostatController(config *controllers.ReconcilerConfig) controllers.CommonReconciler { +func newCryostatController(config *controllers.ReconcilerConfig) (controllers.CommonReconciler, error) { return controllers.NewCryostatReconciler(config) } diff --git a/internal/controllers/insights/apicast.go b/internal/controllers/insights/apicast.go new file mode 100644 index 00000000..4fabf60b --- /dev/null +++ b/internal/controllers/insights/apicast.go @@ -0,0 +1,97 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package insights + +import ( + "bytes" + "text/template" +) + +type apiCastConfigParams struct { + FrontendDomains string + BackendInsightsDomain string + HeaderValue string + ProxyDomain string +} + +var apiCastConfigTemplate = template.Must(template.New("").Parse(`{ + "services": [ + { + "id": "1", + "backend_version": "1", + "proxy": { + "hosts": [{{ .FrontendDomains }}], + "api_backend": "https://{{ .BackendInsightsDomain }}:443/", + "backend": { "endpoint": "http://127.0.0.1:8081", "host": "backend" }, + "policy_chain": [ + { + "name": "default_credentials", + "version": "builtin", + "configuration": { + "auth_type": "user_key", + "user_key": "dummy_key" + } + }, + {{- if .ProxyDomain }} + { + "name": "apicast.policy.http_proxy", + "configuration": { + "https_proxy": "http://{{ .ProxyDomain }}/", + "http_proxy": "http://{{ .ProxyDomain }}/" + } + }, + {{- end }} + { + "name": "headers", + "version": "builtin", + "configuration": { + "request": [ + { + "op": "set", + "header": "Authorization", + "value_type": "plain", + "value": "Bearer {{ .HeaderValue }}" + } + ] + } + }, + { + "name": "apicast.policy.apicast" + } + ], + "proxy_rules": [ + { + "http_method": "POST", + "pattern": "/", + "metric_system_name": "hits", + "delta": 1, + "parameters": [], + "querystring_parameters": {} + } + ] + } + } + ] +}`)) + +func getAPICastConfig(params *apiCastConfigParams) (*string, error) { + buf := &bytes.Buffer{} + err := apiCastConfigTemplate.Execute(buf, params) + if err != nil { + return nil, err + } + result := buf.String() + return &result, nil +} diff --git a/internal/controllers/insights/insights.go b/internal/controllers/insights/insights.go new file mode 100644 index 00000000..0018ec45 --- /dev/null +++ b/internal/controllers/insights/insights.go @@ -0,0 +1,356 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package insights + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/cryostatio/cryostat-operator/internal/controllers/common" + "github.com/cryostatio/cryostat-operator/internal/controllers/constants" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func (r *InsightsReconciler) reconcileInsights(ctx context.Context) error { + err := r.reconcilePullSecret(ctx) + if err != nil { + return err + } + err = r.reconcileProxyDeployment(ctx) + if err != nil { + return err + } + return r.reconcileProxyService(ctx) +} + +func (r *InsightsReconciler) reconcilePullSecret(ctx context.Context) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProxySecretName, + Namespace: r.Namespace, + }, + } + owner := &corev1.ConfigMap{} + err := r.Client.Get(ctx, types.NamespacedName{Name: InsightsConfigMapName, + Namespace: r.Namespace}, owner) + if err != nil { + return err + } + + token, err := r.getTokenFromPullSecret(ctx) + if err != nil { + return err + } + + params := &apiCastConfigParams{ + FrontendDomains: fmt.Sprintf("\"%s\",\"%s.%s.svc.cluster.local\"", ProxyServiceName, ProxyServiceName, r.Namespace), + BackendInsightsDomain: r.backendDomain, + ProxyDomain: r.proxyDomain, + HeaderValue: *token, + } + config, err := getAPICastConfig(params) + if err != nil { + return err + } + + return r.createOrUpdateProxySecret(ctx, secret, owner, *config) +} + +func (r *InsightsReconciler) reconcileProxyDeployment(ctx context.Context) error { + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProxyDeploymentName, + Namespace: r.Namespace, + }, + } + owner := &corev1.ConfigMap{} + err := r.Client.Get(ctx, types.NamespacedName{Name: InsightsConfigMapName, + Namespace: r.Namespace}, owner) + if err != nil { + return err + } + + return r.createOrUpdateProxyDeployment(ctx, deploy, owner) +} + +func (r *InsightsReconciler) reconcileProxyService(ctx context.Context) error { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: ProxyServiceName, + Namespace: r.Namespace, + }, + } + owner := &corev1.ConfigMap{} + err := r.Client.Get(ctx, types.NamespacedName{Name: InsightsConfigMapName, + Namespace: r.Namespace}, owner) + if err != nil { + return err + } + + return r.createOrUpdateProxyService(ctx, svc, owner) +} + +func (r *InsightsReconciler) getTokenFromPullSecret(ctx context.Context) (*string, error) { + // Get the global pull secret + pullSecret := &corev1.Secret{} + err := r.Client.Get(ctx, types.NamespacedName{Namespace: "openshift-config", Name: "pull-secret"}, pullSecret) + if err != nil { + return nil, err + } + + // Look for the .dockerconfigjson key within it + dockerConfigRaw, pres := pullSecret.Data[corev1.DockerConfigJsonKey] + if !pres { + return nil, fmt.Errorf("no %s key present in pull secret", corev1.DockerConfigJsonKey) + } + + // Unmarshal the .dockerconfigjson into a struct + dockerConfig := struct { + Auths map[string]struct { + Auth string `json:"auth"` + } `json:"auths"` + }{} + err = json.Unmarshal(dockerConfigRaw, &dockerConfig) + if err != nil { + return nil, err + } + + // Look for the "cloud.openshift.com" auth + openshiftAuth, pres := dockerConfig.Auths["cloud.openshift.com"] + if !pres { + return nil, errors.New("no \"cloud.openshift.com\" auth within pull secret") + } + + token := strings.TrimSpace(openshiftAuth.Auth) + if strings.Contains(token, "\n") || strings.Contains(token, "\r") { + return nil, fmt.Errorf("invalid cloud.openshift.com token") + } + return &token, nil +} + +func (r *InsightsReconciler) createOrUpdateProxySecret(ctx context.Context, secret *corev1.Secret, owner metav1.Object, + config string) error { + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, secret, func() error { + // Set the config map as controller + if err := controllerutil.SetControllerReference(owner, secret, r.Scheme); err != nil { + return err + } + // Add the APICast config.json + if secret.StringData == nil { + secret.StringData = map[string]string{} + } + secret.StringData["config.json"] = config + return nil + }) + if err != nil { + return err + } + r.Log.Info(fmt.Sprintf("Secret %s", op), "name", secret.Name, "namespace", secret.Namespace) + return nil +} + +func (r *InsightsReconciler) createOrUpdateProxyDeployment(ctx context.Context, deploy *appsv1.Deployment, owner metav1.Object) error { + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, deploy, func() error { + labels := map[string]string{"app": ProxyDeploymentName} + annotations := map[string]string{} + common.MergeLabelsAndAnnotations(&deploy.ObjectMeta, labels, annotations) + // Set the config map as controller + if err := controllerutil.SetControllerReference(owner, deploy, r.Scheme); err != nil { + return err + } + // Immutable, only updated when the deployment is created + if deploy.CreationTimestamp.IsZero() { + // Selector is immutable, avoid modifying if possible + deploy.Spec.Selector = &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": ProxyDeploymentName, + }, + } + } + + // Update pod template spec + r.createOrUpdateProxyPodSpec(deploy) + // Update pod template metadata + common.MergeLabelsAndAnnotations(&deploy.Spec.Template.ObjectMeta, labels, annotations) + return nil + }) + if err != nil { + return err + } + r.Log.Info(fmt.Sprintf("Deployment %s", op), "name", deploy.Name, "namespace", deploy.Namespace) + return nil +} + +func (r *InsightsReconciler) createOrUpdateProxyService(ctx context.Context, svc *corev1.Service, owner metav1.Object) error { + op, err := controllerutil.CreateOrUpdate(ctx, r.Client, svc, func() error { + // Update labels and annotations + labels := map[string]string{"app": ProxyDeploymentName} + annotations := map[string]string{} + common.MergeLabelsAndAnnotations(&svc.ObjectMeta, labels, annotations) + + // Set the config map as controller + if err := controllerutil.SetControllerReference(owner, svc, r.Scheme); err != nil { + return err + } + // Update the service type + svc.Spec.Type = corev1.ServiceTypeClusterIP + svc.Spec.Selector = map[string]string{ + "app": ProxyDeploymentName, + } + svc.Spec.Ports = []corev1.ServicePort{ + { + Name: "proxy", + Port: 8080, + TargetPort: intstr.FromString("proxy"), + }, + { + Name: "management", + Port: 8090, + TargetPort: intstr.FromString("management"), + }, + } + return nil + }) + if err != nil { + return err + } + r.Log.Info(fmt.Sprintf("Service %s", op), "name", svc.Name, "namespace", svc.Namespace) + return nil +} + +const ( + defaultProxyCPURequest = "50m" + defaultProxyCPULimit = "200m" + defaultProxyMemRequest = "64Mi" + defaultProxyMemLimit = "128Mi" +) + +func (r *InsightsReconciler) createOrUpdateProxyPodSpec(deploy *appsv1.Deployment) { + privEscalation := false + nonRoot := true + readOnlyMode := int32(0440) + + podSpec := &deploy.Spec.Template.Spec + // Create the container if it doesn't exist + var container *corev1.Container + if deploy.CreationTimestamp.IsZero() { + podSpec.Containers = []corev1.Container{{}} + } + container = &podSpec.Containers[0] + + // Set fields that are hard-coded by operator + container.Name = ProxyDeploymentName + container.Image = r.proxyImageTag + container.Env = []corev1.EnvVar{ + { + Name: "THREESCALE_CONFIG_FILE", + Value: "/tmp/gateway-configuration-volume/config.json", + }, + } + container.VolumeMounts = []corev1.VolumeMount{ + { + Name: "gateway-configuration-volume", + MountPath: "/tmp/gateway-configuration-volume", + ReadOnly: true, + }, + } + container.Ports = []corev1.ContainerPort{ + { + Name: "proxy", + ContainerPort: 8080, + }, + { + Name: "management", + ContainerPort: 8090, + }, + { + Name: "metrics", + ContainerPort: 9421, + }, + } + container.SecurityContext = &corev1.SecurityContext{ + AllowPrivilegeEscalation: &privEscalation, + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{constants.CapabilityAll}, + }, + } + container.LivenessProbe = &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 5, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/status/live", + Port: intstr.FromInt(8090), + }, + }, + } + container.ReadinessProbe = &corev1.Probe{ + InitialDelaySeconds: 15, + PeriodSeconds: 30, + TimeoutSeconds: 5, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/status/ready", + Port: intstr.FromInt(8090), + }, + }, + } + + // Set resource requirements only on creation, this allows + // the user to modify them if they wish + if deploy.CreationTimestamp.IsZero() { + container.Resources = corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(defaultProxyCPURequest), + corev1.ResourceMemory: resource.MustParse(defaultProxyMemRequest), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse(defaultProxyCPULimit), + corev1.ResourceMemory: resource.MustParse(defaultProxyMemLimit), + }, + } + } + + podSpec.Volumes = []corev1.Volume{ // TODO detect change and redeploy + { + Name: "gateway-configuration-volume", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: ProxySecretName, + Items: []corev1.KeyToPath{ + { + Key: "config.json", + Path: "config.json", + Mode: &readOnlyMode, + }, + }, + }, + }, + }, + } + podSpec.SecurityContext = &corev1.PodSecurityContext{ + RunAsNonRoot: &nonRoot, + SeccompProfile: common.SeccompProfile(true), + } +} diff --git a/internal/controllers/insights/insights_controller.go b/internal/controllers/insights/insights_controller.go new file mode 100644 index 00000000..94950a8e --- /dev/null +++ b/internal/controllers/insights/insights_controller.go @@ -0,0 +1,140 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package insights + +import ( + "context" + "errors" + + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/cryostatio/cryostat-operator/internal/controllers/common" + "github.com/go-logr/logr" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +// InsightsReconciler reconciles the Insights proxy for Cryostat agents +type InsightsReconciler struct { + *InsightsReconcilerConfig + backendDomain string + proxyDomain string + proxyImageTag string +} + +// InsightsReconcilerConfig contains configuration to create an InsightsReconciler +type InsightsReconcilerConfig struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + Namespace string + common.OSUtils +} + +const ( + InsightsConfigMapName = "insights-proxy" + ProxyDeploymentName = InsightsConfigMapName + ProxyServiceName = ProxyDeploymentName + ProxySecretName = "apicastconf" + EnvInsightsBackendDomain = "INSIGHTS_BACKEND_DOMAIN" + EnvInsightsProxyDomain = "INSIGHTS_PROXY_DOMAIN" + EnvInsightsEnabled = "INSIGHTS_ENABLED" + // Environment variable to override the Insights proxy image + EnvInsightsProxyImageTag = "RELATED_IMAGE_INSIGHTS_PROXY" +) + +// NewInsightsReconciler creates an InsightsReconciler using the provided configuration +func NewInsightsReconciler(config *InsightsReconcilerConfig) (*InsightsReconciler, error) { + backendDomain := config.GetEnv(EnvInsightsBackendDomain) + if len(backendDomain) == 0 { + return nil, errors.New("no backend domain provided for Insights") + } + imageTag := config.GetEnv(EnvInsightsProxyImageTag) + if len(imageTag) == 0 { + return nil, errors.New("no proxy image tag provided for Insights") + } + proxyDomain := config.GetEnv(EnvInsightsProxyDomain) + + return &InsightsReconciler{ + InsightsReconcilerConfig: config, + backendDomain: backendDomain, + proxyDomain: proxyDomain, + proxyImageTag: imageTag, + }, nil +} + +// +kubebuilder:rbac:groups=apps,resources=deployments;deployments/finalizers,verbs=* +// +kubebuilder:rbac:groups="",resources=services;secrets;configmaps;configmaps/finalizers,verbs=* + +// Reconcile processes the Insights proxy deployment and configures it accordingly +func (r *InsightsReconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { + reqLogger := r.Log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name) + reqLogger.Info("Reconciling Insights Proxy") + + // Reconcile all Insights support + err := r.reconcileInsights(ctx) + if err != nil { + return reconcile.Result{}, err + } + return reconcile.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *InsightsReconciler) SetupWithManager(mgr ctrl.Manager) error { + c := ctrl.NewControllerManagedBy(mgr). + Named("insights"). + // Filter controller to watch only specific objects we care about + Watches(&source.Kind{Type: &corev1.Secret{}}, + handler.EnqueueRequestsFromMapFunc(r.isPullSecretOrProxyConfig)). + Watches(&source.Kind{Type: &appsv1.Deployment{}}, + handler.EnqueueRequestsFromMapFunc(r.isProxyDeployment)). + Watches(&source.Kind{Type: &corev1.Service{}}, + handler.EnqueueRequestsFromMapFunc(r.isProxyService)) + return c.Complete(r) +} + +func (r *InsightsReconciler) isPullSecretOrProxyConfig(secret client.Object) []reconcile.Request { + if !(secret.GetNamespace() == "openshift-config" && secret.GetName() == "pull-secret") && + !(secret.GetNamespace() == r.Namespace && secret.GetName() == ProxySecretName) { + return nil + } + return r.proxyDeploymentRequest() +} + +func (r *InsightsReconciler) isProxyDeployment(deploy client.Object) []reconcile.Request { + if deploy.GetNamespace() != r.Namespace || deploy.GetName() != ProxyDeploymentName { + return nil + } + return r.proxyDeploymentRequest() +} + +func (r *InsightsReconciler) isProxyService(svc client.Object) []reconcile.Request { + if svc.GetNamespace() != r.Namespace || svc.GetName() != ProxyServiceName { + return nil + } + return r.proxyDeploymentRequest() +} + +func (r *InsightsReconciler) proxyDeploymentRequest() []reconcile.Request { + req := reconcile.Request{NamespacedName: types.NamespacedName{Namespace: r.Namespace, Name: ProxyDeploymentName}} + return []reconcile.Request{req} +} diff --git a/internal/controllers/insights/insights_controller_test.go b/internal/controllers/insights/insights_controller_test.go new file mode 100644 index 00000000..b982bdef --- /dev/null +++ b/internal/controllers/insights/insights_controller_test.go @@ -0,0 +1,265 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package insights_test + +import ( + "context" + + "github.com/cryostatio/cryostat-operator/internal/controllers/insights" + insightstest "github.com/cryostatio/cryostat-operator/internal/controllers/insights/test" + "github.com/cryostatio/cryostat-operator/internal/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type insightsTestInput struct { + client ctrlclient.Client + controller *insights.InsightsReconciler + objs []ctrlclient.Object + *insightstest.TestUtilsConfig + *insightstest.InsightsTestResources +} + +var _ = Describe("InsightsController", func() { + var t *insightsTestInput + + Describe("reconciling a request", func() { + BeforeEach(func() { + t = &insightsTestInput{ + TestUtilsConfig: &insightstest.TestUtilsConfig{ + EnvInsightsEnabled: &[]bool{true}[0], + EnvInsightsBackendDomain: &[]string{"insights.example.com"}[0], + EnvInsightsProxyImageTag: &[]string{"example.com/proxy:latest"}[0], + }, + InsightsTestResources: &insightstest.InsightsTestResources{ + TestResources: &test.TestResources{ + Namespace: "test", + }, + }, + } + t.objs = []ctrlclient.Object{ + t.NewNamespace(), + t.NewGlobalPullSecret(), + t.NewOperatorDeployment(), + t.NewProxyConfigMap(), + } + }) + + JustBeforeEach(func() { + s := test.NewTestScheme() + logger := zap.New() + logf.SetLogger(logger) + + // Set a CreationTimestamp for created objects to match a real API server + // TODO When using envtest instead of fake client, this is probably no longer needed + err := test.SetCreationTimestamp(t.objs...) + Expect(err).ToNot(HaveOccurred()) + t.client = fake.NewClientBuilder().WithScheme(s).WithObjects(t.objs...).Build() + + config := &insights.InsightsReconcilerConfig{ + Client: test.NewClientWithTimestamp(test.NewTestClient(t.client, t.TestResources)), + Scheme: s, + Log: logger, + Namespace: t.Namespace, + OSUtils: insightstest.NewTestOSUtils(t.TestUtilsConfig), + } + t.controller, err = insights.NewInsightsReconciler(config) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("successfully creates required resources", func() { + Context("with defaults", func() { + JustBeforeEach(func() { + result, err := t.reconcile() + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + }) + It("should create the APICast config secret", func() { + expected := t.NewInsightsProxySecret() + actual := &corev1.Secret{} + err := t.client.Get(context.Background(), types.NamespacedName{ + Name: expected.Name, + Namespace: expected.Namespace, + }, actual) + Expect(err).ToNot(HaveOccurred()) + + Expect(actual.Labels).To(Equal(expected.Labels)) + Expect(actual.Annotations).To(Equal(expected.Annotations)) + Expect(metav1.IsControlledBy(actual, t.NewProxyConfigMap())).To(BeTrue()) + Expect(actual.StringData).To(HaveLen(1)) + Expect(actual.StringData).To(HaveKey("config.json")) + Expect(actual.StringData["config.json"]).To(MatchJSON(expected.StringData["config.json"])) + }) + It("should create the proxy deployment", func() { + expected := t.NewInsightsProxyDeployment() + actual := &appsv1.Deployment{} + err := t.client.Get(context.Background(), types.NamespacedName{ + Name: expected.Name, + Namespace: expected.Namespace, + }, actual) + Expect(err).ToNot(HaveOccurred()) + + t.checkProxyDeployment(actual, expected) + }) + It("should create the proxy service", func() { + expected := t.NewInsightsProxyService() + actual := &corev1.Service{} + err := t.client.Get(context.Background(), types.NamespacedName{ + Name: expected.Name, + Namespace: expected.Namespace, + }, actual) + Expect(err).ToNot(HaveOccurred()) + + Expect(actual.Labels).To(Equal(expected.Labels)) + Expect(actual.Annotations).To(Equal(expected.Annotations)) + Expect(metav1.IsControlledBy(actual, t.NewProxyConfigMap())).To(BeTrue()) + + Expect(actual.Spec.Selector).To(Equal(expected.Spec.Selector)) + Expect(actual.Spec.Type).To(Equal(expected.Spec.Type)) + Expect(actual.Spec.Ports).To(ConsistOf(expected.Spec.Ports)) + }) + }) + Context("with a proxy domain", func() { + BeforeEach(func() { + t.EnvInsightsProxyDomain = &[]string{"proxy.example.com"}[0] + }) + JustBeforeEach(func() { + result, err := t.reconcile() + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + }) + It("should create the APICast config secret", func() { + expected := t.NewInsightsProxySecretWithProxyDomain() + actual := &corev1.Secret{} + err := t.client.Get(context.Background(), types.NamespacedName{ + Name: expected.Name, + Namespace: expected.Namespace, + }, actual) + Expect(err).ToNot(HaveOccurred()) + + Expect(actual.Labels).To(Equal(expected.Labels)) + Expect(actual.Annotations).To(Equal(expected.Annotations)) + Expect(metav1.IsControlledBy(actual, t.NewProxyConfigMap())).To(BeTrue()) + Expect(actual.StringData).To(HaveLen(1)) + Expect(actual.StringData).To(HaveKey("config.json")) + Expect(actual.StringData["config.json"]).To(MatchJSON(expected.StringData["config.json"])) + }) + }) + }) + Context("updating the deployment", func() { + BeforeEach(func() { + t.objs = append(t.objs, + t.NewInsightsProxyDeployment(), + t.NewInsightsProxySecret(), + t.NewInsightsProxyService(), + ) + }) + Context("with resource requirements", func() { + var resources *corev1.ResourceRequirements + + BeforeEach(func() { + resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("500m"), + corev1.ResourceMemory: resource.MustParse("256Mi"), + }, + } + }) + JustBeforeEach(func() { + // Fetch the deployment + deploy := t.getProxyDeployment() + + // Change the resource requirements + deploy.Spec.Template.Spec.Containers[0].Resources = *resources + + // Update the deployment + err := t.client.Update(context.Background(), deploy) + Expect(err).ToNot(HaveOccurred()) + + // Reconcile again + result, err := t.reconcile() + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(Equal(reconcile.Result{})) + }) + It("should leave the custom resource requirements", func() { + // Fetch the deployment again + actual := t.getProxyDeployment() + + // Check only resource requirements differ from defaults + t.Resources = resources + expected := t.NewInsightsProxyDeployment() + t.checkProxyDeployment(actual, expected) + }) + }) + }) + }) +}) + +func (t *insightsTestInput) reconcile() (reconcile.Result, error) { + req := reconcile.Request{NamespacedName: types.NamespacedName{Name: "insights-proxy", Namespace: t.Namespace}} + return t.controller.Reconcile(context.Background(), req) +} + +func (t *insightsTestInput) getProxyDeployment() *appsv1.Deployment { + deploy := t.NewInsightsProxyDeployment() + err := t.client.Get(context.Background(), types.NamespacedName{ + Name: deploy.Name, + Namespace: deploy.Namespace, + }, deploy) + Expect(err).ToNot(HaveOccurred()) + return deploy +} + +func (t *insightsTestInput) checkProxyDeployment(actual, expected *appsv1.Deployment) { + Expect(actual.Labels).To(Equal(expected.Labels)) + Expect(actual.Annotations).To(Equal(expected.Annotations)) + Expect(metav1.IsControlledBy(actual, t.NewProxyConfigMap())).To(BeTrue()) + Expect(actual.Spec.Selector).To(Equal(expected.Spec.Selector)) + + expectedTemplate := expected.Spec.Template + actualTemplate := actual.Spec.Template + Expect(actualTemplate.Labels).To(Equal(expectedTemplate.Labels)) + Expect(actualTemplate.Annotations).To(Equal(expectedTemplate.Annotations)) + Expect(actualTemplate.Spec.SecurityContext).To(Equal(expectedTemplate.Spec.SecurityContext)) + Expect(actualTemplate.Spec.Volumes).To(Equal(expectedTemplate.Spec.Volumes)) + + Expect(actualTemplate.Spec.Containers).To(HaveLen(1)) + expectedContainer := expectedTemplate.Spec.Containers[0] + actualContainer := actualTemplate.Spec.Containers[0] + Expect(actualContainer.Ports).To(ConsistOf(expectedContainer.Ports)) + Expect(actualContainer.Env).To(ConsistOf(expectedContainer.Env)) + Expect(actualContainer.EnvFrom).To(ConsistOf(expectedContainer.EnvFrom)) + Expect(actualContainer.VolumeMounts).To(ConsistOf(expectedContainer.VolumeMounts)) + Expect(actualContainer.LivenessProbe).To(Equal(expectedContainer.LivenessProbe)) + Expect(actualContainer.StartupProbe).To(Equal(expectedContainer.StartupProbe)) + Expect(actualContainer.SecurityContext).To(Equal(expectedContainer.SecurityContext)) + + test.ExpectResourceRequirements(&actualContainer.Resources, &expectedContainer.Resources) +} diff --git a/internal/controllers/insights/insights_controller_unit_test.go b/internal/controllers/insights/insights_controller_unit_test.go new file mode 100644 index 00000000..ccaf682e --- /dev/null +++ b/internal/controllers/insights/insights_controller_unit_test.go @@ -0,0 +1,149 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package insights + +import ( + insightstest "github.com/cryostatio/cryostat-operator/internal/controllers/insights/test" + "github.com/cryostatio/cryostat-operator/internal/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +type insightsUnitTestInput struct { + client ctrlclient.Client + controller *InsightsReconciler + objs []ctrlclient.Object + *insightstest.TestUtilsConfig + *insightstest.InsightsTestResources +} + +var _ = Describe("InsightsController", func() { + var t *insightsUnitTestInput + + Describe("configuring watches", func() { + + BeforeEach(func() { + t = &insightsUnitTestInput{ + TestUtilsConfig: &insightstest.TestUtilsConfig{ + EnvInsightsEnabled: &[]bool{true}[0], + EnvInsightsBackendDomain: &[]string{"insights.example.com"}[0], + EnvInsightsProxyImageTag: &[]string{"example.com/proxy:latest"}[0], + EnvNamespace: &[]string{"test"}[0], + }, + InsightsTestResources: &insightstest.InsightsTestResources{ + TestResources: &test.TestResources{ + Namespace: "test", + }, + }, + } + t.objs = []ctrlclient.Object{ + t.NewNamespace(), + t.NewGlobalPullSecret(), + t.NewOperatorDeployment(), + } + }) + + JustBeforeEach(func() { + s := test.NewTestScheme() + logger := zap.New() + logf.SetLogger(logger) + + // Set a CreationTimestamp for created objects to match a real API server + // TODO When using envtest instead of fake client, this is probably no longer needed + err := test.SetCreationTimestamp(t.objs...) + Expect(err).ToNot(HaveOccurred()) + t.client = fake.NewClientBuilder().WithScheme(s).WithObjects(t.objs...).Build() + + config := &InsightsReconcilerConfig{ + Client: test.NewClientWithTimestamp(test.NewTestClient(t.client, t.TestResources)), + Scheme: s, + Log: logger, + Namespace: t.Namespace, + OSUtils: insightstest.NewTestOSUtils(t.TestUtilsConfig), + } + t.controller, err = NewInsightsReconciler(config) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("for secrets", func() { + It("should reconcile global pull secret", func() { + result := t.controller.isPullSecretOrProxyConfig(t.NewGlobalPullSecret()) + Expect(result).To(ConsistOf(t.deploymentReconcileRequest())) + }) + It("should reconcile APICast secret", func() { + result := t.controller.isPullSecretOrProxyConfig(t.NewInsightsProxySecret()) + Expect(result).To(ConsistOf(t.deploymentReconcileRequest())) + }) + It("should not reconcile a secret in another namespace", func() { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: t.NewGlobalPullSecret().Name, + Namespace: "other", + }, + } + result := t.controller.isPullSecretOrProxyConfig(secret) + Expect(result).To(BeEmpty()) + }) + }) + + Context("for deployments", func() { + It("should reconcile proxy deployment", func() { + result := t.controller.isProxyDeployment(t.NewInsightsProxyDeployment()) + Expect(result).To(ConsistOf(t.deploymentReconcileRequest())) + }) + It("should not reconcile a deployment in another namespace", func() { + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: t.NewInsightsProxyDeployment().Name, + Namespace: "other", + }, + } + result := t.controller.isProxyDeployment(deploy) + Expect(result).To(BeEmpty()) + }) + }) + + Context("for services", func() { + It("should reconcile proxy service", func() { + result := t.controller.isProxyService(t.NewInsightsProxyService()) + Expect(result).To(ConsistOf(t.deploymentReconcileRequest())) + }) + It("should not reconcile a service in another namespace", func() { + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: t.NewInsightsProxyService().Name, + Namespace: "other", + }, + } + result := t.controller.isProxyService(svc) + Expect(result).To(BeEmpty()) + }) + }) + }) +}) + +func (t *insightsUnitTestInput) deploymentReconcileRequest() reconcile.Request { + return reconcile.Request{NamespacedName: types.NamespacedName{Name: "insights-proxy", Namespace: t.Namespace}} +} diff --git a/internal/controllers/insights/insights_suite_test.go b/internal/controllers/insights/insights_suite_test.go new file mode 100644 index 00000000..35a3d501 --- /dev/null +++ b/internal/controllers/insights/insights_suite_test.go @@ -0,0 +1,96 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package insights_test + +import ( + "fmt" + "path/filepath" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + + operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1" + + certv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + configv1 "github.com/openshift/api/config/v1" + consolev1 "github.com/openshift/api/console/v1" + routev1 "github.com/openshift/api/route/v1" + // +kubebuilder:scaffold:imports +) + +// These tests use Ginkgo (BDD-style Go testing framework). Refer to +// http://onsi.github.io/ginkgo/ to learn more about Ginkgo. + +var cfg *rest.Config +var k8sClient client.Client +var testEnv *envtest.Environment + +func TestInsights(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Insights Suite") +} + +var _ = BeforeSuite(func() { + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) + + By("bootstrapping test environment") + testEnv = &envtest.Environment{ + CRDDirectoryPaths: []string{ + filepath.Join("..", "..", "..", "config", "crd", "bases"), + }, + ErrorIfCRDPathMissing: true, + } + fmt.Println(testEnv.CRDDirectoryPaths) + + var err error + // cfg is defined in this file globally. + cfg, err = testEnv.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(cfg).NotTo(BeNil()) + + err = operatorv1beta1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = certv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = consolev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = routev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + err = configv1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) + + // +kubebuilder:scaffold:scheme + + k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(k8sClient).NotTo(BeNil()) +}) + +var _ = AfterSuite(func() { + By("tearing down the test environment") + err := testEnv.Stop() + Expect(err).NotTo(HaveOccurred()) +}) diff --git a/internal/controllers/insights/setup.go b/internal/controllers/insights/setup.go new file mode 100644 index 00000000..b5eca7cc --- /dev/null +++ b/internal/controllers/insights/setup.go @@ -0,0 +1,161 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package insights + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/cryostatio/cryostat-operator/internal/controllers/common" + "github.com/cryostatio/cryostat-operator/internal/controllers/constants" + "github.com/go-logr/logr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +type InsightsIntegration struct { + Manager ctrl.Manager + Log *logr.Logger + common.OSUtils +} + +func NewInsightsIntegration(mgr ctrl.Manager, log *logr.Logger) *InsightsIntegration { + return &InsightsIntegration{ + Manager: mgr, + Log: log, + OSUtils: &common.DefaultOSUtils{}, + } +} + +func (i *InsightsIntegration) Setup() (*url.URL, error) { + var proxyUrl *url.URL + namespace := i.getOperatorNamespace() + // This will happen when running the operator locally + if len(namespace) == 0 { + i.Log.Info("Operator namespace not detected") + return nil, nil + } + + ctx := context.Background() + if i.isInsightsEnabled() { + err := i.createInsightsController(namespace) + if err != nil { + i.Log.Error(err, "unable to add controller to manager", "controller", "Insights") + return nil, err + } + // Create a Config Map to be used as a parent of all Insights Proxy related objects + err = i.createConfigMap(ctx, namespace) + if err != nil { + i.Log.Error(err, "failed to create config map for Insights") + return nil, err + } + proxyUrl = i.getProxyURL(namespace) + } else { + // Delete any previously created Config Map (and its children) + err := i.deleteConfigMap(ctx, namespace) + if err != nil { + i.Log.Error(err, "failed to delete config map for Insights") + return nil, err + } + + } + return proxyUrl, nil +} + +func (i *InsightsIntegration) isInsightsEnabled() bool { + return strings.ToLower(i.GetEnv(EnvInsightsEnabled)) == "true" +} + +func (i *InsightsIntegration) getOperatorNamespace() string { + return i.GetEnv("NAMESPACE") +} + +func (i *InsightsIntegration) createInsightsController(namespace string) error { + config := &InsightsReconcilerConfig{ + Client: i.Manager.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("Insights"), + Scheme: i.Manager.GetScheme(), + Namespace: namespace, + OSUtils: i.OSUtils, + } + controller, err := NewInsightsReconciler(config) + if err != nil { + return err + } + if err := controller.SetupWithManager(i.Manager); err != nil { + return err + } + return nil +} + +func (i *InsightsIntegration) createConfigMap(ctx context.Context, namespace string) error { + // The config map should be owned by the operator deployment to ensure it and its descendants are garbage collected + owner := &appsv1.Deployment{} + // Use the APIReader instead of the cache, since the cache may not be synced yet + err := i.Manager.GetAPIReader().Get(ctx, types.NamespacedName{ + Name: constants.OperatorDeploymentName, Namespace: namespace}, owner) + if err != nil { + return err + } + + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: InsightsConfigMapName, + Namespace: namespace, + }, + } + err = controllerutil.SetControllerReference(owner, cm, i.Manager.GetScheme()) + if err != nil { + return err + } + + err = i.Manager.GetClient().Create(ctx, cm, &client.CreateOptions{}) + if err == nil { + i.Log.Info("Config Map for Insights created", "name", cm.Name, "namespace", cm.Namespace) + } + // This may already exist if the pod restarted + return client.IgnoreAlreadyExists(err) +} + +func (i *InsightsIntegration) deleteConfigMap(ctx context.Context, namespace string) error { + // Children will be garbage collected + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: InsightsConfigMapName, + Namespace: namespace, + }, + } + + err := i.Manager.GetClient().Delete(ctx, cm, &client.DeleteOptions{}) + if err == nil { + i.Log.Info("Config Map for Insights deleted", "name", cm.Name, "namespace", cm.Namespace) + } + // This may not exist if no config map was previously created + return client.IgnoreNotFound(err) +} + +func (i *InsightsIntegration) getProxyURL(namespace string) *url.URL { + return &url.URL{ + Scheme: "http", // TODO add https support (r.IsCertManagerInstalled) + Host: fmt.Sprintf("%s.%s.svc.cluster.local", ProxyServiceName, namespace), + } +} diff --git a/internal/controllers/insights/setup_test.go b/internal/controllers/insights/setup_test.go new file mode 100644 index 00000000..216a326d --- /dev/null +++ b/internal/controllers/insights/setup_test.go @@ -0,0 +1,162 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package insights_test + +import ( + "context" + + "github.com/cryostatio/cryostat-operator/internal/controllers/insights" + insightstest "github.com/cryostatio/cryostat-operator/internal/controllers/insights/test" + "github.com/cryostatio/cryostat-operator/internal/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrlclient "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" +) + +var _ = Describe("InsightsIntegration", func() { + var t *insightsTestInput + + Describe("setting up", func() { + var integration *insights.InsightsIntegration + + BeforeEach(func() { + t = &insightsTestInput{ + TestUtilsConfig: &insightstest.TestUtilsConfig{ + EnvInsightsEnabled: &[]bool{true}[0], + EnvInsightsBackendDomain: &[]string{"insights.example.com"}[0], + EnvInsightsProxyImageTag: &[]string{"example.com/proxy:latest"}[0], + EnvNamespace: &[]string{"test"}[0], + }, + InsightsTestResources: &insightstest.InsightsTestResources{ + TestResources: &test.TestResources{ + Namespace: "test", + }, + }, + } + t.objs = []ctrlclient.Object{ + t.NewNamespace(), + t.NewGlobalPullSecret(), + t.NewOperatorDeployment(), + } + }) + + JustBeforeEach(func() { + s := test.NewTestScheme() + logger := zap.New() + logf.SetLogger(logger) + + // Set a CreationTimestamp for created objects to match a real API server + // TODO When using envtest instead of fake client, this is probably no longer needed + err := test.SetCreationTimestamp(t.objs...) + Expect(err).ToNot(HaveOccurred()) + t.client = fake.NewClientBuilder().WithScheme(s).WithObjects(t.objs...).Build() + + manager := insightstest.NewFakeManager(test.NewClientWithTimestamp(test.NewTestClient(t.client, t.TestResources)), + s, &logger) + integration = insights.NewInsightsIntegration(manager, &logger) + integration.OSUtils = insightstest.NewTestOSUtils(t.TestUtilsConfig) + }) + + Context("with defaults", func() { + It("should return proxy URL", func() { + result, err := integration.Setup() + Expect(err).ToNot(HaveOccurred()) + Expect(result).ToNot(BeNil()) + Expect(result.String()).To(Equal("http://insights-proxy.test.svc.cluster.local")) + }) + + It("should create config map", func() { + _, err := integration.Setup() + Expect(err).ToNot(HaveOccurred()) + + expected := t.NewProxyConfigMap() + actual := &corev1.ConfigMap{} + err = t.client.Get(context.Background(), types.NamespacedName{ + Name: expected.Name, + Namespace: expected.Namespace, + }, actual) + Expect(err).ToNot(HaveOccurred()) + + Expect(actual.Labels).To(Equal(expected.Labels)) + Expect(actual.Annotations).To(Equal(expected.Annotations)) + Expect(metav1.IsControlledBy(actual, t.NewOperatorDeployment())).To(BeTrue()) + Expect(actual.Data).To(BeEmpty()) + }) + }) + + Context("with Insights disabled", func() { + BeforeEach(func() { + t.EnvInsightsEnabled = &[]bool{false}[0] + t.objs = append(t.objs, + t.NewProxyConfigMap(), + ) + }) + + It("should return nil", func() { + result, err := integration.Setup() + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeNil()) + }) + + It("should delete config map", func() { + _, err := integration.Setup() + Expect(err).ToNot(HaveOccurred()) + + expected := t.NewProxyConfigMap() + actual := &corev1.ConfigMap{} + err = t.client.Get(context.Background(), types.NamespacedName{ + Name: expected.Name, + Namespace: expected.Namespace, + }, actual) + Expect(err).To(HaveOccurred()) + Expect(kerrors.IsNotFound(err)).To(BeTrue()) + }) + }) + + Context("when run out-of-cluster", func() { + BeforeEach(func() { + t.EnvNamespace = nil + }) + + It("should return nil", func() { + result, err := integration.Setup() + Expect(err).ToNot(HaveOccurred()) + Expect(result).To(BeNil()) + }) + + It("should not create config map", func() { + _, err := integration.Setup() + Expect(err).ToNot(HaveOccurred()) + + expected := t.NewProxyConfigMap() + actual := &corev1.ConfigMap{} + err = t.client.Get(context.Background(), types.NamespacedName{ + Name: expected.Name, + Namespace: expected.Namespace, + }, actual) + Expect(err).To(HaveOccurred()) + Expect(kerrors.IsNotFound(err)).To(BeTrue()) + }) + }) + }) +}) diff --git a/internal/controllers/insights/test/manager.go b/internal/controllers/insights/test/manager.go new file mode 100644 index 00000000..ebcd764e --- /dev/null +++ b/internal/controllers/insights/test/manager.go @@ -0,0 +1,68 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/config/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/manager" +) + +type FakeManager struct { + ctrl.Manager + client client.Client + scheme *runtime.Scheme + logger *logr.Logger +} + +func NewFakeManager(client client.Client, scheme *runtime.Scheme, logger *logr.Logger) *FakeManager { + return &FakeManager{ + client: client, + scheme: scheme, + logger: logger, + } +} + +func (m *FakeManager) GetClient() client.Client { + return m.client +} + +func (m *FakeManager) GetScheme() *runtime.Scheme { + return m.scheme +} + +func (m *FakeManager) GetAPIReader() client.Reader { + // May need to change if not using a fake client + return m.client +} + +func (m *FakeManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec { + return v1alpha1.ControllerConfigurationSpec{} +} + +func (m *FakeManager) GetLogger() logr.Logger { + return *m.logger +} + +func (m *FakeManager) SetFields(interface{}) error { + return nil +} + +func (m *FakeManager) Add(manager.Runnable) error { + return nil +} diff --git a/internal/controllers/insights/test/resources.go b/internal/controllers/insights/test/resources.go new file mode 100644 index 00000000..c8ada15a --- /dev/null +++ b/internal/controllers/insights/test/resources.go @@ -0,0 +1,367 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + + "github.com/cryostatio/cryostat-operator/internal/test" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +type InsightsTestResources struct { + *test.TestResources + Resources *corev1.ResourceRequirements +} + +func (r *InsightsTestResources) NewGlobalPullSecret() *corev1.Secret { + config := `{"auths":{"example.com":{"auth":"hello"},"cloud.openshift.com":{"auth":"world"}}}` + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pull-secret", + Namespace: "openshift-config", + }, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: []byte(config), + }, + } +} + +func (r *InsightsTestResources) NewOperatorDeployment() *appsv1.Deployment { + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cryostat-operator-controller-manager", + Namespace: r.Namespace, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "control-plane": "controller-manager", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "manager", + Image: "example.com/operator:latest", + }, + }, + }, + }, + }, + } +} + +func (r *InsightsTestResources) NewProxyConfigMap() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "insights-proxy", + Namespace: r.Namespace, + }, + } +} + +func (r *InsightsTestResources) NewInsightsProxySecret() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "apicastconf", + Namespace: r.Namespace, + }, + StringData: map[string]string{ + "config.json": fmt.Sprintf(`{ + "services": [ + { + "id": "1", + "backend_version": "1", + "proxy": { + "hosts": ["insights-proxy","insights-proxy.%s.svc.cluster.local"], + "api_backend": "https://insights.example.com:443/", + "backend": { "endpoint": "http://127.0.0.1:8081", "host": "backend" }, + "policy_chain": [ + { + "name": "default_credentials", + "version": "builtin", + "configuration": { + "auth_type": "user_key", + "user_key": "dummy_key" + } + }, + { + "name": "headers", + "version": "builtin", + "configuration": { + "request": [ + { + "op": "set", + "header": "Authorization", + "value_type": "plain", + "value": "Bearer world" + } + ] + } + }, + { + "name": "apicast.policy.apicast" + } + ], + "proxy_rules": [ + { + "http_method": "POST", + "pattern": "/", + "metric_system_name": "hits", + "delta": 1, + "parameters": [], + "querystring_parameters": {} + } + ] + } + } + ] + }`, r.Namespace), + }, + } +} + +func (r *InsightsTestResources) NewInsightsProxySecretWithProxyDomain() *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "apicastconf", + Namespace: r.Namespace, + }, + StringData: map[string]string{ + "config.json": fmt.Sprintf(`{ + "services": [ + { + "id": "1", + "backend_version": "1", + "proxy": { + "hosts": ["insights-proxy","insights-proxy.%s.svc.cluster.local"], + "api_backend": "https://insights.example.com:443/", + "backend": { "endpoint": "http://127.0.0.1:8081", "host": "backend" }, + "policy_chain": [ + { + "name": "default_credentials", + "version": "builtin", + "configuration": { + "auth_type": "user_key", + "user_key": "dummy_key" + } + }, + { + "name": "apicast.policy.http_proxy", + "configuration": { + "https_proxy": "http://proxy.example.com/", + "http_proxy": "http://proxy.example.com/" + } + }, + { + "name": "headers", + "version": "builtin", + "configuration": { + "request": [ + { + "op": "set", + "header": "Authorization", + "value_type": "plain", + "value": "Bearer world" + } + ] + } + }, + { + "name": "apicast.policy.apicast" + } + ], + "proxy_rules": [ + { + "http_method": "POST", + "pattern": "/", + "metric_system_name": "hits", + "delta": 1, + "parameters": [], + "querystring_parameters": {} + } + ] + } + } + ] + }`, r.Namespace), + }, + } +} + +func (r *InsightsTestResources) NewInsightsProxyDeployment() *appsv1.Deployment { + var resources *corev1.ResourceRequirements + if r.Resources != nil { + resources = r.Resources + } else { + resources = &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("50m"), + corev1.ResourceMemory: resource.MustParse("64Mi"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("128Mi"), + }, + } + } + return &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "insights-proxy", + Namespace: r.Namespace, + Labels: map[string]string{ + "app": "insights-proxy", + }, + }, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "insights-proxy", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "insights-proxy", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "insights-proxy", + Image: "example.com/proxy:latest", + Env: []corev1.EnvVar{ + { + Name: "THREESCALE_CONFIG_FILE", + Value: "/tmp/gateway-configuration-volume/config.json", + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "gateway-configuration-volume", + MountPath: "/tmp/gateway-configuration-volume", + ReadOnly: true, + }, + }, + Ports: []corev1.ContainerPort{ + { + Name: "proxy", + ContainerPort: 8080, + }, + { + Name: "management", + ContainerPort: 8090, + }, + { + Name: "metrics", + ContainerPort: 9421, + }, + }, + Resources: *resources, + SecurityContext: &corev1.SecurityContext{ + AllowPrivilegeEscalation: &[]bool{false}[0], + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + }, + LivenessProbe: &corev1.Probe{ + InitialDelaySeconds: 10, + TimeoutSeconds: 5, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/status/live", + Port: intstr.FromInt(8090), + }, + }, + }, + ReadinessProbe: &corev1.Probe{ + InitialDelaySeconds: 15, + PeriodSeconds: 30, + TimeoutSeconds: 5, + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/status/ready", + Port: intstr.FromInt(8090), + }, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: "gateway-configuration-volume", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: "apicastconf", + Items: []corev1.KeyToPath{ + { + Key: "config.json", + Path: "config.json", + Mode: &[]int32{0440}[0], + }, + }, + }, + }, + }, + }, + SecurityContext: &corev1.PodSecurityContext{ + RunAsNonRoot: &[]bool{true}[0], + }, + }, + }, + }, + } +} + +func (r *InsightsTestResources) NewInsightsProxyService() *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "insights-proxy", + Namespace: r.Namespace, + Labels: map[string]string{ + "app": "insights-proxy", + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + "app": "insights-proxy", + }, + Ports: []corev1.ServicePort{ + { + Name: "proxy", + Port: 8080, + TargetPort: intstr.FromString("proxy"), + }, + { + Name: "management", + Port: 8090, + TargetPort: intstr.FromString("management"), + }, + }, + }, + } +} diff --git a/internal/controllers/insights/test/utils.go b/internal/controllers/insights/test/utils.go new file mode 100644 index 00000000..9b27c685 --- /dev/null +++ b/internal/controllers/insights/test/utils.go @@ -0,0 +1,66 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "strconv" +) + +// TestUtilsConfig groups parameters used to create a test OSUtils +type TestUtilsConfig struct { + EnvInsightsEnabled *bool + EnvInsightsProxyImageTag *string + EnvInsightsBackendDomain *string + EnvInsightsProxyDomain *string + EnvNamespace *string +} + +type testOSUtils struct { + envs map[string]string +} + +func NewTestOSUtils(config *TestUtilsConfig) *testOSUtils { + envs := map[string]string{} + if config.EnvInsightsEnabled != nil { + envs["INSIGHTS_ENABLED"] = strconv.FormatBool(*config.EnvInsightsEnabled) + } + if config.EnvInsightsProxyImageTag != nil { + envs["RELATED_IMAGE_INSIGHTS_PROXY"] = *config.EnvInsightsProxyImageTag + } + if config.EnvInsightsBackendDomain != nil { + envs["INSIGHTS_BACKEND_DOMAIN"] = *config.EnvInsightsBackendDomain + } + if config.EnvInsightsProxyDomain != nil { + envs["INSIGHTS_PROXY_DOMAIN"] = *config.EnvInsightsProxyDomain + } + if config.EnvNamespace != nil { + envs["NAMESPACE"] = *config.EnvNamespace + } + return &testOSUtils{envs: envs} +} + +func (o *testOSUtils) GetFileContents(path string) ([]byte, error) { + // Unused + return nil, nil +} + +func (o *testOSUtils) GetEnv(name string) string { + return o.envs[name] +} + +func (o *testOSUtils) GenPasswd(length int) string { + // Unused + return "" +} diff --git a/internal/controllers/openshift.go b/internal/controllers/openshift.go index 4a01d8e6..bdda3d5f 100644 --- a/internal/controllers/openshift.go +++ b/internal/controllers/openshift.go @@ -47,26 +47,25 @@ func (r *Reconciler) finalizeOpenShift(ctx context.Context, cr *model.CryostatIn return nil } reqLogger := r.Log.WithValues("Request.Namespace", cr.InstallNamespace, "Request.Name", cr.Name) - err := r.deleteConsoleLink(ctx, newConsoleLink(cr), reqLogger) + err := r.deleteConsoleLink(ctx, r.newConsoleLink(cr), reqLogger) if err != nil { return err } return r.deleteCorsAllowedOrigins(ctx, cr) } -func newConsoleLink(cr *model.CryostatInstance) *consolev1.ConsoleLink { +func (r *Reconciler) newConsoleLink(cr *model.CryostatInstance) *consolev1.ConsoleLink { // Cluster scoped, so use a unique name to avoid conflicts return &consolev1.ConsoleLink{ ObjectMeta: metav1.ObjectMeta{ - Name: common.ClusterUniqueName(cr.Object.GetObjectKind().GroupVersionKind().Kind, - cr.Name, cr.InstallNamespace), + Name: common.ClusterUniqueName(r.gvk, cr.Name, cr.InstallNamespace), }, } } func (r *Reconciler) reconcileConsoleLink(ctx context.Context, cr *model.CryostatInstance) error { reqLogger := r.Log.WithValues("Request.Namespace", cr.InstallNamespace, "Request.Name", cr.Name) - link := newConsoleLink(cr) + link := r.newConsoleLink(cr) url := cr.Status.ApplicationURL if len(url) == 0 { diff --git a/internal/controllers/pvc.go b/internal/controllers/pvc.go index 49bdc37b..33ee2538 100644 --- a/internal/controllers/pvc.go +++ b/internal/controllers/pvc.go @@ -19,7 +19,7 @@ import ( "fmt" operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1" - common "github.com/cryostatio/cryostat-operator/internal/controllers/common" + "github.com/cryostatio/cryostat-operator/internal/controllers/common" "github.com/cryostatio/cryostat-operator/internal/controllers/model" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" diff --git a/internal/controllers/rbac.go b/internal/controllers/rbac.go index 4ef4da1b..92ab4eab 100644 --- a/internal/controllers/rbac.go +++ b/internal/controllers/rbac.go @@ -53,7 +53,7 @@ func (r *Reconciler) reconcileRBAC(ctx context.Context, cr *model.CryostatInstan } func (r *Reconciler) finalizeRBAC(ctx context.Context, cr *model.CryostatInstance) error { - return r.deleteClusterRoleBinding(ctx, newClusterRoleBinding(cr)) + return r.deleteClusterRoleBinding(ctx, r.newClusterRoleBinding(cr)) } func newServiceAccount(cr *model.CryostatInstance) *corev1.ServiceAccount { @@ -150,11 +150,10 @@ func (r *Reconciler) reconcileRoleBinding(ctx context.Context, cr *model.Cryosta return nil } -func newClusterRoleBinding(cr *model.CryostatInstance) *rbacv1.ClusterRoleBinding { +func (r *Reconciler) newClusterRoleBinding(cr *model.CryostatInstance) *rbacv1.ClusterRoleBinding { return &rbacv1.ClusterRoleBinding{ ObjectMeta: metav1.ObjectMeta{ - Name: common.ClusterUniqueName(cr.Object.GetObjectKind().GroupVersionKind().Kind, - cr.Name, cr.InstallNamespace), + Name: common.ClusterUniqueName(r.gvk, cr.Name, cr.InstallNamespace), }, } } @@ -162,7 +161,7 @@ func newClusterRoleBinding(cr *model.CryostatInstance) *rbacv1.ClusterRoleBindin const clusterRoleName = "cryostat-operator-cryostat" func (r *Reconciler) reconcileClusterRoleBinding(ctx context.Context, cr *model.CryostatInstance) error { - binding := newClusterRoleBinding(cr) + binding := r.newClusterRoleBinding(cr) sa := newServiceAccount(cr) subjects := []rbacv1.Subject{ diff --git a/internal/controllers/reconciler.go b/internal/controllers/reconciler.go index f0fdddc2..30b8f4b3 100644 --- a/internal/controllers/reconciler.go +++ b/internal/controllers/reconciler.go @@ -18,6 +18,7 @@ import ( "context" "errors" "fmt" + "net/url" "regexp" "strconv" "time" @@ -39,11 +40,14 @@ import ( "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -57,6 +61,8 @@ type ReconcilerConfig struct { IsCertManagerInstalled bool EventRecorder record.EventRecorder RESTMapper meta.RESTMapper + Namespaces []string + InsightsProxy *url.URL // Only defined if Insights is enabled common.ReconcilerTLS } @@ -64,11 +70,15 @@ type ReconcilerConfig struct { // between the ClusterCryostat and Cryostat reconcilers type CommonReconciler interface { reconcile.Reconciler + SetupWithManager(mgr ctrl.Manager) error GetConfig() *ReconcilerConfig } type Reconciler struct { *ReconcilerConfig + objectType client.Object + isNamespaced bool + gvk *schema.GroupVersionKind } // Name used for Finalizer that handles Cryostat deletion @@ -115,6 +125,19 @@ var reportsDeploymentConditions = deploymentConditionTypeMap{ operatorv1beta1.ConditionTypeReportsDeploymentReplicaFailure: appsv1.DeploymentReplicaFailure, } +func newReconciler(config *ReconcilerConfig, objType client.Object, isNamespaced bool) (*Reconciler, error) { + gvk, err := apiutil.GVKForObject(objType, config.Scheme) + if err != nil { + return nil, err + } + return &Reconciler{ + ReconcilerConfig: config, + objectType: objType, + isNamespaced: isNamespaced, + gvk: &gvk, + }, nil +} + func (r *Reconciler) reconcileCryostat(ctx context.Context, cr *model.CryostatInstance) (ctrl.Result, error) { result, err := r.reconcile(ctx, cr) return result, r.checkConflicts(cr, err) @@ -213,7 +236,9 @@ func (r *Reconciler) reconcile(ctx context.Context, cr *model.CryostatInstance) return reconcile.Result{}, err } - serviceSpecs := &resources.ServiceSpecs{} + serviceSpecs := &resources.ServiceSpecs{ + InsightsURL: r.InsightsProxy, + } err = r.reconcileGrafanaService(ctx, cr, tlsConfig, serviceSpecs) if err != nil { return requeueIfIngressNotReady(reqLogger, err) @@ -267,10 +292,31 @@ func (r *Reconciler) reconcile(ctx context.Context, cr *model.CryostatInstance) return reconcile.Result{}, nil } -func (r *Reconciler) setupWithManager(mgr ctrl.Manager, obj client.Object, - impl reconcile.Reconciler) error { +func namespaceEventFilter(scheme *runtime.Scheme, namespaceList []string) predicate.Predicate { + namespaces := namespacesToSet(namespaceList) + return predicate.NewPredicateFuncs(func(object client.Object) bool { + // Restrict watch for namespaced objects to specified namespaces + if len(object.GetNamespace()) > 0 { + _, pres := namespaces[object.GetNamespace()] + if !pres { + return false + } + } + return true + }) +} + +func (r *Reconciler) setupWithManager(mgr ctrl.Manager, impl reconcile.Reconciler) error { c := ctrl.NewControllerManagedBy(mgr). - For(obj) + For(r.objectType) + + // Filter watch to specified namespaces only if the CRD is namespaced and + // we're not running in AllNamespace mode + // TODO remove this once only AllNamespace mode is supported + if r.isNamespaced && len(r.Namespaces) > 0 { + r.Log.Info(fmt.Sprintf("Adding EventFilter for namespaces: %v", r.Namespaces)) + c = c.WithEventFilter(namespaceEventFilter(mgr.GetScheme(), r.Namespaces)) + } // Watch for changes to secondary resources and requeue the owner Cryostat resources := []client.Object{&appsv1.Deployment{}, &corev1.Service{}, &corev1.Secret{}, &corev1.PersistentVolumeClaim{}, @@ -542,3 +588,11 @@ func findDeployCondition(conditions []appsv1.DeploymentCondition, condType appsv } return nil } + +func namespacesToSet(namespaces []string) map[string]struct{} { + result := make(map[string]struct{}, len(namespaces)) + for _, namespace := range namespaces { + result[namespace] = struct{}{} + } + return result +} diff --git a/internal/controllers/reconciler_test.go b/internal/controllers/reconciler_test.go index 2358efe6..b02b8d3d 100644 --- a/internal/controllers/reconciler_test.go +++ b/internal/controllers/reconciler_test.go @@ -17,6 +17,7 @@ package controllers_test import ( "context" "fmt" + "net/url" "time" certv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" @@ -53,12 +54,13 @@ import ( type controllerTest struct { clusterScoped bool - constructorFunc func(*controllers.ReconcilerConfig) controllers.CommonReconciler + constructorFunc func(*controllers.ReconcilerConfig) (controllers.CommonReconciler, error) } type cryostatTestInput struct { - controller controllers.CommonReconciler - objs []ctrlclient.Object + controller controllers.CommonReconciler + objs []ctrlclient.Object + watchNamespaces []string test.TestReconcilerConfig *test.TestResources } @@ -81,6 +83,7 @@ func (c *controllerTest) commonBeforeEach() *cryostatTestInput { t.NewNamespace(), t.NewApiServer(), } + t.watchNamespaces = []string{t.Namespace} return t } @@ -92,7 +95,8 @@ func (c *controllerTest) commonJustBeforeEach(t *cryostatTestInput) { err := test.SetCreationTimestamp(t.objs...) Expect(err).ToNot(HaveOccurred()) t.Client = fake.NewClientBuilder().WithScheme(s).WithObjects(t.objs...).Build() - t.controller = c.constructorFunc(t.newReconcilerConfig(s, t.Client)) + t.controller, err = c.constructorFunc(t.newReconcilerConfig(s, t.Client)) + Expect(err).ToNot(HaveOccurred()) } func (c *controllerTest) commonJustAfterEach(t *cryostatTestInput) { @@ -106,6 +110,13 @@ func (t *cryostatTestInput) newReconcilerConfig(scheme *runtime.Scheme, client c logger := zap.New().WithValues("cluster-scoped", t.ClusterScoped) logf.SetLogger(logger) + // Set InsightsURL in config, if provided + var insightsURL *url.URL + if len(t.InsightsURL) > 0 { + url, err := url.Parse(t.InsightsURL) + Expect(err).ToNot(HaveOccurred()) + insightsURL = url + } return &controllers.ReconcilerConfig{ Client: test.NewClientWithTimestamp(test.NewTestClient(client, t.TestResources)), Scheme: scheme, @@ -114,6 +125,8 @@ func (t *cryostatTestInput) newReconcilerConfig(scheme *runtime.Scheme, client c RESTMapper: test.NewTESTRESTMapper(), Log: logger, ReconcilerTLS: test.NewTestReconcilerTLS(&t.TestReconcilerConfig), + Namespaces: t.watchNamespaces, + InsightsProxy: insightsURL, } } @@ -205,20 +218,19 @@ func expectSuccessful(t **cryostatTestInput) { func (c *controllerTest) commonTests() { var t *cryostatTestInput - BeforeEach(func() { - t = c.commonBeforeEach() - t.TargetNamespaces = []string{t.Namespace} - }) - - JustBeforeEach(func() { - c.commonJustBeforeEach(t) - }) + Describe("reconciling a request in OpenShift", func() { + BeforeEach(func() { + t = c.commonBeforeEach() + t.TargetNamespaces = []string{t.Namespace} + }) - JustAfterEach(func() { - c.commonJustAfterEach(t) - }) + JustBeforeEach(func() { + c.commonJustBeforeEach(t) + }) - Describe("reconciling a request in OpenShift", func() { + JustAfterEach(func() { + c.commonJustAfterEach(t) + }) Context("with a default CR", func() { BeforeEach(func() { t.objs = append(t.objs, t.NewCryostat().Object) @@ -1335,6 +1347,17 @@ func (c *controllerTest) commonTests() { }) }) }) + Context("with Insights enabled", func() { + BeforeEach(func() { + t.InsightsURL = "http://insights-proxy.foo.svc.cluster.local" + }) + JustBeforeEach(func() { + t.reconcileCryostatFully() + }) + It("should create deployment", func() { + t.expectMainDeployment() + }) + }) }) Context("with cert-manager disabled in CR", func() { BeforeEach(func() { @@ -1761,10 +1784,14 @@ func (c *controllerTest) commonTests() { JustBeforeEach(func() { other.commonJustBeforeEach(otherInput) + // Controllers need to share client to have shared view of objects otherInput.Client = t.Client config := otherInput.newReconcilerConfig(otherInput.Client.Scheme(), otherInput.Client) - otherInput.controller = other.constructorFunc(config) + controller, err := other.constructorFunc(config) + Expect(err).ToNot(HaveOccurred()) + otherInput.controller = controller + // Reconcile conflicting namespaced Cryostat fully otherInput.reconcileCryostatFully() // Try reconciling ClusterCryostat @@ -1853,8 +1880,18 @@ func (c *controllerTest) commonTests() { Describe("reconciling a request in Kubernetes", func() { BeforeEach(func() { + t = c.commonBeforeEach() + t.TargetNamespaces = []string{t.Namespace} t.OpenShift = false }) + + JustBeforeEach(func() { + c.commonJustBeforeEach(t) + }) + + JustAfterEach(func() { + c.commonJustAfterEach(t) + }) Context("with TLS ingress", func() { BeforeEach(func() { t.objs = append(t.objs, t.NewCryostatWithIngress().Object) @@ -2871,7 +2908,7 @@ func (t *cryostatTestInput) checkCoreContainer(container *corev1.Container, ingr Expect(container.StartupProbe).To(Equal(t.NewCoreStartupProbe())) Expect(container.SecurityContext).To(Equal(securityContext)) - checkResourceRequirements(&container.Resources, resources) + test.ExpectResourceRequirements(&container.Resources, resources) } func (t *cryostatTestInput) checkGrafanaContainer(container *corev1.Container, resources *corev1.ResourceRequirements, securityContext *corev1.SecurityContext) { @@ -2888,7 +2925,7 @@ func (t *cryostatTestInput) checkGrafanaContainer(container *corev1.Container, r Expect(container.LivenessProbe).To(Equal(t.NewGrafanaLivenessProbe())) Expect(container.SecurityContext).To(Equal(securityContext)) - checkResourceRequirements(&container.Resources, resources) + test.ExpectResourceRequirements(&container.Resources, resources) } func (t *cryostatTestInput) checkDatasourceContainer(container *corev1.Container, resources *corev1.ResourceRequirements, securityContext *corev1.SecurityContext) { @@ -2905,7 +2942,7 @@ func (t *cryostatTestInput) checkDatasourceContainer(container *corev1.Container Expect(container.LivenessProbe).To(Equal(t.NewDatasourceLivenessProbe())) Expect(container.SecurityContext).To(Equal(securityContext)) - checkResourceRequirements(&container.Resources, resources) + test.ExpectResourceRequirements(&container.Resources, resources) } func (t *cryostatTestInput) checkReportsContainer(container *corev1.Container, resources *corev1.ResourceRequirements, securityContext *corev1.SecurityContext) { @@ -2921,7 +2958,7 @@ func (t *cryostatTestInput) checkReportsContainer(container *corev1.Container, r Expect(container.LivenessProbe).To(Equal(t.NewReportsLivenessProbe())) Expect(container.SecurityContext).To(Equal(securityContext)) - checkResourceRequirements(&container.Resources, resources) + test.ExpectResourceRequirements(&container.Resources, resources) } func (t *cryostatTestInput) checkCoreHasEnvironmentVariables(expectedEnvVars []corev1.EnvVar) { @@ -2935,43 +2972,6 @@ func (t *cryostatTestInput) checkCoreHasEnvironmentVariables(expectedEnvVars []c Expect(coreContainer.Env).To(ContainElements(expectedEnvVars)) } -func checkResourceRequirements(containerResource, expectedResource *corev1.ResourceRequirements) { - // Containers must have resource requests - Expect(containerResource.Requests).ToNot(BeNil()) - - requestCpu, requestCpuFound := containerResource.Requests[corev1.ResourceCPU] - expectedRequestCpu := expectedResource.Requests[corev1.ResourceCPU] - Expect(requestCpuFound).To(BeTrue()) - Expect(requestCpu.Equal(expectedRequestCpu)).To(BeTrue()) - - requestMemory, requestMemoryFound := containerResource.Requests[corev1.ResourceMemory] - expectedRequestMemory := expectedResource.Requests[corev1.ResourceMemory] - Expect(requestMemoryFound).To(BeTrue()) - Expect(requestMemory.Equal(expectedRequestMemory)).To(BeTrue()) - - if expectedResource.Limits == nil { - Expect(containerResource.Limits).To(BeNil()) - } else { - Expect(containerResource.Limits).ToNot(BeNil()) - - limitCpu, limitCpuFound := containerResource.Limits[corev1.ResourceCPU] - expectedLimitCpu, expectedLimitCpuFound := expectedResource.Limits[corev1.ResourceCPU] - - Expect(limitCpuFound).To(Equal(expectedLimitCpuFound)) - if expectedLimitCpuFound { - Expect(limitCpu.Equal(expectedLimitCpu)).To(BeTrue()) - } - - limitMemory, limitMemoryFound := containerResource.Limits[corev1.ResourceMemory] - expectedlimitMemory, expectedLimitMemoryFound := expectedResource.Limits[corev1.ResourceMemory] - - Expect(limitMemoryFound).To(Equal(expectedLimitMemoryFound)) - if expectedLimitCpuFound { - Expect(limitMemory.Equal(expectedlimitMemory)).To(BeTrue()) - } - } -} - func (t *cryostatTestInput) getCryostatInstance() *model.CryostatInstance { cr, err := t.lookupCryostatInstance() Expect(err).ToNot(HaveOccurred()) @@ -3054,7 +3054,7 @@ func (t *cryostatTestInput) expectResourcesUnaffected() { } } -func getControllerFunc(clusterScoped bool) func(*controllers.ReconcilerConfig) controllers.CommonReconciler { +func getControllerFunc(clusterScoped bool) func(*controllers.ReconcilerConfig) (controllers.CommonReconciler, error) { if clusterScoped { return newClusterCryostatController } diff --git a/internal/controllers/reconciler_unit_test.go b/internal/controllers/reconciler_unit_test.go new file mode 100644 index 00000000..bec0e97d --- /dev/null +++ b/internal/controllers/reconciler_unit_test.go @@ -0,0 +1,79 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controllers + +import ( + "github.com/cryostatio/cryostat-operator/internal/controllers/model" + "github.com/cryostatio/cryostat-operator/internal/test" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +type cryostatUnitTestInput struct { + scheme *runtime.Scheme + watchNamespaces []string + *test.TestResources +} + +var _ = Describe("Reconciler", func() { + Describe("filtering requests", func() { + Context("watches the configured namespace(s)", func() { + var t *cryostatUnitTestInput + var filter predicate.Predicate + var cr *model.CryostatInstance + + BeforeEach(func() { + resources := &test.TestResources{ + Name: "cryostat", + Namespace: "test", + } + t = &cryostatUnitTestInput{ + scheme: test.NewTestScheme(), + watchNamespaces: []string{resources.Namespace}, + TestResources: resources, + } + }) + JustBeforeEach(func() { + filter = namespaceEventFilter(t.scheme, t.watchNamespaces) + }) + Context("creating a CR in the watched namespace", func() { + BeforeEach(func() { + cr = t.NewCryostat() + }) + It("should reconcile the CR", func() { + result := filter.Create(event.CreateEvent{ + Object: cr.Object, + }) + Expect(result).To(BeTrue()) + }) + }) + Context("creating a CR in a non-watched namespace", func() { + BeforeEach(func() { + t.Namespace = "something-else" + cr = t.NewCryostat() + }) + It("should reconcile the CR", func() { + result := filter.Create(event.CreateEvent{ + Object: cr.Object, + }) + Expect(result).To(BeFalse()) + }) + }) + }) + }) +}) diff --git a/internal/main.go b/internal/main.go index 1e014318..efe42ab3 100644 --- a/internal/main.go +++ b/internal/main.go @@ -17,7 +17,9 @@ package main import ( "flag" "fmt" + "net/url" "os" + "strings" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -40,6 +42,7 @@ import ( operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1" "github.com/cryostatio/cryostat-operator/internal/controllers" "github.com/cryostatio/cryostat-operator/internal/controllers/common" + "github.com/cryostatio/cryostat-operator/internal/controllers/insights" // +kubebuilder:scaffold:imports ) @@ -82,6 +85,10 @@ func main() { setupLog.Error(err, "unable to get WatchNamespace, "+ "the manager will watch and manage resources in all namespaces") } + namespaces := []string{} + if len(watchNamespace) > 0 { + namespaces = append(namespaces, strings.Split(watchNamespace, ",")...) + } ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) @@ -89,7 +96,7 @@ func main() { // when used with ClusterCryostat // https://github.com/cryostatio/cryostat-operator/issues/580 disableCache := []client.Object{} - if len(watchNamespace) > 0 { + if len(namespaces) > 0 { disableCache = append(disableCache, &rbacv1.RoleBinding{}) } @@ -103,8 +110,7 @@ func main() { HealthProbeBindAddress: probeAddr, LeaderElection: enableLeaderElection, LeaderElectionID: "d696d7ab.redhat.com", - Namespace: watchNamespace, - ClientDisableCacheFor: disableCache, + ClientDisableCacheFor: disableCache, // TODO can probably remove }) if err != nil { setupLog.Error(err, "unable to start manager") @@ -140,16 +146,34 @@ func main() { setupLog.Info("did not find cert-manager installation") } - config := newReconcilerConfig(mgr, "ClusterCryostat", "clustercryostat-controller", openShift, certManager) - if err = (controllers.NewClusterCryostatReconciler(config)).SetupWithManager(mgr); err != nil { + // Optionally enable Insights integration. Will only be enabled if INSIGHTS_ENABLED is true + insightsURL, err := insights.NewInsightsIntegration(mgr, &setupLog).Setup() + if err != nil { + setupLog.Error(err, "failed to set up Insights integration") + } + + config := newReconcilerConfig(mgr, "ClusterCryostat", "clustercryostat-controller", openShift, + certManager, namespaces, insightsURL) + clusterController, err := controllers.NewClusterCryostatReconciler(config) + if err != nil { setupLog.Error(err, "unable to create controller", "controller", "ClusterCryostat") os.Exit(1) } - config = newReconcilerConfig(mgr, "Cryostat", "cryostat-controller", openShift, certManager) - if err = (controllers.NewCryostatReconciler(config)).SetupWithManager(mgr); err != nil { + if err = clusterController.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to add controller to manager", "controller", "ClusterCryostat") + os.Exit(1) + } + config = newReconcilerConfig(mgr, "Cryostat", "cryostat-controller", openShift, certManager, + namespaces, insightsURL) + controller, err := controllers.NewCryostatReconciler(config) + if err != nil { setupLog.Error(err, "unable to create controller", "controller", "Cryostat") os.Exit(1) } + if err = controller.SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to add controller to manager", "controller", "Cryostat") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { @@ -191,7 +215,7 @@ func isCertManagerInstalled(client discovery.DiscoveryInterface) (bool, error) { } func newReconcilerConfig(mgr ctrl.Manager, logName string, eventRecorderName string, openShift bool, - certManager bool) *controllers.ReconcilerConfig { + certManager bool, namespaces []string, insightsURL *url.URL) *controllers.ReconcilerConfig { return &controllers.ReconcilerConfig{ Client: mgr.GetClient(), Log: ctrl.Log.WithName("controllers").WithName(logName), @@ -200,6 +224,8 @@ func newReconcilerConfig(mgr ctrl.Manager, logName string, eventRecorderName str IsCertManagerInstalled: certManager, EventRecorder: mgr.GetEventRecorderFor(eventRecorderName), RESTMapper: mgr.GetRESTMapper(), + Namespaces: namespaces, + InsightsProxy: insightsURL, ReconcilerTLS: common.NewReconcilerTLS(&common.ReconcilerTLSConfig{ Client: mgr.GetClient(), }), diff --git a/internal/test/expect.go b/internal/test/expect.go new file mode 100644 index 00000000..b77a1996 --- /dev/null +++ b/internal/test/expect.go @@ -0,0 +1,57 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +func ExpectResourceRequirements(containerResource, expectedResource *corev1.ResourceRequirements) { + // Containers must have resource requests + gomega.Expect(containerResource.Requests).ToNot(gomega.BeNil()) + + requestCpu, requestCpuFound := containerResource.Requests[corev1.ResourceCPU] + expectedRequestCpu := expectedResource.Requests[corev1.ResourceCPU] + gomega.Expect(requestCpuFound).To(gomega.BeTrue()) + gomega.Expect(requestCpu.Equal(expectedRequestCpu)).To(gomega.BeTrue()) + + requestMemory, requestMemoryFound := containerResource.Requests[corev1.ResourceMemory] + expectedRequestMemory := expectedResource.Requests[corev1.ResourceMemory] + gomega.Expect(requestMemoryFound).To(gomega.BeTrue()) + gomega.Expect(requestMemory.Equal(expectedRequestMemory)).To(gomega.BeTrue()) + + if expectedResource.Limits == nil { + gomega.Expect(containerResource.Limits).To(gomega.BeNil()) + } else { + gomega.Expect(containerResource.Limits).ToNot(gomega.BeNil()) + + limitCpu, limitCpuFound := containerResource.Limits[corev1.ResourceCPU] + expectedLimitCpu, expectedLimitCpuFound := expectedResource.Limits[corev1.ResourceCPU] + + gomega.Expect(limitCpuFound).To(gomega.Equal(expectedLimitCpuFound)) + if expectedLimitCpuFound { + gomega.Expect(limitCpu.Equal(expectedLimitCpu)).To(gomega.BeTrue()) + } + + limitMemory, limitMemoryFound := containerResource.Limits[corev1.ResourceMemory] + expectedlimitMemory, expectedLimitMemoryFound := expectedResource.Limits[corev1.ResourceMemory] + + gomega.Expect(limitMemoryFound).To(gomega.Equal(expectedLimitMemoryFound)) + if expectedLimitCpuFound { + gomega.Expect(limitMemory.Equal(expectedlimitMemory)).To(gomega.BeTrue()) + } + } +} diff --git a/internal/test/resources.go b/internal/test/resources.go index ad7e454c..79c95726 100644 --- a/internal/test/resources.go +++ b/internal/test/resources.go @@ -51,6 +51,7 @@ type TestResources struct { ReportReplicas int32 ClusterScoped bool TargetNamespaces []string + InsightsURL string } func NewTestScheme() *runtime.Scheme { @@ -184,7 +185,14 @@ func (r *TestResources) NewCryostatWithTemplates() *model.CryostatInstance { } func (r *TestResources) NewCryostatWithIngress() *model.CryostatInstance { - cr := r.NewCryostat() + return r.addIngressToCryostat(r.NewCryostat()) +} + +func (r *TestResources) NewCryostatWithIngressCertManagerDisabled() *model.CryostatInstance { + return r.addIngressToCryostat(r.NewCryostatCertManagerDisabled()) +} + +func (r *TestResources) addIngressToCryostat(cr *model.CryostatInstance) *model.CryostatInstance { networkConfig := r.newNetworkConfigurationList() cr.Spec.NetworkOptions = &networkConfig return cr @@ -1377,6 +1385,14 @@ func (r *TestResources) NewCoreEnvironmentVariables(reportsUrl string, authProps Value: "200", }) } + + if len(r.InsightsURL) > 0 { + envs = append(envs, + corev1.EnvVar{ + Name: "INSIGHTS_PROXY", + Value: r.InsightsURL, + }) + } return envs }