From f347ca982cf232739f7285ef88adb2686b07a906 Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Tue, 6 Aug 2024 11:40:13 +0200 Subject: [PATCH] [release-2.12] Add support for mcoa-managed observability capabilities (#1470) * Adds manifests for multicluster-observability-addon Signed-off-by: Periklis Tsirakidis * Add addon template renderer Signed-off-by: Periklis Tsirakidis * Address code review suggestions Signed-off-by: Periklis Tsirakidis * Fix deep copy Signed-off-by: Periklis Tsirakidis * Add missing operator RBAC Signed-off-by: Periklis Tsirakidis * Fix MCOA component image replacement Signed-off-by: Periklis Tsirakidis * Add update AddOnDeploymentConfig and ClusterManagementAddOn fns Signed-off-by: Periklis Tsirakidis * Fix deployer Signed-off-by: Periklis Tsirakidis * Regenerate bundle Signed-off-by: Periklis Tsirakidis * Apply code review suggestions Signed-off-by: Periklis Tsirakidis * Add support for t-shirt sizes Signed-off-by: Periklis Tsirakidis * Add missing license doc Signed-off-by: Periklis Tsirakidis * Regenerate bundles Signed-off-by: Periklis Tsirakidis * Fix sonar-lint Signed-off-by: Periklis Tsirakidis * Regenerate bundle Signed-off-by: Periklis Tsirakidis * Remove obsolete permissions for secrets Signed-off-by: Periklis Tsirakidis * Fix unit tests Signed-off-by: Periklis Tsirakidis * Fix last rebase Signed-off-by: Periklis Tsirakidis * Fix unit tests and sonar lint Signed-off-by: Periklis Tsirakidis * Update operators/multiclusterobservability/manifests/base/multicluster-observability-addon/deployment.yaml Co-authored-by: Joao Marcal Signed-off-by: Periklis Tsirakidis * Fix e2e tests Signed-off-by: Periklis Tsirakidis * Address suggestions from code review Signed-off-by: Periklis Tsirakidis * Address code review suggestions Signed-off-by: Periklis Tsirakidis * Add missing generated code Signed-off-by: Periklis Tsirakidis * Address sonar lint issues Signed-off-by: Periklis Tsirakidis * Regenerate bundle Signed-off-by: Periklis Tsirakidis * Addressing sonar lint issues Signed-off-by: Periklis Tsirakidis * Replace the mcoa image when running on a separate tagSuffix Signed-off-by: Periklis Tsirakidis --------- Signed-off-by: Periklis Tsirakidis Co-authored-by: Joao Marcal --- go.mod | 4 +- .../multiclusterobservability_types.go | 166 ++++++++++++ .../api/v1beta2/zz_generated.deepcopy.go | 209 ++++++++++++++ ...bility-operator.clusterserviceversion.yaml | 5 +- ...gement.io_multiclusterobservabilities.yaml | 157 +++++++++++ ...gement.io_multiclusterobservabilities.yaml | 157 +++++++++++ .../config/rbac/mco_role.yaml | 3 + .../multiclusterobservability_controller.go | 32 ++- ...lticlusterobservability_controller_test.go | 41 +-- operators/multiclusterobservability/main.go | 1 + .../addon_deployment_config.yaml | 10 + .../cluster_management_addon.yaml | 42 +++ .../cluster_role.yaml | 70 +++++ .../cluster_role_binding.yaml | 16 ++ .../deployment.yaml | 57 ++++ .../kustomization.yaml | 7 + .../service_account.yaml | 5 + .../pkg/config/config.go | 45 ++-- .../pkg/config/resources.go | 16 ++ .../pkg/config/resources_map.go | 34 +++ .../pkg/rendering/renderer.go | 13 + .../pkg/rendering/renderer_mcoa.go | 255 ++++++++++++++++++ .../pkg/rendering/renderer_mcoa_test.go | 174 ++++++++++++ .../pkg/rendering/templates/templates.go | 27 +- operators/pkg/deploying/deployer.go | 43 +++ operators/pkg/deploying/deployer_test.go | 131 ++++++++- tests/pkg/utils/mco_deploy.go | 2 +- 27 files changed, 1667 insertions(+), 55 deletions(-) create mode 100644 operators/multiclusterobservability/manifests/base/multicluster-observability-addon/addon_deployment_config.yaml create mode 100644 operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_management_addon.yaml create mode 100644 operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_role.yaml create mode 100644 operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_role_binding.yaml create mode 100644 operators/multiclusterobservability/manifests/base/multicluster-observability-addon/deployment.yaml create mode 100644 operators/multiclusterobservability/manifests/base/multicluster-observability-addon/kustomization.yaml create mode 100644 operators/multiclusterobservability/manifests/base/multicluster-observability-addon/service_account.yaml create mode 100644 operators/multiclusterobservability/pkg/rendering/renderer_mcoa.go create mode 100644 operators/multiclusterobservability/pkg/rendering/renderer_mcoa_test.go diff --git a/go.mod b/go.mod index 344bb6a17..f410ba1db 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/golang/snappy v0.0.4 github.com/google/go-cmp v0.6.0 github.com/hashicorp/go-version v1.6.0 + github.com/imdario/mergo v0.3.16 github.com/oklog/run v1.1.0 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.33.1 @@ -47,6 +48,7 @@ require ( k8s.io/client-go v0.31.0-alpha.2 k8s.io/klog v1.0.0 k8s.io/kubectl v0.29.3 + k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 open-cluster-management.io/addon-framework v0.9.2 open-cluster-management.io/api v0.13.0 sigs.k8s.io/controller-runtime v0.18.1-0.20240626171621-700befecdffa @@ -134,7 +136,6 @@ require ( github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect - github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jhump/protoreflect v1.9.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -242,7 +243,6 @@ require ( k8s.io/kms v0.31.0-alpha.1 // indirect k8s.io/kube-aggregator v0.30.1 // indirect k8s.io/kube-openapi v0.0.0-20240620174524-b456828f718b // indirect - k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0 // indirect open-cluster-management.io/sdk-go v0.13.1-0.20240416030555-aa744f426379 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.29.0 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect diff --git a/operators/multiclusterobservability/api/v1beta2/multiclusterobservability_types.go b/operators/multiclusterobservability/api/v1beta2/multiclusterobservability_types.go index 10481ded9..519d1119c 100644 --- a/operators/multiclusterobservability/api/v1beta2/multiclusterobservability_types.go +++ b/operators/multiclusterobservability/api/v1beta2/multiclusterobservability_types.go @@ -13,6 +13,16 @@ import ( // MultiClusterObservabilitySpec defines the desired state of MultiClusterObservability. type MultiClusterObservabilitySpec struct { + // Capabilities defines the platform and user workload observabilities capabilities + // managed exclusively by the multicluster-observability-addon. Enabling any of these + // capabilities will result in deploying the following resources: + // - The addon Deployment, ServiceAccount and RBAC. + // - A ClusterManagementAddon managing placement for capability related custom resources. + // - An AddonDeploymentConfig managing the addon feature gates for activated capabilities. + // + // +optional + // +kubebuilder:validation:Optional + Capabilities *CapabilitiesSpec `json:"capabilities,omitempty"` // Advanced configurations for observability // +optional AdvancedConfig *AdvancedConfig `json:"advanced,omitempty"` @@ -48,6 +58,159 @@ type MultiClusterObservabilitySpec struct { // +kubebuilder:validation:Enum:={"default","minimal","small","medium","large","xlarge","2xlarge","4xlarge"} type TShirtSize string +// PlatformLogsCollectionSpec defines the spec for the addon to collect and forward logs +// from fleet managed clusters using the ClusterLogForwarder custom resource. +type PlatformLogsCollectionSpec struct { + // Enabled defines a flag to enable/disable the platform log collection. + // + // +optional + // +kubebuilder:validation:Optional + Enabled bool `json:"enabled,omitempty"` +} + +// PlatformLogsSpec defines the spec for the addon to collect, forward and store logs +// from fleet managed clusters. +type PlatformLogsSpec struct { + // Collection defines the spec for the addon to collect and forward logs + // from fleet managed clusters. + // + // +optional + // +kubebuilder:validation:Optional + Collection PlatformLogsCollectionSpec `json:"collection,omitempty"` +} + +// PlatformCapabilitiesSpec defines the observability capabilities managed by the addon +// for platform components. +type PlatformCapabilitiesSpec struct { + // Logs defines the configuration spec for collecting and storing logs from + // platform components running on fleet managed clusters. + // + // +optional + // +kubebuilder:validation:Optional + Logs PlatformLogsSpec `json:"logs,omitempty"` +} + +// ClusterLogForwarderSpec defines the spec for the addon to collect and forward logs +// using the ClusterLogForwarder custom resource. +type ClusterLogForwarderSpec struct { + // Enabled defines a flag to enable/disable the platform log collection using the ClusterLogForwarder resource. + // + // +optional + // +kubebuilder:validation:Optional + Enabled bool `json:"enabled,omitempty"` +} + +// UserWorkloadLogsCollectionSpec defines the spec for the addon to collect and forward logs +// from user workloads hosted on fleet managed clusters. +type UserWorkloadLogsCollectionSpec struct { + // Enabled defines a flag to enable/disable the platform log collection using the ClusterLogForwarder resource. + // + // +optional + // +kubebuilder:validation:Optional + ClusterLogForwarder ClusterLogForwarderSpec `json:"clusterLogForwarder,omitempty"` +} + +// UserWorkloadLogsSpec defines the spec for the addon to collect,forward and store logs +// from user workloads hosted on fleet managed clusters. +type UserWorkloadLogsSpec struct { + // Collection defines the spec for the addon to collect and forward logs + // from user workloads hosted on fleet managed clusters. + // + // +optional + // +kubebuilder:validation:Optional + Collection UserWorkloadLogsCollectionSpec `json:"collection,omitempty"` +} + +// OpenTelemetryCollectorSpec defines the spec for the addon to collect and forward observability signals +// using the OpenTelemetryCollector custom resource. +type OpenTelemetryCollectorSpec struct { + // Enabled defines a flag to enable/disable the user workload observability collection using the OpenTelemetryCollector resource. + // + // +optional + // +kubebuilder:validation:Optional + Enabled bool `json:"enabled,omitempty"` +} + +// OpenTelemetryCollectorSpec defines the spec for the addon to collect observability signals +// using the Instrumentation custom resource. +type InstrumentationSpec struct { + // Enabled defines a flag to enable/disable the user workload observability collection using the Instrumentation resource. + // + // +optional + // +kubebuilder:validation:Optional + Enabled bool `json:"enabled,omitempty"` +} + +// OpenTelemetryCollectionSpec defines the spec for the addon to collect and forward observability signals +// from user workloads hosted on fleet managed clusters using the OpenTelemetryCollector with or without +// instrumentation. +type OpenTelemetryCollectionSpec struct { + // Collector defines the spec for the user workload observability collection using the OpenTelemetryCollector resource. + // + // +optional + // +kubebuilder:validation:Optional + Collector OpenTelemetryCollectorSpec `json:"collector,omitempty"` + // Instrumentation defines the spec for the user workload observability collection using the Instrumentation resource. + // + // +optional + // +kubebuilder:validation:Optional + Instrumentation InstrumentationSpec `json:"instrumentation,omitempty"` +} + +// UserWorkloadTracesSpec defines the spec for the addon to collect, forward and store traces +// from user workloads hosted on fleet managed clusters. +type UserWorkloadTracesSpec struct { + // Collection defines the spec for the addon to collect and forward traces + // from user workloads hosted on fleet managed clusters. + // + // +optional + // +kubebuilder:validation:Optional + Collection OpenTelemetryCollectionSpec `json:"collection,omitempty"` +} + +// UserWorkloadCapabilitiesSpec defines the spec for user workload observability capabilities managed by the addon. +type UserWorkloadCapabilitiesSpec struct { + // Logs defines the spec for the addon to collect, forward and store logs + // from user workloads hosted on fleet managed clusters. + // + // +optional + // +kubebuilder:validation:Optional + Logs UserWorkloadLogsSpec `json:"logs,omitempty"` + // Traces defines the spec for the addon to collect, forward and store traces + // from user workloads hosted on fleet managed clusters. + // + // +optional + // +kubebuilder:validation:Optional + Traces UserWorkloadTracesSpec `json:"traces,omitempty"` +} + +// CapabilitiesSpec defines the platform and user workload observabilities capabilities +// managed exclusively by the multicluster-observability-addon. Enabling any of these +// capabilities will result in deploying the following resources: +// - The addon Deployment, ServiceAccount and RBAC. +// - A ClusterManagementAddon managing placement for capability related custom resources. +// - An AddonDeploymentConfig managing the addon feature gates for activated capabilities. +type CapabilitiesSpec struct { + // Platform defines the spec for platform observability capabilities managed by the addon. + // The platform is defined as the ACM/OCM/OCP components running on managed clusters to run, + // manage and observe the managed clusters themselves locally as well as remotely from a hub. + // Such components live on namespaces with prefixes for example: + // - openshift- + // - open-cluster-management- + // - default + // + // +optional + // +kubebuilder:validation:Optional + Platform *PlatformCapabilitiesSpec `json:"platform,omitempty"` + // UserWorkloads defines the spec for user workloads observability capabilities managed by the addon. + // As user workloads are defined any containers hosted on spoke clusters and execute any task unrelated + // to managing the fleet or the individual managed cluster itself. + // + // +optional + // +kubebuilder:validation:Optional + UserWorkloads *UserWorkloadCapabilitiesSpec `json:"userWorkloads,omitempty"` +} + type AdvancedConfig struct { // CustomObservabilityHubURL overrides the endpoint used by the metrics-collector to send // metrics to the hub server. @@ -98,6 +261,9 @@ type AdvancedConfig struct { // spec for thanos-store-shard // +optional Store *StoreSpec `json:"store,omitempty"` + // spec for multicluster-obervability-addon + // +optional + MultiClusterObservabilityAddon *CommonSpec `json:"multiClusterObservabilityAddon,omitempty"` } type CommonSpec struct { diff --git a/operators/multiclusterobservability/api/v1beta2/zz_generated.deepcopy.go b/operators/multiclusterobservability/api/v1beta2/zz_generated.deepcopy.go index c629bbd1e..9c3c8448f 100644 --- a/operators/multiclusterobservability/api/v1beta2/zz_generated.deepcopy.go +++ b/operators/multiclusterobservability/api/v1beta2/zz_generated.deepcopy.go @@ -95,6 +95,11 @@ func (in *AdvancedConfig) DeepCopyInto(out *AdvancedConfig) { *out = new(StoreSpec) (*in).DeepCopyInto(*out) } + if in.MultiClusterObservabilityAddon != nil { + in, out := &in.MultiClusterObservabilityAddon, &out.MultiClusterObservabilityAddon + *out = new(CommonSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdvancedConfig. @@ -133,6 +138,46 @@ func (in *CacheConfig) DeepCopy() *CacheConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CapabilitiesSpec) DeepCopyInto(out *CapabilitiesSpec) { + *out = *in + if in.Platform != nil { + in, out := &in.Platform, &out.Platform + *out = new(PlatformCapabilitiesSpec) + **out = **in + } + if in.UserWorkloads != nil { + in, out := &in.UserWorkloads, &out.UserWorkloads + *out = new(UserWorkloadCapabilitiesSpec) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CapabilitiesSpec. +func (in *CapabilitiesSpec) DeepCopy() *CapabilitiesSpec { + if in == nil { + return nil + } + out := new(CapabilitiesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ClusterLogForwarderSpec) DeepCopyInto(out *ClusterLogForwarderSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterLogForwarderSpec. +func (in *ClusterLogForwarderSpec) DeepCopy() *ClusterLogForwarderSpec { + if in == nil { + return nil + } + out := new(ClusterLogForwarderSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CommonSpec) DeepCopyInto(out *CommonSpec) { *out = *in @@ -192,6 +237,21 @@ func (in *CompactSpec) DeepCopy() *CompactSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *InstrumentationSpec) DeepCopyInto(out *InstrumentationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InstrumentationSpec. +func (in *InstrumentationSpec) DeepCopy() *InstrumentationSpec { + if in == nil { + return nil + } + out := new(InstrumentationSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MultiClusterObservability) DeepCopyInto(out *MultiClusterObservability) { *out = *in @@ -254,6 +314,11 @@ func (in *MultiClusterObservabilityList) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MultiClusterObservabilitySpec) DeepCopyInto(out *MultiClusterObservabilitySpec) { *out = *in + if in.Capabilities != nil { + in, out := &in.Capabilities, &out.Capabilities + *out = new(CapabilitiesSpec) + (*in).DeepCopyInto(*out) + } if in.AdvancedConfig != nil { in, out := &in.AdvancedConfig, &out.AdvancedConfig *out = new(AdvancedConfig) @@ -317,6 +382,85 @@ func (in *MultiClusterObservabilityStatus) DeepCopy() *MultiClusterObservability return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenTelemetryCollectionSpec) DeepCopyInto(out *OpenTelemetryCollectionSpec) { + *out = *in + out.Collector = in.Collector + out.Instrumentation = in.Instrumentation +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenTelemetryCollectionSpec. +func (in *OpenTelemetryCollectionSpec) DeepCopy() *OpenTelemetryCollectionSpec { + if in == nil { + return nil + } + out := new(OpenTelemetryCollectionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenTelemetryCollectorSpec) DeepCopyInto(out *OpenTelemetryCollectorSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenTelemetryCollectorSpec. +func (in *OpenTelemetryCollectorSpec) DeepCopy() *OpenTelemetryCollectorSpec { + if in == nil { + return nil + } + out := new(OpenTelemetryCollectorSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlatformCapabilitiesSpec) DeepCopyInto(out *PlatformCapabilitiesSpec) { + *out = *in + out.Logs = in.Logs +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlatformCapabilitiesSpec. +func (in *PlatformCapabilitiesSpec) DeepCopy() *PlatformCapabilitiesSpec { + if in == nil { + return nil + } + out := new(PlatformCapabilitiesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlatformLogsCollectionSpec) DeepCopyInto(out *PlatformLogsCollectionSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlatformLogsCollectionSpec. +func (in *PlatformLogsCollectionSpec) DeepCopy() *PlatformLogsCollectionSpec { + if in == nil { + return nil + } + out := new(PlatformLogsCollectionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PlatformLogsSpec) DeepCopyInto(out *PlatformLogsSpec) { + *out = *in + out.Collection = in.Collection +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PlatformLogsSpec. +func (in *PlatformLogsSpec) DeepCopy() *PlatformLogsSpec { + if in == nil { + return nil + } + out := new(PlatformLogsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QueryFrontendSpec) DeepCopyInto(out *QueryFrontendSpec) { *out = *in @@ -505,3 +649,68 @@ func (in *StoreSpec) DeepCopy() *StoreSpec { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserWorkloadCapabilitiesSpec) DeepCopyInto(out *UserWorkloadCapabilitiesSpec) { + *out = *in + out.Logs = in.Logs + out.Traces = in.Traces +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserWorkloadCapabilitiesSpec. +func (in *UserWorkloadCapabilitiesSpec) DeepCopy() *UserWorkloadCapabilitiesSpec { + if in == nil { + return nil + } + out := new(UserWorkloadCapabilitiesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserWorkloadLogsCollectionSpec) DeepCopyInto(out *UserWorkloadLogsCollectionSpec) { + *out = *in + out.ClusterLogForwarder = in.ClusterLogForwarder +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserWorkloadLogsCollectionSpec. +func (in *UserWorkloadLogsCollectionSpec) DeepCopy() *UserWorkloadLogsCollectionSpec { + if in == nil { + return nil + } + out := new(UserWorkloadLogsCollectionSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserWorkloadLogsSpec) DeepCopyInto(out *UserWorkloadLogsSpec) { + *out = *in + out.Collection = in.Collection +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserWorkloadLogsSpec. +func (in *UserWorkloadLogsSpec) DeepCopy() *UserWorkloadLogsSpec { + if in == nil { + return nil + } + out := new(UserWorkloadLogsSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UserWorkloadTracesSpec) DeepCopyInto(out *UserWorkloadTracesSpec) { + *out = *in + out.Collection = in.Collection +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UserWorkloadTracesSpec. +func (in *UserWorkloadTracesSpec) DeepCopy() *UserWorkloadTracesSpec { + if in == nil { + return nil + } + out := new(UserWorkloadTracesSpec) + in.DeepCopyInto(out) + return out +} diff --git a/operators/multiclusterobservability/bundle/manifests/multicluster-observability-operator.clusterserviceversion.yaml b/operators/multiclusterobservability/bundle/manifests/multicluster-observability-operator.clusterserviceversion.yaml index beb79f096..dd06f1299 100644 --- a/operators/multiclusterobservability/bundle/manifests/multicluster-observability-operator.clusterserviceversion.yaml +++ b/operators/multiclusterobservability/bundle/manifests/multicluster-observability-operator.clusterserviceversion.yaml @@ -49,7 +49,7 @@ metadata: } ] capabilities: Basic Install - createdAt: "2024-07-02T08:16:00Z" + createdAt: "2024-08-01T12:07:10Z" operators.operatorframework.io/builder: operator-sdk-v1.34.2 operators.operatorframework.io/project_layout: go.kubebuilder.io/v3 name: multicluster-observability-operator.v0.1.0 @@ -454,6 +454,9 @@ spec: - get - list - watch + - create + - patch + - delete serviceAccountName: multicluster-observability-operator deployments: - label: diff --git a/operators/multiclusterobservability/bundle/manifests/observability.open-cluster-management.io_multiclusterobservabilities.yaml b/operators/multiclusterobservability/bundle/manifests/observability.open-cluster-management.io_multiclusterobservabilities.yaml index 30881476d..ce73f6f74 100644 --- a/operators/multiclusterobservability/bundle/manifests/observability.open-cluster-management.io_multiclusterobservabilities.yaml +++ b/operators/multiclusterobservability/bundle/manifests/observability.open-cluster-management.io_multiclusterobservabilities.yaml @@ -2031,6 +2031,63 @@ spec: type: object type: object type: object + multiClusterObservabilityAddon: + description: spec for multicluster-obervability-addon + properties: + replicas: + description: Replicas for this component. + format: int32 + type: integer + resources: + description: Compute Resources required by this component. + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where this + field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: object observatoriumAPI: description: Spec of observatorium api properties: @@ -10131,6 +10188,106 @@ spec: type: object type: object type: object + capabilities: + description: 'Capabilities defines the platform and user workload + observabilities capabilities managed exclusively by the multicluster-observability-addon. + Enabling any of these capabilities will result in deploying the + following resources: - The addon Deployment, ServiceAccount and + RBAC. - A ClusterManagementAddon managing placement for capability + related custom resources. - An AddonDeploymentConfig managing the + addon feature gates for activated capabilities.' + properties: + platform: + description: 'Platform defines the spec for platform observability + capabilities managed by the addon. The platform is defined as + the ACM/OCM/OCP components running on managed clusters to run, + manage and observe the managed clusters themselves locally as + well as remotely from a hub. Such components live on namespaces + with prefixes for example: - openshift- - open-cluster-management- + - default' + properties: + logs: + description: Logs defines the configuration spec for collecting + and storing logs from platform components running on fleet + managed clusters. + properties: + collection: + description: Collection defines the spec for the addon + to collect and forward logs from fleet managed clusters. + properties: + enabled: + description: Enabled defines a flag to enable/disable + the platform log collection. + type: boolean + type: object + type: object + type: object + userWorkloads: + description: UserWorkloads defines the spec for user workloads + observability capabilities managed by the addon. As user workloads + are defined any containers hosted on spoke clusters and execute + any task unrelated to managing the fleet or the individual managed + cluster itself. + properties: + logs: + description: Logs defines the spec for the addon to collect, + forward and store logs from user workloads hosted on fleet + managed clusters. + properties: + collection: + description: Collection defines the spec for the addon + to collect and forward logs from user workloads hosted + on fleet managed clusters. + properties: + clusterLogForwarder: + description: Enabled defines a flag to enable/disable + the platform log collection using the ClusterLogForwarder + resource. + properties: + enabled: + description: Enabled defines a flag to enable/disable + the platform log collection using the ClusterLogForwarder + resource. + type: boolean + type: object + type: object + type: object + traces: + description: Traces defines the spec for the addon to collect, + forward and store traces from user workloads hosted on fleet + managed clusters. + properties: + collection: + description: Collection defines the spec for the addon + to collect and forward traces from user workloads hosted + on fleet managed clusters. + properties: + collector: + description: Collector defines the spec for the user + workload observability collection using the OpenTelemetryCollector + resource. + properties: + enabled: + description: Enabled defines a flag to enable/disable + the user workload observability collection using + the OpenTelemetryCollector resource. + type: boolean + type: object + instrumentation: + description: Instrumentation defines the spec for + the user workload observability collection using + the Instrumentation resource. + properties: + enabled: + description: Enabled defines a flag to enable/disable + the user workload observability collection using + the Instrumentation resource. + type: boolean + type: object + type: object + type: object + type: object + type: object enableDownsampling: default: true description: Enable or disable the downsample. diff --git a/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_multiclusterobservabilities.yaml b/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_multiclusterobservabilities.yaml index a320cca46..401ff8ee8 100644 --- a/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_multiclusterobservabilities.yaml +++ b/operators/multiclusterobservability/config/crd/bases/observability.open-cluster-management.io_multiclusterobservabilities.yaml @@ -2015,6 +2015,63 @@ spec: type: object type: object type: object + multiClusterObservabilityAddon: + description: spec for multicluster-obervability-addon + properties: + replicas: + description: Replicas for this component. + format: int32 + type: integer + resources: + description: Compute Resources required by this component. + properties: + claims: + description: "Claims lists the names of resources, defined + in spec.resourceClaims, that are used by this container. + \n This is an alpha field and requires enabling the + DynamicResourceAllocation feature gate. \n This field + is immutable. It can only be set for containers." + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry + in pod.spec.resourceClaims of the Pod where this + field is used. It makes that resource available + inside a container. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: object observatoriumAPI: description: Spec of observatorium api properties: @@ -10115,6 +10172,106 @@ spec: type: object type: object type: object + capabilities: + description: 'Capabilities defines the platform and user workload + observabilities capabilities managed exclusively by the multicluster-observability-addon. + Enabling any of these capabilities will result in deploying the + following resources: - The addon Deployment, ServiceAccount and + RBAC. - A ClusterManagementAddon managing placement for capability + related custom resources. - An AddonDeploymentConfig managing the + addon feature gates for activated capabilities.' + properties: + platform: + description: 'Platform defines the spec for platform observability + capabilities managed by the addon. The platform is defined as + the ACM/OCM/OCP components running on managed clusters to run, + manage and observe the managed clusters themselves locally as + well as remotely from a hub. Such components live on namespaces + with prefixes for example: - openshift- - open-cluster-management- + - default' + properties: + logs: + description: Logs defines the configuration spec for collecting + and storing logs from platform components running on fleet + managed clusters. + properties: + collection: + description: Collection defines the spec for the addon + to collect and forward logs from fleet managed clusters. + properties: + enabled: + description: Enabled defines a flag to enable/disable + the platform log collection. + type: boolean + type: object + type: object + type: object + userWorkloads: + description: UserWorkloads defines the spec for user workloads + observability capabilities managed by the addon. As user workloads + are defined any containers hosted on spoke clusters and execute + any task unrelated to managing the fleet or the individual managed + cluster itself. + properties: + logs: + description: Logs defines the spec for the addon to collect, + forward and store logs from user workloads hosted on fleet + managed clusters. + properties: + collection: + description: Collection defines the spec for the addon + to collect and forward logs from user workloads hosted + on fleet managed clusters. + properties: + clusterLogForwarder: + description: Enabled defines a flag to enable/disable + the platform log collection using the ClusterLogForwarder + resource. + properties: + enabled: + description: Enabled defines a flag to enable/disable + the platform log collection using the ClusterLogForwarder + resource. + type: boolean + type: object + type: object + type: object + traces: + description: Traces defines the spec for the addon to collect, + forward and store traces from user workloads hosted on fleet + managed clusters. + properties: + collection: + description: Collection defines the spec for the addon + to collect and forward traces from user workloads hosted + on fleet managed clusters. + properties: + collector: + description: Collector defines the spec for the user + workload observability collection using the OpenTelemetryCollector + resource. + properties: + enabled: + description: Enabled defines a flag to enable/disable + the user workload observability collection using + the OpenTelemetryCollector resource. + type: boolean + type: object + instrumentation: + description: Instrumentation defines the spec for + the user workload observability collection using + the Instrumentation resource. + properties: + enabled: + description: Enabled defines a flag to enable/disable + the user workload observability collection using + the Instrumentation resource. + type: boolean + type: object + type: object + type: object + type: object + type: object enableDownsampling: default: true description: Enable or disable the downsample. diff --git a/operators/multiclusterobservability/config/rbac/mco_role.yaml b/operators/multiclusterobservability/config/rbac/mco_role.yaml index d55918278..af37cd06b 100644 --- a/operators/multiclusterobservability/config/rbac/mco_role.yaml +++ b/operators/multiclusterobservability/config/rbac/mco_role.yaml @@ -370,3 +370,6 @@ rules: - get - list - watch + - create + - patch + - delete diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go index 04648beea..a4d0c96d1 100644 --- a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller.go @@ -14,6 +14,7 @@ import ( "time" operatorconfig "github.com/stolostron/multicluster-observability-operator/operators/pkg/config" + addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" "github.com/go-logr/logr" routev1 "github.com/openshift/api/route/v1" @@ -372,7 +373,8 @@ func (r *MultiClusterObservabilityReconciler) Reconcile(ctx context.Context, req } func (r *MultiClusterObservabilityReconciler) initFinalization( - mco *mcov1beta2.MultiClusterObservability) (bool, error) { + mco *mcov1beta2.MultiClusterObservability, +) (bool, error) { if mco.GetDeletionTimestamp() != nil && slices.Contains(mco.GetFinalizers(), resFinalizer) { log.Info("To delete resources across namespaces") // clean up the cluster resources, eg. clusterrole, clusterrolebinding, etc @@ -462,6 +464,10 @@ func (r *MultiClusterObservabilityReconciler) SetupWithManager(mgr ctrl.Manager) Owns(&corev1.Service{}). // Watch for changes to secondary Observatorium CR and requeue the owner MultiClusterObservability Owns(&observatoriumv1alpha1.Observatorium{}). + // Watch for changes to secondary AddOnDeploymentConfig CR and requeue the owner MultiClusterObservability + Owns(&addonv1alpha1.AddOnDeploymentConfig{}). + // Watch for changes to secondary ClusterManagementAddOn CR and requeue the owner MultiClusterObservability + Owns(&addonv1alpha1.ClusterManagementAddOn{}). // Watch the configmap for thanos-ruler-custom-rules update Watches(&corev1.ConfigMap{}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(cmPred)). // Watch the secret for deleting event of alertmanager-config @@ -509,7 +515,6 @@ func (r *MultiClusterObservabilityReconciler) SetupWithManager(mgr ctrl.Manager) } func checkStorageChanged(mcoOldConfig, mcoNewConfig *mcov1beta2.StorageConfig) { - if mcoOldConfig.AlertmanagerStorageSize != mcoNewConfig.AlertmanagerStorageSize { isAlertmanagerStorageSizeChanged = true } @@ -532,8 +537,8 @@ func checkStorageChanged(mcoOldConfig, mcoNewConfig *mcov1beta2.StorageConfig) { // 2. Removed StatefulSet and // wait for operator to re-create the StatefulSet with the correct size on the claim func (r *MultiClusterObservabilityReconciler) HandleStorageSizeChange( - mco *mcov1beta2.MultiClusterObservability) (*reconcile.Result, error) { - + mco *mcov1beta2.MultiClusterObservability, +) (*reconcile.Result, error) { if isAlertmanagerStorageSizeChanged { isAlertmanagerStorageSizeChanged = false err := updateStorageSizeChange(r.Client, @@ -634,7 +639,8 @@ func updateStorageSizeChange(c client.Client, matchLabels map[string]string, sto // GenerateAlertmanagerRoute create route for Alertmanager endpoint func GenerateAlertmanagerRoute( runclient client.Client, scheme *runtime.Scheme, - mco *mcov1beta2.MultiClusterObservability) (*ctrl.Result, error) { + mco *mcov1beta2.MultiClusterObservability, +) (*ctrl.Result, error) { amGateway := &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: config.AlertmanagerRouteName, @@ -729,7 +735,8 @@ func GenerateAlertmanagerRoute( // GenerateProxyRoute create route for Proxy endpoint func GenerateProxyRoute( runclient client.Client, scheme *runtime.Scheme, - mco *mcov1beta2.MultiClusterObservability) (*ctrl.Result, error) { + mco *mcov1beta2.MultiClusterObservability, +) (*ctrl.Result, error) { proxyGateway := &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: config.ProxyRouteName, @@ -824,7 +831,8 @@ func GenerateProxyRoute( // The cluster scoped resources need to be deleted manually because they don't have ownerrefenence set as the MCO CR func cleanUpClusterScopedResources( r *MultiClusterObservabilityReconciler, - mco *mcov1beta2.MultiClusterObservability) error { + mco *mcov1beta2.MultiClusterObservability, +) error { matchLabels := map[string]string{config.GetCrLabelKey(): mco.Name} listOpts := []client.ListOption{ client.MatchingLabels(matchLabels), @@ -863,8 +871,8 @@ func cleanUpClusterScopedResources( } func (r *MultiClusterObservabilityReconciler) ensureOpenShiftNamespaceLabel(ctx context.Context, - m *mcov1beta2.MultiClusterObservability) (reconcile.Result, error) { - + m *mcov1beta2.MultiClusterObservability, +) (reconcile.Result, error) { log := logf.FromContext(ctx) existingNs := &corev1.Namespace{} resNS := m.GetNamespace() @@ -899,8 +907,10 @@ func (r *MultiClusterObservabilityReconciler) ensureOpenShiftNamespaceLabel(ctx func (r *MultiClusterObservabilityReconciler) deleteSpecificPrometheusRule(ctx context.Context) error { promRule := &monitoringv1.PrometheusRule{} - err := r.Client.Get(ctx, client.ObjectKey{Name: "acm-observability-alert-rules", - Namespace: "openshift-monitoring"}, promRule) + err := r.Client.Get(ctx, client.ObjectKey{ + Name: "acm-observability-alert-rules", + Namespace: "openshift-monitoring", + }, promRule) if err == nil { err = r.Client.Delete(ctx, promRule) if err != nil { diff --git a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go index 616caf91d..6c00ac92c 100644 --- a/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go +++ b/operators/multiclusterobservability/controllers/multiclusterobservability/multiclusterobservability_controller_test.go @@ -57,7 +57,7 @@ func setupTest(t *testing.T) func() { t.Fatalf("Failed to get work dir: (%v)", err) } t.Log("begin setupTest") - os.MkdirAll(path.Join(wd, "../../tests"), 0755) + os.MkdirAll(path.Join(wd, "../../tests"), 0o755) testManifestsPath := path.Join(wd, "../../tests/manifests") manifestsPath := path.Join(wd, "../../manifests") os.Setenv("TEMPLATES_PATH", testManifestsPath) @@ -100,18 +100,19 @@ func newTestCert(name string, namespace string) *corev1.Secret { } var testImagemanifestsMap = map[string]string{ - "endpoint_monitoring_operator": "test.io/endpoint-monitoring:test", - "grafana": "test.io/origin-grafana:test", - "grafana_dashboard_loader": "test.io/grafana-dashboard-loader:test", - "management_ingress": "test.io/management-ingress:test", - "observatorium": "test.io/observatorium:test", - "observatorium_operator": "test.io/observatorium-operator:test", - "prometheus_alertmanager": "test.io/prometheus-alertmanager:test", - "configmap_reloader": "test.io/configmap-reloader:test", - "rbac_query_proxy": "test.io/rbac-query-proxy:test", - "thanos": "test.io/thanos:test", - "thanos_receive_controller": "test.io/thanos_receive_controller:test", - "kube_rbac_proxy": "test.io/kube-rbac-proxy:test", + "endpoint_monitoring_operator": "test.io/endpoint-monitoring:test", + "grafana": "test.io/origin-grafana:test", + "grafana_dashboard_loader": "test.io/grafana-dashboard-loader:test", + "management_ingress": "test.io/management-ingress:test", + "observatorium": "test.io/observatorium:test", + "observatorium_operator": "test.io/observatorium-operator:test", + "prometheus_alertmanager": "test.io/prometheus-alertmanager:test", + "configmap_reloader": "test.io/configmap-reloader:test", + "rbac_query_proxy": "test.io/rbac-query-proxy:test", + "thanos": "test.io/thanos:test", + "thanos_receive_controller": "test.io/thanos_receive_controller:test", + "kube_rbac_proxy": "test.io/kube-rbac-proxy:test", + "multicluster_observability_addon": "test.io/multicluster-observability-addon:test", } func newTestImageManifestsConfigMap(namespace, version string) *corev1.ConfigMap { @@ -332,8 +333,10 @@ func TestMultiClusterMonitoringCRUpdate(t *testing.T) { }, } - objs := []runtime.Object{mco, svc, serverCACerts, clientCACerts, proxyRouteBYOCACerts, grafanaCert, serverCert, - testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, proxyRouteBYOCert, clustermgmtAddon, extensionApiserverAuthenticationCM} + objs := []runtime.Object{ + mco, svc, serverCACerts, clientCACerts, proxyRouteBYOCACerts, grafanaCert, serverCert, + testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, proxyRouteBYOCert, clustermgmtAddon, extensionApiserverAuthenticationCM, + } // Create a fake client to mock API calls. cl := fake.NewClientBuilder(). WithRuntimeObjects(objs...). @@ -746,8 +749,10 @@ func TestImageReplaceForMCO(t *testing.T) { }, } - objs := []runtime.Object{mco, observatoriumAPIsvc, serverCACerts, clientCACerts, grafanaCert, serverCert, - testMCHInstance, imageManifestsCM, testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, clustermgmtAddon, extensionApiserverAuthenticationCM} + objs := []runtime.Object{ + mco, observatoriumAPIsvc, serverCACerts, clientCACerts, grafanaCert, serverCert, + testMCHInstance, imageManifestsCM, testAmRouteBYOCaSecret, testAmRouteBYOCertSecret, clustermgmtAddon, extensionApiserverAuthenticationCM, + } // Create a fake client to mock API calls. cl := fake.NewClientBuilder().WithRuntimeObjects(objs...).Build() @@ -850,7 +855,6 @@ func TestImageReplaceForMCO(t *testing.T) { } func createSecret(key, name, namespace string) *corev1.Secret { - s3Conf := &config.ObjectStorgeConf{ Type: "s3", Config: config.Config{ @@ -962,7 +966,6 @@ func TestHandleStorageSizeChange(t *testing.T) { } else { t.Errorf("update pvc failed: %v", err) } - } func createStatefulSet(name, namespace, statefulSetName string) *appsv1.StatefulSet { diff --git a/operators/multiclusterobservability/main.go b/operators/multiclusterobservability/main.go index c1ac8b7d1..69e65c8f6 100644 --- a/operators/multiclusterobservability/main.go +++ b/operators/multiclusterobservability/main.go @@ -68,6 +68,7 @@ func init() { utilruntime.Must(observabilityv1beta2.AddToScheme(scheme)) utilruntime.Must(observatoriumAPIs.AddToScheme(scheme)) utilruntime.Must(prometheusv1.AddToScheme(scheme)) + utilruntime.Must(addonv1alpha1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } diff --git a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/addon_deployment_config.yaml b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/addon_deployment_config.yaml new file mode 100644 index 000000000..fe8d773e1 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/addon_deployment_config.yaml @@ -0,0 +1,10 @@ +apiVersion: addon.open-cluster-management.io/v1alpha1 +kind: AddOnDeploymentConfig +metadata: + name: multicluster-observability-addon + namespace: open-cluster-management-observability +spec: + customizedVariables: + # Operator Subscription Channels + - name: openshiftLoggingChannel + value: stable-5.9 diff --git a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_management_addon.yaml b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_management_addon.yaml new file mode 100644 index 000000000..adc03f9de --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_management_addon.yaml @@ -0,0 +1,42 @@ +apiVersion: addon.open-cluster-management.io/v1alpha1 +kind: ClusterManagementAddOn +metadata: + annotations: + addon.open-cluster-management.io/lifecycle: addon-manager + labels: + app.kubernetes.io/component: multicluster-observability-addon-manager + app.kubernetes.io/name: multicluster-observability-addon-manager + app.kubernetes.io/part-of: multicluster-observability-addon + name: multicluster-observability-addon +spec: + addOnMeta: + displayName: Multicluster Observability Addon + description: "multicluster-observability-addon is the addon to configure spoke clusters to collect and forward logs/traces to a given set of outputs" + supportedConfigs: + # Describes the general addon configuration applicable for all managed clusters. It includes: + # - Default subscription channel name for install the `Red Hat OpenShift Logging` operator on each managed cluster. + - group: addon.open-cluster-management.io + resource: addondeploymentconfigs + defaultConfig: + name: multicluster-observability-addon + namespace: open-cluster-management-observability + # Describes the default log forwarding outputs for each log type applied to all managed clusters. + - group: logging.openshift.io + resource: clusterlogforwarders + # Describes the default OpenTelemetryCollector type applied to all managed clusters. + - group: opentelemetry.io + resource: opentelemetrycollectors + installStrategy: + type: Placements + placements: + - name: global + namespace: open-cluster-management-global-set + configs: + - group: logging.openshift.io + resource: clusterlogforwarders + name: instance + namespace: open-cluster-management-observability + - group: opentelemetry.io + resource: opentelemetrycollectors + name: instance + namespace: open-cluster-management-observability diff --git a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_role.yaml b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_role.yaml new file mode 100644 index 000000000..976ebe21f --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_role.yaml @@ -0,0 +1,70 @@ + kind: ClusterRole + apiVersion: rbac.authorization.k8s.io/v1 + metadata: + labels: + app.kubernetes.io/component: multicluster-observability-addon-manager + app.kubernetes.io/name: multicluster-observability-addon-manager + app.kubernetes.io/part-of: multicluster-observability-addon + name: multicluster-observability-addon-manager + rules: + - apiGroups: [""] + resources: ["configmaps", "events"] + verbs: ["get", "list", "watch", "create", "update", "delete", "deletecollection", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["authorization.k8s.io"] + resources: ["subjectaccessreviews"] + verbs: ["get", "create"] + - apiGroups: ["cluster.open-cluster-management.io"] + resources: ["managedclusters", "placements", "placementdecisions"] + verbs: ["get", "list", "watch"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["managedclusteraddons/finalizers"] + verbs: ["update"] + - apiGroups: [ "addon.open-cluster-management.io" ] + resources: [ "clustermanagementaddons/finalizers" ] + verbs: [ "update" ] + - apiGroups: [ "addon.open-cluster-management.io" ] + resources: [ "clustermanagementaddons/status" ] + verbs: ["update", "patch"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["clustermanagementaddons"] + verbs: ["get", "list", "watch"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["managedclusteraddons"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["managedclusteraddons/status"] + verbs: ["update", "patch"] + - apiGroups: ["addon.open-cluster-management.io"] + resources: ["addontemplates", "addondeploymentconfigs"] + verbs: ["get", "list", "watch"] + - apiGroups: ["work.open-cluster-management.io"] + resources: ["manifestworks"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + # addon template controller needs these permissions to approve CSR + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests"] + verbs: ["create", "get", "list", "watch"] + - apiGroups: ["certificates.k8s.io"] + resources: ["certificatesigningrequests/approval", "certificatesigningrequests/status"] + verbs: ["update"] + - apiGroups: ["certificates.k8s.io"] + resources: ["signers"] + verbs: ["approve", "sign"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: ["rolebindings"] + verbs: ["get", "list", "watch", "create", "delete"] + # The addon will need to create secrets to ensure secure communication + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + # Roles for addon to perform logging specific actions + - apiGroups: ["logging.openshift.io"] + resources: ["clusterlogforwarders"] + verbs: ["get", "list", "watch"] + # Role for addon to perform tracing specific actions + - apiGroups: ["opentelemetry.io"] + resources: ["opentelemetrycollectors"] + verbs: ["get", "list", "watch"] diff --git a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_role_binding.yaml b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_role_binding.yaml new file mode 100644 index 000000000..a4563a21b --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/cluster_role_binding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: multicluster-observability-addon-manager + app.kubernetes.io/name: multicluster-observability-addon-manager + app.kubernetes.io/part-of: multicluster-observability-addon + name: multicluster-observability-addon-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: multicluster-observability-addon-manager +subjects: + - kind: ServiceAccount + name: multicluster-observability-addon-manager + namespace: open-cluster-management-observability diff --git a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/deployment.yaml b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/deployment.yaml new file mode 100644 index 000000000..085c87813 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/deployment.yaml @@ -0,0 +1,57 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: multicluster-observability-addon-manager + app.kubernetes.io/name: multicluster-observability-addon-manager + app.kubernetes.io/part-of: multicluster-observability-addon + name: multicluster-observability-addon-manager + namespace: open-cluster-management-observability +spec: + replicas: 1 + selector: + matchLabels: + app: multicluster-observability-addon-manager + template: + metadata: + labels: + app: multicluster-observability-addon-manager + spec: + automountServiceAccountToken: false + serviceAccountName: multicluster-observability-addon-manager + containers: + - name: manager + image: quay.io/rhobs/multicluster-observability-addon:v0.0.1 + imagePullPolicy: IfNotPresent + args: + - "controller" + resources: + limits: + cpu: 200m + memory: 512Mi + ephemeral-storage: 2Gi + requests: + cpu: 100m + memory: 256Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + runAsNonRoot: true + volumeMounts: + - name: sa-token + mountPath: /var/run/secrets/kubernetes.io/serviceaccount + readOnly: true + volumes: + - name: sa-token + projected: + sources: + - serviceAccountToken: + path: token + - configMap: + name: kube-root-ca.crt + items: + - key: ca.crt + path: ca.crt diff --git a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/kustomization.yaml b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/kustomization.yaml new file mode 100644 index 000000000..c40390388 --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- addon_deployment_config.yaml +- cluster_management_addon.yaml +- cluster_role_binding.yaml +- cluster_role.yaml +- deployment.yaml +- service_account.yaml diff --git a/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/service_account.yaml b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/service_account.yaml new file mode 100644 index 000000000..c1119d2cb --- /dev/null +++ b/operators/multiclusterobservability/manifests/base/multicluster-observability-addon/service_account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: multicluster-observability-addon-manager + namespace: open-cluster-management-observability diff --git a/operators/multiclusterobservability/pkg/config/config.go b/operators/multiclusterobservability/pkg/config/config.go index 7e41d8ccb..aab3ed945 100644 --- a/operators/multiclusterobservability/pkg/config/config.go +++ b/operators/multiclusterobservability/pkg/config/config.go @@ -156,26 +156,32 @@ const ( EndpointControllerImgName = "endpoint-monitoring-operator" EndpointControllerKey = "endpoint_monitoring_operator" + MultiClusterObservabilityAddonImgRepo = "quay.io/rhobs" + MultiClusterObservabilityAddonImgName = "multicluster-observability-addon" + MultiClusterObservabilityAddonImgTagSuffix = "v0.0.1" + MultiClusterObservabilityAddonImgKey = "multicluster_observability_addon" + RBACQueryProxyImgName = "rbac-query-proxy" RBACQueryProxyKey = "rbac_query_proxy" - ObservatoriumAPI = "observatorium-api" - ThanosCompact = "thanos-compact" - ThanosQuery = "thanos-query" - ThanosQueryFrontend = "thanos-query-frontend" - ThanosQueryFrontendMemcached = "thanos-query-frontend-memcached" - ThanosRule = "thanos-rule" - ThanosReceive = "thanos-receive-default" - ThanosStoreMemcached = "thanos-store-memcached" - ThanosStoreShard = "thanos-store-shard" - MemcachedExporter = "memcached-exporter" - Grafana = "grafana" - RBACQueryProxy = "rbac-query-proxy" - Alertmanager = "alertmanager" - ThanosReceiveController = "thanos-receive-controller" - ObservatoriumOperator = "observatorium-operator" - MetricsCollector = "metrics-collector" - Observatorium = "observatorium" + ObservatoriumAPI = "observatorium-api" + ThanosCompact = "thanos-compact" + ThanosQuery = "thanos-query" + ThanosQueryFrontend = "thanos-query-frontend" + ThanosQueryFrontendMemcached = "thanos-query-frontend-memcached" + ThanosRule = "thanos-rule" + ThanosReceive = "thanos-receive-default" + ThanosStoreMemcached = "thanos-store-memcached" + ThanosStoreShard = "thanos-store-shard" + MemcachedExporter = "memcached-exporter" + Grafana = "grafana" + RBACQueryProxy = "rbac-query-proxy" + Alertmanager = "alertmanager" + ThanosReceiveController = "thanos-receive-controller" + ObservatoriumOperator = "observatorium-operator" + MetricsCollector = "metrics-collector" + Observatorium = "observatorium" + MultiClusterObservabilityAddon = "multicluster-observability-addon" RetentionResolutionRaw = "365d" RetentionResolution5m = "365d" @@ -689,6 +695,7 @@ func SetOperandNames(c client.Client) error { operandNames[ObservatoriumOperator] = GetOperandNamePrefix() + ObservatoriumOperator operandNames[Observatorium] = GetDefaultCRName() operandNames[ObservatoriumAPI] = GetOperandNamePrefix() + ObservatoriumAPI + operandNames[MultiClusterObservabilityAddon] = GetOperandNamePrefix() + MultiClusterObservabilityAddon // Check if the Observatorium CR already exists opts := &client.ListOptions{ @@ -712,6 +719,7 @@ func SetOperandNames(c client.Client) error { operandNames[ObservatoriumOperator] = ObservatoriumOperator operandNames[Observatorium] = observatorium.Name operandNames[ObservatoriumAPI] = observatorium.Name + "-" + ObservatoriumAPI + operandNames[MultiClusterObservabilityAddon] = MultiClusterObservabilityAddon } break } @@ -794,7 +802,8 @@ func GetMulticloudConsoleHost(client client.Client, isStandalone bool) (string, found := &routev1.Route{} err := client.Get(context.TODO(), types.NamespacedName{ - Name: MulticloudConsoleRouteName, Namespace: namespace}, found) + Name: MulticloudConsoleRouteName, Namespace: namespace, + }, found) if err != nil { return "", err } diff --git a/operators/multiclusterobservability/pkg/config/resources.go b/operators/multiclusterobservability/pkg/config/resources.go index 9778e387a..a89e10286 100644 --- a/operators/multiclusterobservability/pkg/config/resources.go +++ b/operators/multiclusterobservability/pkg/config/resources.go @@ -51,6 +51,8 @@ func getDefaultResourceCPU(component string, tshirtSize observabilityv1beta2.TSh return AlertmanagerCPURequest[tshirtSize] case Grafana: return GrafanaCPURequest[tshirtSize] + case MultiClusterObservabilityAddon: + return MCOACPURequest[tshirtSize] default: return "" } @@ -87,6 +89,8 @@ func getDefaultResourceMemory(component string, tshirtSize observabilityv1beta2. return AlertmanagerMemoryRequest[tshirtSize] case Grafana: return GrafanaMemoryRequest[tshirtSize] + case MultiClusterObservabilityAddon: + return MCOAMemoryRequest[tshirtSize] default: return "" } @@ -97,6 +101,8 @@ func getDefaultResourceMemoryLimit(component string) string { switch component { case Grafana: return GrafanaMemoryLimit + case MultiClusterObservabilityAddon: + return MCOAMemoryLimits default: return "" } @@ -107,6 +113,8 @@ func getDefaultResourceCPULimit(component string) string { switch component { case Grafana: return GrafanaCPULimit + case MultiClusterObservabilityAddon: + return MCOACPULimits default: return "" } @@ -192,6 +200,10 @@ func getAdvancedConfigResourceOverride(component string, tshirtSize observabilit if advanced.Alertmanager != nil { resourcesReq = advanced.Alertmanager.Resources } + case MultiClusterObservabilityAddon: + if advanced.MultiClusterObservabilityAddon != nil { + resourcesReq = advanced.MultiClusterObservabilityAddon.Resources + } } final := corev1.ResourceRequirements{} @@ -348,6 +360,10 @@ func GetReplicas(component string, tshirtSize observabilityv1beta2.TShirtSize, a if advanced.Alertmanager != nil { replicas = advanced.Alertmanager.Replicas } + case MultiClusterObservabilityAddon: + if advanced.MultiClusterObservabilityAddon != nil { + replicas = advanced.MultiClusterObservabilityAddon.Replicas + } } if replicas == nil || *replicas == 0 { diff --git a/operators/multiclusterobservability/pkg/config/resources_map.go b/operators/multiclusterobservability/pkg/config/resources_map.go index 4263cb1a1..4a00c5e34 100644 --- a/operators/multiclusterobservability/pkg/config/resources_map.go +++ b/operators/multiclusterobservability/pkg/config/resources_map.go @@ -361,6 +361,27 @@ var ( TwoXLarge: "100Mi", FourXLarge: "100Mi", } + + MCOACPURequest ResourceSizeMap = map[observabilityv1beta2.TShirtSize]string{ + Minimal: "100m", + Default: "100m", + Small: "100m", + Medium: "100m", + Large: "100m", + XLarge: "100m", + TwoXLarge: "100m", + FourXLarge: "100m", + } + MCOAMemoryRequest ResourceSizeMap = map[observabilityv1beta2.TShirtSize]string{ + Minimal: "256Mi", + Default: "256Mi", + Small: "256Mi", + Medium: "256Mi", + Large: "256Mi", + XLarge: "256Mi", + TwoXLarge: "256Mi", + FourXLarge: "256Mi", + } ) const ( @@ -369,6 +390,9 @@ const ( MetricsCollectorCPULimits = "" MetricsCollectorMemoryLimits = "" + + MCOACPULimits = "200m" + MCOAMemoryLimits = "512Mi" ) type ReplicaMap map[observabilityv1beta2.TShirtSize]*int32 @@ -492,5 +516,15 @@ var ( TwoXLarge: intptr(3), FourXLarge: intptr(3), }, + MultiClusterObservabilityAddon: { + Minimal: intptr(1), + Default: intptr(1), + Small: intptr(1), + Medium: intptr(1), + Large: intptr(1), + XLarge: intptr(1), + TwoXLarge: intptr(1), + FourXLarge: intptr(1), + }, } ) diff --git a/operators/multiclusterobservability/pkg/rendering/renderer.go b/operators/multiclusterobservability/pkg/rendering/renderer.go index 7771fa6dc..0605a72ca 100644 --- a/operators/multiclusterobservability/pkg/rendering/renderer.go +++ b/operators/multiclusterobservability/pkg/rendering/renderer.go @@ -30,6 +30,7 @@ type MCORenderer struct { renderAlertManagerFns map[string]rendererutil.RenderFn renderThanosFns map[string]rendererutil.RenderFn renderProxyFns map[string]rendererutil.RenderFn + renderMCOAFns map[string]rendererutil.RenderFn } func NewMCORenderer(multipleClusterMonitoring *obv1beta2.MultiClusterObservability, kubeClient client.Client) *MCORenderer { @@ -42,6 +43,7 @@ func NewMCORenderer(multipleClusterMonitoring *obv1beta2.MultiClusterObservabili mcoRenderer.newAlertManagerRenderer() mcoRenderer.newThanosRenderer() mcoRenderer.newProxyRenderer() + mcoRenderer.newMCOARenderer() return mcoRenderer } @@ -104,6 +106,17 @@ func (r *MCORenderer) Render() ([]*unstructured.Unstructured, error) { } resources = append(resources, proxyResources...) + // load and render multicluster-observability-addon templates + mcoaTemplates, err := templates.GetOrLoadMCOATemplates(templatesutil.GetTemplateRenderer()) + if err != nil { + return nil, err + } + mcoaResources, err := r.renderMCOATemplates(mcoaTemplates, namespace, labels) + if err != nil { + return nil, err + } + resources = append(resources, mcoaResources...) + for idx := range resources { if resources[idx].GetKind() == "Deployment" { obj := util.GetK8sObj(resources[idx].GetKind()) diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_mcoa.go b/operators/multiclusterobservability/pkg/rendering/renderer_mcoa.go new file mode 100644 index 000000000..48f5446d6 --- /dev/null +++ b/operators/multiclusterobservability/pkg/rendering/renderer_mcoa.go @@ -0,0 +1,255 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package rendering + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + addonapiv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + "sigs.k8s.io/kustomize/api/resource" + + "github.com/imdario/mergo" + mcoconfig "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/pkg/config" + rendererutil "github.com/stolostron/multicluster-observability-operator/operators/pkg/rendering" +) + +const ( + // AODC CustomizedVariable Names + namePlatformLogsCollection = "platformLogsCollection" + nameUserWorkloadLogsCollection = "userWorkloadLogsCollection" + nameUserWorkloadTracesCollection = "userWorkloadTracesCollection" + nameUserWorkloadInstrumentation = "userWorkloadInstrumentation" + + // AODC CustomizedVariable Values + clfV1 = "clusterlogforwarders.v1.logging.openshift.io" + otelV1beta1 = "opentelemetrycollectors.v1beta1.opentelemetry.io" + instrV1alpha1 = "instrumentations.v1alpha1.opentelemetry.io" +) + +func (r *MCORenderer) newMCOARenderer() { + r.renderMCOAFns = map[string]rendererutil.RenderFn{ + "AddOnDeploymentConfig": r.renderAddonDeploymentConfig, + "ClusterManagementAddOn": r.renderClusterManagementAddOn, + "Deployment": r.renderMCOADeployment, + "ServiceAccount": r.renderer.RenderNamespace, + "ClusterRole": r.renderer.RenderClusterRole, + "ClusterRoleBinding": r.renderer.RenderClusterRoleBinding, + } +} + +func (r *MCORenderer) renderMCOADeployment( + res *resource.Resource, + namespace string, + labels map[string]string, +) (*unstructured.Unstructured, error) { + u, err := r.renderer.RenderNamespace(res, namespace, labels) + if err != nil { + return nil, err + } + + obj := &appsv1.Deployment{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, obj); err != nil { + return nil, err + } + crLabelKey := mcoconfig.GetCrLabelKey() + + img := fmt.Sprintf("%s/%s:%s", + mcoconfig.MultiClusterObservabilityAddonImgRepo, + mcoconfig.MultiClusterObservabilityAddonImgName, + mcoconfig.MultiClusterObservabilityAddonImgTagSuffix, + ) + found, replacement := mcoconfig.ReplaceImage( + r.cr.Annotations, + fmt.Sprintf("%s/%s", + mcoconfig.DefaultImgRepository, + mcoconfig.MultiClusterObservabilityAddonImgName, + ), + mcoconfig.MultiClusterObservabilityAddonImgKey, + ) + if found { + img = replacement + } + + patch := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + crLabelKey: r.cr.Name, + }, + Name: mcoconfig.GetOperandName(mcoconfig.MultiClusterObservabilityAddon), + }, + Spec: appsv1.DeploymentSpec{ + Replicas: mcoconfig.GetReplicas( + mcoconfig.MultiClusterObservabilityAddon, + r.cr.Spec.InstanceSize, + r.cr.Spec.AdvancedConfig, + ), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + crLabelKey: r.cr.Name, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + crLabelKey: r.cr.Name, + }, + }, + Spec: corev1.PodSpec{ + NodeSelector: r.cr.Spec.NodeSelector, + Tolerations: r.cr.Spec.Tolerations, + ImagePullSecrets: []corev1.LocalObjectReference{ + { + Name: mcoconfig.GetImagePullSecret(r.cr.Spec), + }, + }, + }, + }, + }, + } + + if err := mergo.Merge(obj, patch); err != nil { + return nil, err + } + + mcoaResources := mcoconfig.GetResources( + mcoconfig.MultiClusterObservabilityAddon, + r.cr.Spec.InstanceSize, + r.cr.Spec.AdvancedConfig, + ) + + patchContainer := &corev1.Container{ + Image: img, + ImagePullPolicy: mcoconfig.GetImagePullPolicy(r.cr.Spec), + Resources: mcoaResources, + } + + if err := mergo.Merge(&obj.Spec.Template.Spec.Containers[0], patchContainer, mergo.WithOverride); err != nil { + return nil, err + } + + uObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj) + if err != nil { + return nil, err + } + + return &unstructured.Unstructured{Object: uObj}, nil +} + +func (r *MCORenderer) renderClusterManagementAddOn( + res *resource.Resource, + namespace string, + labels map[string]string, +) (*unstructured.Unstructured, error) { + m, err := res.Map() + if err != nil { + return nil, err + } + u := &unstructured.Unstructured{Object: m} + + cLabels := u.GetLabels() + if cLabels == nil { + cLabels = make(map[string]string) + } + for k, v := range labels { + cLabels[k] = v + } + u.SetLabels(cLabels) + + return u, nil +} + +func (r *MCORenderer) renderAddonDeploymentConfig( + res *resource.Resource, + namespace string, + labels map[string]string, +) (*unstructured.Unstructured, error) { + u, err := r.renderer.RenderNamespace(res, namespace, labels) + if err != nil { + return nil, err + } + + if cs := r.cr.Spec.Capabilities; cs != nil { + aodc := &addonapiv1alpha1.AddOnDeploymentConfig{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(u.Object, aodc); err != nil { + return nil, err + } + + appendCustomVar := func(aodc *addonapiv1alpha1.AddOnDeploymentConfig, name, value string) { + aodc.Spec.CustomizedVariables = append( + aodc.Spec.CustomizedVariables, + addonapiv1alpha1.CustomizedVariable{Name: name, Value: value}, + ) + } + + if cs.Platform != nil { + if cs.Platform.Logs.Collection.Enabled { + appendCustomVar(aodc, namePlatformLogsCollection, clfV1) + } + } + + if cs.UserWorkloads != nil { + if cs.UserWorkloads.Logs.Collection.ClusterLogForwarder.Enabled { + appendCustomVar(aodc, nameUserWorkloadLogsCollection, clfV1) + } + if cs.UserWorkloads.Traces.Collection.Collector.Enabled { + appendCustomVar(aodc, nameUserWorkloadTracesCollection, otelV1beta1) + } + if cs.UserWorkloads.Traces.Collection.Instrumentation.Enabled { + appendCustomVar(aodc, nameUserWorkloadInstrumentation, instrV1alpha1) + } + } + + u.Object, err = runtime.DefaultUnstructuredConverter.ToUnstructured(aodc) + if err != nil { + return nil, err + } + } + + cLabels := u.GetLabels() + if cLabels == nil { + cLabels = make(map[string]string) + } + for k, v := range labels { + cLabels[k] = v + } + u.SetLabels(cLabels) + + return u, nil +} + +func (r *MCORenderer) renderMCOATemplates( + templates []*resource.Resource, + namespace string, + labels map[string]string, +) ([]*unstructured.Unstructured, error) { + uobjs := []*unstructured.Unstructured{} + for _, template := range templates { + render, ok := r.renderMCOAFns[template.GetKind()] + if !ok { + m, err := template.Map() + if err != nil { + return []*unstructured.Unstructured{}, err + } + uobjs = append(uobjs, &unstructured.Unstructured{Object: m}) + continue + } + uobj, err := render(template.DeepCopy(), namespace, labels) + if err != nil { + return []*unstructured.Unstructured{}, err + } + if uobj == nil { + continue + } + uobjs = append(uobjs, uobj) + + } + + return uobjs, nil +} diff --git a/operators/multiclusterobservability/pkg/rendering/renderer_mcoa_test.go b/operators/multiclusterobservability/pkg/rendering/renderer_mcoa_test.go new file mode 100644 index 000000000..4b33fcd64 --- /dev/null +++ b/operators/multiclusterobservability/pkg/rendering/renderer_mcoa_test.go @@ -0,0 +1,174 @@ +// Copyright (c) Red Hat, Inc. +// Copyright Contributors to the Open Cluster Management project +// Licensed under the Apache License 2.0 + +package rendering + +import ( + "os" + "path/filepath" + "testing" + + mcov1beta2 "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/api/v1beta2" + mcoconfig "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/pkg/config" + "github.com/stolostron/multicluster-observability-operator/operators/multiclusterobservability/pkg/rendering/templates" + templatesutil "github.com/stolostron/multicluster-observability-operator/operators/pkg/rendering/templates" + "github.com/stretchr/testify/assert" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/utils/ptr" + addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" + kustomizeres "sigs.k8s.io/kustomize/api/resource" +) + +func TestRenderMCOADeployment(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + templatesPath := filepath.Join(wd, "..", "..", "manifests") + t.Setenv(templatesutil.TemplatesPathEnvVar, templatesPath) + + tmplRenderer := templatesutil.NewTemplateRenderer(templatesPath) + mcoaTemplates, err := templates.GetOrLoadMCOATemplates(tmplRenderer) + assert.NoError(t, err) + + var dp *kustomizeres.Resource + for _, template := range mcoaTemplates { + if template.GetKind() == "Deployment" { + dp = template.DeepCopy() + break + } + } + + mco := &mcov1beta2.MultiClusterObservability{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "cr-key": "cr-value", + }, + Name: "multicluster-observability", + }, + Spec: mcov1beta2.MultiClusterObservabilitySpec{ + ImagePullPolicy: corev1.PullIfNotPresent, + AdvancedConfig: &mcov1beta2.AdvancedConfig{ + MultiClusterObservabilityAddon: &mcov1beta2.CommonSpec{ + Replicas: ptr.To[int32](1), + Resources: &corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + corev1.ResourceMemory: resource.MustParse("256M"), + }, + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("200m"), + corev1.ResourceMemory: resource.MustParse("512M"), + corev1.ResourceEphemeralStorage: resource.MustParse("4Gi"), + }, + }, + }, + }, + }, + } + + renderer := &MCORenderer{cr: mco} + + uobj, err := renderer.renderMCOADeployment(dp, "test", map[string]string{"key": "value"}) + assert.NoError(t, err) + assert.NotNil(t, uobj) + + got := &appsv1.Deployment{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(uobj.Object, got) + assert.NoError(t, err) + + // Assert labels include the CR label key + assert.Len(t, got.Labels, 4) + assert.Contains(t, got.Labels, mcoconfig.GetCrLabelKey()) + assert.Len(t, got.Spec.Selector.MatchLabels, 2) + assert.Contains(t, got.Spec.Selector.MatchLabels, mcoconfig.GetCrLabelKey()) + assert.Len(t, got.Spec.Template.ObjectMeta.Labels, 2) + assert.Contains(t, got.Spec.Template.ObjectMeta.Labels, mcoconfig.GetCrLabelKey()) + + // Assert pod spec values + assert.Equal(t, mco.Spec.AdvancedConfig.MultiClusterObservabilityAddon.Replicas, got.Spec.Replicas) + + container := got.Spec.Template.Spec.Containers[0] + assert.Contains(t, container.Image, mcoconfig.MultiClusterObservabilityAddonImgRepo) + assert.Contains(t, container.Image, mcoconfig.MultiClusterObservabilityAddonImgName) + assert.Contains(t, container.Image, mcoconfig.MultiClusterObservabilityAddonImgTagSuffix) + assert.Equal(t, corev1.PullIfNotPresent, container.ImagePullPolicy) + assert.Equal(t, *mco.Spec.AdvancedConfig.MultiClusterObservabilityAddon.Resources, container.Resources) + assert.True(t, *container.SecurityContext.RunAsNonRoot) +} + +func TestRenderAddonDeploymentConfig(t *testing.T) { + wd, err := os.Getwd() + assert.NoError(t, err) + templatesPath := filepath.Join(wd, "..", "..", "manifests") + t.Setenv(templatesutil.TemplatesPathEnvVar, templatesPath) + + tmplRenderer := templatesutil.NewTemplateRenderer(templatesPath) + mcoaTemplates, err := templates.GetOrLoadMCOATemplates(tmplRenderer) + assert.NoError(t, err) + + var aodc *kustomizeres.Resource + for _, template := range mcoaTemplates { + if template.GetKind() == "AddOnDeploymentConfig" { + aodc = template.DeepCopy() + break + } + } + + mco := &mcov1beta2.MultiClusterObservability{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "cr-key": "cr-value", + }, + Name: "multicluster-observability", + }, + Spec: mcov1beta2.MultiClusterObservabilitySpec{ + Capabilities: &mcov1beta2.CapabilitiesSpec{ + Platform: &mcov1beta2.PlatformCapabilitiesSpec{ + Logs: mcov1beta2.PlatformLogsSpec{ + Collection: mcov1beta2.PlatformLogsCollectionSpec{ + Enabled: true, + }, + }, + }, + UserWorkloads: &mcov1beta2.UserWorkloadCapabilitiesSpec{ + Logs: mcov1beta2.UserWorkloadLogsSpec{ + Collection: mcov1beta2.UserWorkloadLogsCollectionSpec{ + ClusterLogForwarder: mcov1beta2.ClusterLogForwarderSpec{ + Enabled: true, + }, + }, + }, + Traces: mcov1beta2.UserWorkloadTracesSpec{ + Collection: mcov1beta2.OpenTelemetryCollectionSpec{ + Collector: mcov1beta2.OpenTelemetryCollectorSpec{ + Enabled: true, + }, + Instrumentation: mcov1beta2.InstrumentationSpec{ + Enabled: true, + }, + }, + }, + }, + }, + }, + } + renderer := &MCORenderer{cr: mco} + + uobj, err := renderer.renderAddonDeploymentConfig(aodc, "test", map[string]string{"key": "value"}) + assert.NoError(t, err) + assert.NotNil(t, uobj) + + got := &addonv1alpha1.AddOnDeploymentConfig{} + err = runtime.DefaultUnstructuredConverter.FromUnstructured(uobj.Object, got) + assert.NoError(t, err) + + assert.Len(t, got.Spec.CustomizedVariables, 5) + assert.Contains(t, got.Spec.CustomizedVariables, addonv1alpha1.CustomizedVariable{Name: namePlatformLogsCollection, Value: clfV1}) + assert.Contains(t, got.Spec.CustomizedVariables, addonv1alpha1.CustomizedVariable{Name: nameUserWorkloadLogsCollection, Value: clfV1}) + assert.Contains(t, got.Spec.CustomizedVariables, addonv1alpha1.CustomizedVariable{Name: nameUserWorkloadTracesCollection, Value: otelV1beta1}) + assert.Contains(t, got.Spec.CustomizedVariables, addonv1alpha1.CustomizedVariable{Name: nameUserWorkloadInstrumentation, Value: instrV1alpha1}) +} diff --git a/operators/multiclusterobservability/pkg/rendering/templates/templates.go b/operators/multiclusterobservability/pkg/rendering/templates/templates.go index 525949633..67ee900bc 100644 --- a/operators/multiclusterobservability/pkg/rendering/templates/templates.go +++ b/operators/multiclusterobservability/pkg/rendering/templates/templates.go @@ -13,7 +13,16 @@ import ( ) // *Templates contains all kustomize resources. -var genericTemplates, grafanaTemplates, alertManagerTemplates, thanosTemplates, proxyTemplates, endpointObservabilityTemplates, prometheusTemplates []*resource.Resource +var ( + genericTemplates []*resource.Resource + grafanaTemplates []*resource.Resource + alertManagerTemplates []*resource.Resource + thanosTemplates []*resource.Resource + proxyTemplates []*resource.Resource + endpointObservabilityTemplates []*resource.Resource + prometheusTemplates []*resource.Resource + mcoaTemplates []*resource.Resource +) // GetOrLoadGenericTemplates reads base manifest. func GetOrLoadGenericTemplates(r *templates.TemplateRenderer) ([]*resource.Resource, error) { @@ -96,6 +105,21 @@ func GetOrLoadProxyTemplates(r *templates.TemplateRenderer) ([]*resource.Resourc return proxyTemplates, nil } +// GetOrLoadMCOATemplates reads the multicluster-observability-addon manifests. +func GetOrLoadMCOATemplates(r *templates.TemplateRenderer) ([]*resource.Resource, error) { + if len(mcoaTemplates) > 0 { + return mcoaTemplates, nil + } + + basePath := path.Join(r.GetTemplatesPath(), "base") + + // add mcoa templates + if err := r.AddTemplateFromPath(path.Join(basePath, "multicluster-observability-addon"), &mcoaTemplates); err != nil { + return mcoaTemplates, err + } + return mcoaTemplates, nil +} + // GetEndpointObservabilityTemplates reads endpoint-observability manifest. func GetOrLoadEndpointObservabilityTemplates(r *templates.TemplateRenderer) ([]*resource.Resource, error) { if len(endpointObservabilityTemplates) > 0 { @@ -136,4 +160,5 @@ func ResetTemplates() { thanosTemplates = nil proxyTemplates = nil endpointObservabilityTemplates = nil + mcoaTemplates = nil } diff --git a/operators/pkg/deploying/deployer.go b/operators/pkg/deploying/deployer.go index cae21c867..35bc1bc5c 100644 --- a/operators/pkg/deploying/deployer.go +++ b/operators/pkg/deploying/deployer.go @@ -20,6 +20,7 @@ import ( "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/types" + addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -56,6 +57,8 @@ func NewDeployer(client client.Client) *Deployer { "ServiceAccount": deployer.updateServiceAccount, "DaemonSet": deployer.updateDaemonSet, "ServiceMonitor": deployer.updateServiceMonitor, + "AddOnDeploymentConfig": deployer.updateAddOnDeploymentConfig, + "ClusterManagementAddOn": deployer.updateClusterManagementAddOn, } return deployer } @@ -363,6 +366,46 @@ func (d *Deployer) updateServiceMonitor(ctx context.Context, desiredObj, runtime return nil } +func (d *Deployer) updateAddOnDeploymentConfig( + ctx context.Context, + desiredObj, runtimeObj *unstructured.Unstructured, +) error { + desiredAODC, runtimeAODC, err := unstructuredPairToTyped[addonv1alpha1.AddOnDeploymentConfig](desiredObj, runtimeObj) + if err != nil { + return err + } + + if !apiequality.Semantic.DeepDerivative(desiredAODC.Spec, runtimeAODC.Spec) { + logUpdateInfo(runtimeObj) + if desiredAODC.ResourceVersion != runtimeAODC.ResourceVersion { + desiredAODC.ResourceVersion = runtimeAODC.ResourceVersion + } + return d.client.Update(ctx, desiredAODC) + } + + return nil +} + +func (d *Deployer) updateClusterManagementAddOn( + ctx context.Context, + desiredObj, runtimeObj *unstructured.Unstructured, +) error { + desiredCMAO, runtimeCMAO, err := unstructuredPairToTyped[addonv1alpha1.ClusterManagementAddOn](desiredObj, runtimeObj) + if err != nil { + return err + } + + if !apiequality.Semantic.DeepDerivative(desiredCMAO.Spec, runtimeCMAO.Spec) { + logUpdateInfo(runtimeObj) + if desiredCMAO.ResourceVersion != runtimeCMAO.ResourceVersion { + desiredCMAO.ResourceVersion = runtimeCMAO.ResourceVersion + } + return d.client.Update(ctx, desiredCMAO) + } + + return nil +} + // unstructuredToType converts an unstructured.Unstructured object to a specified type. // It marshals the object to JSON and then unmarshals it into the target type. // The target parameter must be a pointer to the type T. diff --git a/operators/pkg/deploying/deployer_test.go b/operators/pkg/deploying/deployer_test.go index 021a390f2..3cf4b1cae 100644 --- a/operators/pkg/deploying/deployer_test.go +++ b/operators/pkg/deploying/deployer_test.go @@ -17,6 +17,7 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -27,7 +28,6 @@ var ( ) func TestDeploy(t *testing.T) { - cases := []struct { name string createObj runtime.Object @@ -782,6 +782,133 @@ func TestDeploy(t *testing.T) { } }, }, + { + name: "create and update AddOnDeploymentConfig", + createObj: &addonv1alpha1.AddOnDeploymentConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "addon.open-cluster-management.io/v1alpha1", + Kind: "AddOnDeploymentConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-aodc", + Namespace: "ns1", + }, + Spec: addonv1alpha1.AddOnDeploymentConfigSpec{ + CustomizedVariables: []addonv1alpha1.CustomizedVariable{ + { + Name: "test", + Value: "value", + }, + }, + }, + }, + updateObj: &addonv1alpha1.AddOnDeploymentConfig{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "addon.open-cluster-management.io/v1alpha1", + Kind: "AddOnDeploymentConfig", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-aodc", + Namespace: "ns1", + ResourceVersion: "1", + }, + Spec: addonv1alpha1.AddOnDeploymentConfigSpec{ + CustomizedVariables: []addonv1alpha1.CustomizedVariable{ + { + Name: "test", + Value: "value", + }, + { + Name: "other", + Value: "more", + }, + }, + }, + }, + validateResults: func(client client.Client) { + namespacedName := types.NamespacedName{ + Name: "test-aodc", + Namespace: "ns1", + } + obj := &addonv1alpha1.AddOnDeploymentConfig{} + client.Get(context.Background(), namespacedName, obj) + + if len(obj.Spec.CustomizedVariables) != 2 { + t.Fatalf("Missing Customied Variables, got %#v", obj.Spec.CustomizedVariables) + } + }, + }, + { + name: "create and update ClusterManagementAddOn", + createObj: &addonv1alpha1.ClusterManagementAddOn{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "addon.open-cluster-management.io/v1alpha1", + Kind: "ClusterManagementAddOn", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cmao", + Namespace: "ns1", + }, + Spec: addonv1alpha1.ClusterManagementAddOnSpec{ + SupportedConfigs: []addonv1alpha1.ConfigMeta{ + { + ConfigGroupResource: addonv1alpha1.ConfigGroupResource{Group: "test.io", Resource: "TestRes"}, + }, + }, + InstallStrategy: addonv1alpha1.InstallStrategy{ + Type: "Placements", + Placements: []addonv1alpha1.PlacementStrategy{}, + }, + }, + }, + updateObj: &addonv1alpha1.ClusterManagementAddOn{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "addon.open-cluster-management.io/v1alpha1", + Kind: "ClusterManagementAddOn", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-cmao", + Namespace: "ns1", + ResourceVersion: "1", + }, + Spec: addonv1alpha1.ClusterManagementAddOnSpec{ + SupportedConfigs: []addonv1alpha1.ConfigMeta{ + { + ConfigGroupResource: addonv1alpha1.ConfigGroupResource{Group: "test.io", Resource: "TestRes"}, + }, + { + ConfigGroupResource: addonv1alpha1.ConfigGroupResource{Group: "example.io", Resource: "ExampleRes"}, + }, + }, + InstallStrategy: addonv1alpha1.InstallStrategy{ + Type: "Placements", + Placements: []addonv1alpha1.PlacementStrategy{ + { + PlacementRef: addonv1alpha1.PlacementRef{Namespace: "test", Name: "test-res"}, + }, + { + PlacementRef: addonv1alpha1.PlacementRef{Namespace: "example", Name: "exaple-res"}, + }, + }, + }, + }, + }, + validateResults: func(client client.Client) { + namespacedName := types.NamespacedName{ + Name: "test-cmao", + Namespace: "ns1", + } + obj := &addonv1alpha1.ClusterManagementAddOn{} + client.Get(context.Background(), namespacedName, obj) + + if len(obj.Spec.SupportedConfigs) != 2 { + t.Fatalf("Missing Supported Configs, got %#v", obj.Spec.SupportedConfigs) + } + if len(obj.Spec.InstallStrategy.Placements) != 2 { + t.Fatalf("Missing Placements, got %#v", obj.Spec.InstallStrategy.Placements) + } + }, + }, } scheme := runtime.NewScheme() @@ -791,6 +918,7 @@ func TestDeploy(t *testing.T) { rbacv1.AddToScheme(scheme) prometheusv1.AddToScheme(scheme) networkingv1.AddToScheme(scheme) + addonv1alpha1.AddToScheme(scheme) client := fake.NewClientBuilder().WithScheme(scheme).Build() deployer := NewDeployer(client) @@ -813,7 +941,6 @@ func TestDeploy(t *testing.T) { c.validateResults(client) }) } - } func toPtr(t *testing.T, s string) *string { diff --git a/tests/pkg/utils/mco_deploy.go b/tests/pkg/utils/mco_deploy.go index 0b3ddd26f..0a48233c0 100644 --- a/tests/pkg/utils/mco_deploy.go +++ b/tests/pkg/utils/mco_deploy.go @@ -183,7 +183,7 @@ func CheckAllPodsAffinity(opt TestOptions) error { for _, pod := range podList { if pod.Labels["name"] == "endpoint-observability-operator" || pod.Labels["component"] == "metrics-collector" || - pod.Labels["component"] == "uwl-metrics-collector" { + pod.Labels["component"] == "uwl-metrics-collector" || pod.Labels["app"] == "multicluster-observability-addon-manager" { // No affinity set for endpoint-operator and metrics-collector in the hub continue }