From 454edb41475c60eca5ff955029b4e7898fab6bee Mon Sep 17 00:00:00 2001 From: bartoszmajsak Date: Wed, 20 Sep 2023 09:17:45 +0200 Subject: [PATCH 1/4] feat(service-mesh): adds service mesh support - intialization of Service Mesh Auth(z) as part of DSCI - support for Dashboard and Workbenches components --- Makefile | 6 +- .../v1/dscinitialization_types.go | 13 +- .../dscinitialization/v1/servicemesh_types.go | 120 + .../v1/zz_generated.deepcopy.go | 176 +- ...ion.opendatahub.io_dscinitializations.yaml | 90 + ...atahub.io_servicemeshresourcetrackers.yaml | 54 + ...atahub-operator.clusterserviceversion.yaml | 63 +- components/dashboard/dashboard.go | 149 +- components/kserve/kserve.go | 6 +- .../modelmeshserving/modelmeshserving.go | 11 +- components/workbenches/workbenches.go | 50 +- ...ion.opendatahub.io_dscinitializations.yaml | 90 + ...zation.opendatahub.io_featuretrackers.yaml | 46 + config/crd/kustomization.yaml | 1 + ...atahub-operator.clusterserviceversion.yaml | 3 + config/rbac/role.yaml | 55 + ...scinitialization_v1_dscinitialization.yaml | 2 +- .../datasciencecluster/kubebuilder_rbac.go | 26 +- .../dscinitialization_controller.go | 222 +- .../dscinitialization_test.go | 19 +- controllers/dscinitialization/suite_test.go | 12 +- controllers/secretgenerator/secret.go | 40 +- controllers/secretgenerator/secret_test.go | 2 +- .../secretgenerator_controller.go | 14 +- docs/SERVICE-MESH.md | 242 + get_all_manifests.sh | 7 +- go.mod | 7 +- go.sum | 10 +- pkg/cluster/cluster_config.go | 145 + pkg/cluster/cluster_suite_test.go | 14 + .../hostname_and_port_extraction_unit_test.go | 46 + pkg/cluster/operations.go | 100 + pkg/common/common.go | 96 - pkg/deploy/setup.go | 23 +- pkg/feature/builder.go | 186 + pkg/feature/cert.go | 91 + pkg/feature/conditions.go | 83 + pkg/feature/feature.go | 254 + pkg/feature/feature_suite_test.go | 14 + pkg/feature/manifest.go | 79 + pkg/feature/raw_resources.go | 157 + pkg/feature/resources.go | 65 + pkg/feature/servicemesh/cleanup.go | 126 + pkg/feature/servicemesh/conditions.go | 76 + pkg/feature/servicemesh/envoy_secrets.go | 64 + pkg/feature/servicemesh/loaders.go | 54 + pkg/feature/servicemesh/resources.go | 136 + pkg/feature/servicemesh/servicemesh_setup.go | 66 + pkg/feature/template_loader.go | 39 + .../servicemesh/authorino/auth-smm.tmpl | 10 + .../authorino/base/authconfig.tmpl | 46 + .../base/operator-cluster-wide-no-tls.tmpl | 15 + .../mesh-authz-ext-provider.patch.tmpl | 13 + .../servicemesh/authorino/namespace.tmpl | 6 + .../rbac/cluster-monitoring-role-binding.tmpl | 13 + .../authorino/rbac/cluster-role-binding.tmpl | 13 + .../authorino/rbac/cluster-role.tmpl | 160 + .../authorino/rbac/role-binding.tmpl | 12 + .../servicemesh/authorino/rbac/role.tmpl | 125 + .../base/control-plane-ingress.patch.tmpl | 23 + .../components/dashboard/gateway.tmpl | 24 + .../components/dashboard/virtual-service.tmpl | 27 + .../control-plane/control-plane-minimal.tmpl | 18 + .../control-plane/filters/filter-oauth2.tmpl | 90 + .../filters/filter-propagate-token.tmpl | 53 + .../control-plane/namespace.patch.tmpl | 7 + .../control-plane/oauth/auth-policy.tmpl | 26 + .../control-plane/oauth/oauth-client.tmpl | 8 + .../control-plane/routing/gateway-route.tmpl | 21 + .../servicemesh/control-plane/smm.tmpl | 9 + pkg/feature/types.go | 27 + pkg/gvr/gvr.go | 35 + tests/envtestutil/cleaner.go | 131 + tests/envtestutil/name_gen.go | 59 + .../envtestutil/utils.go | 6 +- .../crd/servicemeshcontrolplanes.crd.yaml | 10237 ++++++++++++++++ .../servicemesh/crd/test-resource.yaml | 17 + .../service_mesh_setup_features_int_test.go | 528 + .../service_mesh_suite_int_test.go | 81 + 79 files changed, 15054 insertions(+), 236 deletions(-) create mode 100644 apis/dscinitialization/v1/servicemesh_types.go create mode 100644 bundle/manifests/dscinitialization.opendatahub.io_servicemeshresourcetrackers.yaml create mode 100644 config/crd/bases/dscinitialization.opendatahub.io_featuretrackers.yaml create mode 100644 docs/SERVICE-MESH.md create mode 100644 pkg/cluster/cluster_config.go create mode 100644 pkg/cluster/cluster_suite_test.go create mode 100644 pkg/cluster/hostname_and_port_extraction_unit_test.go create mode 100644 pkg/cluster/operations.go create mode 100644 pkg/feature/builder.go create mode 100644 pkg/feature/cert.go create mode 100644 pkg/feature/conditions.go create mode 100644 pkg/feature/feature.go create mode 100644 pkg/feature/feature_suite_test.go create mode 100644 pkg/feature/manifest.go create mode 100644 pkg/feature/raw_resources.go create mode 100644 pkg/feature/resources.go create mode 100644 pkg/feature/servicemesh/cleanup.go create mode 100644 pkg/feature/servicemesh/conditions.go create mode 100644 pkg/feature/servicemesh/envoy_secrets.go create mode 100644 pkg/feature/servicemesh/loaders.go create mode 100644 pkg/feature/servicemesh/resources.go create mode 100644 pkg/feature/servicemesh/servicemesh_setup.go create mode 100644 pkg/feature/template_loader.go create mode 100644 pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/base/authconfig.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/namespace.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/rbac/cluster-monitoring-role-binding.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/rbac/cluster-role-binding.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/rbac/cluster-role.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/rbac/role-binding.tmpl create mode 100644 pkg/feature/templates/servicemesh/authorino/rbac/role.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/base/control-plane-ingress.patch.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/components/dashboard/gateway.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/components/dashboard/virtual-service.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/control-plane-minimal.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/filters/filter-oauth2.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/filters/filter-propagate-token.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/namespace.patch.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/oauth/auth-policy.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/oauth/oauth-client.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/routing/gateway-route.tmpl create mode 100644 pkg/feature/templates/servicemesh/control-plane/smm.tmpl create mode 100644 pkg/feature/types.go create mode 100644 pkg/gvr/gvr.go create mode 100644 tests/envtestutil/cleaner.go create mode 100644 tests/envtestutil/name_gen.go rename controllers/test/envtest_setup.go => tests/envtestutil/utils.go (94%) create mode 100644 tests/integration/servicemesh/crd/servicemeshcontrolplanes.crd.yaml create mode 100644 tests/integration/servicemesh/crd/test-resource.yaml create mode 100644 tests/integration/servicemesh/service_mesh_setup_features_int_test.go create mode 100644 tests/integration/servicemesh/service_mesh_suite_int_test.go diff --git a/Makefile b/Makefile index bb9b21af9f2..2eff4efe071 100644 --- a/Makefile +++ b/Makefile @@ -126,7 +126,7 @@ endef manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. # TODO: enable below when we do webhook # $(CONTROLLER_GEN) rbac:roleName=controller-manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases - $(CONTROLLER_GEN) rbac:roleName=controller-manager-role crd paths="./..." output:crd:artifacts:config=config/crd/bases + $(CONTROLLER_GEN) rbac:roleName=controller-manager-role crd:ignoreUnexportedFields=true paths="./..." output:crd:artifacts:config=config/crd/bases $(call fetch-external-crds,github.com/openshift/api,route/v1) $(call fetch-external-crds,github.com/openshift/api,user/v1) @@ -308,6 +308,8 @@ toolbox: ## Create a toolbox instance with the proper Golang and Operator SDK ve toolbox create opendatahub-toolbox --image localhost/opendatahub-toolbox:latest # Run tests. +TEST_SRC=./controllers/... ./tests/integration/servicemesh/... + .PHONY: envtest envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. $(ENVTEST): $(LOCALBIN) @@ -318,7 +320,7 @@ test: unit-test e2e-test .PHONY: unit-test unit-test: envtest - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test ./controllers/... -v -coverprofile cover.out + KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $(TEST_SRC) -v -coverprofile cover.out .PHONY: e2e-test e2e-test: ## Run e2e tests for the controller diff --git a/apis/dscinitialization/v1/dscinitialization_types.go b/apis/dscinitialization/v1/dscinitialization_types.go index 4deb280bd65..9c160645276 100644 --- a/apis/dscinitialization/v1/dscinitialization_types.go +++ b/apis/dscinitialization/v1/dscinitialization_types.go @@ -35,9 +35,13 @@ type DSCInitializationSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,order=2 // +optional Monitoring Monitoring `json:"monitoring,omitempty"` + // Enable Service Mesh for Data Science Clusters + // +operator-sdk:csv:customresourcedefinitions:type=spec,order=3 + // +optional + ServiceMesh ServiceMeshSpec `json:"serviceMesh,omitempty"` // Internal development useful field to test customizations. // This is not recommended to be used in production environment. - // +operator-sdk:csv:customresourcedefinitions:type=spec,order=3 + // +operator-sdk:csv:customresourcedefinitions:type=spec,order=4 // +optional DevFlags DevFlags `json:"devFlags,omitempty"` } @@ -108,5 +112,10 @@ type DSCInitializationList struct { } func init() { - SchemeBuilder.Register(&DSCInitialization{}, &DSCInitializationList{}) + SchemeBuilder.Register( + &DSCInitialization{}, + &DSCInitializationList{}, + &FeatureTracker{}, + &FeatureTrackerList{}, + ) } diff --git a/apis/dscinitialization/v1/servicemesh_types.go b/apis/dscinitialization/v1/servicemesh_types.go new file mode 100644 index 00000000000..c32625d7ccc --- /dev/null +++ b/apis/dscinitialization/v1/servicemesh_types.go @@ -0,0 +1,120 @@ +package v1 + +import ( + operatorv1 "github.com/openshift/api/operator/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ServiceMeshSpec configures Service Mesh. +type ServiceMeshSpec struct { + // +kubebuilder:validation:Enum=Managed;Removed + // +kubebuilder:default=Removed + ManagementState operatorv1.ManagementState `json:"managementState,omitempty"` + // Mesh holds configuration of Service Mesh used by Opendatahub. + Mesh MeshSpec `json:"mesh,omitempty"` + // Auth holds configuration of authentication and authorization services + // used by Service Mesh in Opendatahub. + Auth AuthSpec `json:"auth,omitempty"` +} + +type MeshSpec struct { + // Name is a name Service Mesh Control Plan. Defaults to "basic". + // +kubebuilder:default=basic + Name string `json:"name,omitempty"` + // Namespace is a namespace where Service Mesh is deployed. Defaults to "istio-system". + // +kubebuilder:default=istio-system + Namespace string `json:"namespace,omitempty"` + // Certificate allows to define how to use certificates for the Service Mesh communication. + Certificate CertSpec `json:"certificate,omitempty"` +} + +type CertSpec struct { + // Name of the certificate to be used by Service Mesh. + // +kubebuilder:default=opendatahub-dashboard-cert + Name string `json:"name,omitempty"` + // Generate indicates if the certificate should be generated. If set to false + // it will assume certificate with the given name is made available as a secret + // in Service Mesh namespace. + // +kubebuilder:default=true + Generate bool `json:"generate,omitempty"` +} + +type AuthSpec struct { + // Name of the authorization provider used for Service Mesh. + // +kubebuilder:default=authorino + Name string `json:"name,omitempty"` + // Namespace where it is deployed. + // +kubebuilder:default=auth-provider + Namespace string `json:"namespace,omitempty"` + // Authorino holds configuration of Authorino service used as external authorization provider. + Authorino AuthorinoSpec `json:"authorino,omitempty"` +} + +type AuthorinoSpec struct { + // Name specifies how external authorization provider should be called. + // +kubebuilder:default=authorino-mesh-authz-provider + Name string `json:"name,omitempty"` + // Audiences is a list of the identifiers that the resource server presented + // with the token identifies as. Audience-aware token authenticators will verify + // that the token was intended for at least one of the audiences in this list. + // If no audiences are provided, the audience will default to the audience of the + // Kubernetes apiserver (kubernetes.default.svc). + // +kubebuilder:default={"https://kubernetes.default.svc"} + Audiences []string `json:"audiences,omitempty"` + // Label narrows amount of AuthConfigs to process by Authorino service. + // +kubebuilder:default=authorino/topic=odh + Label string `json:"label,omitempty"` + // Image allows to define a custom container image to be used when deploying Authorino's instance. + // +kubebuilder:default="quay.io/kuadrant/authorino:v0.13.0" + Image string `json:"image,omitempty"` +} + +// FeatureTracker is a cluster-scoped resource for tracking objects +// created through Features API for Data Science Platform. +// It's primarily used as owner reference for resources created across namespaces so that they can be +// garbage collected by Kubernetes when they're not needed anymore. +// +kubebuilder:object:root=true +// +kubebuilder:resource:scope=Cluster +type FeatureTracker struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec FeatureTrackerSpec `json:"spec,omitempty"` + Status FeatureTrackerStatus `json:"status,omitempty"` +} + +func (s *FeatureTracker) ToOwnerReference() metav1.OwnerReference { + return metav1.OwnerReference{ + APIVersion: s.APIVersion, + Kind: s.Kind, + Name: s.Name, + UID: s.UID, + } +} + +// FeatureTrackerSpec defines the desired state of FeatureTracker. +type FeatureTrackerSpec struct { +} + +// FeatureTrackerStatus defines the observed state of FeatureTracker. +type FeatureTrackerStatus struct { +} + +// +kubebuilder:object:root=true + +// FeatureTrackerList contains a list of FeatureTracker. +type FeatureTrackerList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []FeatureTracker `json:"items"` +} + +// IsValid returns true if the spec is a valid and complete. +// If invalid it provides message with the reasons. +func (s *ServiceMeshSpec) IsValid() (bool, string) { + if s.Auth.Name != "authorino" { + return false, "currently only Authorino is available as authorization layer" + } + + return true, "" +} diff --git a/apis/dscinitialization/v1/zz_generated.deepcopy.go b/apis/dscinitialization/v1/zz_generated.deepcopy.go index c885191f04b..a7a815ea879 100644 --- a/apis/dscinitialization/v1/zz_generated.deepcopy.go +++ b/apis/dscinitialization/v1/zz_generated.deepcopy.go @@ -27,12 +27,63 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthSpec) DeepCopyInto(out *AuthSpec) { + *out = *in + in.Authorino.DeepCopyInto(&out.Authorino) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthSpec. +func (in *AuthSpec) DeepCopy() *AuthSpec { + if in == nil { + return nil + } + out := new(AuthSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AuthorinoSpec) DeepCopyInto(out *AuthorinoSpec) { + *out = *in + if in.Audiences != nil { + in, out := &in.Audiences, &out.Audiences + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AuthorinoSpec. +func (in *AuthorinoSpec) DeepCopy() *AuthorinoSpec { + if in == nil { + return nil + } + out := new(AuthorinoSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CertSpec) DeepCopyInto(out *CertSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CertSpec. +func (in *CertSpec) DeepCopy() *CertSpec { + if in == nil { + return nil + } + out := new(CertSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DSCInitialization) DeepCopyInto(out *DSCInitialization) { *out = *in out.TypeMeta = in.TypeMeta in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - out.Spec = in.Spec + in.Spec.DeepCopyInto(&out.Spec) in.Status.DeepCopyInto(&out.Status) } @@ -90,6 +141,7 @@ func (in *DSCInitializationList) DeepCopyObject() runtime.Object { func (in *DSCInitializationSpec) DeepCopyInto(out *DSCInitializationSpec) { *out = *in out.Monitoring = in.Monitoring + in.ServiceMesh.DeepCopyInto(&out.ServiceMesh) out.DevFlags = in.DevFlags } @@ -145,6 +197,111 @@ func (in *DevFlags) DeepCopy() *DevFlags { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeatureTracker) DeepCopyInto(out *FeatureTracker) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + out.Status = in.Status +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureTracker. +func (in *FeatureTracker) DeepCopy() *FeatureTracker { + if in == nil { + return nil + } + out := new(FeatureTracker) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FeatureTracker) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeatureTrackerList) DeepCopyInto(out *FeatureTrackerList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]FeatureTracker, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureTrackerList. +func (in *FeatureTrackerList) DeepCopy() *FeatureTrackerList { + if in == nil { + return nil + } + out := new(FeatureTrackerList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *FeatureTrackerList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeatureTrackerSpec) DeepCopyInto(out *FeatureTrackerSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureTrackerSpec. +func (in *FeatureTrackerSpec) DeepCopy() *FeatureTrackerSpec { + if in == nil { + return nil + } + out := new(FeatureTrackerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FeatureTrackerStatus) DeepCopyInto(out *FeatureTrackerStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FeatureTrackerStatus. +func (in *FeatureTrackerStatus) DeepCopy() *FeatureTrackerStatus { + if in == nil { + return nil + } + out := new(FeatureTrackerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MeshSpec) DeepCopyInto(out *MeshSpec) { + *out = *in + out.Certificate = in.Certificate +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MeshSpec. +func (in *MeshSpec) DeepCopy() *MeshSpec { + if in == nil { + return nil + } + out := new(MeshSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Monitoring) DeepCopyInto(out *Monitoring) { *out = *in @@ -159,3 +316,20 @@ func (in *Monitoring) DeepCopy() *Monitoring { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ServiceMeshSpec) DeepCopyInto(out *ServiceMeshSpec) { + *out = *in + out.Mesh = in.Mesh + in.Auth.DeepCopyInto(&out.Auth) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ServiceMeshSpec. +func (in *ServiceMeshSpec) DeepCopy() *ServiceMeshSpec { + if in == nil { + return nil + } + out := new(ServiceMeshSpec) + in.DeepCopyInto(out) + return out +} diff --git a/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml b/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml index 2110872b9e1..d00539e6f89 100644 --- a/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml +++ b/bundle/manifests/dscinitialization.opendatahub.io_dscinitializations.yaml @@ -80,6 +80,96 @@ spec: description: Namespace for monitoring if it is enabled type: string type: object + serviceMesh: + description: Enable Service Mesh for Data Science Clusters + properties: + auth: + description: Auth holds configuration of authentication and authorization + services used by Service Mesh in Opendatahub. + properties: + authorino: + description: Authorino holds configuration of Authorino service + used as external authorization provider. + properties: + audiences: + default: + - https://kubernetes.default.svc + description: Audiences is a list of the identifiers that + the resource server presented with the token identifies + as. Audience-aware token authenticators will verify + that the token was intended for at least one of the + audiences in this list. If no audiences are provided, + the audience will default to the audience of the Kubernetes + apiserver (kubernetes.default.svc). + items: + type: string + type: array + image: + default: quay.io/kuadrant/authorino:v0.13.0 + description: Image allows to define a custom container + image to be used when deploying Authorino's instance. + type: string + label: + default: authorino/topic=odh + description: Label narrows amount of AuthConfigs to process + by Authorino service. + type: string + name: + default: authorino-mesh-authz-provider + description: Name specifies how external authorization + provider should be called. + type: string + type: object + name: + default: authorino + description: Name of the authorization provider used for Service + Mesh. + type: string + namespace: + default: auth-provider + description: Namespace where it is deployed. + type: string + type: object + managementState: + default: Removed + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + mesh: + description: Mesh holds configuration of Service Mesh used by + Opendatahub. + properties: + certificate: + description: Certificate allows to define how to use certificates + for the Service Mesh communication. + properties: + generate: + default: true + description: Generate indicates if the certificate should + be generated. If set to false it will assume certificate + with the given name is made available as a secret in + Service Mesh namespace. + type: boolean + name: + default: opendatahub-dashboard-cert + description: Name of the certificate to be used by Service + Mesh. + type: string + type: object + name: + default: basic + description: Name is a name Service Mesh Control Plan. Defaults + to "basic". + type: string + namespace: + default: istio-system + description: Namespace is a namespace where Service Mesh is + deployed. Defaults to "istio-system". + type: string + type: object + type: object required: - applicationsNamespace type: object diff --git a/bundle/manifests/dscinitialization.opendatahub.io_servicemeshresourcetrackers.yaml b/bundle/manifests/dscinitialization.opendatahub.io_servicemeshresourcetrackers.yaml new file mode 100644 index 00000000000..37067165282 --- /dev/null +++ b/bundle/manifests/dscinitialization.opendatahub.io_servicemeshresourcetrackers.yaml @@ -0,0 +1,54 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: servicemeshresourcetrackers.dscinitialization.opendatahub.io +spec: + group: dscinitialization.opendatahub.io + names: + kind: ServiceMeshResourceTracker + listKind: ServiceMeshResourceTrackerList + plural: servicemeshresourcetrackers + singular: servicemeshresourcetracker + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: ServiceMeshResourceTracker is a cluster-scoped resource for tracking + objects created by Service Mesh initialization for Data Science Platform. + It's primarily used as owner reference for resources created across namespaces + so that they can be garbage collected by Kubernetes when they're not needed + anymore. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: ServiceMeshResourceTrackerSpec defines the desired state + of ServiceMeshResourceTracker. + type: object + status: + description: ServiceMeshResourceTrackerStatus defines the observed state + of ServiceMeshResourceTracker. + type: object + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: null + storedVersions: null diff --git a/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml b/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml index efb250c4569..de9ae8a8dab 100644 --- a/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml +++ b/bundle/manifests/opendatahub-operator.clusterserviceversion.yaml @@ -72,7 +72,7 @@ metadata: categories: AI/Machine Learning, Big Data certified: "False" containerImage: quay.io/opendatahub/opendatahub-operator:v2.1.0 - createdAt: "2023-8-23T00:00:00Z" + createdAt: "2023-10-05T16:20:15Z" olm.skipRange: '>=1.0.0 <2.0.0' operatorframework.io/initialization-resource: |- { @@ -148,6 +148,9 @@ spec: - description: Enable monitoring on specified namespace displayName: Monitoring path: monitoring + - description: Enable Service Mesh for Data Science Clusters + displayName: Service Mesh + path: serviceMesh - description: Internal development useful field to test customizations. This is not recommended to be used in production environment. displayName: Dev Flags @@ -158,6 +161,9 @@ spec: displayName: Conditions path: conditions version: v1 + - kind: ServiceMeshResourceTracker + name: servicemeshresourcetrackers.dscinitialization.opendatahub.io + version: v1 description: "The Open Data Hub is a machine-learning-as-a-service platform built on Red Hat's Kubernetes-based OpenShift® Container Platform. Open Data Hub integrates multiple AI/ML open source components into one operator that can easily be downloaded @@ -325,6 +331,12 @@ spec: - tokenreviews verbs: - create + - apiGroups: + - authorino.kuadrant.io + resources: + - authconfigs + verbs: + - '*' - apiGroups: - authorization.k8s.io resources: @@ -482,6 +494,12 @@ spec: verbs: - list - watch + - apiGroups: + - config.openshift.io + resources: + - ingresses + verbs: + - get - apiGroups: - console.openshift.io resources: @@ -804,6 +822,18 @@ spec: - get - patch - update + - apiGroups: + - dscinitialization.opendatahub.io + resources: + - servicemeshresourcetrackers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - events.k8s.io resources: @@ -1001,6 +1031,18 @@ spec: - patch - update - watch + - apiGroups: + - networking.istio.io + resources: + - envoyfilters + verbs: + - '*' + - apiGroups: + - networking.istio.io + resources: + - gateways + verbs: + - '*' - apiGroups: - networking.istio.io resources: @@ -1072,6 +1114,12 @@ spec: - patch - update - watch + - apiGroups: + - operator.authorino.kuadrant.io + resources: + - authorinos + verbs: + - '*' - apiGroups: - operator.openshift.io resources: @@ -1194,6 +1242,19 @@ spec: - patch - update - watch + - apiGroups: + - route.openshift.io + resources: + - routes/custom-host + verbs: + - create + - get + - apiGroups: + - security.istio.io + resources: + - authorizationpolicies + verbs: + - '*' - apiGroups: - security.openshift.io resources: diff --git a/components/dashboard/dashboard.go b/components/dashboard/dashboard.go index fa2e86f6804..25d7816595c 100644 --- a/components/dashboard/dashboard.go +++ b/components/dashboard/dashboard.go @@ -5,6 +5,8 @@ package dashboard import ( "context" "fmt" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" + "path" "path/filepath" "strings" @@ -12,30 +14,38 @@ import ( dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/common" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" - + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" routev1 "github.com/openshift/api/route/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) var ( - ComponentName = "dashboard" - ComponentNameSupported = "rhods-dashboard" - Path = deploy.DefaultManifestPath + "/" + ComponentName + "/base" - PathSupported = deploy.DefaultManifestPath + "/" + ComponentName + "/overlays/rhods" - PathISVSM = deploy.DefaultManifestPath + "/" + ComponentName + "/apps/apps-onprem" - PathISVAddOn = deploy.DefaultManifestPath + "/" + ComponentName + "/apps/apps-addon" - PathOVMS = deploy.DefaultManifestPath + "/" + ComponentName + "/modelserving" - PathODHDashboardConfig = deploy.DefaultManifestPath + "/" + ComponentName + "/odhdashboardconfig" - PathConsoleLink = deploy.DefaultManifestPath + "/" + ComponentName + "/consolelink" - PathCRDs = deploy.DefaultManifestPath + "/" + ComponentName + "/crd" - NameConsoleLink = "console" - NamespaceConsoleLink = "openshift-console" - PathAnaconda = deploy.DefaultManifestPath + "/partners/anaconda/base/" + ComponentName = "dashboard" + ComponentNameSupported = "rhods-dashboard" + Path = deploy.DefaultManifestPath + "/" + ComponentName + "/base" + PathServiceMesh = deploy.DefaultManifestPath + "/" + ComponentName + "/overlays/service-mesh" + PathSupported = deploy.DefaultManifestPath + "/" + ComponentName + "/overlays/rhods" + PathISVSM = deploy.DefaultManifestPath + "/" + ComponentName + "/apps/apps-onprem" + PathISVAddOn = deploy.DefaultManifestPath + "/" + ComponentName + "/apps/apps-addon" + PathOVMS = deploy.DefaultManifestPath + "/" + ComponentName + "/modelserving" + PathODHDashboardConfig = deploy.DefaultManifestPath + "/" + ComponentName + "/odhdashboardconfig" + PathODHProjectController = deploy.DefaultManifestPath + "/" + ProjectController + "/base" + PathConsoleLink = deploy.DefaultManifestPath + "/" + ComponentName + "/consolelink" + PathCRDs = deploy.DefaultManifestPath + "/" + ComponentName + "/crd" + NameConsoleLink = "console" + NamespaceConsoleLink = "openshift-console" + PathAnaconda = deploy.DefaultManifestPath + "/partners/anaconda/base/" + ProjectController = "odh-project-controller" ) +// Verifies that Dashboard implements ComponentInterface +var _ components.ComponentInterface = (*Dashboard)(nil) + type Dashboard struct { components.Component `json:""` } @@ -69,9 +79,6 @@ func (d *Dashboard) GetComponentName() string { return ComponentName } -// Verifies that Dashboard implements ComponentInterface. -var _ components.ComponentInterface = (*Dashboard)(nil) - func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsci.DSCInitializationSpec) error { var imageParamMap = map[string]string{ "odh-dashboard-image": "RELATED_IMAGE_ODH_DASHBOARD_IMAGE", @@ -87,9 +94,8 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d return err } - if platform == deploy.OpenDataHub || platform == "" { - err := common.UpdatePodSecurityRolebinding(cli, []string{"odh-dashboard"}, dscispec.ApplicationsNamespace) - if err != nil { + if platform == deploy.OpenDataHub || platform == deploy.Unknown { + if err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, "odh-dashboard"); err != nil { return err } @@ -99,8 +105,7 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d } if platform == deploy.SelfManagedRhods || platform == deploy.ManagedRhods { - err := common.UpdatePodSecurityRolebinding(cli, []string{"rhods-dashboard"}, dscispec.ApplicationsNamespace) - if err != nil { + if err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, "rhods-dashboard"); err != nil { return err } @@ -122,11 +127,17 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d } // Deploy odh-dashboard manifests - if platform == deploy.OpenDataHub || platform == "" { - if err = deploy.DeployManifestsFromPath(cli, owner, Path, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { + switch { + case platform == deploy.OpenDataHub, platform == deploy.Unknown: + base := Path + if dscispec.ServiceMesh.ManagementState == operatorv1.Managed { + base = PathServiceMesh + } + if err = deploy.DeployManifestsFromPath(cli, owner, base, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { return err } - } else if platform == deploy.SelfManagedRhods || platform == deploy.ManagedRhods { + + case platform == deploy.SelfManagedRhods, platform == deploy.ManagedRhods: // Apply authentication overlay if err := deploy.DeployManifestsFromPath(cli, owner, PathSupported, dscispec.ApplicationsNamespace, ComponentNameSupported, enabled); err != nil { return err @@ -149,6 +160,33 @@ func (d *Dashboard) ReconcileComponent(cli client.Client, owner metav1.Object, d } } + if enabled { + if err := d.configureServiceMesh(cli, owner, dscispec); err != nil { + return err + } + } + + return nil +} + +func (d *Dashboard) Cleanup(cli client.Client, dscispec *dsci.DSCInitializationSpec) error { + shouldConfigureServiceMesh, err := deploy.ShouldConfigureServiceMesh(cli, dscispec) + if err != nil { + return err + } + + if shouldConfigureServiceMesh { + serviceMeshInitializer := servicemesh.NewServiceMeshInitializer(dscispec, d.defineServiceMeshFeatures(dscispec)) + + if err := serviceMeshInitializer.Prepare(); err != nil { + return err + } + + if err := serviceMeshInitializer.Delete(); err != nil { + return err + } + } + return nil } @@ -187,7 +225,7 @@ func (d *Dashboard) applyRhodsSpecificConfigs(cli client.Client, owner metav1.Ob return fmt.Errorf("failed to set dashboard OVMS from %s: %w", PathOVMS, err) } - if err := common.CreateSecret(cli, "anaconda-ce-access", namespace); err != nil { + if err := cluster.CreateSecret(cli, "anaconda-ce-access", namespace); err != nil { return fmt.Errorf("failed to create access-secret for anaconda: %w", err) } @@ -239,3 +277,62 @@ func (d *Dashboard) deployConsoleLink(cli client.Client, owner metav1.Object, na return nil } + +func (d *Dashboard) configureServiceMesh(cli client.Client, owner metav1.Object, dscispec *dsci.DSCInitializationSpec) error { + shouldConfigureServiceMesh, err := deploy.ShouldConfigureServiceMesh(cli, dscispec) + if err != nil { + return err + } + + if shouldConfigureServiceMesh { + serviceMeshInitializer := servicemesh.NewServiceMeshInitializer(dscispec, d.defineServiceMeshFeatures(dscispec)) + + if err := serviceMeshInitializer.Prepare(); err != nil { + return err + } + + if err := serviceMeshInitializer.Apply(); err != nil { + return err + } + + enabled := d.GetManagementState() == operatorv1.Managed + if err := deploy.DeployManifestsFromPath(cli, owner, PathODHProjectController, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { + return err + } + } + + return nil +} + +func (d *Dashboard) defineServiceMeshFeatures(dscispec *dsci.DSCInitializationSpec) servicemesh.DefineFeatures { + return func(s *servicemesh.ServiceMeshInitializer) error { + var rootDir = filepath.Join(feature.BaseOutputDir, dscispec.ApplicationsNamespace) + if err := feature.CopyEmbeddedFiles("templates", rootDir); err != nil { + return err + } + + createMeshResources, err := feature.CreateFeature("create-service-mesh-routing-resources-for-dashboard"). + For(dscispec). + Manifests( + path.Join(rootDir, feature.ControlPlaneDir, "components", d.GetComponentName()), + ). + WithResources(servicemesh.EnabledInDashboard). + WithData(servicemesh.ClusterDetails). + PreConditions( + feature.WaitForResourceToBeCreated(dscispec.ApplicationsNamespace, gvr.ODHDashboardConfigGVR), + ). + PostConditions( + feature.WaitForPodsToBeReady(dscispec.ServiceMesh.Mesh.Namespace), + ). + OnDelete(servicemesh.DisabledInDashboard). + Load() + + if err != nil { + return err + } + + s.Features = append(s.Features, createMeshResources) + + return nil + } +} diff --git a/components/kserve/kserve.go b/components/kserve/kserve.go index cc90815565d..0027967b776 100644 --- a/components/kserve/kserve.go +++ b/components/kserve/kserve.go @@ -3,13 +3,12 @@ package kserve import ( "fmt" - "path/filepath" "strings" dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/common" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -122,8 +121,7 @@ func (k *Kserve) ReconcileComponent(cli client.Client, owner metav1.Object, dsci // For odh-model-controller if enabled { - err := common.UpdatePodSecurityRolebinding(cli, []string{"odh-model-controller"}, dscispec.ApplicationsNamespace) - if err != nil { + if err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, "odh-model-controller"); err != nil { return err } // Update image parameters for odh-maodel-controller diff --git a/components/modelmeshserving/modelmeshserving.go b/components/modelmeshserving/modelmeshserving.go index 13b6d7e70e2..7eae438d3ad 100644 --- a/components/modelmeshserving/modelmeshserving.go +++ b/components/modelmeshserving/modelmeshserving.go @@ -8,7 +8,7 @@ import ( dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/common" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -93,8 +93,11 @@ func (m *ModelMeshServing) ReconcileComponent(cli client.Client, owner metav1.Ob return err } - err := common.UpdatePodSecurityRolebinding(cli, []string{"modelmesh", "modelmesh-controller", "odh-prometheus-operator", "prometheus-custom"}, dscispec.ApplicationsNamespace) - if err != nil { + if err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, + "modelmesh", + "modelmesh-controller", + "odh-prometheus-operator", + "prometheus-custom"); err != nil { return err } // Update image parameters @@ -112,7 +115,7 @@ func (m *ModelMeshServing) ReconcileComponent(cli client.Client, owner metav1.Ob // For odh-model-controller if enabled { - err := common.UpdatePodSecurityRolebinding(cli, []string{"odh-model-controller"}, dscispec.ApplicationsNamespace) + err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, "odh-model-controller") if err != nil { return err } diff --git a/components/workbenches/workbenches.go b/components/workbenches/workbenches.go index 86475ddfb1d..01648f328aa 100644 --- a/components/workbenches/workbenches.go +++ b/components/workbenches/workbenches.go @@ -7,7 +7,7 @@ import ( dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/components" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/common" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -18,13 +18,18 @@ var ( ComponentName = "workbenches" DependentComponentName = "notebooks" // manifests for nbc in ODH and downstream + downstream use it for imageparams - notebookControllerPath = deploy.DefaultManifestPath + "/odh-notebook-controller/odh-notebook-controller/base" + notebookControllerPath = deploy.DefaultManifestPath + "/odh-notebook-controller/odh-notebook-controller/base" + notebookControllerServiceMeshPath = deploy.DefaultManifestPath + "/odh-notebook-controller/odh-notebook-controller/overlays/service-mesh" // manifests for ODH nbc - kfnotebookControllerPath = deploy.DefaultManifestPath + "/odh-notebook-controller/kf-notebook-controller/overlays/openshift" - notebookImagesPath = deploy.DefaultManifestPath + "/notebooks/overlays/additional" - notebookImagesPathSupported = deploy.DefaultManifestPath + "/jupyterhub/notebook-images/overlays/additional" + kfnotebookControllerPath = deploy.DefaultManifestPath + "/odh-notebook-controller/kf-notebook-controller/overlays/openshift" + kfnotebookControllerServiceMeshPath = deploy.DefaultManifestPath + "/odh-notebook-controller/kf-notebook-controller/overlays/service-mesh" + notebookImagesPath = deploy.DefaultManifestPath + "/notebooks/overlays/additional" + notebookImagesPathSupported = deploy.DefaultManifestPath + "/jupyterhub/notebook-images/overlays/additional" ) +// Verifies that Workbenches implements ComponentInterface. +var _ components.ComponentInterface = (*Workbenches)(nil) + type Workbenches struct { components.Component `json:""` } @@ -32,7 +37,7 @@ type Workbenches struct { func (w *Workbenches) OverrideManifests(platform string) error { // Download manifests if defined by devflags if len(w.DevFlags.Manifests) != 0 { - // Go through each manifests and set the overlays if defined + // Go through each manifest and set the overlays if defined for _, subcomponent := range w.DevFlags.Manifests { if strings.Contains(subcomponent.URI, DependentComponentName) { // Download subcomponent @@ -87,9 +92,6 @@ func (w *Workbenches) GetComponentName() string { return ComponentName } -// Verifies that Dashboard implements ComponentInterface. -var _ components.ComponentInterface = (*Workbenches)(nil) - func (w *Workbenches) ReconcileComponent(cli client.Client, owner metav1.Object, dscispec *dsci.DSCInitializationSpec) error { var imageParamMap = map[string]string{ "odh-notebook-controller-image": "RELATED_IMAGE_ODH_NOTEBOOK_CONTROLLER_IMAGE", @@ -111,39 +113,48 @@ func (w *Workbenches) ReconcileComponent(cli client.Client, owner metav1.Object, } if platform == deploy.SelfManagedRhods || platform == deploy.ManagedRhods { - err := common.CreateNamespace(cli, "rhods-notebooks") - if err != nil { + if err := cluster.CreateNamespace(cli, "rhods-notebooks"); err != nil { // no need to log error as it was already logged in createOdhNamespace return err } } - // Update Default rolebinding - err = common.UpdatePodSecurityRolebinding(cli, []string{"notebook-controller-service-account"}, dscispec.ApplicationsNamespace) - if err != nil { + if err := cluster.UpdatePodSecurityRolebinding(cli, dscispec.ApplicationsNamespace, "notebook-controller-service-account"); err != nil { return err } } - err = deploy.DeployManifestsFromPath(cli, owner, notebookControllerPath, dscispec.ApplicationsNamespace, ComponentName, enabled) + actualNbCtrlPath := notebookControllerPath + shouldConfigureServiceMesh, err := deploy.ShouldConfigureServiceMesh(cli, dscispec) if err != nil { return err } + if shouldConfigureServiceMesh { + actualNbCtrlPath = notebookControllerServiceMeshPath + } + + if err := deploy.DeployManifestsFromPath(cli, owner, actualNbCtrlPath, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { + return err + } // Update image parameters for nbc in downstream if enabled { if dscispec.DevFlags.ManifestsUri == "" && len(w.DevFlags.Manifests) == 0 { if platform == deploy.ManagedRhods || platform == deploy.SelfManagedRhods { - if err := deploy.ApplyParams(notebookControllerPath, w.SetImageParamsMap(imageParamMap), false); err != nil { + if err := deploy.ApplyParams(actualNbCtrlPath, w.SetImageParamsMap(imageParamMap), false); err != nil { return err } } } } - if platform == deploy.OpenDataHub || platform == "" { + if platform == deploy.OpenDataHub || platform == deploy.Unknown { // only for ODH after transit to kubeflow repo + path := kfnotebookControllerPath + if shouldConfigureServiceMesh { + path = kfnotebookControllerServiceMeshPath + } err = deploy.DeployManifestsFromPath(cli, owner, - kfnotebookControllerPath, + path, dscispec.ApplicationsNamespace, ComponentName, enabled) if err != nil { @@ -157,8 +168,7 @@ func (w *Workbenches) ReconcileComponent(cli client.Client, owner metav1.Object, enabled) return err } else { - err = deploy.DeployManifestsFromPath(cli, owner, notebookImagesPathSupported, dscispec.ApplicationsNamespace, ComponentName, enabled) - return err + return deploy.DeployManifestsFromPath(cli, owner, notebookImagesPathSupported, dscispec.ApplicationsNamespace, ComponentName, enabled) } } diff --git a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml index 84abc1fa1e0..5f762857b0c 100644 --- a/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml +++ b/config/crd/bases/dscinitialization.opendatahub.io_dscinitializations.yaml @@ -81,6 +81,96 @@ spec: description: Namespace for monitoring if it is enabled type: string type: object + serviceMesh: + description: Enable Service Mesh for Data Science Clusters + properties: + auth: + description: Auth holds configuration of authentication and authorization + services used by Service Mesh in Opendatahub. + properties: + authorino: + description: Authorino holds configuration of Authorino service + used as external authorization provider. + properties: + audiences: + default: + - https://kubernetes.default.svc + description: Audiences is a list of the identifiers that + the resource server presented with the token identifies + as. Audience-aware token authenticators will verify + that the token was intended for at least one of the + audiences in this list. If no audiences are provided, + the audience will default to the audience of the Kubernetes + apiserver (kubernetes.default.svc). + items: + type: string + type: array + image: + default: quay.io/kuadrant/authorino:v0.13.0 + description: Image allows to define a custom container + image to be used when deploying Authorino's instance. + type: string + label: + default: authorino/topic=odh + description: Label narrows amount of AuthConfigs to process + by Authorino service. + type: string + name: + default: authorino-mesh-authz-provider + description: Name specifies how external authorization + provider should be called. + type: string + type: object + name: + default: authorino + description: Name of the authorization provider used for Service + Mesh. + type: string + namespace: + default: auth-provider + description: Namespace where it is deployed. + type: string + type: object + managementState: + default: Removed + enum: + - Managed + - Removed + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + mesh: + description: Mesh holds configuration of Service Mesh used by + Opendatahub. + properties: + certificate: + description: Certificate allows to define how to use certificates + for the Service Mesh communication. + properties: + generate: + default: true + description: Generate indicates if the certificate should + be generated. If set to false it will assume certificate + with the given name is made available as a secret in + Service Mesh namespace. + type: boolean + name: + default: opendatahub-dashboard-cert + description: Name of the certificate to be used by Service + Mesh. + type: string + type: object + name: + default: basic + description: Name is a name Service Mesh Control Plan. Defaults + to "basic". + type: string + namespace: + default: istio-system + description: Namespace is a namespace where Service Mesh is + deployed. Defaults to "istio-system". + type: string + type: object + type: object required: - applicationsNamespace type: object diff --git a/config/crd/bases/dscinitialization.opendatahub.io_featuretrackers.yaml b/config/crd/bases/dscinitialization.opendatahub.io_featuretrackers.yaml new file mode 100644 index 00000000000..3ad5b72560f --- /dev/null +++ b/config/crd/bases/dscinitialization.opendatahub.io_featuretrackers.yaml @@ -0,0 +1,46 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.9.2 + creationTimestamp: null + name: featuretrackers.dscinitialization.opendatahub.io +spec: + group: dscinitialization.opendatahub.io + names: + kind: FeatureTracker + listKind: FeatureTrackerList + plural: featuretrackers + singular: featuretracker + scope: Cluster + versions: + - name: v1 + schema: + openAPIV3Schema: + description: FeatureTracker is a cluster-scoped resource for tracking objects + created through Features API for Data Science Platform. It's primarily used + as owner reference for resources created across namespaces so that they + can be garbage collected by Kubernetes when they're not needed anymore. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: FeatureTrackerSpec defines the desired state of FeatureTracker. + type: object + status: + description: FeatureTrackerStatus defines the observed state of FeatureTracker. + type: object + type: object + served: true + storage: true diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 110438d509c..6adcedb9a88 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -3,6 +3,7 @@ # It should be run by config/default resources: - bases/dscinitialization.opendatahub.io_dscinitializations.yaml +- bases/dscinitialization.opendatahub.io_featuretrackers.yaml - bases/datasciencecluster.opendatahub.io_datascienceclusters.yaml #+kubebuilder:scaffold:crdkustomizeresource diff --git a/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml b/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml index ac81810dc37..15c53bd3a8a 100644 --- a/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/opendatahub-operator.clusterserviceversion.yaml @@ -81,6 +81,9 @@ spec: - description: Enable monitoring on specified namespace displayName: Monitoring path: monitoring + - description: Enable Service Mesh for Data Science Clusters + displayName: Service Mesh + path: serviceMesh - description: Internal development useful field to test customizations. This is not recommended to be used in production environment. displayName: Dev Flags diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index c5a39ae2848..f3a223cb4a4 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -143,6 +143,12 @@ rules: - tokenreviews verbs: - create +- apiGroups: + - authorino.kuadrant.io + resources: + - authconfigs + verbs: + - '*' - apiGroups: - authorization.k8s.io resources: @@ -300,6 +306,12 @@ rules: verbs: - list - watch +- apiGroups: + - config.openshift.io + resources: + - ingresses + verbs: + - get - apiGroups: - console.openshift.io resources: @@ -622,6 +634,18 @@ rules: - get - patch - update +- apiGroups: + - dscinitialization.opendatahub.io + resources: + - servicemeshresourcetrackers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - events.k8s.io resources: @@ -819,6 +843,18 @@ rules: - patch - update - watch +- apiGroups: + - networking.istio.io + resources: + - envoyfilters + verbs: + - '*' +- apiGroups: + - networking.istio.io + resources: + - gateways + verbs: + - '*' - apiGroups: - networking.istio.io resources: @@ -890,6 +926,12 @@ rules: - patch - update - watch +- apiGroups: + - operator.authorino.kuadrant.io + resources: + - authorinos + verbs: + - '*' - apiGroups: - operator.openshift.io resources: @@ -1012,6 +1054,19 @@ rules: - patch - update - watch +- apiGroups: + - route.openshift.io + resources: + - routes/custom-host + verbs: + - create + - get +- apiGroups: + - security.istio.io + resources: + - authorizationpolicies + verbs: + - '*' - apiGroups: - security.openshift.io resources: diff --git a/config/samples/dscinitialization_v1_dscinitialization.yaml b/config/samples/dscinitialization_v1_dscinitialization.yaml index 745876813a1..cb0d24da046 100644 --- a/config/samples/dscinitialization_v1_dscinitialization.yaml +++ b/config/samples/dscinitialization_v1_dscinitialization.yaml @@ -12,4 +12,4 @@ spec: monitoring: managementState: "Managed" namespace: 'opendatahub' - applicationsNamespace: 'opendatahub' \ No newline at end of file + applicationsNamespace: 'opendatahub' diff --git a/controllers/datasciencecluster/kubebuilder_rbac.go b/controllers/datasciencecluster/kubebuilder_rbac.go index ce9a6dc2973..a7c13910e6d 100644 --- a/controllers/datasciencecluster/kubebuilder_rbac.go +++ b/controllers/datasciencecluster/kubebuilder_rbac.go @@ -4,6 +4,20 @@ package datasciencecluster //+kubebuilder:rbac:groups="datasciencecluster.opendatahub.io",resources=datascienceclusters/finalizers,verbs=update;patch //+kubebuilder:rbac:groups="datasciencecluster.opendatahub.io",resources=datascienceclusters,verbs=get;list;watch;create;update;patch;delete +/* Service Mesh Integration */ +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmemberrolls,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers/finalizers,verbs=create;get;list;patch;update;use;watch +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/status,verbs=update;patch;delete +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs=* +// +kubebuilder:rbac:groups="networking.istio.io",resources=gateways,verbs=* +// +kubebuilder:rbac:groups="networking.istio.io",resources=envoyfilters,verbs=* +// +kubebuilder:rbac:groups="security.istio.io",resources=authorizationpolicies,verbs=* +// +kubebuilder:rbac:groups="authorino.kuadrant.io",resources=authconfigs,verbs=* +// +kubebuilder:rbac:groups="operator.authorino.kuadrant.io",resources=authorinos,verbs=* + /* This is for DSP */ //+kubebuilder:rbac:groups="datasciencepipelinesapplications.opendatahub.io",resources=datasciencepipelinesapplications/status,verbs=update;patch;get //+kubebuilder:rbac:groups="datasciencepipelinesapplications.opendatahub.io",resources=datasciencepipelinesapplications/finalizers,verbs=update;patch @@ -62,7 +76,8 @@ package datasciencecluster // +kubebuilder:rbac:groups="security.openshift.io",resources=securitycontextconstraints,verbs=*,resourceNames=anyuid // +kubebuilder:rbac:groups="security.openshift.io",resources=securitycontextconstraints,verbs=* -// +kubebuilder:rbac:groups="route.openshift.io",resources=routes,verbs=get;list;watch;create;delete;update;patch +// +kubebuilder:rbac:groups="route.openshift.io",resources=routes,verbs=create;delete;list;update;watch;patch;get +// +kubebuilder:rbac:groups="route.openshift.io",resources=routes/custom-host,verbs=create;get // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=* @@ -83,10 +98,6 @@ package datasciencecluster // +kubebuilder:rbac:groups="networking.k8s.io",resources=networkpolicies,verbs=get;create;list;watch;delete;update;patch // +kubebuilder:rbac:groups="networking.k8s.io",resources=ingresses,verbs=create;delete;list;update;watch;patch;get -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/status,verbs=update;patch;delete -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices/finalizers,verbs=get;list;watch;create;update;patch;delete -// +kubebuilder:rbac:groups="networking.istio.io",resources=virtualservices,verbs=* - // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=servicemonitors,verbs=get;create;delete;update;watch;list;patch // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=podmonitors,verbs=get;create;delete;update;watch;list;patch // +kubebuilder:rbac:groups="monitoring.coreos.com",resources=prometheusrules,verbs=get;create;patch;delete @@ -211,11 +222,6 @@ package datasciencecluster // +kubebuilder:rbac:groups="*",resources=customresourcedefinitions,verbs=get;list;watch -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshcontrolplanes,verbs=create;get;list;patch;update;use;watch -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmemberrolls,verbs=create;get;list;patch;update;use;watch -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers,verbs=create;get;list;patch;update;use;watch -// +kubebuilder:rbac:groups="maistra.io",resources=servicemeshmembers/finalizers,verbs=create;get;list;patch;update;use;watch - /* Only for RHODS */ // +kubebuilder:rbac:groups="user.openshift.io",resources=groups,verbs=get;create;list;watch;patch;delete // +kubebuilder:rbac:groups="console.openshift.io",resources=consolelinks,verbs=create;get;patch;delete diff --git a/controllers/dscinitialization/dscinitialization_controller.go b/controllers/dscinitialization/dscinitialization_controller.go index 1fbd44411c1..3b9715a8042 100644 --- a/controllers/dscinitialization/dscinitialization_controller.go +++ b/controllers/dscinitialization/dscinitialization_controller.go @@ -21,6 +21,7 @@ import ( "context" "errors" "fmt" + "path" "path/filepath" "github.com/go-logr/logr" @@ -29,6 +30,8 @@ import ( dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" "github.com/opendatahub-io/opendatahub-operator/v2/controllers/status" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/upgrade" appsv1 "k8s.io/api/apps/v1" @@ -61,6 +64,7 @@ type DSCInitializationReconciler struct { // +kubebuilder:rbac:groups="dscinitialization.opendatahub.io",resources=dscinitializations/status,verbs=get;update;patch;delete // +kubebuilder:rbac:groups="dscinitialization.opendatahub.io",resources=dscinitializations/finalizers,verbs=get;update;patch;delete // +kubebuilder:rbac:groups="dscinitialization.opendatahub.io",resources=dscinitializations,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="dscinitialization.opendatahub.io",resources=servicemeshresourcetrackers,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups="kfdef.apps.kubeflow.org",resources=kfdefs,verbs=get;list;watch;create;update;patch;delete // Reconcile contains controller logic specific to DSCInitialization instance updates. @@ -96,7 +100,9 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re } } else { r.Log.Info("Finalization DSCInitialization start deleting instance", "name", instance.Name, "namespace", instance.Namespace, "finalizer", finalizerName) - // Add cleanup logic here + if err := r.cleanupServiceMesh(instance); err != nil { + return ctrl.Result{}, err + } if controllerutil.ContainsFinalizer(instance, finalizerName) { controllerutil.RemoveFinalizer(instance, finalizerName) if err := r.Update(ctx, instance); err != nil { @@ -126,8 +132,7 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re // Check namespace namespace := instance.Spec.ApplicationsNamespace - err = r.createOdhNamespace(ctx, instance, namespace) - if err != nil { + if err := r.createOdhNamespace(ctx, instance, namespace); err != nil { // no need to log error as it was already logged in createOdhNamespace return reconcile.Result{}, err } @@ -146,23 +151,203 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re return reconcile.Result{}, err } + if errRHODS := r.applyRHODSConfig(ctx, instance, platform); errRHODS != nil { + return reconcile.Result{}, errRHODS + } + + if monitoringErr := r.handleMonitoring(ctx, instance, platform); monitoringErr != nil { + return reconcile.Result{}, monitoringErr + } + + if errServiceMesh := r.handleServiceMesh(instance); errServiceMesh != nil { + return reconcile.Result{}, errServiceMesh + } + + // Finish reconciling + _, err = r.updateStatus(ctx, instance, func(saved *dsci.DSCInitialization) { + status.SetCompleteCondition(&saved.Status.Conditions, status.ReconcileCompleted, status.ReconcileCompletedMessage) + saved.Status.Phase = status.PhaseReady + }) + if err != nil { + r.Log.Error(err, "failed to update DSCInitialization status after successfully completed reconciliation") + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "Failed to update DSCInitialization status") + } + return ctrl.Result{}, nil +} + +func (r *DSCInitializationReconciler) handleServiceMesh(instance *dsci.DSCInitialization) error { + shouldConfigureServiceMesh, err := deploy.ShouldConfigureServiceMesh(r.Client, &instance.Spec) + if err != nil { + return err + } + + if shouldConfigureServiceMesh { + serviceMeshInitializer := servicemesh.NewServiceMeshInitializer(&instance.Spec, configureServiceMeshFeatures) + + if err := serviceMeshInitializer.Prepare(); err != nil { + r.Log.Error(err, "failed configuring service mesh resources") + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "failed configuring service mesh resources") + + return err + } + + if err := serviceMeshInitializer.Apply(); err != nil { + r.Log.Error(err, "failed applying service mesh resources") + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "failed applying service mesh resources") + + return err + } + } + + return nil +} + +func (r *DSCInitializationReconciler) cleanupServiceMesh(instance *dsci.DSCInitialization) error { + shouldConfigureServiceMesh, err := deploy.ShouldConfigureServiceMesh(r.Client, &instance.Spec) + if err != nil { + return err + } + + if shouldConfigureServiceMesh { + serviceMeshInitializer := servicemesh.NewServiceMeshInitializer(&instance.Spec, configureServiceMeshFeatures) + if err := serviceMeshInitializer.Prepare(); err != nil { + return err + } + if err := serviceMeshInitializer.Delete(); err != nil { + return err + } + } + + return nil +} + +func configureServiceMeshFeatures(s *servicemesh.ServiceMeshInitializer) error { + var rootDir = filepath.Join(feature.BaseOutputDir, s.DSCInitializationSpec.ApplicationsNamespace) + if err := feature.CopyEmbeddedFiles("templates", rootDir); err != nil { + return err + } + + serviceMeshSpec := s.ServiceMesh + + if oauth, err := feature.CreateFeature("control-plane-configure-oauth"). + For(s.DSCInitializationSpec). + Manifests( + path.Join(rootDir, feature.ControlPlaneDir, "base"), + path.Join(rootDir, feature.ControlPlaneDir, "oauth"), + path.Join(rootDir, feature.ControlPlaneDir, "filters"), + ). + WithResources( + feature.SelfSignedCertificate, + servicemesh.EnvoyOAuthSecrets, + ). + WithData(servicemesh.ClusterDetails, servicemesh.OAuthConfig). + PreConditions( + servicemesh.EnsureServiceMeshInstalled, + ). + PostConditions( + feature.WaitForPodsToBeReady(serviceMeshSpec.Mesh.Namespace), + ). + OnDelete( + servicemesh.RemoveOAuthClient, + servicemesh.RemoveTokenVolumes, + ).Load(); err != nil { + return err + } else { + s.Features = append(s.Features, oauth) + } + + if cfMaps, err := feature.CreateFeature("shared-config-maps"). + For(s.DSCInitializationSpec). + WithResources(servicemesh.ConfigMaps). + Load(); err != nil { + return err + } else { + s.Features = append(s.Features, cfMaps) + } + + if serviceMesh, err := feature.CreateFeature("app-add-namespace-to-service-mesh"). + For(s.DSCInitializationSpec). + Manifests( + path.Join(rootDir, feature.ControlPlaneDir, "smm.tmpl"), + path.Join(rootDir, feature.ControlPlaneDir, "namespace.patch.tmpl"), + ). + WithData(servicemesh.ClusterDetails). + Load(); err != nil { + return err + } else { + s.Features = append(s.Features, serviceMesh) + } + + if gatewayRoute, err := feature.CreateFeature("create-gateway-route"). + For(s.DSCInitializationSpec). + Manifests( + path.Join(rootDir, feature.ControlPlaneDir, "routing"), + ). + WithData(servicemesh.ClusterDetails). + PostConditions( + feature.WaitForPodsToBeReady(serviceMeshSpec.Mesh.Namespace), + ). + Load(); err != nil { + return err + } else { + s.Features = append(s.Features, gatewayRoute) + } + + if dataScienceProjects, err := feature.CreateFeature("app-migrate-data-science-projects"). + For(s.DSCInitializationSpec). + WithResources(servicemesh.MigratedDataScienceProjects). + Load(); err != nil { + return err + } else { + s.Features = append(s.Features, dataScienceProjects) + } + + if extAuthz, err := feature.CreateFeature("control-plane-setup-external-authorization"). + For(s.DSCInitializationSpec). + Manifests( + path.Join(rootDir, feature.AuthDir, "auth-smm.tmpl"), + path.Join(rootDir, feature.AuthDir, "base"), + path.Join(rootDir, feature.AuthDir, "rbac"), + path.Join(rootDir, feature.AuthDir, "mesh-authz-ext-provider.patch.tmpl"), + ). + WithData(servicemesh.ClusterDetails). + PreConditions( + feature.CreateNamespace(serviceMeshSpec.Auth.Namespace), + feature.EnsureCRDIsInstalled("authconfigs.authorino.kuadrant.io"), + servicemesh.EnsureServiceMeshInstalled, + ). + PostConditions( + feature.WaitForPodsToBeReady(serviceMeshSpec.Mesh.Namespace), + feature.WaitForPodsToBeReady(serviceMeshSpec.Auth.Namespace), + ). + OnDelete(servicemesh.RemoveExtensionProvider). + Load(); err != nil { + return err + } else { + s.Features = append(s.Features, extAuthz) + } + + return nil +} + +func (r *DSCInitializationReconciler) applyRHODSConfig(ctx context.Context, instance *dsci.DSCInitialization, platform deploy.Platform) error { // Apply Rhods specific configs if platform == deploy.ManagedRhods || platform == deploy.SelfManagedRhods { // Apply osd specific permissions if platform == deploy.ManagedRhods { osdConfigsPath := filepath.Join(deploy.DefaultManifestPath, "osd-configs") - err = deploy.DeployManifestsFromPath(r.Client, instance, osdConfigsPath, r.ApplicationsNamespace, "osd", true) + err := deploy.DeployManifestsFromPath(r.Client, instance, osdConfigsPath, r.ApplicationsNamespace, "osd", true) if err != nil { r.Log.Error(err, "Failed to apply osd specific configs from manifests", "Manifests path", osdConfigsPath) r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "Failed to apply "+osdConfigsPath) - return reconcile.Result{}, err + return err } } else { // Apply self-managed rhods config // Create rhods-admins Group if it doesn't exist err := r.createUserGroup(ctx, instance, "rhods-admins") if err != nil { - return reconcile.Result{}, err + return err } } // Apply common rhods-specific config @@ -170,46 +355,39 @@ func (r *DSCInitializationReconciler) Reconcile(ctx context.Context, req ctrl.Re // Create odh-admins Group if it doesn't exist err := r.createUserGroup(ctx, instance, "odh-admins") if err != nil { - return reconcile.Result{}, err + return err } } - // If monitoring enabled + return nil +} + +func (r *DSCInitializationReconciler) handleMonitoring(ctx context.Context, instance *dsci.DSCInitialization, platform deploy.Platform) error { if instance.Spec.Monitoring.ManagementState == operatorv1.Managed { switch platform { case deploy.SelfManagedRhods: r.Log.Info("Monitoring enabled, won't apply changes", "cluster", "Self-Managed RHODS Mode") err := r.configureCommonMonitoring(instance) if err != nil { - return reconcile.Result{}, err + return err } case deploy.ManagedRhods: r.Log.Info("Monitoring enabled", "cluster", "Managed Service Mode") err := r.configureManagedMonitoring(ctx, instance) if err != nil { // no need to log error as it was already logged in configureManagedMonitoring - return reconcile.Result{}, err + return err } err = r.configureCommonMonitoring(instance) if err != nil { - return reconcile.Result{}, err + return err } default: // TODO: ODH specific monitoring logic r.Log.Info("Monitoring enabled, won't apply changes", "cluster", "ODH Mode") } } - - // Finish reconciling - _, err = r.updateStatus(ctx, instance, func(saved *dsci.DSCInitialization) { - status.SetCompleteCondition(&saved.Status.Conditions, status.ReconcileCompleted, status.ReconcileCompletedMessage) - saved.Status.Phase = status.PhaseReady - }) - if err != nil { - r.Log.Error(err, "failed to update DSCInitialization status after successfully completed reconciliation") - r.Recorder.Eventf(instance, corev1.EventTypeWarning, "DSCInitializationReconcileError", "Failed to update DSCInitialization status") - } - return ctrl.Result{}, nil + return nil } // SetupWithManager sets up the controller with the Manager. diff --git a/controllers/dscinitialization/dscinitialization_test.go b/controllers/dscinitialization/dscinitialization_test.go index 5ac9e525d50..2f0e0f15a9c 100644 --- a/controllers/dscinitialization/dscinitialization_test.go +++ b/controllers/dscinitialization/dscinitialization_test.go @@ -2,10 +2,10 @@ package dscinitialization_test import ( "context" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/tests/envtestutil" operatorv1 "github.com/openshift/api/operator/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" @@ -24,9 +24,10 @@ const ( var _ = Describe("DataScienceCluster initialization", func() { Context("Creation of related resources", func() { - applicationName := "default-test" + var applicationName string BeforeEach(func() { // when + applicationName = envtestutil.AppendRandomNameTo("default-test") desiredDsci := createDSCI(applicationName) Expect(k8sClient.Create(context.Background(), desiredDsci)).Should(Succeed()) foundDsci := &dsci.DSCInitialization{} @@ -95,7 +96,7 @@ var _ = Describe("DataScienceCluster initialization", func() { AfterEach(cleanupResources) It("Should not update rolebinding if it exists", func() { - applicationName := "rolebinding-test" + applicationName := envtestutil.AppendRandomNameTo("rolebinding-test") // given desiredRoleBinding := &authv1.RoleBinding{ @@ -132,7 +133,7 @@ var _ = Describe("DataScienceCluster initialization", func() { }) It("Should not update configmap if it exists", func() { - applicationName := "configmap-test" + applicationName := envtestutil.AppendRandomNameTo("configmap-test") // given desiredConfigMap := &corev1.ConfigMap{ @@ -165,7 +166,7 @@ var _ = Describe("DataScienceCluster initialization", func() { }) It("Should not update namespace if it exists", func() { - applicationName := "configmap-test" + applicationName := envtestutil.AppendRandomNameTo("configmap-test") anotherNamespace := "test-another-ns" // given @@ -197,10 +198,10 @@ var _ = Describe("DataScienceCluster initialization", func() { func cleanupResources() { defaultNamespace := client.InNamespace(workingNamespace) appNamespace := client.InNamespace(applicationNamespace) - Expect(k8sClient.DeleteAllOf(context.TODO(), &dsci.DSCInitialization{}, defaultNamespace)).ToNot(HaveOccurred()) - Expect(k8sClient.DeleteAllOf(context.TODO(), &netv1.NetworkPolicy{}, appNamespace)).ToNot(HaveOccurred()) - Expect(k8sClient.DeleteAllOf(context.TODO(), &corev1.ConfigMap{}, appNamespace)).ToNot(HaveOccurred()) - Expect(k8sClient.DeleteAllOf(context.TODO(), &authv1.RoleBinding{}, appNamespace)).ToNot(HaveOccurred()) + Expect(k8sClient.DeleteAllOf(context.TODO(), &dsci.DSCInitialization{}, defaultNamespace)).To(Succeed()) + Expect(k8sClient.DeleteAllOf(context.TODO(), &netv1.NetworkPolicy{}, appNamespace)).To(Succeed()) + Expect(k8sClient.DeleteAllOf(context.TODO(), &corev1.ConfigMap{}, appNamespace)).To(Succeed()) + Expect(k8sClient.DeleteAllOf(context.TODO(), &authv1.RoleBinding{}, appNamespace)).To(Succeed()) } func createDSCI(appName string) *dsci.DSCInitialization { diff --git a/controllers/dscinitialization/suite_test.go b/controllers/dscinitialization/suite_test.go index 4c15492a4f4..a94a7dc3291 100644 --- a/controllers/dscinitialization/suite_test.go +++ b/controllers/dscinitialization/suite_test.go @@ -18,8 +18,7 @@ package dscinitialization_test import ( "context" - kfdefv1 "github.com/opendatahub-io/opendatahub-operator/apis/kfdef.apps.kubeflow.org/v1" - "k8s.io/apimachinery/pkg/runtime" + "path/filepath" "testing" "time" @@ -27,18 +26,21 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - util "github.com/opendatahub-io/opendatahub-operator/v2/controllers/test" - + kfdefv1 "github.com/opendatahub-io/opendatahub-operator/apis/kfdef.apps.kubeflow.org/v1" dscinitializationv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" dsci "github.com/opendatahub-io/opendatahub-operator/v2/controllers/dscinitialization" + "github.com/opendatahub-io/opendatahub-operator/v2/tests/envtestutil" + routev1 "github.com/openshift/api/route/v1" userv1 "github.com/openshift/api/user/v1" ofapi "github.com/operator-framework/api/pkg/operators/v1alpha1" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" authv1 "k8s.io/api/rbac/v1" apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" @@ -78,7 +80,7 @@ var _ = BeforeSuite(func() { ctx, cancel = context.WithCancel(context.TODO()) logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseDevMode(true))) By("bootstrapping test environment") - rootPath, pathErr := util.FindProjectRoot() + rootPath, pathErr := envtestutil.FindProjectRoot() Expect(pathErr).ToNot(HaveOccurred(), pathErr) testEnv = &envtest.Environment{ diff --git a/controllers/secretgenerator/secret.go b/controllers/secretgenerator/secret.go index 521fb7fef28..fe91f42ea37 100644 --- a/controllers/secretgenerator/secret.go +++ b/controllers/secretgenerator/secret.go @@ -31,7 +31,7 @@ type Secret struct { OAuthClientRoute string } -func newSecret(annotations map[string]string) (*Secret, error) { +func NewSecretFrom(annotations map[string]string) (*Secret, error) { // Check if annotations is not empty if len(annotations) == 0 { return nil, errors.New(errEmptyAnnotation) @@ -64,14 +64,37 @@ func newSecret(annotations map[string]string) (*Secret, error) { secret.Complexity = SECRET_DEFAULT_COMPLEXITY } - // Generate a random value based on the secret type + if secretOAuthClientRoute, found := annotations[SECRET_OAUTH_CLIENT_ANNOTATION]; found { + secret.OAuthClientRoute = secretOAuthClientRoute + } + + if err := generateSecretValue(&secret); err != nil { + return nil, err + } + + return &secret, nil +} + +func NewSecret(name, secretType string, complexity int) (*Secret, error) { + secret := &Secret{ + Name: name, + Type: secretType, + Complexity: complexity, + } + + err := generateSecretValue(secret) + + return secret, err +} + +func generateSecretValue(secret *Secret) error { switch secret.Type { case "random": randomValue := make([]byte, secret.Complexity) for i := 0; i < secret.Complexity; i++ { num, err := rand.Int(rand.Reader, big.NewInt(int64(len(letterRunes)))) if err != nil { - return nil, err + return err } randomValue[i] = letterRunes[num.Int64()] } @@ -79,16 +102,13 @@ func newSecret(annotations map[string]string) (*Secret, error) { case "oauth": randomValue := make([]byte, secret.Complexity) if _, err := rand.Read(randomValue); err != nil { - return nil, err + return err } secret.Value = base64.StdEncoding.EncodeToString( []byte(base64.StdEncoding.EncodeToString(randomValue))) default: - return nil, errors.New(errUnsupportedType) + return errors.New(errUnsupportedType) } - // Get OAuthClient route name from annotation - if secretOAuthClientRoute, found := annotations[SECRET_OAUTH_CLIENT_ANNOTATION]; found { - secret.OAuthClientRoute = secretOAuthClientRoute - } - return &secret, nil + + return nil } diff --git a/controllers/secretgenerator/secret_test.go b/controllers/secretgenerator/secret_test.go index b3c90ee9c81..240986bb974 100644 --- a/controllers/secretgenerator/secret_test.go +++ b/controllers/secretgenerator/secret_test.go @@ -84,7 +84,7 @@ func TestNewSecret(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { - secret, err := newSecret(tc.annotations) + secret, err := NewSecretFrom(tc.annotations) if err != nil { if err.Error() != tc.err.Error() { t.Errorf("Expected error: %v, got: %v\n", diff --git a/controllers/secretgenerator/secretgenerator_controller.go b/controllers/secretgenerator/secretgenerator_controller.go index b650011e0d7..2561e6caeb3 100644 --- a/controllers/secretgenerator/secretgenerator_controller.go +++ b/controllers/secretgenerator/secretgenerator_controller.go @@ -25,7 +25,7 @@ import ( ocv1 "github.com/openshift/api/oauth/v1" routev1 "github.com/openshift/api/route/v1" v1 "k8s.io/api/core/v1" - k8serrors "k8s.io/apimachinery/pkg/api/errors" + apierrs "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -104,7 +104,7 @@ func (r *SecretGeneratorReconciler) Reconcile(ctx context.Context, request ctrl. foundSecret := &v1.Secret{} err := r.Client.Get(ctx, request.NamespacedName, foundSecret) if err != nil { - if k8serrors.IsNotFound(err) { + if apierrs.IsNotFound(err) { // If Secret is deleted, delete OAuthClient if exists err = r.deleteOAuthClient(ctx, request.Name) } @@ -129,12 +129,12 @@ func (r *SecretGeneratorReconciler) Reconcile(ctx context.Context, request ctrl. } err = r.Client.Get(ctx, generatedSecretKey, generatedSecret) if err != nil { - if k8serrors.IsNotFound(err) { + if apierrs.IsNotFound(err) { // Generate secret random value secGenLog.Info("Generating a random value for a secret in a namespace", "secret", generatedSecret.Name, "namespace", generatedSecret.Namespace) - secret, err := newSecret(foundSecret.GetAnnotations()) + secret, err := NewSecretFrom(foundSecret.GetAnnotations()) if err != nil { secGenLog.Error(err, "error creating secret") return ctrl.Result{}, err @@ -183,7 +183,7 @@ func (r *SecretGeneratorReconciler) getRoute(ctx context.Context, name string, n Namespace: namespace, }, route) if err != nil { - if k8serrors.IsNotFound(err) { + if apierrs.IsNotFound(err) { return false, nil } return false, err @@ -217,7 +217,7 @@ func (r *SecretGeneratorReconciler) createOAuthClient(ctx context.Context, name err := r.Client.Create(ctx, oauthClient) if err != nil { - if k8serrors.IsAlreadyExists(err) { + if apierrs.IsAlreadyExists(err) { secGenLog.Info("OAuth client resource already exists", "name", oauthClient.Name) return nil } @@ -232,7 +232,7 @@ func (r *SecretGeneratorReconciler) deleteOAuthClient(ctx context.Context, secre Name: secretName, }, oauthClient) if err != nil { - if k8serrors.IsNotFound(err) { + if apierrs.IsNotFound(err) { return nil } return err diff --git a/docs/SERVICE-MESH.md b/docs/SERVICE-MESH.md new file mode 100644 index 00000000000..60824c33459 --- /dev/null +++ b/docs/SERVICE-MESH.md @@ -0,0 +1,242 @@ +# Open Data Hub (ODH) Installation Guide with OpenShift Service Mesh (OSSM) + +This guide will walk you through the installation of Open Data Hub with OpenShift Service Mesh. + +## Prerequisites + +* OpenShift cluster +* Command Line Interface (CLI) tools + * `kubectl` + * `operator-sdk` v1.24.1 (until operator changes are merged - to build new bundles) + +* Pre-installed operators + * Openshift Service Mesh + * Authorino + * Open Data Hub + +* Service Mesh Control Plane configured + +### Check Installed Operators + +You can use the following command to verify that all required operators are installed: + +```sh +kubectl get operators | awk -v RS= '/servicemesh/ && /opendatahub/ && /authorino/ {exit 0} {exit 1}' || echo "Please install all required operators." +``` + +#### Install Required Operators + +The `createSubscription` function can be used to simplify the installation of required operators: + +```sh +createSubscription() { + local name=$1 + local source=${2:-"redhat-operators"} + local channel=${3:-"stable"} + + echo "Create Subscription resource for $name" + eval "kubectl apply -f - < **Warning** +> +> You may need to manually update the installation of the Authorino operator via the Installed Operators tab in the OpenShift Console. + + +> **Warning** +> +> Please ensure that the Service Mesh Control Plane is properly configured as we apply patches to it. It is assumed that the installation has already been done. + + +For example, the following commands configure a slimmed-down profile: + +```sh +kubectl create ns istio-system +kubectl apply -n istio-system -f -< dsci.ign.yaml +apiVersion: dscinitialization.opendatahub.io/v1 +kind: DSCInitialization +metadata: + name: default +spec: + applicationsNamespace: opendatahub + monitoring: + managementState: Managed + namespace: opendatahub + serviceMesh: + managementState: Managed # (1) + mesh: + name: minimal # (2) + certificate: + generate: true # (3) +EOF +``` + +* **(1)**: setting this value will enable Service Mesh support for Open Data Hub +* **(2)**: name of Service Mesh Control Plane (defaults to `basic`) +* **(3)**: instructs operator to generate self-signed certificate + +This will instruct the DSCI controller to perform following steps: + +- Deploys an instance of Authorino controller which will handle all `AuthConfig`s for ODH (using certain label) +- Registers Authorino as external authorization provider for Istio +- Configure Envoy filters to handle OAuth2 flow for Dashboard +- Create Gateway route for Dashboard +- Create necessary Istio resources such as Virtual Services, Gateways and Policies + +Next, create DataScienceCluster with managed `Dashboard` and `Workbenches` components: + +```sh +apiVersion: datasciencecluster.opendatahub.io/v1 +kind: DataScienceCluster +metadata: + name: default +spec: + components: + dashboard: + managementState: "Managed" + workbenches: + managementState: "Managed" +``` +` +> **Warning** +> +> Other components are not supported yet. + +Go to the Open Data Hub dashboard in the browser: + +```sh +export ODH_ROUTE=$(kubectl get route --all-namespaces -l maistra.io/gateway-name=odh-gateway -o yaml | yq '.items[].spec.host') + +xdg-open https://$ODH_ROUTE > /dev/null 2>&1 & +``` + +## Troubleshooting + +### Audience-aware tokens + +Audience-aware token authenticators will verify that the token was intended for at least one of the audiences in this list. This can be crucial for environments such as ROSA. If no audiences are provided, the audience will default to the audience of the Kubernetes apiserver (`kubernetes.default.svc`). In order to change it, you should first know the token audience of your `serviceaccount`. + +```shell +TOKEN=YOUR_USER_TOKEN +ODH_NS=opendatahub +kubectl create -o jsonpath='{.status.audiences[0]}' -f -< get_client=client check failed, client_id=${ODH_NS}-oauth2-client`)`, check if the token is the same everywhere by comparing the output of the following commands: + + +```sh +kubectl get oauthclient.oauth.openshift.io ${ODH_NS}-oauth2-client +kubectl exec $(kubectl get pods -n istio-system -l app=istio-ingressgateway -o jsonpath='{.items[*].metadata.name}') -n istio-system -c istio-proxy -- cat /etc/istio/${ODH_NS}-oauth2-tokens/token-secret.yaml +kubectl get secret ${ODH_NS}-oauth2-tokens -n istio-system -o yaml +``` +To read the actual value of secrets you could use a [`kubectl` plugin](https://github.com/elsesiy/kubectl-view-secret) instead. Then the last line would look as follows `kubectl view-secret ${ODH_NS}-oauth2-tokens -n istio-system -a`. + +The `i`stio-ingressgateway` pod might be out of sync (and so `EnvoyFilter` responsible for OAuth2 flow). Check its logs and consider restarting it: + +```sh +kubectl rollout restart deployment -n istio-system istio-ingressgateway +``` diff --git a/get_all_manifests.sh b/get_all_manifests.sh index fce24ad2aa7..8b60a8ed049 100755 --- a/get_all_manifests.sh +++ b/get_all_manifests.sh @@ -9,18 +9,19 @@ MANIFEST_RELEASE="master" MANIFESTS_TARBALL_URL="${GITHUB_URL}/${MANIFEST_ORG}/odh-manifests/tarball/${MANIFEST_RELEASE}" # component: dsp, kserve, dashbaord, cf/ray. in the format of "repo-org:repo-name:branch-name:source-folder:target-folder" -# TODO: workbench, modelmesh, monitoring, etc +# TODO: workbench, monitoring, etc declare -A COMPONENT_MANIFESTS=( ["codeflare"]="opendatahub-io:codeflare-operator:main:config:codeflare" ["ray"]="opendatahub-io:kuberay:master:ray-operator/config:ray" ["data-science-pipelines-operator"]="opendatahub-io:data-science-pipelines-operator:main:config:data-science-pipelines-operator" ["odh-dashboard"]="opendatahub-io:odh-dashboard:incubation:manifests:dashboard" - ["kf-notebook-controller"]="opendatahub-io:kubeflow:v1.7-branch:components/notebook-controller/config:odh-notebook-controller/kf-notebook-controller" - ["odh-notebook-controller"]="opendatahub-io:kubeflow:v1.7-branch:components/odh-notebook-controller/config:odh-notebook-controller/odh-notebook-controller" + ["kf-notebook-controller"]="cam-garrison:kubeflow:add-sm-overlay:components/notebook-controller/config:odh-notebook-controller/kf-notebook-controller" + ["odh-notebook-controller"]="cam-garrison:kubeflow:add-sm-overlay:components/odh-notebook-controller/config:odh-notebook-controller/odh-notebook-controller" ["notebooks"]="opendatahub-io:notebooks:main:manifests:notebooks" ["trustyai"]="trustyai-explainability:trustyai-service-operator:release/1.10.2:config:trustyai-service-operator" ["model-mesh"]="opendatahub-io:modelmesh-serving:release-0.11.0:config:model-mesh" ["odh-model-controller"]="opendatahub-io:odh-model-controller:release-0.11.0:config:odh-model-controller" + ["odh-project-controller"]="maistra:odh-project-controller:main:config:odh-project-controller" ) # Allow overwriting repo using flags component=repo diff --git a/go.mod b/go.mod index 02cf3354918..556cd9cab29 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,14 @@ module github.com/opendatahub-io/opendatahub-operator/v2 go 1.19 require ( + github.com/bitly/go-simplejson v0.5.1 + github.com/ghodss/yaml v1.0.0 github.com/go-logr/logr v1.2.4 github.com/hashicorp/go-multierror v1.1.1 - github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/ginkgo/v2 v2.12.1 github.com/onsi/gomega v1.27.10 github.com/opendatahub-io/opendatahub-operator v1.7.0 - github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd + github.com/openshift/addon-operator/apis v0.0.0-20230919043633-820afed15881 github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 github.com/openshift/custom-resource-status v1.1.2 github.com/operator-framework/api v0.17.6 @@ -34,7 +36,6 @@ require ( github.com/evanphx/json-patch v4.12.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect - github.com/ghodss/yaml v1.0.0 // indirect github.com/go-errors/errors v1.4.2 // indirect github.com/go-logr/zapr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect diff --git a/go.sum b/go.sum index 4b34c925a5e..09541934338 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow= +github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q= github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -684,8 +686,8 @@ github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47 github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.12.1 h1:uHNEO1RP2SpuZApSkel9nEh1/Mu+hmQe7Q+Pepg5OYA= +github.com/onsi/ginkgo/v2 v2.12.1/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -718,8 +720,8 @@ github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.m github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opendatahub-io/opendatahub-operator v1.7.0 h1:Pn76VWCUHeqD5E0L94t5FtZ2OU6ZaWk/IwbEqXGQ4Gs= github.com/opendatahub-io/opendatahub-operator v1.7.0/go.mod h1:XsgkXbGjJoVeZOkmRztemhY5ppX7puzuqDw9oUdzNZk= -github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd h1:6elrLdOa+BRHJVaHnZAHltufWk0pzPZYF67fX9aFCjU= -github.com/openshift/addon-operator/apis v0.0.0-20230616140313-b6e2f736fdcd/go.mod h1:cDMtOZx741HfmmUMmT09PWM8cOBxEJp3ipUHeHPr8F4= +github.com/openshift/addon-operator/apis v0.0.0-20230919043633-820afed15881 h1:d0hmj9Is2sCLkNYWtBicZV0Ft8+Os+w+RUkFRjie0VI= +github.com/openshift/addon-operator/apis v0.0.0-20230919043633-820afed15881/go.mod h1:2hsK4sYLKcjVJ8SziFrzr/c/Tmp5zBDy8aYvrFaRm2o= github.com/openshift/api v0.0.0-20200326152221-912866ddb162/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE= github.com/openshift/api v0.0.0-20200331152225-585af27e34fd/go.mod h1:RKMJ5CBnljLfnej+BJ/xnOWc3kZDvJUaIAEq2oKSPtE= github.com/openshift/api v0.0.0-20230823114715-5fdd7511b790 h1:e3zIxk67/kiABxGFfFVECqJ4FcQRG5DPF8lgDV9f+MM= diff --git a/pkg/cluster/cluster_config.go b/pkg/cluster/cluster_config.go new file mode 100644 index 00000000000..7b2f83cce7d --- /dev/null +++ b/pkg/cluster/cluster_config.go @@ -0,0 +1,145 @@ +package cluster + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "github.com/bitly/go-simplejson" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" + "io" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/client-go/dynamic" + "net/http" + "net/url" + "os" + "strconv" + "strings" +) + +// +kubebuilder:rbac:groups="config.openshift.io",resources=ingresses,verbs=get + +func GetDomain(dynamicClient dynamic.Interface) (string, error) { + cluster, err := dynamicClient.Resource(gvr.OpenshiftIngress).Get(context.TODO(), "cluster", metav1.GetOptions{}) + if err != nil { + panic(err.Error()) + } + + domain, found, err := unstructured.NestedString(cluster.Object, "spec", "domain") + if !found { + return "", errors.New("spec.domain not found") + } + return domain, err +} + +func GetOAuthServerDetails() (*simplejson.Json, error) { + response, err := request(http.MethodGet, "/.well-known/oauth-authorization-server") + if err != nil { + return nil, err + } + + return simplejson.NewJson(response) +} + +const saCert = "/run/secrets/kubernetes.io/serviceaccount/ca.crt" + +func request(method string, url string) ([]byte, error) { + certPool, err := createCertPool() + if err != nil { + return nil, err + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certPool, + }, + }} + + request, err := http.NewRequest(method, getKubeAPIURLWithPath(url).String(), nil) + if err != nil { + return nil, fmt.Errorf("failed to get api endpoint %s, error: %s", url, err) + } + + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("failed to get api endpoint %s, error: %s", url, err) + } + + defer response.Body.Close() + + body, err := io.ReadAll(response.Body) + if err != nil || response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to get api endpoint %s, error: %s", url, err) + } + + return body, nil +} + +func createCertPool() (*x509.CertPool, error) { + certPool := x509.NewCertPool() + cert, err := os.ReadFile(saCert) + + if err != nil { + return nil, fmt.Errorf("failed to get root CA certificates: %s", err) + } + + certPool.AppendCertsFromPEM(cert) + return certPool, err +} + +func getKubernetesServiceHost() string { + if host := os.Getenv("KUBERNETES_SERVICE_HOST"); len(host) > 0 { + // assume IPv6 if host contains colons + if strings.IndexByte(host, ':') != -1 { + host = "[" + host + "]" + } + + return host + } + + return "kubernetes.default.svc" +} + +func getKubeAPIURLWithPath(path string) *url.URL { + return &url.URL{ + Scheme: "https", + Host: getKubernetesServiceHost(), + Path: path, + } +} + +// ExtractHostNameAndPort strips given URL in string from http(s):// prefix and subsequent path, +// returning host name and port if defined (otherwise defaults to 443). +// +// This is useful when getting value from http headers (such as origin). +// If given string does not start with http(s) prefix it will be returned as is. +func ExtractHostNameAndPort(s string) (string, string, error) { + u, err := url.Parse(s) + if err != nil { + return "", "", err + } + + if u.Scheme != "http" && u.Scheme != "https" { + return s, "", nil + } + + hostname := u.Hostname() + + port := "443" // default for https + if u.Scheme == "http" { + port = "80" + } + + if u.Port() != "" { + port = u.Port() + _, err := strconv.Atoi(port) + if err != nil { + return "", "", errors.New("invalid port number: " + port) + } + } + + return hostname, port, nil +} diff --git a/pkg/cluster/cluster_suite_test.go b/pkg/cluster/cluster_suite_test.go new file mode 100644 index 00000000000..d666ea1ad5d --- /dev/null +++ b/pkg/cluster/cluster_suite_test.go @@ -0,0 +1,14 @@ +package cluster_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestClusterHelpers(t *testing.T) { + RegisterFailHandler(Fail) + // for integration tests see tests/integration directory + RunSpecs(t, "Cluster helper funcs unit tests") +} diff --git a/pkg/cluster/hostname_and_port_extraction_unit_test.go b/pkg/cluster/hostname_and_port_extraction_unit_test.go new file mode 100644 index 00000000000..78e10133e9c --- /dev/null +++ b/pkg/cluster/hostname_and_port_extraction_unit_test.go @@ -0,0 +1,46 @@ +package cluster_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" +) + +var _ = Describe("extracting hostname and port from URL", func() { + + It("should extract hostname and port for HTTP URL", func() { + hostname, port, err := cluster.ExtractHostNameAndPort("http://opendatahub.io:8080/path") + Expect(err).ToNot(HaveOccurred()) + Expect(hostname).To(Equal("opendatahub.io")) + Expect(port).To(Equal("8080")) + }) + + It("should return original URL if it does not start with http(s) but with other valid protocol", func() { + originalURL := "gopher://opendatahub.io" + hostname, port, err := cluster.ExtractHostNameAndPort(originalURL) + Expect(err).ToNot(HaveOccurred()) + Expect(hostname).To(Equal(originalURL)) + Expect(port).To(Equal("")) + }) + + It("should handle invalid URLs by returning corresponding error", func() { + invalidURL := ":opendatahub.io" + _, _, err := cluster.ExtractHostNameAndPort(invalidURL) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("missing protocol scheme"))) + }) + + It("should handle URLs without port and default to 443 for HTTPS", func() { + hostname, port, err := cluster.ExtractHostNameAndPort("https://opendatahub.io") + Expect(err).ToNot(HaveOccurred()) + Expect(hostname).To(Equal("opendatahub.io")) + Expect(port).To(Equal("443")) + }) + + It("should handle URLs without port and default to 80 for HTTP", func() { + hostname, port, err := cluster.ExtractHostNameAndPort("http://opendatahub.io") + Expect(err).ToNot(HaveOccurred()) + Expect(hostname).To(Equal("opendatahub.io")) + Expect(port).To(Equal("80")) + }) +}) diff --git a/pkg/cluster/operations.go b/pkg/cluster/operations.go new file mode 100644 index 00000000000..b00f2069b47 --- /dev/null +++ b/pkg/cluster/operations.go @@ -0,0 +1,100 @@ +package cluster + +import ( + "context" + corev1 "k8s.io/api/core/v1" + authv1 "k8s.io/api/rbac/v1" + apierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + // odhGeneratedNamespaceLabel is the label added to all the namespaces genereated by odh-deployer + odhGeneratedNamespaceLabel = "opendatahub.io/generated-namespace" +) + +// UpdatePodSecurityRolebinding update default rolebinding which is created in applications namespace by manifests +// being used by different components. +func UpdatePodSecurityRolebinding(cli client.Client, namespace string, serviceAccountsList ...string) error { + foundRoleBinding := &authv1.RoleBinding{} + err := cli.Get(context.TODO(), client.ObjectKey{Name: namespace, Namespace: namespace}, foundRoleBinding) + if err != nil { + return err + } + + for _, sa := range serviceAccountsList { + // Append serviceAccount if not added already + if !subjectExistInRoleBinding(foundRoleBinding.Subjects, sa, namespace) { + foundRoleBinding.Subjects = append(foundRoleBinding.Subjects, authv1.Subject{ + Kind: authv1.ServiceAccountKind, + Name: sa, + Namespace: namespace, + }) + } + } + + return cli.Update(context.TODO(), foundRoleBinding) +} + +// Internal function used by UpdatePodSecurityRolebinding() +// Return whether Rolebinding matching service account and namespace exists or not. +func subjectExistInRoleBinding(subjectList []authv1.Subject, serviceAccountName, namespace string) bool { + for _, subject := range subjectList { + if subject.Name == serviceAccountName && subject.Namespace == namespace { + return true + } + } + return false +} + +// CreateSecret creates secrets required by dashboard component in downstream. +func CreateSecret(cli client.Client, name, namespace string) error { + desiredSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Type: corev1.SecretTypeOpaque, + } + + foundSecret := &corev1.Secret{} + err := cli.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: namespace}, foundSecret) + if err != nil { + if apierrs.IsNotFound(err) { + err = cli.Create(context.TODO(), desiredSecret) + if err != nil && !apierrs.IsAlreadyExists(err) { + return err + } + } else { + return err + } + } + return nil +} + +// CreateNamespace creates namespace required by workbenches component in downstream. +func CreateNamespace(cli client.Client, namespace string) error { + desiredNamespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + Labels: map[string]string{ + odhGeneratedNamespaceLabel: "true", + }, + }, + } + + foundNamespace := &corev1.Namespace{} + err := cli.Get(context.TODO(), client.ObjectKey{Name: namespace}, foundNamespace) + if err != nil { + if apierrs.IsNotFound(err) { + err = cli.Create(context.TODO(), desiredNamespace) + if err != nil && !apierrs.IsAlreadyExists(err) { + return err + } + } else { + return err + } + } + return nil +} diff --git a/pkg/common/common.go b/pkg/common/common.go index 600bc56f3b6..80cfa9b717e 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -18,56 +18,11 @@ limitations under the License. package common import ( - "context" "fmt" - corev1 "k8s.io/api/core/v1" - authv1 "k8s.io/api/rbac/v1" - apierrs "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "os" - "sigs.k8s.io/controller-runtime/pkg/client" "strings" ) -const ( - // odhGeneratedNamespaceLabel is the label added to all the namespaces genereated by odh-deployer - odhGeneratedNamespaceLabel = "opendatahub.io/generated-namespace" -) - -// UpdatePodSecurityRolebinding update default rolebinding which is created in applications namespace by manifests -// being used by different components. -func UpdatePodSecurityRolebinding(cli client.Client, serviceAccountsList []string, namespace string) error { - foundRoleBinding := &authv1.RoleBinding{} - err := cli.Get(context.TODO(), client.ObjectKey{Name: namespace, Namespace: namespace}, foundRoleBinding) - if err != nil { - return err - } - - for _, sa := range serviceAccountsList { - // Append serviceAccount if not added already - if !subjectExistInRoleBinding(foundRoleBinding.Subjects, sa, namespace) { - foundRoleBinding.Subjects = append(foundRoleBinding.Subjects, authv1.Subject{ - Kind: authv1.ServiceAccountKind, - Name: sa, - Namespace: namespace, - }) - } - } - - return cli.Update(context.TODO(), foundRoleBinding) -} - -// Internal function used by UpdatePodSecurityRolebinding() -// Return whether Rolebinding matching service account and namespace exists or not. -func subjectExistInRoleBinding(subjectList []authv1.Subject, serviceAccountName, namespace string) bool { - for _, subject := range subjectList { - if subject.Name == serviceAccountName && subject.Namespace == namespace { - return true - } - } - return false -} - // ReplaceStringsInFile replaces variable with value in manifests during runtime. func ReplaceStringsInFile(fileName string, replacements map[string]string) error { // Read the contents of the file @@ -90,54 +45,3 @@ func ReplaceStringsInFile(fileName string, replacements map[string]string) error return nil } - -// CreateSecret creates secrets required by dashboard component in downstream. -func CreateSecret(cli client.Client, name, namespace string) error { - desiredSecret := &corev1.Secret{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Type: corev1.SecretTypeOpaque, - } - - foundSecret := &corev1.Secret{} - err := cli.Get(context.TODO(), client.ObjectKey{Name: name, Namespace: namespace}, foundSecret) - if err != nil { - if apierrs.IsNotFound(err) { - err = cli.Create(context.TODO(), desiredSecret) - if err != nil && !apierrs.IsAlreadyExists(err) { - return err - } - } else { - return err - } - } - return nil -} - -// CreateNamespace creates namespace required by workbenches component in downstream. -func CreateNamespace(cli client.Client, namespace string) error { - desiredNamespace := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespace, - Labels: map[string]string{ - odhGeneratedNamespaceLabel: "true", - }, - }, - } - - foundNamespace := &corev1.Namespace{} - err := cli.Get(context.TODO(), client.ObjectKey{Name: namespace}, foundNamespace) - if err != nil { - if apierrs.IsNotFound(err) { - err = cli.Create(context.TODO(), desiredNamespace) - if err != nil && !apierrs.IsAlreadyExists(err) { - return err - } - } else { - return err - } - } - return nil -} diff --git a/pkg/deploy/setup.go b/pkg/deploy/setup.go index 131a6d29527..f9eca6cf162 100644 --- a/pkg/deploy/setup.go +++ b/pkg/deploy/setup.go @@ -2,6 +2,8 @@ package deploy import ( "context" + dsci "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + operatorv1 "github.com/openshift/api/operator/v1" "strings" ofapi "github.com/operator-framework/api/pkg/operators/v1alpha1" @@ -17,6 +19,8 @@ const ( SelfManagedRhods Platform = "Red Hat OpenShift Data Science" // OpenDataHub defines display name in csv. OpenDataHub Platform = "Open Data Hub Operator" + // Unknown indicates that operator is not deployed using OLM + Unknown Platform = "" ) type Platform string @@ -41,7 +45,7 @@ func isSelfManaged(cli client.Client) (Platform, error) { } } } - return "", nil + return Unknown, nil } // isManagedRHODS checks if CRD add-on exists and contains string ManagedRhods. @@ -59,9 +63,9 @@ func isManagedRHODS(cli client.Client) (Platform, error) { err := cli.List(context.TODO(), expectedCatlogSource) if err != nil { if apierrs.IsNotFound(err) { - return "", nil + return Unknown, nil } else { - return "", err + return Unknown, err } } if len(expectedCatlogSource.Items) > 0 { @@ -78,7 +82,7 @@ func isManagedRHODS(cli client.Client) (Platform, error) { func GetPlatform(cli client.Client) (Platform, error) { // First check if its addon installation to return 'ManagedRhods, nil' if platform, err := isManagedRHODS(cli); err != nil { - return "", err + return Unknown, err } else if platform == ManagedRhods { return ManagedRhods, nil } @@ -86,3 +90,14 @@ func GetPlatform(cli client.Client) (Platform, error) { // check and return whether ODH or self-managed platform return isSelfManaged(cli) } + +// ShouldConfigureServiceMesh determines if the operator should invoke service-mesh specific setup. +func ShouldConfigureServiceMesh(cli client.Client, dscispec *dsci.DSCInitializationSpec) (bool, error) { + platform, err := GetPlatform(cli) + if err != nil { + return false, err + } + + supportedPlatforms := platform == OpenDataHub || platform == Unknown + return dscispec.ServiceMesh.ManagementState == operatorv1.Managed && supportedPlatforms, nil +} diff --git a/pkg/feature/builder.go b/pkg/feature/builder.go new file mode 100644 index 00000000000..98b7c4db830 --- /dev/null +++ b/pkg/feature/builder.go @@ -0,0 +1,186 @@ +package feature + +import ( + v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/pkg/errors" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type partialBuilder func(f *Feature) error + +type featureBuilder struct { + name string + builders []partialBuilder +} + +func CreateFeature(name string) *featureBuilder { + return &featureBuilder{name: name} +} + +func (fb *featureBuilder) For(spec *v1.DSCInitializationSpec) *featureBuilder { + createSpec := func(f *Feature) error { + f.Spec = &Spec{ + AppNamespace: spec.ApplicationsNamespace, + ServiceMeshSpec: &spec.ServiceMesh, + } + + return nil + } + + // Ensures creation of .Spec object is always invoked first + fb.builders = append([]partialBuilder{createSpec}, fb.builders...) + + return fb +} + +func (fb *featureBuilder) UsingConfig(config *rest.Config) *featureBuilder { + fb.builders = append(fb.builders, createClients(config)) + + return fb +} + +func createClients(config *rest.Config) partialBuilder { + return func(f *Feature) error { + var err error + f.Clientset, err = kubernetes.NewForConfig(config) + if err != nil { + return err + } + + f.DynamicClient, err = dynamic.NewForConfig(config) + if err != nil { + return err + } + + f.Client, err = client.New(config, client.Options{}) + if err != nil { + return errors.WithStack(err) + } + + if err := apiextv1.AddToScheme(f.Client.Scheme()); err != nil { + return err + } + + return nil + } +} + +func (fb *featureBuilder) Manifests(paths ...string) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + var err error + var manifests []manifest + + for _, path := range paths { + manifests, err = loadManifestsFrom(path) + if err != nil { + return errors.WithStack(err) + } + + f.manifests = append(f.manifests, manifests...) + } + + return nil + }) + + return fb +} + +func (fb *featureBuilder) WithData(loader ...Action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.loaders = append(f.loaders, loader...) + + return nil + }) + + return fb +} + +func (fb *featureBuilder) PreConditions(preconditions ...Action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.preconditions = append(f.preconditions, preconditions...) + + return nil + }) + + return fb +} + +func (fb *featureBuilder) PostConditions(postconditions ...Action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.postconditions = append(f.postconditions, postconditions...) + + return nil + }) + + return fb +} + +func (fb *featureBuilder) OnDelete(cleanups ...Action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.addCleanup(cleanups...) + + return nil + }) + + return fb +} + +func (fb *featureBuilder) WithResources(resources ...Action) *featureBuilder { + fb.builders = append(fb.builders, func(f *Feature) error { + f.resources = resources + + return nil + }) + + return fb +} + +func (fb *featureBuilder) Load() (*Feature, error) { + feature := &Feature{ + Name: fb.name, + Enabled: true, + } + + for i := range fb.builders { + if err := fb.builders[i](feature); err != nil { + return nil, err + } + } + + // UsingConfig builder wasn't called while constructing this feature. + // Get default settings and create needed clients. + if feature.Client == nil { + config, err := rest.InClusterConfig() + if errors.Is(err, rest.ErrNotInCluster) { + // rollback to local kubeconfig - this can be helpful when running the process locally i.e. while debugging + kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ExplicitPath: clientcmd.RecommendedHomeFile}, + &clientcmd.ConfigOverrides{}, + ) + + config, err = kubeconfig.ClientConfig() + if err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + + if err := createClients(config)(feature); err != nil { + return nil, err + } + } + + if feature.Enabled { + if err := feature.createResourceTracker(); err != nil { + return feature, err + } + } + + return feature, nil +} diff --git a/pkg/feature/cert.go b/pkg/feature/cert.go new file mode 100644 index 00000000000..8c3716d953e --- /dev/null +++ b/pkg/feature/cert.go @@ -0,0 +1,91 @@ +package feature + +import ( + "bytes" + cryptorand "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "math/big" + "math/rand" + "net" + "time" +) + +var seededRand = rand.New(rand.NewSource(time.Now().UnixNano())) + +func GenerateSelfSignedCertificateAsSecret(addr string, objectMeta metav1.ObjectMeta) (*corev1.Secret, error) { + cert, key, err := generateCertificate(addr) + if err != nil { + return nil, errors.WithStack(err) + } + + return &corev1.Secret{ + ObjectMeta: objectMeta, + Data: map[string][]byte{ + corev1.TLSCertKey: cert, + corev1.TLSPrivateKeyKey: key, + }, + }, nil +} + +func generateCertificate(addr string) ([]byte, []byte, error) { + key, err := rsa.GenerateKey(cryptorand.Reader, 2048) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: new(big.Int).SetInt64(seededRand.Int63()), + Subject: pkix.Name{ + CommonName: addr, + Organization: []string{"opendatahub-self-signed"}, + }, + NotBefore: now.UTC(), + NotAfter: now.Add(time.Second * 60 * 60 * 24 * 365).UTC(), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + + if ip := net.ParseIP(addr); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } else { + tmpl.DNSNames = append(tmpl.DNSNames, addr) + } + + tmpl.DNSNames = append(tmpl.DNSNames, "localhost") + + certDERBytes, err := x509.CreateCertificate(cryptorand.Reader, &tmpl, &tmpl, key.Public(), key) + if err != nil { + return nil, nil, errors.WithStack(err) + } + certificate, err := x509.ParseCertificate(certDERBytes) + if err != nil { + return nil, nil, errors.WithStack(err) + } + + certBuffer := bytes.Buffer{} + if err := pem.Encode(&certBuffer, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certificate.Raw, + }); err != nil { + return nil, nil, errors.WithStack(err) + } + + keyBuffer := bytes.Buffer{} + if err := pem.Encode(&keyBuffer, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }); err != nil { + return nil, nil, errors.WithStack(err) + } + + return certBuffer.Bytes(), keyBuffer.Bytes(), nil +} diff --git a/pkg/feature/conditions.go b/pkg/feature/conditions.go new file mode 100644 index 00000000000..c58d5c6959f --- /dev/null +++ b/pkg/feature/conditions.go @@ -0,0 +1,83 @@ +package feature + +import ( + "context" + corev1 "k8s.io/api/core/v1" + apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/wait" + "sigs.k8s.io/controller-runtime/pkg/client" + "time" +) + +const ( + interval = 2 * time.Second + duration = 5 * time.Minute +) + +func EnsureCRDIsInstalled(name string) Action { + return func(f *Feature) error { + return f.Client.Get(context.TODO(), client.ObjectKey{Name: name}, &apiextv1.CustomResourceDefinition{}) + } +} + +func WaitForPodsToBeReady(namespace string) Action { + return func(feature *Feature) error { + return wait.PollUntilContextTimeout(context.TODO(), interval, duration, false, func(ctx context.Context) (bool, error) { + log.Info("waiting for pods to become ready", "feature", feature.Name, "namespace", namespace, "duration (s)", duration.Seconds()) + podList, err := feature.Clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return false, err + } + + readyPods := 0 + totalPods := len(podList.Items) + + for _, pod := range podList.Items { + podReady := true + for _, condition := range pod.Status.Conditions { + if condition.Type == corev1.PodReady { + if condition.Status != corev1.ConditionTrue { + podReady = false + break + } + } + } + if podReady { + readyPods++ + } + } + + done := readyPods == totalPods + + if done { + log.Info("done waiting for pods to become ready", "feature", feature.Name, "namespace", namespace) + } + + return done, nil + }) + } +} + +func WaitForResourceToBeCreated(namespace string, gvr schema.GroupVersionResource) Action { + return func(feature *Feature) error { + return wait.PollUntilContextTimeout(context.TODO(), interval, duration, false, func(ctx context.Context) (bool, error) { + log.Info("waiting for resource to be created", "namespace", namespace, "resource", gvr) + + resources, err := feature.DynamicClient.Resource(gvr).Namespace(namespace).List(context.TODO(), metav1.ListOptions{Limit: 1}) + if err != nil { + log.Error(err, "failed waiting for resource", "namespace", namespace, "resource", gvr) + return false, err + } + + if len(resources.Items) > 0 { + log.Info("resource created", "namespace", namespace, "resource", gvr) + return true, nil + } + + log.Info("still waiting for resource", "namespace", namespace, "resource", gvr) + return false, nil + }) + } +} diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go new file mode 100644 index 00000000000..caa6b4a4722 --- /dev/null +++ b/pkg/feature/feature.go @@ -0,0 +1,254 @@ +package feature + +import ( + "context" + "github.com/hashicorp/go-multierror" + v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "net/url" + "regexp" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" + "strings" +) + +var log = ctrlLog.Log.WithName("features") + +type Feature struct { + Name string + Spec *Spec + Enabled bool + + Clientset *kubernetes.Clientset + DynamicClient dynamic.Interface + Client client.Client + + manifests []manifest + cleanups []Action + resources []Action + preconditions []Action + postconditions []Action + loaders []Action +} + +// Action is a func type which can be used for different purposes while having access to Feature struct +type Action func(feature *Feature) error + +func (f *Feature) Apply() error { + if !f.Enabled { + log.Info("feature is disabled, skipping.", "feature", f.Name) + + return nil + } + + // Verify all precondition and collect errors + var multiErr *multierror.Error + for _, precondition := range f.preconditions { + multiErr = multierror.Append(multiErr, precondition(f)) + } + + if multiErr.ErrorOrNil() != nil { + return multiErr.ErrorOrNil() + } + + // Load necessary data + for _, loader := range f.loaders { + multiErr = multierror.Append(multiErr, loader(f)) + } + if multiErr.ErrorOrNil() != nil { + return multiErr.ErrorOrNil() + } + + // create or update resources + for _, resource := range f.resources { + if err := resource(f); err != nil { + return err + } + } + + // Process and apply manifests + for _, m := range f.manifests { + if err := m.processTemplate(f.Spec); err != nil { + return errors.WithStack(err) + } + + log.Info("applying manifest", "feature", f.Name, "path", m.targetPath()) + } + + if err := f.applyManifests(); err != nil { + return err + } + + for _, postcondition := range f.postconditions { + multiErr = multierror.Append(multiErr, postcondition(f)) + } + + if multiErr.ErrorOrNil() != nil { + return multiErr.ErrorOrNil() + } + + return nil +} + +func (f *Feature) Cleanup() error { + if !f.Enabled { + log.Info("feature is disabled, skipping.", "feature", f.Name) + + return nil + } + + var cleanupErrors *multierror.Error + for _, cleanupFunc := range f.cleanups { + cleanupErrors = multierror.Append(cleanupErrors, cleanupFunc(f)) + } + + return cleanupErrors.ErrorOrNil() +} + +func (f *Feature) applyManifests() error { + var applyErrors *multierror.Error + for _, m := range f.manifests { + err := f.apply(m) + applyErrors = multierror.Append(applyErrors, err) + } + + return applyErrors.ErrorOrNil() +} + +func (f *Feature) CreateConfigMap(cfgMapName string, data map[string]string) error { + configMap := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: cfgMapName, + Namespace: f.Spec.AppNamespace, + OwnerReferences: []metav1.OwnerReference{ + f.OwnerReference(), + }, + }, + Data: data, + } + + configMaps := f.Clientset.CoreV1().ConfigMaps(configMap.Namespace) + _, err := configMaps.Get(context.TODO(), configMap.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { //nolint:gocritic + _, err = configMaps.Create(context.TODO(), configMap, metav1.CreateOptions{}) + if err != nil { + return err + } + } else if k8serrors.IsAlreadyExists(err) { + _, err = configMaps.Update(context.TODO(), configMap, metav1.UpdateOptions{}) + if err != nil { + return err + } + } else { + return err + } + + return nil +} + +func (f *Feature) addCleanup(cleanupFuncs ...Action) { + f.cleanups = append(f.cleanups, cleanupFuncs...) +} + +type apply func(filename string) error + +func (f *Feature) apply(m manifest) error { + var applier apply + targetPath := m.targetPath() + + if m.patch { + applier = func(filename string) error { + log.Info("patching using manifest", "feature", f.Name, "name", m.name, "path", targetPath) + + return f.patchResourceFromFile(filename) + } + } else { + applier = func(filename string) error { + log.Info("applying manifest", "feature", f.Name, "name", m.name, "path", targetPath) + + return f.createResourceFromFile(filename) + } + } + + if err := applier(targetPath); err != nil { + log.Error(err, "failed to create resource", "feature", f.Name, "name", m.name, "path", targetPath) + + return err + } + + return nil +} + +func (f *Feature) OwnerReference() metav1.OwnerReference { + return f.Spec.Tracker.ToOwnerReference() +} + +// createResourceTracker instantiates FeatureTracker for a given Feature. All resources created when applying +// it will have this object attached as an OwnerReference. +// It's a cluster-scoped resource. Once created, there's a cleanup hook added which will be invoked on deletion, resulting +// in removal of all owned resources which belong to this Feature. +func (f *Feature) createResourceTracker() error { + tracker := &v1.FeatureTracker{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "dscinitialization.opendatahub.io/v1", + Kind: "FeatureTracker", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: f.Spec.AppNamespace + "-" + convertToRFC1123Subdomain(f.Name), + }, + } + + foundTracker, err := f.DynamicClient.Resource(gvr.ResourceTracker).Get(context.TODO(), tracker.Name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + unstructuredTracker, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tracker) + if err != nil { + return err + } + + u := unstructured.Unstructured{Object: unstructuredTracker} + + foundTracker, err = f.DynamicClient.Resource(gvr.ResourceTracker).Create(context.TODO(), &u, metav1.CreateOptions{}) + if err != nil { + return err + } + } else if err != nil { + return err + } + + f.Spec.Tracker = &v1.FeatureTracker{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(foundTracker.Object, f.Spec.Tracker); err != nil { + return err + } + + // Register its own cleanup + f.addCleanup(func(feature *Feature) error { + if err := f.DynamicClient.Resource(gvr.ResourceTracker).Delete(context.TODO(), f.Spec.Tracker.Name, metav1.DeleteOptions{}); err != nil && !k8serrors.IsNotFound(err) { + return err + } + + return nil + }) + + return nil +} + +func convertToRFC1123Subdomain(input string) string { + escaped := url.PathEscape(input) + + // Define a regular expression to match characters that need to be replaced + regex := regexp.MustCompile(`[^A-Za-z0-9.\-_]+`) + + // Replace non-alphanumeric characters with a hyphen + replaced := regex.ReplaceAllString(escaped, "-") + + // Convert the result to lowercase + return strings.ToLower(replaced) +} diff --git a/pkg/feature/feature_suite_test.go b/pkg/feature/feature_suite_test.go new file mode 100644 index 00000000000..ac87f11549f --- /dev/null +++ b/pkg/feature/feature_suite_test.go @@ -0,0 +1,14 @@ +package feature_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestOssmFeatures(t *testing.T) { + RegisterFailHandler(Fail) + // for integration tests see tests/integration directory + RunSpecs(t, "Openshift Service Mesh features unit tests") +} diff --git a/pkg/feature/manifest.go b/pkg/feature/manifest.go new file mode 100644 index 00000000000..a821fa82cbc --- /dev/null +++ b/pkg/feature/manifest.go @@ -0,0 +1,79 @@ +package feature + +import ( + "fmt" + "github.com/pkg/errors" + "html/template" + "os" + "path/filepath" + "strings" +) + +const ( + BaseDir = "templates/servicemesh/" + ControlPlaneDir = BaseDir + "control-plane" + AuthDir = BaseDir + "authorino" + BaseOutputDir = "/tmp/servicemesh-manifests/" +) + +type manifest struct { + name, + path string + template, + patch, + processed bool +} + +func loadManifestsFrom(path string) ([]manifest, error) { + var manifests []manifest + if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + basePath := filepath.Base(path) + manifests = append(manifests, manifest{ + name: basePath, + path: path, + patch: strings.Contains(basePath, ".patch"), + template: filepath.Ext(path) == ".tmpl", + }) + return nil + }); err != nil { + return nil, errors.WithStack(err) + } + + return manifests, nil +} + +func (m *manifest) targetPath() string { + return fmt.Sprintf("%s%s", m.path[:len(m.path)-len(filepath.Ext(m.path))], ".yaml") +} + +func (m *manifest) processTemplate(data interface{}) error { + if !m.template { + return nil + } + path := m.targetPath() + + f, err := os.Create(path) + if err != nil { + log.Error(err, "Failed to create file") + + return err + } + + tmpl := template.New(m.name).Funcs(template.FuncMap{"ReplaceChar": ReplaceChar}) + + tmpl, err = tmpl.ParseFiles(m.path) + if err != nil { + return err + } + + err = tmpl.Execute(f, data) + m.processed = err == nil + + return err +} diff --git a/pkg/feature/raw_resources.go b/pkg/feature/raw_resources.go new file mode 100644 index 00000000000..4ea955c222e --- /dev/null +++ b/pkg/feature/raw_resources.go @@ -0,0 +1,157 @@ +/* +Copyright (c) 2016-2017 Bitnami +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package feature + +import ( + "context" + "fmt" + "github.com/ghodss/yaml" + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + "os" + "regexp" + "strings" +) + +const ( + YamlSeparator = "(?m)^---[ \t]*$" +) + +type NameValue struct { + Name string + Value string +} + +func (f *Feature) createResourceFromFile(filename string, elems ...NameValue) error { + elemsMap := make(map[string]NameValue) + for _, nv := range elems { + elemsMap[nv.Name] = nv + } + + data, err := os.ReadFile(filename) + if err != nil { + return errors.WithStack(err) + } + splitter := regexp.MustCompile(YamlSeparator) + objectStrings := splitter.Split(string(data), -1) + for _, str := range objectStrings { + if strings.TrimSpace(str) == "" { + continue + } + u := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(str), u); err != nil { + return errors.WithStack(err) + } + + name := u.GetName() + namespace := u.GetNamespace() + if namespace == "" { + if val, exists := elemsMap["namespace"]; exists { + u.SetNamespace(val.Value) + } else { + u.SetNamespace("default") + } + } + + u.SetOwnerReferences([]metav1.OwnerReference{ + f.OwnerReference(), + }) + + log.Info("Creating resource", "name", name) + + err := f.Client.Get(context.TODO(), k8stypes.NamespacedName{Name: name, Namespace: namespace}, u.DeepCopy()) + if err == nil { + log.Info("Object already exists...") + continue + } + if !k8serrors.IsNotFound(err) { + return errors.WithStack(err) + } + + err = f.Client.Create(context.TODO(), u) + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func (f *Feature) patchResourceFromFile(filename string, elems ...NameValue) error { + elemsMap := make(map[string]NameValue) + for _, nv := range elems { + elemsMap[nv.Name] = nv + } + + data, err := os.ReadFile(filename) + if err != nil { + return errors.WithStack(err) + } + splitter := regexp.MustCompile(YamlSeparator) + objectStrings := splitter.Split(string(data), -1) + for _, str := range objectStrings { + if strings.TrimSpace(str) == "" { + continue + } + p := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(str), p); err != nil { + log.Error(err, "error unmarshalling yaml") + return errors.WithStack(err) + } + + // Adding `namespace:` to Namespace resource doesn't make sense + if p.GetKind() != "Namespace" { + namespace := p.GetNamespace() + if namespace == "" { + if val, exists := elemsMap["namespace"]; exists { + p.SetNamespace(val.Value) + } else { + p.SetNamespace("default") + } + } + } + + gvr := schema.GroupVersionResource{ + Group: strings.ToLower(p.GroupVersionKind().Group), + Version: p.GroupVersionKind().Version, + Resource: strings.ToLower(p.GroupVersionKind().Kind) + "s", + } + + // Convert the patch from YAML to JSON + patchAsJSON, err := yaml.YAMLToJSON(data) + if err != nil { + log.Error(err, "error converting yaml to json") + return errors.WithStack(err) + } + + _, err = f.DynamicClient.Resource(gvr). + Namespace(p.GetNamespace()). + Patch(context.TODO(), p.GetName(), k8stypes.MergePatchType, patchAsJSON, metav1.PatchOptions{}) + if err != nil { + log.Error(err, "error patching resource", + "gvr", fmt.Sprintf("%+v\n", gvr), + "patch", fmt.Sprintf("%+v\n", p), + "json", fmt.Sprintf("%+v\n", patchAsJSON)) + return errors.WithStack(err) + } + + if err != nil { + return errors.WithStack(err) + } + } + return nil +} diff --git a/pkg/feature/resources.go b/pkg/feature/resources.go new file mode 100644 index 00000000000..3636d41d05c --- /dev/null +++ b/pkg/feature/resources.go @@ -0,0 +1,65 @@ +package feature + +import ( + "context" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// CreateNamespace will create namespace with the given name if it does not exist yet and sets owner, so it will be deleted +// when a feature is cleaned up. +func CreateNamespace(namespace string) Action { + return func(f *Feature) error { + nsClient := f.Clientset.CoreV1().Namespaces() + + _, err := nsClient.Get(context.TODO(), namespace, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + _, err := nsClient.Create(context.TODO(), &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespace, + OwnerReferences: []metav1.OwnerReference{ + f.OwnerReference(), + }, + }, + }, metav1.CreateOptions{}) + + // we either successfully created new namespace or failed during the process + // returning err which indicates the state + return err + } + + return err + } +} + +func SelfSignedCertificate(feature *Feature) error { + if feature.Spec.Mesh.Certificate.Generate { + meta := metav1.ObjectMeta{ + Name: feature.Spec.Mesh.Certificate.Name, + Namespace: feature.Spec.Mesh.Namespace, + OwnerReferences: []metav1.OwnerReference{ + feature.OwnerReference(), + }, + } + + cert, err := GenerateSelfSignedCertificateAsSecret(feature.Spec.Domain, meta) + if err != nil { + return errors.WithStack(err) + } + + if err != nil { + return errors.WithStack(err) + } + + _, err = feature.Clientset.CoreV1(). + Secrets(feature.Spec.Mesh.Namespace). + Create(context.TODO(), cert, metav1.CreateOptions{}) + if err != nil && !k8serrors.IsAlreadyExists(err) { + return errors.WithStack(err) + } + } + + return nil +} diff --git a/pkg/feature/servicemesh/cleanup.go b/pkg/feature/servicemesh/cleanup.go new file mode 100644 index 00000000000..b9ce41d968a --- /dev/null +++ b/pkg/feature/servicemesh/cleanup.go @@ -0,0 +1,126 @@ +package servicemesh + +import ( + "context" + "fmt" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func RemoveTokenVolumes(f *feature.Feature) error { + tokenVolume := fmt.Sprintf("%s-oauth2-tokens", f.Spec.AppNamespace) + + meshNs := f.Spec.Mesh.Namespace + meshName := f.Spec.Mesh.Name + + smcp, err := f.DynamicClient.Resource(gvr.SMCP).Namespace(meshNs).Get(context.TODO(), meshName, metav1.GetOptions{}) + if err != nil { + return err + } + volumes, found, err := unstructured.NestedSlice(smcp.Object, "spec", "gateways", "ingress", "volumes") + if err != nil { + return err + } + if !found { + log.Info("no volumes found", "f", f.Name, "control-plane", meshName, "istio-ns", meshNs) + return nil + } + + for i, v := range volumes { + volume, ok := v.(map[string]interface{}) + if !ok { + log.Info("unexpected type for volume", "f", f.Name, "type", fmt.Sprintf("%T", volume)) + continue + } + + volumeMount, found, err := unstructured.NestedMap(volume, "volumeMount") + if err != nil { + return err + } + if !found { + log.Info("no volumeMount found in the volume", "f", f.Name) + continue + } + + if volumeMount["name"] == tokenVolume { + volumes = append(volumes[:i], volumes[i+1:]...) + err = unstructured.SetNestedSlice(smcp.Object, volumes, "spec", "gateways", "ingress", "volumes") + if err != nil { + return err + } + break + } + } + + _, err = f.DynamicClient.Resource(gvr.SMCP).Namespace(meshNs).Update(context.TODO(), smcp, metav1.UpdateOptions{}) + + return err +} + +func RemoveOAuthClient(f *feature.Feature) error { + oauthClientName := fmt.Sprintf("%s-oauth2-client", f.Spec.AppNamespace) + + if _, err := f.DynamicClient.Resource(gvr.OAuthClient).Get(context.TODO(), oauthClientName, metav1.GetOptions{}); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } + + return err + } + + if err := f.DynamicClient.Resource(gvr.OAuthClient).Delete(context.TODO(), oauthClientName, metav1.DeleteOptions{}); err != nil { + log.Error(err, "failed deleting OAuthClient", "f", f.Name, "name", oauthClientName) + + return err + } + + return nil +} + +func RemoveExtensionProvider(f *feature.Feature) error { + ossmAuthzProvider := fmt.Sprintf("%s-odh-auth-provider", f.Spec.AppNamespace) + + mesh := f.Spec.Mesh + + smcp, err := f.DynamicClient.Resource(gvr.SMCP). + Namespace(mesh.Namespace). + Get(context.TODO(), mesh.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + extensionProviders, found, err := unstructured.NestedSlice(smcp.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + if !found { + log.Info("no extension providers found", "f", f.Name, "control-plane", mesh.Name, "namespace", mesh.Namespace) + return nil + } + + for i, v := range extensionProviders { + extensionProvider, ok := v.(map[string]interface{}) + if !ok { + fmt.Println("Unexpected type for extensionProvider") + continue + } + + if extensionProvider["name"] == ossmAuthzProvider { + extensionProviders = append(extensionProviders[:i], extensionProviders[i+1:]...) + err = unstructured.SetNestedSlice(smcp.Object, extensionProviders, "spec", "techPreview", "meshConfig", "extensionProviders") + if err != nil { + return err + } + break + } + } + + _, err = f.DynamicClient.Resource(gvr.SMCP). + Namespace(mesh.Namespace). + Update(context.TODO(), smcp, metav1.UpdateOptions{}) + + return err +} diff --git a/pkg/feature/servicemesh/conditions.go b/pkg/feature/servicemesh/conditions.go new file mode 100644 index 00000000000..3c2cd304681 --- /dev/null +++ b/pkg/feature/servicemesh/conditions.go @@ -0,0 +1,76 @@ +package servicemesh + +import ( + "context" + "github.com/hashicorp/go-multierror" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" + "github.com/pkg/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/dynamic" + "time" +) + +const ( + interval = 2 * time.Second + duration = 5 * time.Minute +) + +func EnsureServiceMeshInstalled(f *feature.Feature) error { + if err := feature.EnsureCRDIsInstalled("servicemeshcontrolplanes.maistra.io")(f); err != nil { + log.Info("Failed to find the pre-requisite Service Mesh Control Plane CRD, please ensure Service Mesh Operator is installed.", "feature", f.Name) + + return err + } + + smcp := f.Spec.Mesh.Name + smcpNs := f.Spec.Mesh.Namespace + + if err := WaitForControlPlaneToBeReady(f); err != nil { + log.Error(err, "failed waiting for control plane being ready", "feature", f.Name, "control-plane", smcp, "namespace", smcpNs) + + return multierror.Append(err, errors.New("service mesh control plane is not ready")).ErrorOrNil() + } + + return nil +} + +func WaitForControlPlaneToBeReady(feature *feature.Feature) error { + return wait.PollUntilContextTimeout(context.TODO(), interval, duration, false, func(ctx context.Context) (bool, error) { + smcp := feature.Spec.Mesh.Name + smcpNs := feature.Spec.Mesh.Namespace + + log.Info("waiting for control plane components to be ready", "feature", feature.Name, "control-plane", smcp, "namespace", smcpNs, "duration (s)", duration.Seconds()) + ready, err := CheckControlPlaneComponentReadiness(feature.DynamicClient, smcp, smcpNs) + + if ready { + log.Info("done waiting for control plane components to be ready", "feature", feature.Name, "control-plane", smcp, "namespace", smcpNs) + } + + return ready, err + }) +} + +func CheckControlPlaneComponentReadiness(dynamicClient dynamic.Interface, smcp, smcpNs string) (bool, error) { + unstructObj, err := dynamicClient.Resource(gvr.SMCP).Namespace(smcpNs).Get(context.TODO(), smcp, metav1.GetOptions{}) + if err != nil { + log.Info("failed to find Service Mesh Control Plane", "control-plane", smcp, "namespace", smcpNs) + + return false, err + } + + components, found, err := unstructured.NestedMap(unstructObj.Object, "status", "readiness", "components") + if err != nil || !found { + log.Info("status conditions not found or error in parsing of Service Mesh Control Plane") + + return false, err + } + + readyComponents := len(components["ready"].([]interface{})) + pendingComponents := len(components["pending"].([]interface{})) + unreadyComponents := len(components["unready"].([]interface{})) + + return pendingComponents == 0 && unreadyComponents == 0 && readyComponents > 0, nil +} diff --git a/pkg/feature/servicemesh/envoy_secrets.go b/pkg/feature/servicemesh/envoy_secrets.go new file mode 100644 index 00000000000..ca083f3d0fe --- /dev/null +++ b/pkg/feature/servicemesh/envoy_secrets.go @@ -0,0 +1,64 @@ +package servicemesh + +import ( + "bytes" + "fmt" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "text/template" +) + +const tokenSecret = ` +resources: +- "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" + name: token + generic_secret: + secret: + inline_string: "{{ .Secret }}" +` + +const hmacSecret = ` +resources: +- "@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.Secret" + name: hmac + generic_secret: + secret: + inline_bytes: "{{ .Secret }}" +` + +func createEnvoySecret(oAuth feature.OAuth, objectMeta metav1.ObjectMeta) (*corev1.Secret, error) { + clientSecret, err := processInlineTemplate(tokenSecret, struct{ Secret string }{Secret: oAuth.ClientSecret}) + if err != nil { + return nil, errors.WithStack(err) + } + + hmacSecret, err := processInlineTemplate(hmacSecret, struct{ Secret string }{Secret: oAuth.Hmac}) + if err != nil { + return nil, errors.WithStack(err) + } + + return &corev1.Secret{ + ObjectMeta: objectMeta, + Data: map[string][]byte{ + "token-secret.yaml": clientSecret, + "hmac-secret.yaml": hmacSecret, + }, + }, nil +} + +func processInlineTemplate(templateString string, data interface{}) ([]byte, error) { + tmpl, err := template.New("inline-template").Parse(templateString) + if err != nil { + return nil, fmt.Errorf("error parsing template: %v", err) + } + + var output bytes.Buffer + err = tmpl.Execute(&output, data) + if err != nil { + return nil, fmt.Errorf("error executing template: %v", err) + } + + return output.Bytes(), nil +} diff --git a/pkg/feature/servicemesh/loaders.go b/pkg/feature/servicemesh/loaders.go new file mode 100644 index 00000000000..507bbe9d66c --- /dev/null +++ b/pkg/feature/servicemesh/loaders.go @@ -0,0 +1,54 @@ +package servicemesh + +import ( + "github.com/opendatahub-io/opendatahub-operator/v2/controllers/secretgenerator" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/pkg/errors" +) + +func ClusterDetails(f *feature.Feature) error { + data := f.Spec + + if domain, err := cluster.GetDomain(f.DynamicClient); err == nil { + data.Domain = domain + } else { + return errors.WithStack(err) + } + + return nil +} + +func OAuthConfig(f *feature.Feature) error { + data := f.Spec + + var err error + var clientSecret, hmac *secretgenerator.Secret + if clientSecret, err = secretgenerator.NewSecret("ossm-odh-oauth", "random", 32); err != nil { + return errors.WithStack(err) + } + + if hmac, err = secretgenerator.NewSecret("ossm-odh-hmac", "random", 32); err != nil { + return errors.WithStack(err) + } + + if oauthServerDetailsJSON, err := cluster.GetOAuthServerDetails(); err == nil { + hostName, port, errURLParsing := cluster.ExtractHostNameAndPort(oauthServerDetailsJSON.Get("issuer").MustString("issuer")) + if errURLParsing != nil { + return errURLParsing + } + + data.OAuth = feature.OAuth{ + AuthzEndpoint: oauthServerDetailsJSON.Get("authorization_endpoint").MustString("authorization_endpoint"), + TokenEndpoint: oauthServerDetailsJSON.Get("token_endpoint").MustString("token_endpoint"), + Route: hostName, + Port: port, + ClientSecret: clientSecret.Value, + Hmac: hmac.Value, + } + } else { + return errors.WithStack(err) + } + + return nil +} diff --git a/pkg/feature/servicemesh/resources.go b/pkg/feature/servicemesh/resources.go new file mode 100644 index 00000000000..ee426e09f19 --- /dev/null +++ b/pkg/feature/servicemesh/resources.go @@ -0,0 +1,136 @@ +package servicemesh + +import ( + "context" + "fmt" + "github.com/hashicorp/go-multierror" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "strings" +) + +func EnvoyOAuthSecrets(feature *feature.Feature) error { + objectMeta := metav1.ObjectMeta{ + Name: feature.Spec.AppNamespace + "-oauth2-tokens", + Namespace: feature.Spec.Mesh.Namespace, + OwnerReferences: []metav1.OwnerReference{ + feature.OwnerReference(), + }, + } + + envoySecret, err := createEnvoySecret(feature.Spec.OAuth, objectMeta) + if err != nil { + return errors.WithStack(err) + } + + _, err = feature.Clientset.CoreV1(). + Secrets(objectMeta.Namespace). + Create(context.TODO(), envoySecret, metav1.CreateOptions{}) + if err != nil && !k8serrors.IsAlreadyExists(err) { + return errors.WithStack(err) + } + + return nil +} + +func ConfigMaps(feature *feature.Feature) error { + meshConfig := feature.Spec.Mesh + if err := feature.CreateConfigMap("service-mesh-refs", + map[string]string{ + "CONTROL_PLANE_NAME": meshConfig.Name, + "MESH_NAMESPACE": meshConfig.Namespace, + }); err != nil { + return errors.WithStack(err) + } + + authorinoConfig := feature.Spec.Auth.Authorino + if err := feature.CreateConfigMap("auth-refs", + map[string]string{ + "AUTHORINO_LABEL": authorinoConfig.Label, + "AUTH_AUDIENCE": strings.Join(authorinoConfig.Audiences, ","), + }); err != nil { + return errors.WithStack(err) + } + + return nil +} + +func EnabledInDashboard(feature *feature.Feature) error { + return setServiceMeshDisabledFlag(false)(feature) +} + +func DisabledInDashboard(feature *feature.Feature) error { + return setServiceMeshDisabledFlag(true)(feature) +} + +func setServiceMeshDisabledFlag(disabled bool) feature.Action { + return func(feature *feature.Feature) error { + configs, err := feature.DynamicClient. + Resource(gvr.ODHDashboardConfigGVR). + Namespace(feature.Spec.AppNamespace). + List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return err + } + + if len(configs.Items) == 0 { + log.Info("No odhdashboardconfig found in namespace, doing nothing") + return nil + } + + // Assuming there is only one odhdashboardconfig in the namespace, patching the first one + config := configs.Items[0] + if config.Object["spec"] == nil { + config.Object["spec"] = map[string]interface{}{} + } + spec := config.Object["spec"].(map[string]interface{}) + if spec["dashboardConfig"] == nil { + spec["dashboardConfig"] = map[string]interface{}{} + } + dashboardConfig := spec["dashboardConfig"].(map[string]interface{}) + dashboardConfig["disableServiceMesh"] = disabled + + if _, err := feature.DynamicClient.Resource(gvr.ODHDashboardConfigGVR). + Namespace(feature.Spec.AppNamespace). + Update(context.TODO(), &config, metav1.UpdateOptions{}); err != nil { + log.Error(err, "Failed to update odhdashboardconfig") + + return err + } + + log.Info("Successfully patched odhdashboardconfig") + return nil + } +} + +func MigratedDataScienceProjects(feature *feature.Feature) error { + selector := labels.SelectorFromSet(labels.Set{"opendatahub.io/dashboard": "true"}) + + namespaceClient := feature.Clientset.CoreV1().Namespaces() + + namespaces, err := namespaceClient.List(context.TODO(), metav1.ListOptions{LabelSelector: selector.String()}) + if err != nil { + return fmt.Errorf("failed to get namespaces: %v", err) + } + + var result *multierror.Error + + for _, namespace := range namespaces.Items { + annotations := namespace.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations["opendatahub.io/service-mesh"] = "true" + namespace.SetAnnotations(annotations) + + if _, err := namespaceClient.Update(context.TODO(), &namespace, metav1.UpdateOptions{}); err != nil { + result = multierror.Append(result, err) + } + } + + return result.ErrorOrNil() +} diff --git a/pkg/feature/servicemesh/servicemesh_setup.go b/pkg/feature/servicemesh/servicemesh_setup.go new file mode 100644 index 00000000000..d9da6e070db --- /dev/null +++ b/pkg/feature/servicemesh/servicemesh_setup.go @@ -0,0 +1,66 @@ +package servicemesh + +import ( + "github.com/hashicorp/go-multierror" + v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/pkg/errors" + ctrlLog "sigs.k8s.io/controller-runtime/pkg/log" +) + +var log = ctrlLog.Log.WithName("service-mesh") + +type ServiceMeshInitializer struct { + *v1.DSCInitializationSpec + defineFeatures DefineFeatures + Features []*feature.Feature +} + +type DefineFeatures func(s *ServiceMeshInitializer) error + +func NewServiceMeshInitializer(spec *v1.DSCInitializationSpec, def DefineFeatures) *ServiceMeshInitializer { + return &ServiceMeshInitializer{ + DSCInitializationSpec: spec, + defineFeatures: def, + } +} + +// Prepare performs validation of the spec and ensures all resources, +// such as Features and their templates, are processed and initialized +// before proceeding with the actual cluster set-up. +func (s *ServiceMeshInitializer) Prepare() error { + log.Info("Initializing Service Mesh configuration") + + serviceMeshSpec := &s.DSCInitializationSpec.ServiceMesh + + if valid, reason := serviceMeshSpec.IsValid(); !valid { + return errors.New(reason) + } + + return s.defineFeatures(s) +} + +func (s *ServiceMeshInitializer) Apply() error { + var applyErrors *multierror.Error + + for _, f := range s.Features { + err := f.Apply() + applyErrors = multierror.Append(applyErrors, err) + } + + return applyErrors.ErrorOrNil() +} + +// Delete executes registered clean-up tasks in the opposite order they were initiated (following a stack structure). +// For instance, this allows for the unpatching of Service Mesh Control Plane before its deletion. +// This approach assumes that Features are either instantiated in the correct sequence +// or are self-contained. +func (s *ServiceMeshInitializer) Delete() error { + var cleanupErrors *multierror.Error + for i := len(s.Features) - 1; i >= 0; i-- { + log.Info("cleanup", "name", s.Features[i].Name) + cleanupErrors = multierror.Append(cleanupErrors, s.Features[i].Cleanup()) + } + + return cleanupErrors.ErrorOrNil() +} diff --git a/pkg/feature/template_loader.go b/pkg/feature/template_loader.go new file mode 100644 index 00000000000..9d2fd53cf1e --- /dev/null +++ b/pkg/feature/template_loader.go @@ -0,0 +1,39 @@ +package feature + +import ( + "embed" + "io/fs" + "os" + "path/filepath" +) + +//go:embed templates +var embeddedFiles embed.FS + +// CopyEmbeddedFiles ensures that files embedded using go:embed are populated +// to dest directory. In order to process the templates, we need to create a tmp directory +// to store the files. This is because embedded files are read only. +func CopyEmbeddedFiles(src, dest string) error { + return fs.WalkDir(embeddedFiles, src, func(path string, dir fs.DirEntry, err error) error { + if err != nil { + return err + } + + destPath := filepath.Join(dest, path) + if dir.IsDir() { + if err := os.MkdirAll(destPath, 0755); err != nil { + return err + } + } else { + data, err := fs.ReadFile(embeddedFiles, path) + if err != nil { + return err + } + if err := os.WriteFile(destPath, data, 0644); err != nil { + return err + } + } + + return nil + }) +} diff --git a/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl b/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl new file mode 100644 index 00000000000..ba6e9a6ea0b --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/auth-smm.tmpl @@ -0,0 +1,10 @@ +apiVersion: maistra.io/v1 +kind: ServiceMeshMember +metadata: + name: default + namespace: {{ .Auth.Namespace }} +spec: + controlPlaneRef: + namespace: {{ .Mesh.Namespace }} + name: {{ .Mesh.Name }} + diff --git a/pkg/feature/templates/servicemesh/authorino/base/authconfig.tmpl b/pkg/feature/templates/servicemesh/authorino/base/authconfig.tmpl new file mode 100644 index 00000000000..981ec8875e6 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/base/authconfig.tmpl @@ -0,0 +1,46 @@ +apiVersion: authorino.kuadrant.io/v1beta1 +kind: AuthConfig +metadata: + name: odh-dashboard-protection + namespace: {{ .AppNamespace }} + labels: + {{ ReplaceChar .Auth.Authorino.Label "=" ": " }} +spec: + hosts: + - {{.AppNamespace}}.{{ .Domain }} + identity: + - name: kubernetes-users + kubernetes: + audiences: +{{- range .Auth.Authorino.Audiences }} + - "{{ . }}" +{{- end }} + authorization: + - name: k8s-rbac-only-service-viewers + kubernetes: + user: + valueFrom: { authJSON: auth.identity.username } + resourceAttributes: + namespace: + value: {{ .AppNamespace }} + group: + value: "" + resource: + value: services + name: + value: odh-dashboard + verb: + value: get + response: + - name: x-auth-data + json: + properties: + - name: username + valueFrom: { authJSON: auth.identity.username } + denyWith: + unauthenticated: + message: + value: "Access denied" + unauthorized: + message: + value: "Unauthorized" diff --git a/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl b/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl new file mode 100644 index 00000000000..95b1a8fb63b --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/base/operator-cluster-wide-no-tls.tmpl @@ -0,0 +1,15 @@ +apiVersion: operator.authorino.kuadrant.io/v1beta1 +kind: Authorino +metadata: + name: {{ .Auth.Authorino.Name }} + namespace: {{ .Auth.Namespace }} +spec: + image: {{ .Auth.Authorino.Image }} + authConfigLabelSelectors: {{ .Auth.Authorino.Label }} + clusterWide: true + listener: + tls: + enabled: false + oidcServer: + tls: + enabled: false diff --git a/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl b/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl new file mode 100644 index 00000000000..4dee47f7e0c --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/mesh-authz-ext-provider.patch.tmpl @@ -0,0 +1,13 @@ +apiVersion: maistra.io/v2 +kind: ServiceMeshControlPlane +metadata: + name: {{ .Mesh.Name }} + namespace: {{ .Mesh.Namespace }} +spec: + techPreview: + meshConfig: + extensionProviders: + - name: {{ .AppNamespace }}-odh-auth-provider + envoyExtAuthzGrpc: + service: {{ .Auth.Authorino.Name }}-authorino-authorization.{{ .Auth.Namespace }}.svc.cluster.local + port: 50051 diff --git a/pkg/feature/templates/servicemesh/authorino/namespace.tmpl b/pkg/feature/templates/servicemesh/authorino/namespace.tmpl new file mode 100644 index 00000000000..e243fd18fa3 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/namespace.tmpl @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .Auth.Namespace }} + labels: + control-plane: authorino-operator diff --git a/pkg/feature/templates/servicemesh/authorino/rbac/cluster-monitoring-role-binding.tmpl b/pkg/feature/templates/servicemesh/authorino/rbac/cluster-monitoring-role-binding.tmpl new file mode 100644 index 00000000000..cf4114eac88 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/rbac/cluster-monitoring-role-binding.tmpl @@ -0,0 +1,13 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: auth-service-monitoring + namespace: {{ .Auth.Namespace }} +subjects: + - kind: ServiceAccount + name: auth-service + namespace: {{ .Auth.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-monitoring-view diff --git a/pkg/feature/templates/servicemesh/authorino/rbac/cluster-role-binding.tmpl b/pkg/feature/templates/servicemesh/authorino/rbac/cluster-role-binding.tmpl new file mode 100644 index 00000000000..f54ca7c7703 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/rbac/cluster-role-binding.tmpl @@ -0,0 +1,13 @@ +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: auth-service + namespace: {{ .Auth.Namespace }} +subjects: + - kind: ServiceAccount + name: auth-service + namespace: {{ .Auth.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: auth-service diff --git a/pkg/feature/templates/servicemesh/authorino/rbac/cluster-role.tmpl b/pkg/feature/templates/servicemesh/authorino/rbac/cluster-role.tmpl new file mode 100644 index 00000000000..bf47469fb4c --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/rbac/cluster-role.tmpl @@ -0,0 +1,160 @@ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: auth-service + namespace: {{ .Auth.Namespace }} +rules: + - verbs: + - get + - list + apiGroups: + - machine.openshift.io + - autoscaling.openshift.io + resources: + - machineautoscalers + - machinesets + - verbs: + - get + - watch + - list + apiGroups: + - '' + - config.openshift.io + resources: + - clusterversions + - verbs: + - get + - list + - watch + apiGroups: + - operators.coreos.com + resources: + - clusterserviceversions + - subscriptions + - apiGroups: + - '' + - image.openshift.io + resources: + - imagestreams/layers + verbs: + - get + - apiGroups: + - '' + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + resources: + - configmaps + - persistentvolumeclaims + - secrets + - verbs: + - get + - list + - watch + apiGroups: + - route.openshift.io + resources: + - routes + - verbs: + - get + - list + - watch + apiGroups: + - console.openshift.io + resources: + - consolelinks + - verbs: + - get + - list + - watch + apiGroups: + - operator.openshift.io + resources: + - consoles + - verbs: + - get + - watch + - list + apiGroups: + - '' + - integreatly.org + resources: + - rhmis + - verbs: + - get + - list + - watch + apiGroups: + - user.openshift.io + resources: + - groups + - verbs: + - get + - list + - watch + apiGroups: + - user.openshift.io + resources: + - users + - verbs: + - get + - list + - watch + apiGroups: + - '' + resources: + - pods + - serviceaccounts + - services + - apiGroups: + - '' + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + resources: + - namespaces + - apiGroups: + - rbac.authorization.k8s.io + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + resources: + - rolebindings + - clusterrolebindings + - roles + - apiGroups: + - '' + - events.k8s.io + resources: + - events + verbs: + - get + - list + - watch + - apiGroups: + - kubeflow.org + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + resources: + - notebooks diff --git a/pkg/feature/templates/servicemesh/authorino/rbac/role-binding.tmpl b/pkg/feature/templates/servicemesh/authorino/rbac/role-binding.tmpl new file mode 100644 index 00000000000..70d1f9316ed --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/rbac/role-binding.tmpl @@ -0,0 +1,12 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: auth-service + namespace: {{ .Auth.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: auth-service +subjects: + - kind: ServiceAccount + name: auth-service diff --git a/pkg/feature/templates/servicemesh/authorino/rbac/role.tmpl b/pkg/feature/templates/servicemesh/authorino/rbac/role.tmpl new file mode 100644 index 00000000000..9afc140f814 --- /dev/null +++ b/pkg/feature/templates/servicemesh/authorino/rbac/role.tmpl @@ -0,0 +1,125 @@ +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: auth-service + namespace: {{ .Auth.Namespace }} +rules: + - apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - get + - list + - watch + - apiGroups: + - kfdef.apps.kubeflow.org + resources: + - kfdefs + verbs: + - get + - list + - watch + - apiGroups: + - batch + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + resources: + - cronjobs + - jobs + - jobs/status + - apiGroups: + - image.openshift.io + verbs: + - create + - get + - list + - update + - patch + - delete + resources: + - imagestreams + - apiGroups: + - build.openshift.io + verbs: + - get + - list + - watch + - create + - patch + - delete + resources: + - builds + - buildconfigs + - buildconfigs/instantiate + - apiGroups: + - apps + verbs: + - patch + - update + resources: + - deployments + - apiGroups: + - apps.openshift.io + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + resources: + - deploymentconfigs + - deploymentconfigs/instantiate + - apiGroups: + - opendatahub.io + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + resources: + - odhdashboardconfigs + - apiGroups: + - kubeflow.org + verbs: + - get + - list + - watch + - create + - update + - patch + - delete + resources: + - notebooks + - verbs: + - get + - list + apiGroups: + - dashboard.opendatahub.io + resources: + - odhapplications + - verbs: + - get + - list + apiGroups: + - dashboard.opendatahub.io + resources: + - odhdocuments + - verbs: + - get + - list + apiGroups: + - console.openshift.io + resources: + - odhquickstarts diff --git a/pkg/feature/templates/servicemesh/control-plane/base/control-plane-ingress.patch.tmpl b/pkg/feature/templates/servicemesh/control-plane/base/control-plane-ingress.patch.tmpl new file mode 100644 index 00000000000..595f0ed98e7 --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/base/control-plane-ingress.patch.tmpl @@ -0,0 +1,23 @@ +apiVersion: maistra.io/v2 +kind: ServiceMeshControlPlane +metadata: + name: {{ .Mesh.Name }} + namespace: {{ .Mesh.Namespace }} +spec: + proxy: + injection: + autoInject: false + gateways: + openshiftRoute: + enabled: false + ingress: + volumes: + - volume: + secret: + secretName: {{ .AppNamespace }}-oauth2-tokens + optional: true + volumeMount: + name: {{ .AppNamespace }}-oauth2-tokens + mountPath: "/etc/istio/{{ .AppNamespace }}-oauth2-tokens" + readOnly: true + diff --git a/pkg/feature/templates/servicemesh/control-plane/components/dashboard/gateway.tmpl b/pkg/feature/templates/servicemesh/control-plane/components/dashboard/gateway.tmpl new file mode 100644 index 00000000000..d5a13c188e5 --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/components/dashboard/gateway.tmpl @@ -0,0 +1,24 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: odh-gateway + namespace: {{ .AppNamespace }} +spec: + selector: + istio: ingressgateway + servers: + - port: + number: 443 + name: https + protocol: HTTPS + tls: + mode: SIMPLE + credentialName: {{ .Mesh.Certificate.Name }} + hosts: + - "{{ .AppNamespace }}.{{ .Domain }}" + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "{{ .AppNamespace }}.{{ .Domain }}" \ No newline at end of file diff --git a/pkg/feature/templates/servicemesh/control-plane/components/dashboard/virtual-service.tmpl b/pkg/feature/templates/servicemesh/control-plane/components/dashboard/virtual-service.tmpl new file mode 100644 index 00000000000..b05f1152d75 --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/components/dashboard/virtual-service.tmpl @@ -0,0 +1,27 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: odh-dashboard + namespace: {{ .AppNamespace }} +spec: + gateways: + - odh-gateway + hosts: + - "{{ .AppNamespace }}.{{ .Domain }}" + http: + - match: + - uri: + # match host.com/notebook/ns/username/logout + regex: "^/notebook/.*/.*/logout" + route: + - destination: + host: odh-dashboard + port: + number: 80 + rewrite: + uri: "/oauth/sign_out" + - route: + - destination: + host: odh-dashboard + port: + number: 80 diff --git a/pkg/feature/templates/servicemesh/control-plane/control-plane-minimal.tmpl b/pkg/feature/templates/servicemesh/control-plane/control-plane-minimal.tmpl new file mode 100644 index 00000000000..bfe3b64af5b --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/control-plane-minimal.tmpl @@ -0,0 +1,18 @@ +apiVersion: maistra.io/v2 +kind: ServiceMeshControlPlane +metadata: + name: {{ .Mesh.Name }} + namespace: {{ .Mesh.Namespace }} +spec: + tracing: + type: None + addons: + prometheus: + enabled: false + grafana: + enabled: false + jaeger: + name: jaeger + kiali: + name: kiali + enabled: false diff --git a/pkg/feature/templates/servicemesh/control-plane/filters/filter-oauth2.tmpl b/pkg/feature/templates/servicemesh/control-plane/filters/filter-oauth2.tmpl new file mode 100644 index 00000000000..8139730989b --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/filters/filter-oauth2.tmpl @@ -0,0 +1,90 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: oauth2-ingress + namespace: {{ .Mesh.Namespace }} + labels: + name: oauth2-envoy + app: odh +spec: + priority: 10 + workloadSelector: + labels: + istio: ingressgateway # this is all or nothing option - we should narrow to more sane labels / ns? + configPatches: + - applyTo: CLUSTER + match: + cluster: + service: oauth-openshift + patch: + operation: ADD + value: + name: oauth-openshift + dns_lookup_family: V4_ONLY + type: LOGICAL_DNS + connect_timeout: 10s + lb_policy: ROUND_ROBIN + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + sni: {{ .OAuth.Route }} + load_assignment: + cluster_name: oauth-openshift + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: {{ .OAuth.Route }} + port_value: {{ .OAuth.Port }} + - applyTo: HTTP_FILTER + match: + context: GATEWAY + listener: + filterChain: + filter: + name: "envoy.filters.network.http_connection_manager" + # subFilter: + # name: "envoy.filters.http.jwt_authn" + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.oauth2 + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.oauth2.v3.OAuth2 + config: + token_endpoint: + cluster: oauth-openshift + uri: {{ .OAuth.TokenEndpoint }} + timeout: 3s + authorization_endpoint: {{ .OAuth.AuthzEndpoint }} + redirect_uri: "https://%REQ(:authority)%/callback" + redirect_path_matcher: + path: + exact: /callback + signout_path: + path: + exact: /oauth/sign_out + credentials: + client_id: {{ .AppNamespace }}-oauth2-client + token_secret: + name: token + sds_config: + path: "/etc/istio/{{ .AppNamespace }}-oauth2-tokens/token-secret.yaml" + hmac_secret: + name: hmac + sds_config: + path: "/etc/istio/{{ .AppNamespace }}-oauth2-tokens/hmac-secret.yaml" + auth_scopes: + - user:full + forward_bearer_token: true + # FIXME: This always bypasses authN for ModelMesh. It should not if + # the model is flagged to be auth-protected. + pass_through_matcher: + - name: ":path" + prefix_match: "/modelmesh/" + - name: ":path" + prefix_match: "/vmodel-route/" + - name: content-type + prefix_match: application/grpc diff --git a/pkg/feature/templates/servicemesh/control-plane/filters/filter-propagate-token.tmpl b/pkg/feature/templates/servicemesh/control-plane/filters/filter-propagate-token.tmpl new file mode 100644 index 00000000000..173dbf96c92 --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/filters/filter-propagate-token.tmpl @@ -0,0 +1,53 @@ +apiVersion: networking.istio.io/v1alpha3 +kind: EnvoyFilter +metadata: + name: propagate-token + namespace: {{ .Mesh.Namespace }} + labels: + name: oauth2-envoy + app: odh +spec: + workloadSelector: + labels: + istio: ingressgateway + priority: 20 # after oauth2 filter + configPatches: + - applyTo: HTTP_FILTER + match: + context: GATEWAY + listener: + filterChain: + filter: + name: envoy.filters.network.http_connection_manager + patch: + operation: INSERT_BEFORE + value: + name: envoy.filters.http.lua + typed_config: + "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua + inlineCode: | + function envoy_on_request(request_handle) + local headers = request_handle:headers() + if not headers then + return + end + + local bearer_token = nil + local cookies = headers:get("cookie") + -- Extracting token value from the header which is in form of + -- cookie: OauthHMAC=HMAC_VALUE;OauthExpires=1679994106; BearerToken=TOKEN_VALUE` + if cookies then + for cookie in cookies:gmatch("([^;]+)") do + local name, value = cookie:match("^%s*([^=]+)=(.*)$") + if name and value and name:lower() == "bearertoken" then + bearer_token = value + break + end + end + end + + -- Set the "x-forwarded-access-token" header if the bearer token was found + if bearer_token then + headers:add("x-forwarded-access-token", bearer_token) + end + end diff --git a/pkg/feature/templates/servicemesh/control-plane/namespace.patch.tmpl b/pkg/feature/templates/servicemesh/control-plane/namespace.patch.tmpl new file mode 100644 index 00000000000..98b12916ebd --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/namespace.patch.tmpl @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: {{ .AppNamespace }} + annotations: + opendatahub.io/service-mesh: "true" + \ No newline at end of file diff --git a/pkg/feature/templates/servicemesh/control-plane/oauth/auth-policy.tmpl b/pkg/feature/templates/servicemesh/control-plane/oauth/auth-policy.tmpl new file mode 100644 index 00000000000..486f55b3d7b --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/oauth/auth-policy.tmpl @@ -0,0 +1,26 @@ +kind: AuthorizationPolicy +apiVersion: security.istio.io/v1beta1 +metadata: + name: {{ .AppNamespace }}-odh-auth-policy + namespace: {{ .Mesh.Namespace }} +spec: + selector: + matchLabels: + app: istio-ingressgateway + action: CUSTOM + provider: + name: {{ .AppNamespace }}-odh-auth-provider + rules: + - to: + - operation: + notPaths: # todo: see if this is necessary + - "/auth/*" + - "/metrics/*" + # FIXME: This always bypasses authZ for ModelMesh. It should not if + # the model is flagged to be auth-protected. + - "/modelmesh/*" + - "/vmodel-route/*" + when: + - key: request.headers[Content-Type] + notValues: + - "application/grpc*" diff --git a/pkg/feature/templates/servicemesh/control-plane/oauth/oauth-client.tmpl b/pkg/feature/templates/servicemesh/control-plane/oauth/oauth-client.tmpl new file mode 100644 index 00000000000..5ca66161d24 --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/oauth/oauth-client.tmpl @@ -0,0 +1,8 @@ +kind: OAuthClient +apiVersion: oauth.openshift.io/v1 +metadata: + name: {{ .AppNamespace }}-oauth2-client +secret: "{{ .OAuth.ClientSecret }}" +redirectURIs: +- "https://{{ .AppNamespace }}.{{ .Domain}}" +grantMethod: prompt diff --git a/pkg/feature/templates/servicemesh/control-plane/routing/gateway-route.tmpl b/pkg/feature/templates/servicemesh/control-plane/routing/gateway-route.tmpl new file mode 100644 index 00000000000..b3f9a2ffe4d --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/routing/gateway-route.tmpl @@ -0,0 +1,21 @@ +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: {{ .AppNamespace }}-odh-gateway + namespace: {{ .Mesh.Namespace }} + labels: + app: odh-dashboard + app.kubernetes.io/part-of: odh-dashboard + maistra.io/gateway-name: {{ .AppNamespace }}-odh-gateway + maistra.io/gateway-namespace: {{ .AppNamespace }} +spec: + host: {{ .AppNamespace }}.{{ .Domain }} + to: + kind: Service + name: istio-ingressgateway + weight: 100 + port: + targetPort: https + tls: + termination: passthrough + wildcardPolicy: None diff --git a/pkg/feature/templates/servicemesh/control-plane/smm.tmpl b/pkg/feature/templates/servicemesh/control-plane/smm.tmpl new file mode 100644 index 00000000000..7dd5cfc5157 --- /dev/null +++ b/pkg/feature/templates/servicemesh/control-plane/smm.tmpl @@ -0,0 +1,9 @@ +apiVersion: maistra.io/v1 +kind: ServiceMeshMember +metadata: + name: default + namespace: {{ .AppNamespace }} +spec: + controlPlaneRef: + namespace: {{ .Mesh.Namespace }} + name: {{ .Mesh.Name }} diff --git a/pkg/feature/types.go b/pkg/feature/types.go new file mode 100644 index 00000000000..d1ace6edcf6 --- /dev/null +++ b/pkg/feature/types.go @@ -0,0 +1,27 @@ +package feature + +import ( + v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "strings" +) + +type Spec struct { + *v1.ServiceMeshSpec + OAuth OAuth + AppNamespace string + Domain string + Tracker *v1.FeatureTracker +} + +type OAuth struct { + AuthzEndpoint, + TokenEndpoint, + Route, + Port, + ClientSecret, + Hmac string +} + +func ReplaceChar(s string, oldChar, newChar string) string { + return strings.ReplaceAll(s, oldChar, newChar) +} diff --git a/pkg/gvr/gvr.go b/pkg/gvr/gvr.go new file mode 100644 index 00000000000..93a8bb57a07 --- /dev/null +++ b/pkg/gvr/gvr.go @@ -0,0 +1,35 @@ +package gvr + +import "k8s.io/apimachinery/pkg/runtime/schema" + +var ( + OpenshiftIngress = schema.GroupVersionResource{ + Group: "config.openshift.io", + Version: "v1", + Resource: "ingresses", + } + + ResourceTracker = schema.GroupVersionResource{ + Group: "dscinitialization.opendatahub.io", + Version: "v1", + Resource: "featuretrackers", + } + + ODHDashboardConfigGVR = schema.GroupVersionResource{ + Group: "opendatahub.io", + Version: "v1alpha", + Resource: "odhdashboardconfigs", + } + + SMCP = schema.GroupVersionResource{ + Group: "maistra.io", + Version: "v2", + Resource: "servicemeshcontrolplanes", + } + + OAuthClient = schema.GroupVersionResource{ + Group: "oauth.openshift.io", + Version: "v1", + Resource: "oauthclients", + } +) diff --git a/tests/envtestutil/cleaner.go b/tests/envtestutil/cleaner.go new file mode 100644 index 00000000000..8c7a1a82152 --- /dev/null +++ b/tests/envtestutil/cleaner.go @@ -0,0 +1,131 @@ +package envtestutil + +import ( + "context" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + + . "github.com/onsi/gomega" //nolint +) + +// Cleaner is a struct to perform deletion of resources, +// enforcing removal of finalizers. Otherwise deletion of namespaces wouldn't be possible. +// See: https://book.kubebuilder.io/reference/envtest.html#namespace-usage-limitation +// Based on https://github.com/kubernetes-sigs/controller-runtime/issues/880#issuecomment-749742403 +type Cleaner struct { + clientset *kubernetes.Clientset + client client.Client + timeout, interval time.Duration +} + +func CreateCleaner(c client.Client, config *rest.Config, timeout, interval time.Duration) *Cleaner { + k8sClient, err := kubernetes.NewForConfig(config) + if err != nil { + panic(err) + } + return &Cleaner{ + clientset: k8sClient, + client: c, + timeout: timeout, + interval: interval, + } +} + +func (c *Cleaner) DeleteAll(objects ...client.Object) { + for _, obj := range objects { + Expect(client.IgnoreNotFound(c.client.Delete(context.Background(), obj))).Should(Succeed()) + + if ns, ok := obj.(*corev1.Namespace); ok { + // Normally the kube-controller-manager would handle finalization + // and garbage collection of namespaces, but with envtest, we aren't + // running a kube-controller-manager. Instead, we're going to approximate + // (poorly) the kube-controller-manager by explicitly deleting some + // resources within the namespace and then removing the `kubernetes` + // finalizer from the namespace resource, so it can finish deleting. + // Note that any resources within the namespace that we don't + // successfully delete could reappear if the namespace is ever + // recreated with the same name. + + // Look up all namespaced resources under the discovery API + _, apiResources, err := c.clientset.DiscoveryClient.ServerGroupsAndResources() + Expect(err).ShouldNot(HaveOccurred()) + namespacedGVKs := make(map[string]schema.GroupVersionKind) + for _, apiResourceList := range apiResources { + defaultGV, err := schema.ParseGroupVersion(apiResourceList.GroupVersion) + Expect(err).ShouldNot(HaveOccurred()) + for _, r := range apiResourceList.APIResources { + if !r.Namespaced || strings.Contains(r.Name, "/") { + // skip non-namespaced and subresources + continue + } + gvk := schema.GroupVersionKind{ + Group: defaultGV.Group, + Version: defaultGV.Version, + Kind: r.Kind, + } + if r.Group != "" { + gvk.Group = r.Group + } + if r.Version != "" { + gvk.Version = r.Version + } + namespacedGVKs[gvk.String()] = gvk + } + } + + // Delete all namespaced resources in this namespace + for _, gvk := range namespacedGVKs { + var u unstructured.Unstructured + u.SetGroupVersionKind(gvk) + err := c.client.DeleteAllOf(context.Background(), &u, client.InNamespace(ns.Name)) + Expect(client.IgnoreNotFound(ignoreMethodNotAllowed(err))).ShouldNot(HaveOccurred()) + } + + Eventually(func() error { + key := client.ObjectKeyFromObject(ns) + + if err := c.client.Get(context.Background(), key, ns); err != nil { + return client.IgnoreNotFound(err) + } + // remove `kubernetes` finalizer + const k8s = "kubernetes" + finalizers := []corev1.FinalizerName{} + for _, f := range ns.Spec.Finalizers { + if f != k8s { + finalizers = append(finalizers, f) + } + } + ns.Spec.Finalizers = finalizers + + // We have to use the k8s.io/client-go library here to expose + // ability to patch the /finalize subresource on the namespace + _, err = c.clientset.CoreV1().Namespaces().Finalize(context.Background(), ns, metav1.UpdateOptions{}) + return err + }, c.timeout, c.interval).Should(Succeed()) + } + + Eventually(func() metav1.StatusReason { + key := client.ObjectKeyFromObject(obj) + if err := c.client.Get(context.Background(), key, obj); err != nil { + return apierrors.ReasonForError(err) + } + return "" + }, c.timeout, c.interval).Should(Equal(metav1.StatusReasonNotFound)) + } +} + +func ignoreMethodNotAllowed(err error) error { + if apierrors.ReasonForError(err) == metav1.StatusReasonMethodNotAllowed { + return nil + } + return err +} diff --git a/tests/envtestutil/name_gen.go b/tests/envtestutil/name_gen.go new file mode 100644 index 00000000000..68ccdc00bdb --- /dev/null +++ b/tests/envtestutil/name_gen.go @@ -0,0 +1,59 @@ +package envtestutil + +import ( + "crypto/rand" + "encoding/hex" + "math" + "math/big" +) + +var letters = []rune("abcdefghijklmnopqrstuvwxyz") + +func RandomUUIDName(len int) string { + uuidBytes := make([]byte, len) + _, _ = rand.Read(uuidBytes) + return hex.EncodeToString(uuidBytes)[:len] +} + +func AppendRandomNameTo(prefix string) string { + return ConcatToMax(63, prefix, GenerateString(16)) +} + +// GenerateString generates random alphabetical name which can be used for example as application or namespace name. +// Maximum length is capped at 63 characters. +// +// Don't forget to seed before using this function, e.g. rand.Seed(time.Now().UTC().UnixNano()) +// otherwise you will always get the same value. +func GenerateString(length int) string { + if length == 0 { + return "" + } + + if length > 63 { + length = 63 + } + + b := make([]rune, length) + for i := range b { + ri, _ := rand.Int(rand.Reader, big.NewInt(int64(len(letters)))) + b[i] = letters[ri.Int64()] + } + + return string(b) +} + +// ConcatToMax will cut each section to length based on number of sections to not go beyond max and separate the sections with -. +func ConcatToMax(max int, sections ...string) string { + sectionLength := (max - len(sections) - 1) / len(sections) + name := "" + + for i, section := range sections { + s := section[:int32(math.Min(float64(len(section)), float64(sectionLength)))] + name = name + "-" + s + if i+1 != len(sections) { + sectionLength = (max - len(name) - 1) / (len(sections) - (i + 1)) + } + } + + return name[1:] +} diff --git a/controllers/test/envtest_setup.go b/tests/envtestutil/utils.go similarity index 94% rename from controllers/test/envtest_setup.go rename to tests/envtestutil/utils.go index f786952c03e..3a6d2396479 100644 --- a/controllers/test/envtest_setup.go +++ b/tests/envtestutil/utils.go @@ -1,4 +1,4 @@ -package controllers_test +package envtestutil import ( "fmt" @@ -11,15 +11,19 @@ func FindProjectRoot() (string, error) { if err != nil { return "", err } + for { if _, err := os.Stat(filepath.Join(currentDir, "go.mod")); err == nil { return filepath.FromSlash(currentDir), nil } + parentDir := filepath.Dir(currentDir) if parentDir == currentDir { break } + currentDir = parentDir } + return "", fmt.Errorf("project root not found") } diff --git a/tests/integration/servicemesh/crd/servicemeshcontrolplanes.crd.yaml b/tests/integration/servicemesh/crd/servicemeshcontrolplanes.crd.yaml new file mode 100644 index 00000000000..ecf91cb49cb --- /dev/null +++ b/tests/integration/servicemesh/crd/servicemeshcontrolplanes.crd.yaml @@ -0,0 +1,10237 @@ + +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + labels: + maistra-version: 2.4.2 + annotations: + service.beta.openshift.io/inject-cabundle: "true" + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: servicemeshcontrolplanes.maistra.io +spec: + group: maistra.io + names: + categories: + - maistra-io + kind: ServiceMeshControlPlane + listKind: ServiceMeshControlPlaneList + plural: servicemeshcontrolplanes + shortNames: + - smcp + singular: servicemeshcontrolplane + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: How many of the total number of components are ready + jsonPath: .status.annotations.readyComponentCount + name: Ready + type: string + - description: Whether or not the control plane installation is up to date. + jsonPath: .status.conditions[?(@.type=="Reconciled")].reason + name: Status + type: string + - description: The configuration template to use as the base. + jsonPath: .status.lastAppliedConfiguration.template + name: Template + type: string + - description: The actual current version of the control plane installation. + jsonPath: .status.lastAppliedConfiguration.version + name: Version + type: string + - description: The age of the object + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: The image hub used as the base for all component images. + jsonPath: .status.lastAppliedConfiguration.istio.global.hub + name: Image HUB + priority: 1 + type: string + name: v1 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + istio: + type: object + x-kubernetes-preserve-unknown-fields: true + networkType: + type: string + profiles: + items: + type: string + type: array + template: + type: string + threeScale: + type: object + x-kubernetes-preserve-unknown-fields: true + version: + type: string + type: object + status: + nullable: true + properties: + annotations: + additionalProperties: + type: string + type: object + components: + items: + properties: + children: + items: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + type: object + type: array + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + resource: + type: string + type: object + type: array + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + lastAppliedConfiguration: + properties: + istio: + type: object + x-kubernetes-preserve-unknown-fields: true + networkType: + type: string + profiles: + items: + type: string + type: array + template: + type: string + threeScale: + type: object + x-kubernetes-preserve-unknown-fields: true + version: + type: string + type: object + observedGeneration: + format: int64 + type: integer + reconciledVersion: + type: string + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - description: How many of the total number of components are ready + jsonPath: .status.annotations.readyComponentCount + name: Ready + type: string + - description: Whether or not the control plane installation is up to date and ready to handle requests. + jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Status + type: string + - description: The configuration profiles applied to the configuration. + jsonPath: .status.appliedSpec.profiles + name: Profiles + type: string + - description: The actual current version of the control plane installation. + jsonPath: .status.chartVersion + name: Version + type: string + - description: The age of the object + jsonPath: .metadata.creationTimestamp + name: Age + type: date + - description: The image registry used as the base for all component images. + jsonPath: .status.appliedSpec.runtime.defaults.container.registry + name: Image Registry + priority: 1 + type: string + name: v2 + schema: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + addons: + properties: + 3scale: + properties: + backend: + properties: + cache_flush_interval: + format: int32 + type: integer + enable_cache: + type: boolean + policy_fail_closed: + type: boolean + type: object + client: + properties: + allow_insecure_connections: + type: boolean + timeout: + format: int32 + type: integer + type: object + enabled: + type: boolean + grpc: + properties: + max_conn_timeout: + format: int32 + type: integer + type: object + listen_addr: + format: int32 + type: integer + log_grpc: + type: boolean + log_json: + type: boolean + log_level: + type: string + metrics: + properties: + port: + format: int32 + type: integer + report: + type: boolean + type: object + system: + properties: + cache_max_size: + format: int64 + type: integer + cache_refresh_interval: + format: int32 + type: integer + cache_refresh_retries: + format: int32 + type: integer + cache_ttl: + format: int32 + type: integer + type: object + type: object + grafana: + properties: + address: + type: string + enabled: + type: boolean + install: + properties: + config: + properties: + env: + additionalProperties: + type: string + type: object + envSecrets: + additionalProperties: + type: string + type: object + type: object + persistence: + properties: + accessMode: + type: string + capacity: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enabled: + type: boolean + storageClassName: + type: string + type: object + security: + properties: + enabled: + type: boolean + passphraseKey: + type: string + secretName: + type: string + usernameKey: + type: string + type: object + selfManaged: + type: boolean + service: + properties: + ingress: + properties: + contextPath: + type: string + enabled: + type: boolean + hosts: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + tls: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodePort: + format: int32 + type: integer + type: object + type: object + type: object + jaeger: + properties: + install: + properties: + ingress: + properties: + enabled: + type: boolean + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object + storage: + properties: + elasticsearch: + properties: + indexCleaner: + type: object + x-kubernetes-preserve-unknown-fields: true + nodeCount: + format: int32 + type: integer + redundancyPolicy: + type: string + storage: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + memory: + properties: + maxTraces: + format: int64 + type: integer + type: object + type: + type: string + type: object + type: object + name: + type: string + type: object + kiali: + properties: + enabled: + type: boolean + install: + properties: + dashboard: + properties: + enableGrafana: + type: boolean + enablePrometheus: + type: boolean + enableTracing: + type: boolean + viewOnly: + type: boolean + type: object + deployment: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + service: + properties: + ingress: + properties: + contextPath: + type: string + enabled: + type: boolean + hosts: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + tls: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodePort: + format: int32 + type: integer + type: object + type: object + name: + type: string + type: object + prometheus: + properties: + address: + type: string + enabled: + type: boolean + install: + properties: + retention: + type: string + scrapeInterval: + type: string + selfManaged: + type: boolean + service: + properties: + ingress: + properties: + contextPath: + type: string + enabled: + type: boolean + hosts: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + tls: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodePort: + format: int32 + type: integer + type: object + useTLS: + type: boolean + type: object + metricsExpiryDuration: + type: string + scrape: + type: boolean + type: object + stackdriver: + properties: + telemetry: + properties: + accessLogging: + properties: + enabled: + type: boolean + logWindowDuration: + type: string + type: object + auth: + properties: + apiKey: + type: string + appCredentials: + type: boolean + serviceAccountPath: + type: string + type: object + configOverride: + type: object + x-kubernetes-preserve-unknown-fields: true + enableContextGraph: + type: boolean + enableLogging: + type: boolean + enableMetrics: + type: boolean + enabled: + type: boolean + type: object + tracer: + properties: + debug: + type: boolean + maxNumberOfAnnotations: + format: int64 + type: integer + maxNumberOfAttributes: + format: int64 + type: integer + maxNumberOfMessageEvents: + format: int64 + type: integer + type: object + type: object + type: object + cluster: + properties: + meshExpansion: + properties: + enabled: + type: boolean + ilbGateway: + properties: + enabled: + type: boolean + namespace: + type: string + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + type: object + multiCluster: + properties: + enabled: + type: boolean + meshNetworks: + additionalProperties: + properties: + endpoints: + items: + properties: + fromCIDR: + type: string + fromRegistry: + type: string + type: object + type: array + gateways: + items: + properties: + address: + type: string + port: + format: int32 + type: integer + registryServiceName: + type: string + service: + type: string + type: object + type: array + type: object + type: object + type: object + name: + type: string + network: + type: string + type: object + gateways: + properties: + additionalEgress: + additionalProperties: + properties: + enabled: + type: boolean + namespace: + type: string + requestedNetworkView: + items: + type: string + type: array + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + type: object + additionalIngress: + additionalProperties: + properties: + enabled: + type: boolean + namespace: + type: string + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + sds: + properties: + enabled: + type: boolean + runtime: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + type: object + egress: + properties: + enabled: + type: boolean + namespace: + type: string + requestedNetworkView: + items: + type: string + type: array + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + enabled: + type: boolean + ingress: + properties: + enabled: + type: boolean + ingress: + type: boolean + meshExpansionPorts: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + namespace: + type: string + routeConfig: + properties: + enabled: + type: boolean + type: object + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + sds: + properties: + enabled: + type: boolean + runtime: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + openshiftRoute: + properties: + enabled: + type: boolean + type: object + type: object + general: + properties: + logging: + properties: + componentLevels: + additionalProperties: + type: string + type: object + logAsJSON: + type: boolean + type: object + validationMessages: + type: boolean + type: object + meshConfig: + properties: + discoverySelectors: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + extensionProviders: + items: + properties: + envoyExtAuthzHttp: + properties: + failOpen: + type: boolean + includeAdditionalHeadersInCheck: + additionalProperties: + type: string + type: object + includeRequestBodyInCheck: + properties: + allowPartialMessage: + type: boolean + maxRequestBytes: + format: int64 + type: integer + packAsBytes: + type: boolean + type: object + includeRequestHeadersInCheck: + items: + type: string + type: array + pathPrefix: + type: string + port: + format: int64 + type: integer + service: + type: string + statusOnError: + type: string + timeout: + type: string + required: + - port + - service + type: object + name: + type: string + prometheus: + type: object + required: + - name + type: object + type: array + type: object + mode: + enum: + - MultiTenant + - ClusterWide + type: string + policy: + properties: + mixer: + properties: + adapters: + properties: + kubernetesenv: + type: boolean + useAdapterCRDs: + type: boolean + type: object + enableChecks: + type: boolean + failOpen: + type: boolean + sessionAffinity: + type: boolean + type: object + remote: + properties: + address: + type: string + createService: + type: boolean + enableChecks: + type: boolean + failOpen: + type: boolean + type: object + type: + type: string + type: object + profiles: + items: + type: string + type: array + proxy: + properties: + accessLogging: + properties: + envoyService: + properties: + address: + type: string + enabled: + type: boolean + tcpKeepalive: + properties: + interval: + type: string + probes: + format: int32 + type: integer + time: + type: string + type: object + tlsSettings: + properties: + caCertificates: + type: string + clientCertificate: + type: string + mode: + type: string + privateKey: + type: string + sni: + type: string + subjectAltNames: + items: + type: string + type: array + type: object + type: object + file: + properties: + encoding: + type: string + format: + type: string + name: + type: string + type: object + type: object + adminPort: + format: int32 + type: integer + concurrency: + format: int32 + type: integer + envoyMetricsService: + properties: + address: + type: string + enabled: + type: boolean + tcpKeepalive: + properties: + interval: + type: string + probes: + format: int32 + type: integer + time: + type: string + type: object + tlsSettings: + properties: + caCertificates: + type: string + clientCertificate: + type: string + mode: + type: string + privateKey: + type: string + sni: + type: string + subjectAltNames: + items: + type: string + type: array + type: object + type: object + injection: + properties: + alwaysInjectSelector: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + autoInject: + type: boolean + injectedAnnotations: + additionalProperties: + type: string + type: object + neverInjectSelector: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + type: object + logging: + properties: + componentLevels: + additionalProperties: + type: string + type: object + level: + type: string + type: object + networking: + properties: + clusterDomain: + type: string + connectionTimeout: + type: string + dns: + properties: + refreshRate: + type: string + searchSuffixes: + items: + type: string + type: array + type: object + initialization: + properties: + initContainer: + properties: + runtime: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: object + type: + type: string + type: object + maxConnectionAge: + type: string + protocol: + properties: + autoDetect: + properties: + inbound: + type: boolean + outbound: + type: boolean + timeout: + type: string + type: object + type: object + trafficControl: + properties: + inbound: + properties: + excludedPorts: + items: + format: int32 + type: integer + type: array + includedPorts: + items: + type: string + type: array + interceptionMode: + type: string + type: object + outbound: + properties: + excludedIPRanges: + items: + type: string + type: array + excludedPorts: + items: + format: int32 + type: integer + type: array + includedIPRanges: + items: + type: string + type: array + policy: + type: string + type: object + type: object + type: object + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + readiness: + properties: + failureThreshold: + format: int32 + type: integer + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + rewriteApplicationProbes: + type: boolean + statusPort: + format: int32 + type: integer + type: object + type: object + type: object + runtime: + properties: + components: + additionalProperties: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + type: object + defaults: + properties: + container: + properties: + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + podDisruption: + properties: + enabled: + type: boolean + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + minAvailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: object + pod: + properties: + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + type: object + security: + properties: + certificateAuthority: + properties: + cert-manager: + properties: + address: + type: string + pilotSecretName: + type: string + rootCAConfigMapName: + type: string + type: object + custom: + properties: + address: + type: string + type: object + istiod: + properties: + privateKey: + properties: + rootCADir: + type: string + type: object + selfSigned: + properties: + checkPeriod: + type: string + enableJitter: + type: boolean + gracePeriod: + type: string + ttl: + type: string + type: object + type: + type: string + workloadCertTTLDefault: + type: string + workloadCertTTLMax: + type: string + type: object + type: + type: string + type: object + controlPlane: + properties: + certProvider: + type: string + mtls: + type: boolean + tls: + properties: + cipherSuites: + items: + type: string + type: array + ecdhCurves: + items: + type: string + type: array + maxProtocolVersion: + type: string + minProtocolVersion: + type: string + type: object + type: object + dataPlane: + properties: + automtls: + type: boolean + mtls: + type: boolean + type: object + identity: + properties: + thirdParty: + properties: + audience: + type: string + issuer: + type: string + type: object + type: + type: string + type: object + jwksResolverCA: + type: string + manageNetworkPolicy: + type: boolean + trust: + properties: + additionalDomains: + items: + type: string + type: array + domain: + type: string + type: object + type: object + techPreview: + type: object + x-kubernetes-preserve-unknown-fields: true + telemetry: + properties: + mixer: + properties: + adapters: + properties: + kubernetesenv: + type: boolean + stdio: + properties: + enabled: + type: boolean + outputAsJSON: + type: boolean + type: object + useAdapterCRDs: + type: boolean + type: object + batching: + properties: + maxEntries: + format: int32 + type: integer + maxTime: + type: string + type: object + loadshedding: + properties: + latencyThreshold: + type: string + mode: + type: string + type: object + sessionAffinity: + type: boolean + type: object + remote: + properties: + address: + type: string + batching: + properties: + maxEntries: + format: int32 + type: integer + maxTime: + type: string + type: object + createService: + type: boolean + type: object + type: + type: string + type: object + tracing: + properties: + sampling: + format: int32 + maximum: 10000 + minimum: 0 + type: integer + type: + type: string + type: object + version: + type: string + type: object + status: + properties: + annotations: + additionalProperties: + type: string + type: object + appliedSpec: + properties: + addons: + properties: + 3scale: + properties: + backend: + properties: + cache_flush_interval: + format: int32 + type: integer + enable_cache: + type: boolean + policy_fail_closed: + type: boolean + type: object + client: + properties: + allow_insecure_connections: + type: boolean + timeout: + format: int32 + type: integer + type: object + enabled: + type: boolean + grpc: + properties: + max_conn_timeout: + format: int32 + type: integer + type: object + listen_addr: + format: int32 + type: integer + log_grpc: + type: boolean + log_json: + type: boolean + log_level: + type: string + metrics: + properties: + port: + format: int32 + type: integer + report: + type: boolean + type: object + system: + properties: + cache_max_size: + format: int64 + type: integer + cache_refresh_interval: + format: int32 + type: integer + cache_refresh_retries: + format: int32 + type: integer + cache_ttl: + format: int32 + type: integer + type: object + type: object + grafana: + properties: + address: + type: string + enabled: + type: boolean + install: + properties: + config: + properties: + env: + additionalProperties: + type: string + type: object + envSecrets: + additionalProperties: + type: string + type: object + type: object + persistence: + properties: + accessMode: + type: string + capacity: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + enabled: + type: boolean + storageClassName: + type: string + type: object + security: + properties: + enabled: + type: boolean + passphraseKey: + type: string + secretName: + type: string + usernameKey: + type: string + type: object + selfManaged: + type: boolean + service: + properties: + ingress: + properties: + contextPath: + type: string + enabled: + type: boolean + hosts: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + tls: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodePort: + format: int32 + type: integer + type: object + type: object + type: object + jaeger: + properties: + install: + properties: + ingress: + properties: + enabled: + type: boolean + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + type: object + storage: + properties: + elasticsearch: + properties: + indexCleaner: + type: object + x-kubernetes-preserve-unknown-fields: true + nodeCount: + format: int32 + type: integer + redundancyPolicy: + type: string + storage: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + memory: + properties: + maxTraces: + format: int64 + type: integer + type: object + type: + type: string + type: object + type: object + name: + type: string + type: object + kiali: + properties: + enabled: + type: boolean + install: + properties: + dashboard: + properties: + enableGrafana: + type: boolean + enablePrometheus: + type: boolean + enableTracing: + type: boolean + viewOnly: + type: boolean + type: object + deployment: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + service: + properties: + ingress: + properties: + contextPath: + type: string + enabled: + type: boolean + hosts: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + tls: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodePort: + format: int32 + type: integer + type: object + type: object + name: + type: string + type: object + prometheus: + properties: + address: + type: string + enabled: + type: boolean + install: + properties: + retention: + type: string + scrapeInterval: + type: string + selfManaged: + type: boolean + service: + properties: + ingress: + properties: + contextPath: + type: string + enabled: + type: boolean + hosts: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + tls: + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodePort: + format: int32 + type: integer + type: object + useTLS: + type: boolean + type: object + metricsExpiryDuration: + type: string + scrape: + type: boolean + type: object + stackdriver: + properties: + telemetry: + properties: + accessLogging: + properties: + enabled: + type: boolean + logWindowDuration: + type: string + type: object + auth: + properties: + apiKey: + type: string + appCredentials: + type: boolean + serviceAccountPath: + type: string + type: object + configOverride: + type: object + x-kubernetes-preserve-unknown-fields: true + enableContextGraph: + type: boolean + enableLogging: + type: boolean + enableMetrics: + type: boolean + enabled: + type: boolean + type: object + tracer: + properties: + debug: + type: boolean + maxNumberOfAnnotations: + format: int64 + type: integer + maxNumberOfAttributes: + format: int64 + type: integer + maxNumberOfMessageEvents: + format: int64 + type: integer + type: object + type: object + type: object + cluster: + properties: + meshExpansion: + properties: + enabled: + type: boolean + ilbGateway: + properties: + enabled: + type: boolean + namespace: + type: string + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + type: object + multiCluster: + properties: + enabled: + type: boolean + meshNetworks: + additionalProperties: + properties: + endpoints: + items: + properties: + fromCIDR: + type: string + fromRegistry: + type: string + type: object + type: array + gateways: + items: + properties: + address: + type: string + port: + format: int32 + type: integer + registryServiceName: + type: string + service: + type: string + type: object + type: array + type: object + type: object + type: object + name: + type: string + network: + type: string + type: object + gateways: + properties: + additionalEgress: + additionalProperties: + properties: + enabled: + type: boolean + namespace: + type: string + requestedNetworkView: + items: + type: string + type: array + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + type: object + additionalIngress: + additionalProperties: + properties: + enabled: + type: boolean + namespace: + type: string + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + sds: + properties: + enabled: + type: boolean + runtime: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + type: object + egress: + properties: + enabled: + type: boolean + namespace: + type: string + requestedNetworkView: + items: + type: string + type: array + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + enabled: + type: boolean + ingress: + properties: + enabled: + type: boolean + ingress: + type: boolean + meshExpansionPorts: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + namespace: + type: string + routeConfig: + properties: + enabled: + type: boolean + type: object + routerMode: + type: string + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + sds: + properties: + enabled: + type: boolean + runtime: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: object + service: + properties: + clusterIP: + type: string + externalIPs: + items: + type: string + type: array + externalName: + type: string + externalTrafficPolicy: + type: string + healthCheckNodePort: + format: int32 + type: integer + ipFamily: + type: string + loadBalancerIP: + type: string + loadBalancerSourceRanges: + items: + type: string + type: array + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array + x-kubernetes-list-map-keys: + - port + x-kubernetes-list-type: map + publishNotReadyAddresses: + type: boolean + selector: + additionalProperties: + type: string + type: object + sessionAffinity: + type: string + sessionAffinityConfig: + properties: + clientIP: + properties: + timeoutSeconds: + format: int32 + type: integer + type: object + type: object + topologyKeys: + items: + type: string + type: array + type: + type: string + type: object + volumes: + items: + properties: + volume: + properties: + configMap: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + name: + type: string + optional: + type: boolean + type: object + secret: + properties: + defaultMode: + format: int32 + type: integer + items: + items: + properties: + key: + type: string + mode: + format: int32 + type: integer + path: + type: string + required: + - key + - path + type: object + type: array + optional: + type: boolean + secretName: + type: string + type: object + type: object + volumeMount: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: object + type: array + type: object + openshiftRoute: + properties: + enabled: + type: boolean + type: object + type: object + general: + properties: + logging: + properties: + componentLevels: + additionalProperties: + type: string + type: object + logAsJSON: + type: boolean + type: object + validationMessages: + type: boolean + type: object + meshConfig: + properties: + discoverySelectors: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + extensionProviders: + items: + properties: + envoyExtAuthzHttp: + properties: + failOpen: + type: boolean + includeAdditionalHeadersInCheck: + additionalProperties: + type: string + type: object + includeRequestBodyInCheck: + properties: + allowPartialMessage: + type: boolean + maxRequestBytes: + format: int64 + type: integer + packAsBytes: + type: boolean + type: object + includeRequestHeadersInCheck: + items: + type: string + type: array + pathPrefix: + type: string + port: + format: int64 + type: integer + service: + type: string + statusOnError: + type: string + timeout: + type: string + required: + - port + - service + type: object + name: + type: string + prometheus: + type: object + required: + - name + type: object + type: array + type: object + mode: + enum: + - MultiTenant + - ClusterWide + type: string + policy: + properties: + mixer: + properties: + adapters: + properties: + kubernetesenv: + type: boolean + useAdapterCRDs: + type: boolean + type: object + enableChecks: + type: boolean + failOpen: + type: boolean + sessionAffinity: + type: boolean + type: object + remote: + properties: + address: + type: string + createService: + type: boolean + enableChecks: + type: boolean + failOpen: + type: boolean + type: object + type: + type: string + type: object + profiles: + items: + type: string + type: array + proxy: + properties: + accessLogging: + properties: + envoyService: + properties: + address: + type: string + enabled: + type: boolean + tcpKeepalive: + properties: + interval: + type: string + probes: + format: int32 + type: integer + time: + type: string + type: object + tlsSettings: + properties: + caCertificates: + type: string + clientCertificate: + type: string + mode: + type: string + privateKey: + type: string + sni: + type: string + subjectAltNames: + items: + type: string + type: array + type: object + type: object + file: + properties: + encoding: + type: string + format: + type: string + name: + type: string + type: object + type: object + adminPort: + format: int32 + type: integer + concurrency: + format: int32 + type: integer + envoyMetricsService: + properties: + address: + type: string + enabled: + type: boolean + tcpKeepalive: + properties: + interval: + type: string + probes: + format: int32 + type: integer + time: + type: string + type: object + tlsSettings: + properties: + caCertificates: + type: string + clientCertificate: + type: string + mode: + type: string + privateKey: + type: string + sni: + type: string + subjectAltNames: + items: + type: string + type: array + type: object + type: object + injection: + properties: + alwaysInjectSelector: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + autoInject: + type: boolean + injectedAnnotations: + additionalProperties: + type: string + type: object + neverInjectSelector: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + type: object + logging: + properties: + componentLevels: + additionalProperties: + type: string + type: object + level: + type: string + type: object + networking: + properties: + clusterDomain: + type: string + connectionTimeout: + type: string + dns: + properties: + refreshRate: + type: string + searchSuffixes: + items: + type: string + type: array + type: object + initialization: + properties: + initContainer: + properties: + runtime: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + type: object + type: + type: string + type: object + maxConnectionAge: + type: string + protocol: + properties: + autoDetect: + properties: + inbound: + type: boolean + outbound: + type: boolean + timeout: + type: string + type: object + type: object + trafficControl: + properties: + inbound: + properties: + excludedPorts: + items: + format: int32 + type: integer + type: array + includedPorts: + items: + type: string + type: array + interceptionMode: + type: string + type: object + outbound: + properties: + excludedIPRanges: + items: + type: string + type: array + excludedPorts: + items: + format: int32 + type: integer + type: array + includedIPRanges: + items: + type: string + type: array + policy: + type: string + type: object + type: object + type: object + runtime: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + readiness: + properties: + failureThreshold: + format: int32 + type: integer + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + rewriteApplicationProbes: + type: boolean + statusPort: + format: int32 + type: integer + type: object + type: object + type: object + runtime: + properties: + components: + additionalProperties: + properties: + container: + properties: + env: + additionalProperties: + type: string + type: object + imageName: + type: string + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + autoScaling: + properties: + enabled: + type: boolean + maxReplicas: + format: int32 + type: integer + minReplicas: + format: int32 + type: integer + targetCPUUtilizationPercentage: + format: int32 + type: integer + type: object + replicas: + format: int32 + type: integer + strategy: + properties: + rollingUpdate: + properties: + maxSurge: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: + type: string + type: object + type: object + pod: + properties: + affinity: + properties: + nodeAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + preference: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + properties: + nodeSelectorTerms: + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringScheduling: + items: + properties: + key: + type: string + operator: + type: string + topologyKey: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + metadata: + properties: + annotations: + additionalProperties: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + type: object + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + type: object + defaults: + properties: + container: + properties: + imagePullPolicy: + type: string + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + imageRegistry: + type: string + imageTag: + type: string + resources: + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + type: object + deployment: + properties: + podDisruption: + properties: + enabled: + type: boolean + maxUnavailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + minAvailable: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + type: object + type: object + pod: + properties: + nodeSelector: + additionalProperties: + type: string + type: object + priorityClassName: + type: string + tolerations: + items: + properties: + effect: + type: string + key: + type: string + operator: + type: string + tolerationSeconds: + format: int64 + type: integer + value: + type: string + type: object + type: array + type: object + type: object + type: object + security: + properties: + certificateAuthority: + properties: + cert-manager: + properties: + address: + type: string + pilotSecretName: + type: string + rootCAConfigMapName: + type: string + type: object + custom: + properties: + address: + type: string + type: object + istiod: + properties: + privateKey: + properties: + rootCADir: + type: string + type: object + selfSigned: + properties: + checkPeriod: + type: string + enableJitter: + type: boolean + gracePeriod: + type: string + ttl: + type: string + type: object + type: + type: string + workloadCertTTLDefault: + type: string + workloadCertTTLMax: + type: string + type: object + type: + type: string + type: object + controlPlane: + properties: + certProvider: + type: string + mtls: + type: boolean + tls: + properties: + cipherSuites: + items: + type: string + type: array + ecdhCurves: + items: + type: string + type: array + maxProtocolVersion: + type: string + minProtocolVersion: + type: string + type: object + type: object + dataPlane: + properties: + automtls: + type: boolean + mtls: + type: boolean + type: object + identity: + properties: + thirdParty: + properties: + audience: + type: string + issuer: + type: string + type: object + type: + type: string + type: object + jwksResolverCA: + type: string + manageNetworkPolicy: + type: boolean + trust: + properties: + additionalDomains: + items: + type: string + type: array + domain: + type: string + type: object + type: object + techPreview: + type: object + x-kubernetes-preserve-unknown-fields: true + telemetry: + properties: + mixer: + properties: + adapters: + properties: + kubernetesenv: + type: boolean + stdio: + properties: + enabled: + type: boolean + outputAsJSON: + type: boolean + type: object + useAdapterCRDs: + type: boolean + type: object + batching: + properties: + maxEntries: + format: int32 + type: integer + maxTime: + type: string + type: object + loadshedding: + properties: + latencyThreshold: + type: string + mode: + type: string + type: object + sessionAffinity: + type: boolean + type: object + remote: + properties: + address: + type: string + batching: + properties: + maxEntries: + format: int32 + type: integer + maxTime: + type: string + type: object + createService: + type: boolean + type: object + type: + type: string + type: object + tracing: + properties: + sampling: + format: int32 + maximum: 10000 + minimum: 0 + type: integer + type: + type: string + type: object + version: + type: string + type: object + appliedValues: + properties: + istio: + type: object + x-kubernetes-preserve-unknown-fields: true + networkType: + type: string + profiles: + items: + type: string + type: array + template: + type: string + threeScale: + type: object + x-kubernetes-preserve-unknown-fields: true + version: + type: string + type: object + chartVersion: + type: string + components: + items: + properties: + children: + items: + properties: + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + type: object + type: array + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + resource: + type: string + type: object + type: array + conditions: + items: + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + type: string + type: object + type: array + observedGeneration: + format: int64 + type: integer + operatorVersion: + type: string + readiness: + properties: + components: + additionalProperties: + items: + type: string + type: array + type: object + type: object + required: + - readiness + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/tests/integration/servicemesh/crd/test-resource.yaml b/tests/integration/servicemesh/crd/test-resource.yaml new file mode 100644 index 00000000000..02464948226 --- /dev/null +++ b/tests/integration/servicemesh/crd/test-resource.yaml @@ -0,0 +1,17 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: test-resources.openshift.io +spec: + group: openshift.io + versions: + - name: test-version + served: true + storage: true + schema: + openAPIV3Schema: + type: "object" + names: + plural: test-resources + kind: "testCRD" + scope: Namespaced diff --git a/tests/integration/servicemesh/service_mesh_setup_features_int_test.go b/tests/integration/servicemesh/service_mesh_setup_features_int_test.go new file mode 100644 index 00000000000..b195fa139f3 --- /dev/null +++ b/tests/integration/servicemesh/service_mesh_setup_features_int_test.go @@ -0,0 +1,528 @@ +package servicemesh_test + +import ( + "context" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + dscv1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" + "github.com/opendatahub-io/opendatahub-operator/v2/tests/envtestutil" + "io" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/rest" + "os" + "path" + "path/filepath" + "time" +) + +const ( + timeout = 5 * time.Second + interval = 250 * time.Millisecond +) + +var _ = Describe("preconditions", func() { + + Context("namespace existence", func() { + + var ( + objectCleaner *envtestutil.Cleaner + testFeature *feature.Feature + namespace string + ) + + BeforeEach(func() { + objectCleaner = envtestutil.CreateCleaner(envTestClient, envTest.Config, timeout, interval) + + testFeatureName := "test-ns-creation" + namespace = envtestutil.AppendRandomNameTo(testFeatureName) + + dsciSpec := newDSCInitializationSpec(namespace) + var err error + testFeature, err = feature.CreateFeature(testFeatureName). + For(dsciSpec). + UsingConfig(envTest.Config). + Load() + Expect(err).ToNot(HaveOccurred()) + }) + + It("should create namespace if it does not exist", func() { + // given + _, err := getNamespace(namespace) + Expect(errors.IsNotFound(err)).To(BeTrue()) + defer objectCleaner.DeleteAll(&v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}) + + // when + err = feature.CreateNamespace(namespace)(testFeature) + + // then + Expect(err).ToNot(HaveOccurred()) + }) + + It("should not try to create namespace if it does already exist", func() { + // given + ns := createNamespace(namespace) + Expect(envTestClient.Create(context.Background(), ns)).To(Succeed()) + defer objectCleaner.DeleteAll(ns) + + // when + err := feature.CreateNamespace(namespace)(testFeature) + + // then + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("ensuring custom resource definitions are installed", func() { + + var ( + dsciSpec *dscv1.DSCInitializationSpec + verificationFeature *feature.Feature + ) + + BeforeEach(func() { + dsciSpec = newDSCInitializationSpec("default") + }) + + It("should successfully check for existing CRD", func() { + // given example CRD installed into env from /ossm/test/crd/ + name := "test-resources.openshift.io" + + var err error + verificationFeature, err = feature.CreateFeature("CRD verification"). + For(dsciSpec). + UsingConfig(envTest.Config). + PreConditions(feature.EnsureCRDIsInstalled(name)). + Load() + Expect(err).ToNot(HaveOccurred()) + + // when + err = verificationFeature.Apply() + + // then + Expect(err).ToNot(HaveOccurred()) + }) + + It("should fail to check non-existing CRD", func() { + // given + name := "non-existing-resource.non-existing-group.io" + + var err error + verificationFeature, err = feature.CreateFeature("CRD verification"). + For(dsciSpec). + UsingConfig(envTest.Config). + PreConditions(feature.EnsureCRDIsInstalled(name)). + Load() + Expect(err).ToNot(HaveOccurred()) + + // when + err = verificationFeature.Apply() + + // then + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("\"non-existing-resource.non-existing-group.io\" not found")) + }) + }) + +}) + +var _ = Describe("Ensuring service mesh is set up correctly", func() { + + var ( + objectCleaner *envtestutil.Cleaner + dsciSpec *dscv1.DSCInitializationSpec + serviceMeshSpec *dscv1.ServiceMeshSpec + serviceMeshCheck *feature.Feature + name = "test-name" + namespace = "test-namespace" + ) + + BeforeEach(func() { + dsciSpec = newDSCInitializationSpec(namespace) + var err error + serviceMeshSpec = &dsciSpec.ServiceMesh + + serviceMeshSpec.Mesh.Name = name + serviceMeshSpec.Mesh.Namespace = namespace + + serviceMeshCheck, err = feature.CreateFeature("datascience-project-migration"). + For(dsciSpec). + UsingConfig(envTest.Config). + PreConditions(servicemesh.EnsureServiceMeshInstalled).Load() + + Expect(err).ToNot(HaveOccurred()) + + objectCleaner = envtestutil.CreateCleaner(envTestClient, envTest.Config, timeout, interval) + }) + + It("should find installed Service Mesh Control Plane", func() { + ns := createNamespace(namespace) + Expect(envTestClient.Create(context.Background(), ns)).To(Succeed()) + defer objectCleaner.DeleteAll(ns) + + createServiceMeshControlPlane(name, namespace) + + // when + err := serviceMeshCheck.Apply() + + // then + Expect(err).ToNot(HaveOccurred()) + }) + + It("should fail to find Service Mesh Control Plane if not present", func() { + Expect(serviceMeshCheck.Apply()).ToNot(Succeed()) + }) + +}) + +var _ = Describe("Data Science Project Migration", func() { + + var ( + objectCleaner *envtestutil.Cleaner + dsciSpec *dscv1.DSCInitializationSpec + migrationFeature *feature.Feature + ) + + BeforeEach(func() { + objectCleaner = envtestutil.CreateCleaner(envTestClient, envTest.Config, timeout, interval) + + dsciSpec = newDSCInitializationSpec("default") + + var err error + migrationFeature, err = feature.CreateFeature("datascience-project-migration"). + For(dsciSpec). + UsingConfig(envTest.Config). + WithResources(servicemesh.MigratedDataScienceProjects).Load() + + Expect(err).ToNot(HaveOccurred()) + + }) + + It("should migrate single namespace", func() { + // given + dataScienceNs := createDataScienceProject("dsp-01") + regularNs := createNamespace("non-dsp") + Expect(envTestClient.Create(context.Background(), dataScienceNs)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), regularNs)).To(Succeed()) + defer objectCleaner.DeleteAll(dataScienceNs, regularNs) + + // when + Expect(migrationFeature.Apply()).To(Succeed()) + + // then + Eventually(findMigratedNamespaces, timeout, interval).Should( + And( + HaveLen(1), + ContainElement("dsp-01"), + ), + ) + }) + + It("should not migrate any non-datascience namespace", func() { + // given + regularNs := createNamespace("non-dsp") + Expect(envTestClient.Create(context.Background(), regularNs)).To(Succeed()) + defer objectCleaner.DeleteAll(regularNs) + + // when + Expect(migrationFeature.Apply()).To(Succeed()) + + // then + Consistently(findMigratedNamespaces, timeout, interval).Should(BeEmpty()) // we can't wait forever, but this should be good enough ;) + }) + + It("should migrate multiple namespaces", func() { + // given + dataScienceNs01 := createDataScienceProject("dsp-01") + dataScienceNs02 := createDataScienceProject("dsp-02") + dataScienceNs03 := createDataScienceProject("dsp-03") + regularNs := createNamespace("non-dsp") + Expect(envTestClient.Create(context.Background(), dataScienceNs01)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), dataScienceNs02)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), dataScienceNs03)).To(Succeed()) + Expect(envTestClient.Create(context.Background(), regularNs)).To(Succeed()) + defer objectCleaner.DeleteAll(dataScienceNs01, dataScienceNs02, dataScienceNs03, regularNs) + + // when + Expect(migrationFeature.Apply()).To(Succeed()) + + // then + Eventually(findMigratedNamespaces, timeout, interval).Should( + And( + HaveLen(3), + ContainElements("dsp-01", "dsp-02", "dsp-03"), + ), + ) + }) + +}) + +var _ = Describe("Cleanup operations", func() { + + Context("configuring control plane for auth(z)", func() { + + var ( + objectCleaner *envtestutil.Cleaner + dsciSpec *dscv1.DSCInitializationSpec + serviceMeshSpec *dscv1.ServiceMeshSpec + namespace = "test" + name = "minimal" + ) + + BeforeEach(func() { + objectCleaner = envtestutil.CreateCleaner(envTestClient, envTest.Config, timeout, interval) + + dsciSpec = newDSCInitializationSpec(namespace) + + serviceMeshSpec = &dsciSpec.ServiceMesh + + serviceMeshSpec.Mesh.Name = name + serviceMeshSpec.Mesh.Namespace = namespace + }) + + It("should be able to remove mounted secret volumes on cleanup", func() { + // given + ns := createNamespace(namespace) + Expect(envTestClient.Create(context.Background(), ns)).To(Succeed()) + defer objectCleaner.DeleteAll(ns) + + createServiceMeshControlPlane(name, namespace) + + controlPlaneWithSecretVolumes, err := feature.CreateFeature("control-plane-with-secret-volumes"). + For(dsciSpec). + Manifests(fromTestTmpDir(path.Join(feature.ControlPlaneDir, "base/control-plane-ingress.patch.tmpl"))). + UsingConfig(envTest.Config). + Load() + + Expect(err).ToNot(HaveOccurred()) + + // when + Expect(controlPlaneWithSecretVolumes.Apply()).To(Succeed()) + // Testing removal function on its own relying on feature setup + Expect(servicemesh.RemoveTokenVolumes(controlPlaneWithSecretVolumes)).To(Succeed()) + + // then + serviceMeshControlPlane, err := getServiceMeshControlPlane(envTest.Config, namespace, name) + Expect(err).ToNot(HaveOccurred()) + + volumes, found, err := unstructured.NestedSlice(serviceMeshControlPlane.Object, "spec", "gateways", "ingress", "volumes") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(volumes).To(BeEmpty()) + }) + + It("should be able to remove external provider on cleanup", func() { + // given + ns := createNamespace(namespace) + Expect(envTestClient.Create(context.Background(), ns)).To(Succeed()) + defer objectCleaner.DeleteAll(ns) + + serviceMeshSpec.Auth.Namespace = "test-provider" + serviceMeshSpec.Auth.Authorino.Name = "authorino" + + createServiceMeshControlPlane(name, namespace) + + controlPlaneWithExtAuthzProvider, err := feature.CreateFeature("control-plane-with-external-authz-provider"). + For(dsciSpec). + Manifests(fromTestTmpDir(path.Join(feature.AuthDir, "mesh-authz-ext-provider.patch.tmpl"))). + UsingConfig(envTest.Config). + Load() + + Expect(err).ToNot(HaveOccurred()) + + // when + By("verifying extension provider has been added after applying feature", func() { + Expect(controlPlaneWithExtAuthzProvider.Apply()).To(Succeed()) + serviceMeshControlPlane, err := getServiceMeshControlPlane(envTest.Config, namespace, name) + Expect(err).ToNot(HaveOccurred()) + + extensionProviders, found, err := unstructured.NestedSlice(serviceMeshControlPlane.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + + extensionProvider := extensionProviders[0].(map[string]interface{}) + Expect(extensionProvider["name"]).To(Equal("test-odh-auth-provider")) + Expect(extensionProvider["envoyExtAuthzGrpc"].(map[string]interface{})["service"]).To(Equal("authorino-authorino-authorization.test-provider.svc.cluster.local")) + }) + + // then + By("verifying that extension provider has been removed", func() { + err = servicemesh.RemoveExtensionProvider(controlPlaneWithExtAuthzProvider) + serviceMeshControlPlane, err := getServiceMeshControlPlane(envTest.Config, namespace, name) + Expect(err).ToNot(HaveOccurred()) + + extensionProviders, found, err := unstructured.NestedSlice(serviceMeshControlPlane.Object, "spec", "techPreview", "meshConfig", "extensionProviders") + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(extensionProviders).To(BeEmpty()) + }) + + }) + + }) + +}) + +func createServiceMeshControlPlane(name, namespace string) { + serviceMeshControlPlane := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "maistra.io/v2", + "kind": "ServiceMeshControlPlane", + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + }, + "spec": map[string]interface{}{}, + }, + } + Expect(createSMCPInCluster(envTest.Config, serviceMeshControlPlane, namespace)).To(Succeed()) +} + +func createDataScienceProject(name string) *v1.Namespace { + namespace := createNamespace(name) + namespace.Labels = map[string]string{ + "opendatahub.io/dashboard": "true", + } + return namespace +} + +func createNamespace(name string) *v1.Namespace { + return &v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + } +} + +func findMigratedNamespaces() []string { + namespaces := &v1.NamespaceList{} + var ns []string + if err := envTestClient.List(context.Background(), namespaces); err != nil && !errors.IsNotFound(err) { + Fail(err.Error()) + } + for _, namespace := range namespaces.Items { + if _, ok := namespace.ObjectMeta.Annotations["opendatahub.io/service-mesh"]; ok { + ns = append(ns, namespace.Name) + } + } + return ns +} + +func newDSCInitializationSpec(ns string) *dscv1.DSCInitializationSpec { + spec := dscv1.DSCInitializationSpec{} + spec.ApplicationsNamespace = ns + return &spec +} + +// createSMCPInCluster uses dynamic client to create a dummy SMCP resource for testing +func createSMCPInCluster(cfg *rest.Config, smcpObj *unstructured.Unstructured, namespace string) error { + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return err + } + + result, err := dynamicClient.Resource(gvr.SMCP).Namespace(namespace).Create(context.TODO(), smcpObj, metav1.CreateOptions{}) + if err != nil { + return err + } + + statusConditions := []interface{}{ + map[string]interface{}{ + "type": "Ready", + "status": "True", + }, + } + + // Since we don't have actual service mesh operator deployed, we simulate the status + status := map[string]interface{}{ + "conditions": statusConditions, + "readiness": map[string]interface{}{ + "components": map[string]interface{}{ + "pending": []interface{}{}, + "ready": []interface{}{ + "istiod", + "ingress-gateway", + }, + "unready": []interface{}{}, + }, + }, + } + + if err := unstructured.SetNestedField(result.Object, status, "status"); err != nil { + return err + } + + _, err = dynamicClient.Resource(gvr.SMCP).Namespace(namespace).UpdateStatus(context.TODO(), result, metav1.UpdateOptions{}) + if err != nil { + return err + } + + return nil +} + +func getServiceMeshControlPlane(cfg *rest.Config, namespace, name string) (*unstructured.Unstructured, error) { + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return nil, err + } + + smcp, err := dynamicClient.Resource(gvr.SMCP).Namespace(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + + return smcp, nil +} + +func getNamespace(namespace string) (*v1.Namespace, error) { + ns := createNamespace(namespace) + err := envTestClient.Get(context.Background(), types.NamespacedName{Name: namespace}, ns) + + return ns, err +} + +func fromTestTmpDir(fileName string) string { + root, err := envtestutil.FindProjectRoot() + Expect(err).ToNot(HaveOccurred()) + + tmpDir := filepath.Join(os.TempDir(), envtestutil.RandomUUIDName(16)) + if err := os.Mkdir(tmpDir, os.ModePerm); err != nil { + Fail(err.Error()) + } + + src := path.Join(root, "pkg", "feature", fileName) + dest := path.Join(tmpDir, fileName) + if err := copyFile(src, dest); err != nil { + Fail(err.Error()) + } + + return dest +} + +func copyFile(src, dst string) error { + source, err := os.Open(src) + if err != nil { + return err + } + defer source.Close() + + if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return err + } + + destination, err := os.Create(dst) + if err != nil { + return err + } + defer destination.Close() + + _, err = io.Copy(destination, source) + return err +} diff --git a/tests/integration/servicemesh/service_mesh_suite_int_test.go b/tests/integration/servicemesh/service_mesh_suite_int_test.go new file mode 100644 index 00000000000..95570a5daa1 --- /dev/null +++ b/tests/integration/servicemesh/service_mesh_suite_int_test.go @@ -0,0 +1,81 @@ +package servicemesh_test + +import ( + "context" + "fmt" + "github.com/opendatahub-io/opendatahub-operator/v2/tests/envtestutil" + v1 "k8s.io/api/core/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "math/rand" + "path/filepath" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/log/zap" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/runtime" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/envtest" +) + +var ( + envTestClient client.Client + envTest *envtest.Environment + ctx context.Context + cancel context.CancelFunc +) + +var testScheme = runtime.NewScheme() + +func TestServiceMeshSetupIntegration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Openshift Service Mesh infra setup integration") +} + +var _ = BeforeSuite(func() { + rand.Seed(time.Now().UTC().UnixNano()) + + ctx, cancel = context.WithCancel(context.TODO()) + + opts := zap.Options{Development: true} + logf.SetLogger(zap.New(zap.WriteTo(GinkgoWriter), zap.UseFlagOptions(&opts))) + + By("Bootstrapping k8s test environment") + projectDir, err := envtestutil.FindProjectRoot() + if err != nil { + fmt.Printf("Error finding project root: %v\n", err) + return + } + + utilruntime.Must(v1.AddToScheme(testScheme)) + + envTest = &envtest.Environment{ + CRDInstallOptions: envtest.CRDInstallOptions{ + Scheme: testScheme, + Paths: []string{ + filepath.Join(projectDir, "config", "crd", "bases"), + filepath.Join(projectDir, "config", "crd", "dashboard-crds"), + filepath.Join(projectDir, "tests", "integration", "servicemesh", "crd"), + }, + ErrorIfPathMissing: true, + CleanUpAfterUse: false, + }, + } + + config, err := envTest.Start() + Expect(err).NotTo(HaveOccurred()) + Expect(config).NotTo(BeNil()) + + envTestClient, err = client.New(config, client.Options{Scheme: testScheme}) + Expect(err).NotTo(HaveOccurred()) + Expect(envTestClient).NotTo(BeNil()) +}) + +var _ = AfterSuite(func() { + By("Tearing down the test environment") + cancel() + Expect(envTest.Stop()).To(Succeed()) +}) From 317faa3164732bae61161d545137ab37c99a8056 Mon Sep 17 00:00:00 2001 From: Cameron Garrison Date: Tue, 17 Oct 2023 16:39:38 -0400 Subject: [PATCH 2/4] Parameterize gateway namespace used by nb-ctrlr Update components/workbenches/workbenches.go Co-authored-by: Bartosz Majsak consolidate regex and string replace funcs into one w helper move to package, add testing move out gateway const rename to ossmcommon clean up unit tests rename pkg, move io funcs, rename gw func --- components/dashboard/dashboard.go | 4 +- components/workbenches/workbenches.go | 4 + controllers/dscinitialization/monitoring.go | 6 +- pkg/common/common.go | 16 +- pkg/feature/servicemesh/io/io_suite_test.go | 14 ++ pkg/feature/servicemesh/io/raw_resources.go | 170 ++++++++++++++++++ .../servicemesh/io/resources_unit_test.go | 64 +++++++ 7 files changed, 267 insertions(+), 11 deletions(-) create mode 100644 pkg/feature/servicemesh/io/io_suite_test.go create mode 100644 pkg/feature/servicemesh/io/raw_resources.go create mode 100644 pkg/feature/servicemesh/io/resources_unit_test.go diff --git a/components/dashboard/dashboard.go b/components/dashboard/dashboard.go index 25d7816595c..b862ed5e8ed 100644 --- a/components/dashboard/dashboard.go +++ b/components/dashboard/dashboard.go @@ -212,7 +212,7 @@ func (d *Dashboard) applyRhodsSpecificConfigs(cli client.Client, owner metav1.Ob deploy.ManagedRhods: "dedicated-admins", }[platform] - if err := common.ReplaceStringsInFile(dashboardConfig, map[string]string{"": adminGroups}); err != nil { + if err := common.ReplaceInFile(dashboardConfig, map[string]string{"": adminGroups}); err != nil { return err } @@ -261,7 +261,7 @@ func (d *Dashboard) deployConsoleLink(cli client.Client, owner metav1.Object, na domainIndex := strings.Index(consoleRoute.Spec.Host, ".") consoleLinkDomain := consoleRoute.Spec.Host[domainIndex+1:] - err := common.ReplaceStringsInFile(pathConsoleLink, map[string]string{ + err := common.ReplaceInFile(pathConsoleLink, map[string]string{ "": "https://rhods-dashboard-" + namespace + "." + consoleLinkDomain, "": sectionTitle, }) diff --git a/components/workbenches/workbenches.go b/components/workbenches/workbenches.go index 01648f328aa..24a2e150572 100644 --- a/components/workbenches/workbenches.go +++ b/components/workbenches/workbenches.go @@ -9,6 +9,7 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/servicemesh/io" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -130,6 +131,9 @@ func (w *Workbenches) ReconcileComponent(cli client.Client, owner metav1.Object, } if shouldConfigureServiceMesh { actualNbCtrlPath = notebookControllerServiceMeshPath + if err := io.OverwriteGatewayName(dscispec.ApplicationsNamespace, kfnotebookControllerServiceMeshPath); err != nil { + return err + } } if err := deploy.DeployManifestsFromPath(cli, owner, actualNbCtrlPath, dscispec.ApplicationsNamespace, ComponentName, enabled); err != nil { diff --git a/controllers/dscinitialization/monitoring.go b/controllers/dscinitialization/monitoring.go index 5355f58d896..39545aa3e28 100644 --- a/controllers/dscinitialization/monitoring.go +++ b/controllers/dscinitialization/monitoring.go @@ -89,7 +89,7 @@ func configurePrometheus(ctx context.Context, dsciInit *dsci.DSCInitialization, r.Log.Info("Success: read prometheus data from prometheus.yaml from CM") // Update prometheus manifests - err = common.ReplaceStringsInFile(filepath.Join(prometheusManifestsPath, "prometheus.yaml"), + err = common.ReplaceInFile(filepath.Join(prometheusManifestsPath, "prometheus.yaml"), map[string]string{ "": alertmanagerRoute.Spec.Host, "": alertmanagerData, @@ -100,7 +100,7 @@ func configurePrometheus(ctx context.Context, dsciInit *dsci.DSCInitialization, return err } - err = common.ReplaceStringsInFile(filepath.Join(prometheusManifestsPath, "prometheus-viewer-rolebinding.yaml"), + err = common.ReplaceInFile(filepath.Join(prometheusManifestsPath, "prometheus-viewer-rolebinding.yaml"), map[string]string{ "": dsciInit.Spec.Monitoring.Namespace, }) @@ -163,7 +163,7 @@ func configureAlertManager(ctx context.Context, dsciInit *dsci.DSCInitialization // Replace variables in alertmanager configmap // TODO: Following variables can later be exposed by the API - err = common.ReplaceStringsInFile(filepath.Join(deploy.DefaultManifestPath, "monitoring", "alertmanager", "alertmanager-configs.yaml"), + err = common.ReplaceInFile(filepath.Join(deploy.DefaultManifestPath, "monitoring", "alertmanager", "alertmanager-configs.yaml"), map[string]string{ "": b64.StdEncoding.EncodeToString(deadmansnitchSecret.Data["SNITCH_URL"]), "": b64.StdEncoding.EncodeToString(pagerDutySecret.Data["PAGERDUTY_KEY"]), diff --git a/pkg/common/common.go b/pkg/common/common.go index 80cfa9b717e..53d41b72957 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -20,21 +20,25 @@ package common import ( "fmt" "os" - "strings" + "regexp" ) -// ReplaceStringsInFile replaces variable with value in manifests during runtime. -func ReplaceStringsInFile(fileName string, replacements map[string]string) error { +// ReplaceInFile replaces content in the given file either by plain strings or regex patterns based on the content. +func ReplaceInFile(fileName string, replacements map[string]string) error { // Read the contents of the file fileContent, err := os.ReadFile(fileName) if err != nil { return fmt.Errorf("failed to read file: %w", err) } - // Replace all occurrences of the strings in the map + // Replace content using string or regex newContent := string(fileContent) - for string1, string2 := range replacements { - newContent = strings.ReplaceAll(newContent, string1, string2) + for pattern, replacement := range replacements { + regexPattern, err := regexp.Compile(pattern) + if err != nil { + return fmt.Errorf("failed to compile pattern: %w", err) + } + newContent = regexPattern.ReplaceAllString(newContent, replacement) } // Write the modified content back to the file diff --git a/pkg/feature/servicemesh/io/io_suite_test.go b/pkg/feature/servicemesh/io/io_suite_test.go new file mode 100644 index 00000000000..a0466209725 --- /dev/null +++ b/pkg/feature/servicemesh/io/io_suite_test.go @@ -0,0 +1,14 @@ +package io_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestOssmCommon(t *testing.T) { + RegisterFailHandler(Fail) + // for integration tests see tests/integration directory + RunSpecs(t, "Openshift Service Mesh io unit tests") +} diff --git a/pkg/feature/servicemesh/io/raw_resources.go b/pkg/feature/servicemesh/io/raw_resources.go new file mode 100644 index 00000000000..6844ec76589 --- /dev/null +++ b/pkg/feature/servicemesh/io/raw_resources.go @@ -0,0 +1,170 @@ +/* +Copyright (c) 2016-2017 Bitnami +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package io + +import ( + "context" + "fmt" + "github.com/ghodss/yaml" + "github.com/go-logr/logr" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/common" + "github.com/pkg/errors" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + k8stypes "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" + "os" + "path/filepath" + "regexp" + "sigs.k8s.io/controller-runtime/pkg/client" + "strings" +) + +const ( + YamlSeparator = "(?m)^---[ \t]*$" + gatewayPattern = `ISTIO_GATEWAY=(.*)/odh-gateway` +) + +type NameValue struct { + Name string + Value string +} + +func CreateResourceFromFile(client client.Client, log logr.Logger, ownerRef metav1.OwnerReference, filename string, elems ...NameValue) error { + elemsMap := make(map[string]NameValue) + for _, nv := range elems { + elemsMap[nv.Name] = nv + } + + data, err := os.ReadFile(filename) + if err != nil { + return errors.WithStack(err) + } + splitter := regexp.MustCompile(YamlSeparator) + objectStrings := splitter.Split(string(data), -1) + for _, str := range objectStrings { + if strings.TrimSpace(str) == "" { + continue + } + u := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(str), u); err != nil { + return errors.WithStack(err) + } + + name := u.GetName() + namespace := u.GetNamespace() + if namespace == "" { + if val, exists := elemsMap["namespace"]; exists { + u.SetNamespace(val.Value) + } else { + u.SetNamespace("default") + } + } + + u.SetOwnerReferences([]metav1.OwnerReference{ + ownerRef, + }) + + log.Info("Creating resource", "name", name) + + err := client.Get(context.TODO(), k8stypes.NamespacedName{Name: name, Namespace: namespace}, u.DeepCopy()) + if err == nil { + log.Info("Object already exists...") + continue + } + if !k8serrors.IsNotFound(err) { + return errors.WithStack(err) + } + + err = client.Create(context.TODO(), u) + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func PatchResourceFromFile(dynamicClient dynamic.Interface, log logr.Logger, filename string, elems ...NameValue) error { + elemsMap := make(map[string]NameValue) + for _, nv := range elems { + elemsMap[nv.Name] = nv + } + + data, err := os.ReadFile(filename) + if err != nil { + return errors.WithStack(err) + } + splitter := regexp.MustCompile(YamlSeparator) + objectStrings := splitter.Split(string(data), -1) + for _, str := range objectStrings { + if strings.TrimSpace(str) == "" { + continue + } + p := &unstructured.Unstructured{} + if err := yaml.Unmarshal([]byte(str), p); err != nil { + log.Error(err, "error unmarshalling yaml") + return errors.WithStack(err) + } + + // Adding `namespace:` to Namespace resource doesn't make sense + if p.GetKind() != "Namespace" { + namespace := p.GetNamespace() + if namespace == "" { + if val, exists := elemsMap["namespace"]; exists { + p.SetNamespace(val.Value) + } else { + p.SetNamespace("default") + } + } + } + + gvr := schema.GroupVersionResource{ + Group: strings.ToLower(p.GroupVersionKind().Group), + Version: p.GroupVersionKind().Version, + Resource: strings.ToLower(p.GroupVersionKind().Kind) + "s", + } + + // Convert the patch from YAML to JSON + patchAsJSON, err := yaml.YAMLToJSON(data) + if err != nil { + log.Error(err, "error converting yaml to json") + return errors.WithStack(err) + } + + _, err = dynamicClient.Resource(gvr). + Namespace(p.GetNamespace()). + Patch(context.TODO(), p.GetName(), k8stypes.MergePatchType, patchAsJSON, metav1.PatchOptions{}) + if err != nil { + log.Error(err, "error patching resource", + "gvr", fmt.Sprintf("%+v\n", gvr), + "patch", fmt.Sprintf("%+v\n", p), + "json", fmt.Sprintf("%+v\n", patchAsJSON)) + return errors.WithStack(err) + } + + if err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func OverwriteGatewayName(namespace string, kfnotebookControllerServiceMeshPath string) error { + ossmEnv := filepath.Join(kfnotebookControllerServiceMeshPath, "ossm.env") + replacement := fmt.Sprintf("ISTIO_GATEWAY=%s/odh-gateway", namespace) + + return common.ReplaceInFile(ossmEnv, map[string]string{gatewayPattern: replacement}) +} diff --git a/pkg/feature/servicemesh/io/resources_unit_test.go b/pkg/feature/servicemesh/io/resources_unit_test.go new file mode 100644 index 00000000000..311240c3a13 --- /dev/null +++ b/pkg/feature/servicemesh/io/resources_unit_test.go @@ -0,0 +1,64 @@ +package io_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/servicemesh/io" + "os" + "path/filepath" +) + +var _ = Describe("Overwriting gateway namespace in ossm env file", func() { + var ( + tempDir string + namespace string + KfnotebookControllerServiceMeshPath string + ) + + BeforeEach(func() { + var err error + + tempDir = GinkgoT().TempDir() + Expect(err).NotTo(HaveOccurred()) + + tempFilePath := filepath.Join(tempDir, "ossm.env") + tempFile, err := os.Create(tempFilePath) + Expect(err).NotTo(HaveOccurred()) + + mockContents := "ISTIO_GATEWAY=defaultnamespace/odh-gateway\nAnotherSetting=value" + _, err = tempFile.WriteString(mockContents) + Expect(err).NotTo(HaveOccurred()) + tempFile.Close() + + // Mock needed vars + KfnotebookControllerServiceMeshPath = tempDir + namespace = "testnamespace" + }) + + It("should replace gateway name in the file", func() { + err := io.OverwriteGatewayName(namespace, KfnotebookControllerServiceMeshPath) + Expect(err).NotTo(HaveOccurred()) + + updatedContents, err := os.ReadFile(filepath.Join(KfnotebookControllerServiceMeshPath, "ossm.env")) + Expect(err).NotTo(HaveOccurred()) + + expected := "ISTIO_GATEWAY=testnamespace/odh-gateway" + Expect(string(updatedContents)).To(ContainSubstring(expected), "Expected content to contain %q, got %q", expected, updatedContents) + }) + + It("should fail if the file does not exist", func() { + err := io.OverwriteGatewayName(namespace, "wrong_directory") + Expect(err).To(HaveOccurred()) + }) + + It("should not modify other text in the file", func() { + err := io.OverwriteGatewayName(namespace, KfnotebookControllerServiceMeshPath) + Expect(err).NotTo(HaveOccurred()) + + updatedContents, err := os.ReadFile(filepath.Join(KfnotebookControllerServiceMeshPath, "ossm.env")) + Expect(err).NotTo(HaveOccurred()) + + expected := "AnotherSetting=value" + Expect(string(updatedContents)).To(ContainSubstring(expected), "Expected content to contain %q, got %q", expected, updatedContents) + }) +}) From e16c8120c7a50092e384ad8f26e9441f083dc6e8 Mon Sep 17 00:00:00 2001 From: Cameron Garrison Date: Wed, 25 Oct 2023 14:48:42 -0400 Subject: [PATCH 3/4] update imports, feature io --- components/workbenches/workbenches.go | 2 +- pkg/feature/feature.go | 5 +++-- pkg/feature/servicemesh/io/resources_unit_test.go | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/workbenches/workbenches.go b/components/workbenches/workbenches.go index 24a2e150572..41598bf8dbb 100644 --- a/components/workbenches/workbenches.go +++ b/components/workbenches/workbenches.go @@ -2,6 +2,7 @@ package workbenches import ( + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh/io" "path/filepath" "strings" @@ -9,7 +10,6 @@ import ( "github.com/opendatahub-io/opendatahub-operator/v2/components" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/cluster" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/deploy" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/servicemesh/io" operatorv1 "github.com/openshift/api/operator/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index caa6b4a4722..28d2da492f8 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -4,6 +4,7 @@ import ( "context" "github.com/hashicorp/go-multierror" v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh/io" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -168,13 +169,13 @@ func (f *Feature) apply(m manifest) error { applier = func(filename string) error { log.Info("patching using manifest", "feature", f.Name, "name", m.name, "path", targetPath) - return f.patchResourceFromFile(filename) + return io.PatchResourceFromFile(f.DynamicClient, log, filename) } } else { applier = func(filename string) error { log.Info("applying manifest", "feature", f.Name, "name", m.name, "path", targetPath) - return f.createResourceFromFile(filename) + return io.CreateResourceFromFile(f.Client, log, f.OwnerReference(), filename) } } diff --git a/pkg/feature/servicemesh/io/resources_unit_test.go b/pkg/feature/servicemesh/io/resources_unit_test.go index 311240c3a13..fc10b442cd3 100644 --- a/pkg/feature/servicemesh/io/resources_unit_test.go +++ b/pkg/feature/servicemesh/io/resources_unit_test.go @@ -3,7 +3,7 @@ package io_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/servicemesh/io" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh/io" "os" "path/filepath" ) From 92af072a843e3ac61dbe0bccb6fb3e12aad91cca Mon Sep 17 00:00:00 2001 From: Cameron Garrison Date: Wed, 25 Oct 2023 15:03:24 -0400 Subject: [PATCH 4/4] move io pkg out to feature level, rm raw_resource --- components/workbenches/workbenches.go | 2 +- pkg/feature/feature.go | 2 +- .../{servicemesh => }/io/io_suite_test.go | 2 +- .../{servicemesh => }/io/raw_resources.go | 0 .../io/resources_unit_test.go | 2 +- pkg/feature/raw_resources.go | 157 ------------------ 6 files changed, 4 insertions(+), 161 deletions(-) rename pkg/feature/{servicemesh => }/io/io_suite_test.go (80%) rename pkg/feature/{servicemesh => }/io/raw_resources.go (100%) rename pkg/feature/{servicemesh => }/io/resources_unit_test.go (99%) delete mode 100644 pkg/feature/raw_resources.go diff --git a/components/workbenches/workbenches.go b/components/workbenches/workbenches.go index 41598bf8dbb..55e572eeba6 100644 --- a/components/workbenches/workbenches.go +++ b/components/workbenches/workbenches.go @@ -2,7 +2,7 @@ package workbenches import ( - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh/io" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/io" "path/filepath" "strings" diff --git a/pkg/feature/feature.go b/pkg/feature/feature.go index 28d2da492f8..9e3c50e8231 100644 --- a/pkg/feature/feature.go +++ b/pkg/feature/feature.go @@ -4,7 +4,7 @@ import ( "context" "github.com/hashicorp/go-multierror" v1 "github.com/opendatahub-io/opendatahub-operator/v2/apis/dscinitialization/v1" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh/io" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/io" "github.com/opendatahub-io/opendatahub-operator/v2/pkg/gvr" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" diff --git a/pkg/feature/servicemesh/io/io_suite_test.go b/pkg/feature/io/io_suite_test.go similarity index 80% rename from pkg/feature/servicemesh/io/io_suite_test.go rename to pkg/feature/io/io_suite_test.go index a0466209725..3c506dcd84f 100644 --- a/pkg/feature/servicemesh/io/io_suite_test.go +++ b/pkg/feature/io/io_suite_test.go @@ -10,5 +10,5 @@ import ( func TestOssmCommon(t *testing.T) { RegisterFailHandler(Fail) // for integration tests see tests/integration directory - RunSpecs(t, "Openshift Service Mesh io unit tests") + RunSpecs(t, "Feature's io unit tests") } diff --git a/pkg/feature/servicemesh/io/raw_resources.go b/pkg/feature/io/raw_resources.go similarity index 100% rename from pkg/feature/servicemesh/io/raw_resources.go rename to pkg/feature/io/raw_resources.go diff --git a/pkg/feature/servicemesh/io/resources_unit_test.go b/pkg/feature/io/resources_unit_test.go similarity index 99% rename from pkg/feature/servicemesh/io/resources_unit_test.go rename to pkg/feature/io/resources_unit_test.go index fc10b442cd3..f6400362248 100644 --- a/pkg/feature/servicemesh/io/resources_unit_test.go +++ b/pkg/feature/io/resources_unit_test.go @@ -3,7 +3,7 @@ package io_test import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/servicemesh/io" + "github.com/opendatahub-io/opendatahub-operator/v2/pkg/feature/io" "os" "path/filepath" ) diff --git a/pkg/feature/raw_resources.go b/pkg/feature/raw_resources.go deleted file mode 100644 index 4ea955c222e..00000000000 --- a/pkg/feature/raw_resources.go +++ /dev/null @@ -1,157 +0,0 @@ -/* -Copyright (c) 2016-2017 Bitnami -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package feature - -import ( - "context" - "fmt" - "github.com/ghodss/yaml" - "github.com/pkg/errors" - k8serrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" - k8stypes "k8s.io/apimachinery/pkg/types" - "os" - "regexp" - "strings" -) - -const ( - YamlSeparator = "(?m)^---[ \t]*$" -) - -type NameValue struct { - Name string - Value string -} - -func (f *Feature) createResourceFromFile(filename string, elems ...NameValue) error { - elemsMap := make(map[string]NameValue) - for _, nv := range elems { - elemsMap[nv.Name] = nv - } - - data, err := os.ReadFile(filename) - if err != nil { - return errors.WithStack(err) - } - splitter := regexp.MustCompile(YamlSeparator) - objectStrings := splitter.Split(string(data), -1) - for _, str := range objectStrings { - if strings.TrimSpace(str) == "" { - continue - } - u := &unstructured.Unstructured{} - if err := yaml.Unmarshal([]byte(str), u); err != nil { - return errors.WithStack(err) - } - - name := u.GetName() - namespace := u.GetNamespace() - if namespace == "" { - if val, exists := elemsMap["namespace"]; exists { - u.SetNamespace(val.Value) - } else { - u.SetNamespace("default") - } - } - - u.SetOwnerReferences([]metav1.OwnerReference{ - f.OwnerReference(), - }) - - log.Info("Creating resource", "name", name) - - err := f.Client.Get(context.TODO(), k8stypes.NamespacedName{Name: name, Namespace: namespace}, u.DeepCopy()) - if err == nil { - log.Info("Object already exists...") - continue - } - if !k8serrors.IsNotFound(err) { - return errors.WithStack(err) - } - - err = f.Client.Create(context.TODO(), u) - if err != nil { - return errors.WithStack(err) - } - } - return nil -} - -func (f *Feature) patchResourceFromFile(filename string, elems ...NameValue) error { - elemsMap := make(map[string]NameValue) - for _, nv := range elems { - elemsMap[nv.Name] = nv - } - - data, err := os.ReadFile(filename) - if err != nil { - return errors.WithStack(err) - } - splitter := regexp.MustCompile(YamlSeparator) - objectStrings := splitter.Split(string(data), -1) - for _, str := range objectStrings { - if strings.TrimSpace(str) == "" { - continue - } - p := &unstructured.Unstructured{} - if err := yaml.Unmarshal([]byte(str), p); err != nil { - log.Error(err, "error unmarshalling yaml") - return errors.WithStack(err) - } - - // Adding `namespace:` to Namespace resource doesn't make sense - if p.GetKind() != "Namespace" { - namespace := p.GetNamespace() - if namespace == "" { - if val, exists := elemsMap["namespace"]; exists { - p.SetNamespace(val.Value) - } else { - p.SetNamespace("default") - } - } - } - - gvr := schema.GroupVersionResource{ - Group: strings.ToLower(p.GroupVersionKind().Group), - Version: p.GroupVersionKind().Version, - Resource: strings.ToLower(p.GroupVersionKind().Kind) + "s", - } - - // Convert the patch from YAML to JSON - patchAsJSON, err := yaml.YAMLToJSON(data) - if err != nil { - log.Error(err, "error converting yaml to json") - return errors.WithStack(err) - } - - _, err = f.DynamicClient.Resource(gvr). - Namespace(p.GetNamespace()). - Patch(context.TODO(), p.GetName(), k8stypes.MergePatchType, patchAsJSON, metav1.PatchOptions{}) - if err != nil { - log.Error(err, "error patching resource", - "gvr", fmt.Sprintf("%+v\n", gvr), - "patch", fmt.Sprintf("%+v\n", p), - "json", fmt.Sprintf("%+v\n", patchAsJSON)) - return errors.WithStack(err) - } - - if err != nil { - return errors.WithStack(err) - } - } - return nil -}