From 75d40282f7187e3398e76c0b2ea4dc8d1693ae0d Mon Sep 17 00:00:00 2001 From: Pedro Juarez Date: Tue, 5 Mar 2024 00:13:13 -0800 Subject: [PATCH] add securityContext for KES pods in Tenant CRD (#2010) * add securityContext for KES pods in Tenant CRD Signed-off-by: pjuarezd * bugfix: wrong key name Signed-off-by: pjuarezd * Kes not nil Signed-off-by: pjuarezd --------- Signed-off-by: pjuarezd --- docs/job_crd.adoc | 32 ++++++++-- docs/tenant_crd.adoc | 17 ++---- .../templates/minio.min.io_tenants.yaml | 61 +++++++++++++++++++ helm/tenant/templates/tenant.yaml | 6 ++ helm/tenant/values.yaml | 5 ++ pkg/apis/minio.min.io/v2/types.go | 2 + .../minio.min.io/v2/zz_generated.deepcopy.go | 5 ++ .../minio.min.io/v2/kesconfig.go | 9 +++ .../informers/externalversions/factory.go | 10 +++ pkg/resources/statefulsets/kes-statefulset.go | 44 +++++++++++++ resources/base/crds/minio.min.io_tenants.yaml | 61 +++++++++++++++++++ 11 files changed, 236 insertions(+), 16 deletions(-) diff --git a/docs/job_crd.adoc b/docs/job_crd.adoc index e5d4c5ce63a..1c2a5c061da 100644 --- a/docs/job_crd.adoc +++ b/docs/job_crd.adoc @@ -4,8 +4,8 @@ [id="{p}-api-reference"] == API Reference -:minio-image: https://hub.docker.com/r/minio/minio/tags[minio/minio:RELEASE.2023-11-15T20-43-25Z] -:kes-image: https://hub.docker.com/r/minio/kes/tags[minio/kes:2023-11-10T10-44-28Z] +:minio-image: https://hub.docker.com/r/minio/minio/tags[minio/minio:RELEASE.2024-02-09T21-25-16Z] +:kes-image: https://hub.docker.com/r/minio/kes/tags[minio/kes:2024-01-11T13-09-29Z] [id="{anchor_prefix}-job-min-io-v1alpha1"] @@ -73,6 +73,30 @@ CommandStatus Status of MinioJob command execution |=== +[id="{anchor_prefix}-github-com-minio-operator-pkg-apis-job-min-io-v1alpha1-execution"] +==== Execution (string) + +Execution is the MinIO Job level execution policy + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-minio-operator-pkg-apis-job-min-io-v1alpha1-miniojobspec[$$MinIOJobSpec$$] +**** + + + +[id="{anchor_prefix}-github-com-minio-operator-pkg-apis-job-min-io-v1alpha1-failurestrategy"] +==== FailureStrategy (string) + +FailureStrategy is the failure strategy at MinIO Job level + +.Appears In: +**** +- xref:{anchor_prefix}-github-com-minio-operator-pkg-apis-job-min-io-v1alpha1-miniojobspec[$$MinIOJobSpec$$] +**** + + + [id="{anchor_prefix}-github-com-minio-operator-pkg-apis-job-min-io-v1alpha1-miniojob"] ==== MinIOJob @@ -122,10 +146,10 @@ MinIOJobSpec (`spec`) defines the configuration of a MinIOJob object. + |*Required* + TenantRef Reference for minio Tenant to eun the jobs against -|*`execution`* __Execution__ +|*`execution`* __xref:{anchor_prefix}-github-com-minio-operator-pkg-apis-job-min-io-v1alpha1-execution[$$Execution$$]__ |Execution order of the jobs, either `parallel` or `sequential`. Defaults to `parallel` if not provided. -|*`failureStrategy`* __FailureStrategy__ +|*`failureStrategy`* __xref:{anchor_prefix}-github-com-minio-operator-pkg-apis-job-min-io-v1alpha1-failurestrategy[$$FailureStrategy$$]__ |FailureStrategy is the forward plan in case of the failure of one or more MinioJob pods Either `stopOnFailure` or `continueOnFailure`, defaults to `continueOnFailure`. |*`commands`* __xref:{anchor_prefix}-github-com-minio-operator-pkg-apis-job-min-io-v1alpha1-commandspec[$$CommandSpec$$] array__ diff --git a/docs/tenant_crd.adoc b/docs/tenant_crd.adoc index 345d8ba1c16..1b06dc8d13c 100644 --- a/docs/tenant_crd.adoc +++ b/docs/tenant_crd.adoc @@ -101,16 +101,6 @@ CertificateStatus keeps track of all the certificates managed by the operator |=== -[id="{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificateconfig"] -==== CustomCertificateConfig - -CustomCertificateConfig (`customCertificateConfig`) provides attributes associated of the TLS certificates manually added to the Operator as part of tenant creation. These fields contain no data if there are no custom TLS certificates. - -.Appears In: -**** -- xref:{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificates[$$CustomCertificates$$] -**** - [id="{anchor_prefix}-github-com-minio-operator-pkg-apis-minio-min-io-v2-customcertificates"] @@ -265,11 +255,11 @@ KESConfig (`kes`) defines the configuration of the https://github.com/minio/kes[ |*`gcpCredentialSecretName`* __string__ |*Optional* + - Specify the GCP default credentials to be used for KES to authenticate to GCP key store + Specify the GCP default credentials to be used for KES to authenticate to GCP key store |*`gcpWorkloadIdentityPool`* __string__ |*Optional* + - Specify the name of the workload identity pool (This is required for generating service account token) + Specify the name of the workload identity pool (This is required for generating service account token) |*`annotations`* __object (keys:string, values:string)__ |*Optional* + @@ -313,6 +303,9 @@ KESConfig (`kes`) defines the configuration of the https://github.com/minio/kes[ * `runAsUser` + * `seLinuxOptions` + +|*`containerSecurityContext`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#securitycontext-v1-core[$$SecurityContext$$]__ +|Specify the https://kubernetes.io/docs/tasks/configure-pod-container/security-context/[Security Context] of MinIO KES pods. + |*`env`* __link:https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.23/#envvar-v1-core[$$EnvVar$$] array__ |*Optional* + If provided, the MinIO Operator adds the specified environment variables when deploying the KES resource. diff --git a/helm/operator/templates/minio.min.io_tenants.yaml b/helm/operator/templates/minio.min.io_tenants.yaml index e3bf49be816..12c0db12c86 100644 --- a/helm/operator/templates/minio.min.io_tenants.yaml +++ b/helm/operator/templates/minio.min.io_tenants.yaml @@ -2048,6 +2048,67 @@ spec: required: - name type: object + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object env: items: properties: diff --git a/helm/tenant/templates/tenant.yaml b/helm/tenant/templates/tenant.yaml index e84ebe1a378..56116b90771 100644 --- a/helm/tenant/templates/tenant.yaml +++ b/helm/tenant/templates/tenant.yaml @@ -187,5 +187,11 @@ spec: {{- toYaml . | nindent 6 }} {{- end }} {{- end }} + {{- if hasKey .kes "containerSecurityContext" }} + containerSecurityContext: {{- if eq (len .kes.containerSecurityContext) 0 }} { } {{- end }} + {{- with (dig "kes" "containerSecurityContext" (dict) .) }} + {{- toYaml . | nindent 6 }} + {{- end }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/tenant/values.yaml b/helm/tenant/values.yaml index 3bf895138d7..ec01bf54450 100644 --- a/helm/tenant/values.yaml +++ b/helm/tenant/values.yaml @@ -415,6 +415,11 @@ tenant: # runAsGroup: 1000 # runAsNonRoot: true # fsGroup: 1000 + # containerSecurityContext: + # runAsUser: 1000 + # runAsGroup: 1000 + # runAsNonRoot: true + # allowPrivilegeEscalation: false ### # Configures `Ingress `__ for the Tenant S3 API and Console. # diff --git a/pkg/apis/minio.min.io/v2/types.go b/pkg/apis/minio.min.io/v2/types.go index 674a2e9dc6d..70fb434342e 100644 --- a/pkg/apis/minio.min.io/v2/types.go +++ b/pkg/apis/minio.min.io/v2/types.go @@ -856,6 +856,8 @@ type KESConfig struct { // * `seLinuxOptions` + // +optional SecurityContext *corev1.PodSecurityContext `json:"securityContext,omitempty"` + // Specify the https://kubernetes.io/docs/tasks/configure-pod-container/security-context/[Security Context] of MinIO KES pods. + ContainerSecurityContext *corev1.SecurityContext `json:"containerSecurityContext,omitempty"` // *Optional* + // // If provided, the MinIO Operator adds the specified environment variables when deploying the KES resource. diff --git a/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go b/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go index 69d775ac116..d0ea81be025 100644 --- a/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go +++ b/pkg/apis/minio.min.io/v2/zz_generated.deepcopy.go @@ -291,6 +291,11 @@ func (in *KESConfig) DeepCopyInto(out *KESConfig) { *out = new(v1.PodSecurityContext) (*in).DeepCopyInto(*out) } + if in.ContainerSecurityContext != nil { + in, out := &in.ContainerSecurityContext, &out.ContainerSecurityContext + *out = new(v1.SecurityContext) + (*in).DeepCopyInto(*out) + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make([]v1.EnvVar, len(*in)) diff --git a/pkg/client/applyconfiguration/minio.min.io/v2/kesconfig.go b/pkg/client/applyconfiguration/minio.min.io/v2/kesconfig.go index ca4c70808ff..c9b4ce7b83c 100644 --- a/pkg/client/applyconfiguration/minio.min.io/v2/kesconfig.go +++ b/pkg/client/applyconfiguration/minio.min.io/v2/kesconfig.go @@ -43,6 +43,7 @@ type KESConfigApplyConfiguration struct { TopologySpreadConstraints []v1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"` KeyName *string `json:"keyName,omitempty"` SecurityContext *v1.PodSecurityContext `json:"securityContext,omitempty"` + ContainerSecurityContext *v1.SecurityContext `json:"containerSecurityContext,omitempty"` Env []v1.EnvVar `json:"env,omitempty"` } @@ -218,6 +219,14 @@ func (b *KESConfigApplyConfiguration) WithSecurityContext(value v1.PodSecurityCo return b } +// WithContainerSecurityContext sets the ContainerSecurityContext field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the ContainerSecurityContext field is set to the value of the last call. +func (b *KESConfigApplyConfiguration) WithContainerSecurityContext(value v1.SecurityContext) *KESConfigApplyConfiguration { + b.ContainerSecurityContext = &value + return b +} + // WithEnv adds the given value to the Env field in the declarative configuration // and returns the receiver, so that objects can be build by chaining "With" function invocations. // If called multiple times, values provided by each call will be appended to the Env field. diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go index 375914b8b8e..f099bdc7f9e 100644 --- a/pkg/client/informers/externalversions/factory.go +++ b/pkg/client/informers/externalversions/factory.go @@ -44,6 +44,7 @@ type sharedInformerFactory struct { lock sync.Mutex defaultResync time.Duration customResync map[reflect.Type]time.Duration + transform cache.TransformFunc informers map[reflect.Type]cache.SharedIndexInformer // startedInformers is used for tracking which informers have been started. @@ -82,6 +83,14 @@ func WithNamespace(namespace string) SharedInformerOption { } } +// WithTransform sets a transform on all informers. +func WithTransform(transform cache.TransformFunc) SharedInformerOption { + return func(factory *sharedInformerFactory) *sharedInformerFactory { + factory.transform = transform + return factory + } +} + // NewSharedInformerFactory constructs a new instance of sharedInformerFactory for all namespaces. func NewSharedInformerFactory(client versioned.Interface, defaultResync time.Duration) SharedInformerFactory { return NewSharedInformerFactoryWithOptions(client, defaultResync) @@ -186,6 +195,7 @@ func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internal } informer = newFunc(f.client, resyncPeriod) + informer.SetTransform(f.transform) f.informers[informerType] = informer return informer diff --git a/pkg/resources/statefulsets/kes-statefulset.go b/pkg/resources/statefulsets/kes-statefulset.go index a1adf5b9f90..49b04091602 100644 --- a/pkg/resources/statefulsets/kes-statefulset.go +++ b/pkg/resources/statefulsets/kes-statefulset.go @@ -118,6 +118,7 @@ func KESServerContainer(t *miniov2.Tenant) corev1.Container { Args: args, Env: KESEnvironmentVars(t), Resources: t.Spec.KES.Resources, + SecurityContext: kesContainerSecurityContext(t), } } @@ -142,6 +143,49 @@ func kesSecurityContext(t *miniov2.Tenant) *corev1.PodSecurityContext { return &securityContext } +// Builds the security context for kes containers +func kesContainerSecurityContext(t *miniov2.Tenant) *corev1.SecurityContext { + // Default values: + // By default, values should be totally empty if not provided + // This is specially needed in OpenShift where Security Context Constraints restrict them + // if let empty then OCP can pick the values from the constraints defined. + containerSecurityContext := corev1.SecurityContext{} + runAsNonRoot := true + var runAsUser int64 = 1000 + var runAsGroup int64 = 1000 + poolSCSet := false + + // Values from pool.SecurityContext ONLY if provided + if t.Spec.KES != nil && t.Spec.KES.SecurityContext != nil { + if t.Spec.KES.SecurityContext.RunAsNonRoot != nil { + runAsNonRoot = *t.Spec.KES.SecurityContext.RunAsNonRoot + poolSCSet = true + } + if t.Spec.KES.SecurityContext.RunAsUser != nil { + runAsUser = *t.Spec.KES.SecurityContext.RunAsUser + poolSCSet = true + } + if t.Spec.KES.SecurityContext.RunAsGroup != nil { + runAsGroup = *t.Spec.KES.SecurityContext.RunAsGroup + poolSCSet = true + } + if poolSCSet { + // Only set values if one of above is set otherwise let it empty + containerSecurityContext = corev1.SecurityContext{ + RunAsNonRoot: &runAsNonRoot, + RunAsUser: &runAsUser, + RunAsGroup: &runAsGroup, + } + } + } + + // Values from kes.ContainerSecurityContext if provided + if t.Spec.KES != nil && t.Spec.KES.ContainerSecurityContext != nil { + containerSecurityContext = *t.Spec.KES.ContainerSecurityContext + } + return &containerSecurityContext +} + // NewForKES creates a new KES StatefulSet for the given Cluster. func NewForKES(t *miniov2.Tenant, serviceName string) *appsv1.StatefulSet { replicas := t.KESReplicas() diff --git a/resources/base/crds/minio.min.io_tenants.yaml b/resources/base/crds/minio.min.io_tenants.yaml index e3bf49be816..12c0db12c86 100644 --- a/resources/base/crds/minio.min.io_tenants.yaml +++ b/resources/base/crds/minio.min.io_tenants.yaml @@ -2048,6 +2048,67 @@ spec: required: - name type: object + containerSecurityContext: + properties: + allowPrivilegeEscalation: + type: boolean + capabilities: + properties: + add: + items: + type: string + type: array + drop: + items: + type: string + type: array + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object env: items: properties: