From 270ffee0f6f03e2c196fd1d9d052becfa04a33b6 Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Wed, 16 Nov 2022 23:47:13 -0800 Subject: [PATCH 01/11] TLS methods refactor to allow create several Certificates --- main.go | 13 +- pkg/controller/cluster/console.go | 25 ++ pkg/controller/cluster/main-controller.go | 19 +- pkg/controller/cluster/minio.go | 9 +- pkg/controller/cluster/operator.go | 337 ++++------------------ pkg/controller/cluster/tls.go | 302 +++++++++++++++++++ 6 files changed, 394 insertions(+), 311 deletions(-) create mode 100644 pkg/controller/cluster/tls.go diff --git a/main.go b/main.go index 1f7055b681e..ae60c2e9189 100644 --- a/main.go +++ b/main.go @@ -44,6 +44,13 @@ import ( "k8s.io/client-go/kubernetes" ) +const ( + // OperatorWatchedNamespaceEnv Env variable name, the namespaces which the operator watches for MinIO tenants. Defaults to "" for all namespaces. + OperatorWatchedNamespaceEnv = "WATCHED_NAMESPACE" + // HostnameEnv Host name env variable + HostnameEnv = "HOSTNAME" +) + // version provides the version of this operator var version = "DEVELOPMENT.GOGET" @@ -110,7 +117,7 @@ func main() { } // Get a comma separated list of namespaces to watch - namespacesENv, isNamespaced := os.LookupEnv("WATCHED_NAMESPACE") + namespacesENv, isNamespaced := os.LookupEnv(OperatorWatchedNamespaceEnv) var namespaces set.StringSet if isNamespaced { namespaces = set.NewStringSet() @@ -174,9 +181,9 @@ func main() { kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeClient, time.Second*30) minioInformerFactory := informers.NewSharedInformerFactory(controllerClient, time.Second*30) - podName := os.Getenv("HOSTNAME") + podName := os.Getenv(HostnameEnv) if podName == "" { - klog.Info("Could not determine $HOSTNAME, defaulting to pod name: operator-pod") + klog.Info("Could not determine $%s, defaulting to pod name: operator-pod", HostnameEnv) podName = "operator-pod" } diff --git a/pkg/controller/cluster/console.go b/pkg/controller/cluster/console.go index e5b3f79109b..d9dc87f0803 100644 --- a/pkg/controller/cluster/console.go +++ b/pkg/controller/cluster/console.go @@ -29,6 +29,15 @@ import ( "k8s.io/klog/v2" ) +const ( + // ConsoleTLSEnv Env variable to turn on / off Console TLS. + ConsoleTLSEnv = "MINIO_CONSOLE_TLS_ENABLE" + // DefaultConsoleDeploymentName is the default name of the console deployment + DefaultConsoleDeploymentName = "console" + // OperatorConsoleTLSSecretName is the name of secret created with TLS certs for Operator console + OperatorConsoleTLSSecretName = "operator-console-tls" +) + // checkConsoleSvc validates the existence of the MinIO service and validate it's status against what the specification // states func (c *Controller) checkConsoleSvc(ctx context.Context, tenant *miniov2.Tenant, nsName types.NamespacedName) error { @@ -87,3 +96,19 @@ func (c *Controller) checkConsoleSvc(ctx context.Context, tenant *miniov2.Tenant } return err } + +// isConsoleTLS Internal func, reads MINIO_OPERATOR_TLS_ENABLE ENV to identify if Operator TLS is enabled, default "off" +// **WARNING** This will change and will be default to "on" in operator v5 +// func isConsoleTLS() bool { +// value, set := os.LookupEnv(ConsoleTLSEnv) +// // By default, Console TLS is NOT used. +// return (set && value == "on") || set +// } + +// func getConsoleDeploymentName() string { +// return env.Get("MINIO_CONSOLE_DEPLOYMENT_NAME", DefaultConsoleDeploymentName) +// } + +// func (c *Controller) generateConsoleTLSCert() (*string, *string) { +// return c.generateTLSCert("console", OperatorConsoleTLSSecretName, getConsoleDeploymentName()) +// } diff --git a/pkg/controller/cluster/main-controller.go b/pkg/controller/cluster/main-controller.go index a6d569732f4..af2f6ac8d93 100644 --- a/pkg/controller/cluster/main-controller.go +++ b/pkg/controller/cluster/main-controller.go @@ -16,7 +16,6 @@ package cluster import ( "context" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -351,7 +350,7 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error { klog.Infof("Using Kubernetes CSR Version: %s", apiCsrVersion) if isOperatorTLS() { - publicCertPath, publicKeyPath := c.generateTLSCert() + publicCertPath, publicKeyPath := c.generateOperatorTLSCert() klog.Infof("Starting HTTPS API server") close(apiServerWillStart) certsManager, err := xcerts.NewManager(ctx, *publicCertPath, *publicKeyPath, LoadX509KeyPair) @@ -360,21 +359,7 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error { panic(err) } serverCertsManager = certsManager - c.ws.TLSConfig = &tls.Config{ - PreferServerCipherSuites: true, - CurvePreferences: []tls.CurveID{tls.CurveP256}, - NextProtos: []string{"h2", "http/1.1"}, - MinVersion: tls.VersionTLS12, - CipherSuites: []uint16{ - tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, - tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, - tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, - }, - GetCertificate: certsManager.GetCertificate, - } + c.ws.TLSConfig = c.createTLSConfig(serverCertsManager) if err := c.ws.ListenAndServeTLS("", ""); err != http.ErrServerClosed { klog.Infof("HTTPS server ListenAndServeTLS failed: %v", err) panic(err) diff --git a/pkg/controller/cluster/minio.go b/pkg/controller/cluster/minio.go index 230419627ef..5ed4601e35a 100644 --- a/pkg/controller/cluster/minio.go +++ b/pkg/controller/cluster/minio.go @@ -66,6 +66,7 @@ func (c *Controller) checkAndCreateMinIOCSR(ctx context.Context, nsName types.Na return nil } +// deleteCSR Removes a CSR func (c *Controller) deleteCSR(ctx context.Context, csrName string) error { if certificates.GetCertificatesAPIVersion(c.kubeClientSet) == certificates.CSRV1 { if err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}); err != nil { @@ -298,13 +299,7 @@ func (c *Controller) certNeedsRenewal(tlsSecret *corev1.Secret) (bool, error) { var certPublicKey []byte var certPrivateKey []byte - publicKey := "public.crt" - privateKey := "private.key" - - if tlsSecret.Type == "kubernetes.io/tls" || tlsSecret.Type == "cert-manager.io/v1alpha2" || tlsSecret.Type == "cert-manager.io/v1" { - publicKey = "tls.crt" - privateKey = "tls.key" - } + publicKey, privateKey := c.getKeyNames(tlsSecret) if _, exist := tlsSecret.Data[publicKey]; !exist { return false, fmt.Errorf("missing '%s' in %s/%s secret", publicKey, tlsSecret.Namespace, tlsSecret.Name) diff --git a/pkg/controller/cluster/operator.go b/pkg/controller/cluster/operator.go index 5db7d74bfd1..ded09d0ac7d 100644 --- a/pkg/controller/cluster/operator.go +++ b/pkg/controller/cluster/operator.go @@ -18,252 +18,92 @@ package cluster import ( "context" - "crypto/rand" "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "errors" - "fmt" "net" "net/http" "os" "time" - xcerts "github.com/minio/pkg/certs" - - "github.com/minio/operator/pkg/controller/cluster/certificates" - - "github.com/minio/pkg/env" - - "k8s.io/apimachinery/pkg/runtime/schema" - - k8serrors "k8s.io/apimachinery/pkg/api/errors" - - corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" + xcerts "github.com/minio/pkg/certs" + "github.com/minio/pkg/env" "k8s.io/klog/v2" ) const ( - // OperatorTLS is the ENV var to turn on / off Operator TLS. - OperatorTLS = "MINIO_OPERATOR_TLS_ENABLE" - // EnvCertPassword ENV var is used to decrypt the private key in the TLS certificate for operator if need it - EnvCertPassword = "OPERATOR_CERT_PASSWD" - // OperatorTLSSecretName is the name of secret created with Operator TLS certs - OperatorTLSSecretName = "operator-tls" + // CertPasswordEnv Env variable is used to decrypt the private key in the TLS certificate for operator if need it + CertPasswordEnv = "OPERATOR_CERT_PASSWD" + // OperatorDeplymentNameEnv Env variable to specify a custom deployment name for Operator + OperatorDeplymentNameEnv = "MINIO_OPERATOR_DEPLOYMENT_NAME" + // OperatorTLSEnv Env variable to turn on / off Operator TLS. + OperatorTLSEnv = "MINIO_OPERATOR_TLS_ENABLE" // OperatorCATLSSecretName is the name of the secret for the operator CA OperatorCATLSSecretName = "operator-ca-tls" + // OperatorTLSSecretName is the name of secret created with TLS certs for Operator + OperatorTLSSecretName = "operator-tls" // DefaultDeploymentName is the default name of the operator deployment DefaultDeploymentName = "minio-operator" ) -var errOperatorWaitForTLS = errors.New("waiting for Operator cert") +var serverCertsManager *xcerts.Manager -func getOperatorDeploymentName() string { - return env.Get("MINIO_OPERATOR_DEPLOYMENT_NAME", DefaultDeploymentName) +// generateOperatorTLSCert Issues the Operator TLS Certificate +func (c *Controller) generateOperatorTLSCert() (*string, *string) { + return c.generateTLSCert("operator", OperatorTLSSecretName, getOperatorDeploymentName()) } -func isOperatorTLS() bool { - value, set := os.LookupEnv(OperatorTLS) - // By default, Operator TLS is used. - return (set && value == "on") || !set -} - -func (c *Controller) generateTLSCert() (*string, *string) { - ctx := context.Background() +// recreateOperatorCertsIfRequired - Generate Operator TLS certs if not present or if is expired +func (c *Controller) recreateOperatorCertsIfRequired(ctx context.Context) error { namespace := miniov2.GetNSFromFile() - // operator deployment for owner reference - operatorDeployment, err := c.kubeClientSet.AppsV1().Deployments(namespace).Get(ctx, getOperatorDeploymentName(), metav1.GetOptions{}) + operatorTLSSecret, err := c.getTLSSecret(ctx, namespace, OperatorTLSSecretName) if err != nil { - panic(err) - } - - publicCertPath := "/tmp/public.crt" - publicKeyPath := "/tmp/private.key" - - for { - // operator TLS certificates - operatorTLSCert, err := c.kubeClientSet.CoreV1().Secrets(namespace).Get(ctx, OperatorTLSSecretName, metav1.GetOptions{}) - if err != nil { - if k8serrors.IsNotFound(err) { - klog.Infof("operator TLS secret not found: %v", err) - if err = c.checkAndCreateOperatorCSR(ctx, operatorDeployment); err != nil { - klog.Infof("Waiting for the operator certificates to be issued %v", err.Error()) - time.Sleep(time.Second * 10) - } else { - err = c.deleteCSR(ctx, c.operatorCSRName()) - if err != nil { - klog.Infof(err.Error()) - } - } - } - } else { - // default secret keys for Opaque k8s secret - publicCertKey := "public.crt" - privateKeyKey := "private.key" - // if secret type is k8s tls or cert-manager use the right secret keys - if operatorTLSCert.Type == "kubernetes.io/tls" || operatorTLSCert.Type == "cert-manager.io/v1alpha2" || operatorTLSCert.Type == "cert-manager.io/v1" { - publicCertKey = "tls.crt" - privateKeyKey = "tls.key" - } - if val, ok := operatorTLSCert.Data[publicCertKey]; ok { - err := os.WriteFile(publicCertPath, val, 0o644) - if err != nil { - panic(err) - } - } else { - panic(fmt.Errorf("missing '%s' in %s/%s", publicCertKey, operatorTLSCert.Namespace, operatorTLSCert.Name)) - } - if val, ok := operatorTLSCert.Data[privateKeyKey]; ok { - err := os.WriteFile(publicKeyPath, val, 0o644) - if err != nil { - panic(err) - } - } else { - panic(fmt.Errorf("missing '%s' in %s/%s", publicCertKey, operatorTLSCert.Namespace, operatorTLSCert.Name)) + if k8serrors.IsNotFound(err) { + klog.V(2).Info("TLS certificate not found. Generating one.") + // Generate new certificate KeyPair for Operator server + c.generateOperatorTLSCert() + // reload in memory certificate for the operator server + if serverCertsManager != nil { + serverCertsManager.ReloadCerts() } - break - } - } - // validate certificates if they are valid, if not panic right here. - if _, err = tls.LoadX509KeyPair(publicCertPath, publicKeyPath); err != nil { - panic(err) - } - - return &publicCertPath, &publicKeyPath -} - -func generateOperatorCryptoData() ([]byte, []byte, error) { - privateKey, err := newPrivateKey(miniov2.DefaultEllipticCurve) - if err != nil { - klog.Errorf("Unexpected error during the ECDSA Key generation: %v", err) - return nil, nil, err - } - - privKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) - if err != nil { - klog.Errorf("Unexpected error during encoding the ECDSA Private Key: %v", err) - return nil, nil, err - } - - opCommon := fmt.Sprintf("operator.%s.svc.%s", miniov2.GetNSFromFile(), miniov2.GetClusterDomain()) - opCommonNoDomain := fmt.Sprintf("operator.%s.svc", miniov2.GetNSFromFile()) - - csrTemplate := x509.CertificateRequest{ - Subject: pkix.Name{ - CommonName: fmt.Sprintf("system:node:%s", opCommonNoDomain), - Organization: []string{"system:nodes"}, - }, - Extensions: []pkix.Extension{ - { - Id: nil, - Critical: false, - Value: []byte("operator"), - }, - { - Id: nil, - Critical: false, - Value: []byte(opCommonNoDomain), - }, - { - Id: nil, - Critical: false, - Value: []byte(opCommon), - }, - }, - SignatureAlgorithm: x509.ECDSAWithSHA512, - DNSNames: []string{"operator", opCommonNoDomain, opCommon}, + return nil + } + return err } - csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &csrTemplate, privateKey) + needsRenewal, err := c.certNeedsRenewal(operatorTLSSecret) if err != nil { - klog.Errorf("Unexpected error during creating the CSR: %v", err) - return nil, nil, err + return err } - return privKeyBytes, csrBytes, nil -} -func (c *Controller) createOperatorSecret(ctx context.Context, operator metav1.Object, labels map[string]string, secretName string, pkBytes, certBytes []byte) error { - secret := &corev1.Secret{ - Type: "Opaque", - ObjectMeta: metav1.ObjectMeta{ - Name: secretName, - Namespace: miniov2.GetNSFromFile(), - Labels: labels, - OwnerReferences: []metav1.OwnerReference{ - *metav1.NewControllerRef(operator, schema.GroupVersionKind{ - Group: "apps", - Version: "v1", - Kind: "Deployment", - }), - }, - }, - Data: map[string][]byte{ - "private.key": pkBytes, - "public.crt": certBytes, - }, + if !needsRenewal { + return nil } - _, err := c.kubeClientSet.CoreV1().Secrets(miniov2.GetNSFromFile()).Create(ctx, secret, metav1.CreateOptions{}) - return err -} -// createOperatorCSR handles all the steps required to create the CSR: from creation of keys, submitting CSR and -// finally creating a secret that Operator deployment will use to mount private key and certificate for TLS -// This Method Blocks till the CSR Request is approved via kubectl approve -func (c *Controller) createOperatorCSR(ctx context.Context, operator metav1.Object) error { - privKeysBytes, csrBytes, err := generateOperatorCryptoData() - if err != nil { - klog.Errorf("Private Key and CSR generation failed with error: %v", err) - return err - } - namespace := miniov2.GetNSFromFile() - err = c.createCertificateSigningRequest(ctx, map[string]string{}, c.operatorCSRName(), namespace, csrBytes) + // Expired cert. Delete the secret + CSR and re-create the cert + err = c.deleteCSR(ctx, operatorCSRName()) if err != nil { - klog.Errorf("Unexpected error during the creation of the csr/%s: %v", c.operatorCSRName(), err) return err } - - // fetch certificate from CSR - certBytes, err := c.fetchCertificate(ctx, c.operatorCSRName()) + klog.V(2).Info("Deleting the TLS secret of expired cert on operator") + err = c.kubeClientSet.CoreV1().Secrets(namespace).Delete(ctx, OperatorTLSSecretName, metav1.DeleteOptions{}) if err != nil { - klog.Errorf("Unexpected error during the creation of the csr/%s: %v", c.operatorCSRName(), err) return err } - // PEM encode private ECDSA key - encodedPrivateKey := pem.EncodeToMemory(&pem.Block{Type: privateKeyType, Bytes: privKeysBytes}) + klog.V(2).Info("Generating a fresh TLS certificate for Operator") + // Generate new certificate KeyPair for Operator server + c.generateOperatorTLSCert() - // Create secret for operator to use - err = c.createOperatorSecret(ctx, operator, map[string]string{}, OperatorTLSSecretName, encodedPrivateKey, certBytes) - if err != nil { - klog.Errorf("Unexpected error during the creation of the %s/%s secret: %v", operator.GetNamespace(), OperatorTLSSecretName, err) - return err + // reload in memory certificate for the operator server + if serverCertsManager != nil { + serverCertsManager.ReloadCerts() } - return nil -} -func (c *Controller) checkAndCreateOperatorCSR(ctx context.Context, operator metav1.Object) error { - var err error - if certificates.GetCertificatesAPIVersion(c.kubeClientSet) == certificates.CSRV1 { - _, err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, c.operatorCSRName(), metav1.GetOptions{}) - } else { - _, err = c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, c.operatorCSRName(), metav1.GetOptions{}) - } - if err != nil { - if k8serrors.IsNotFound(err) { - klog.V(2).Infof("Creating a new Certificate Signing Request for Operator Server Certs, cluster %q") - if err = c.createOperatorCSR(ctx, operator); err != nil { - return err - } - return errOperatorWaitForTLS - } - return err - } return nil } @@ -411,90 +251,19 @@ func (c *Controller) createBuckets(ctx context.Context, tenant *miniov2.Tenant, return nil } -func (c *Controller) operatorCSRName() string { - namespace := miniov2.GetNSFromFile() - return fmt.Sprintf("operator-%s-csr", namespace) +// getOperatorDeploymentName Internal func returns the Operator deployment name from MINIO_OPERATOR_DEPLOYMENT_NAME ENV variable or the default name +func getOperatorDeploymentName() string { + return env.Get(OperatorDeplymentNameEnv, DefaultDeploymentName) } -// recreateOperatorCertsIfRequired - Generate Operator TLS certs if not present or if is expired -func (c *Controller) recreateOperatorCertsIfRequired(ctx context.Context) error { - namespace := miniov2.GetNSFromFile() - operatorTLSSecret, err := c.getTLSSecret(ctx, namespace, OperatorTLSSecretName) - if err != nil { - if k8serrors.IsNotFound(err) { - klog.V(2).Info("TLS certificate not found. Generating one.") - // Generate new certificate KeyPair for Operator server - c.generateTLSCert() - // reload in memory certificate for the operator server - if serverCertsManager != nil { - serverCertsManager.ReloadCerts() - } - - return nil - } - return err - } - - needsRenewal, err := c.certNeedsRenewal(operatorTLSSecret) - if err != nil { - return err - } - - if !needsRenewal { - return nil - } - - // Expired cert. Delete the secret + CSR and re-create the cert - err = c.deleteCSR(ctx, c.operatorCSRName()) - if err != nil { - return err - } - klog.V(2).Info("Deleting the TLS secret of expired cert on operator") - err = c.kubeClientSet.CoreV1().Secrets(namespace).Delete(ctx, OperatorTLSSecretName, metav1.DeleteOptions{}) - if err != nil { - return err - } - - klog.V(2).Info("Generating a fresh TLS certificate for Operator") - // Generate new certificate KeyPair for Operator server - c.generateTLSCert() - - // reload in memory certificate for the operator server - if serverCertsManager != nil { - serverCertsManager.ReloadCerts() - } - - return nil +// isOperatorTLS Internal func, reads MINIO_OPERATOR_TLS_ENABLE ENV to identify if Operator TLS is enabled, default "on" +func isOperatorTLS() bool { + value, set := os.LookupEnv(OperatorTLSEnv) + // By default, Operator TLS is used. + return (set && value == "on") || !set } -var serverCertsManager *xcerts.Manager - -// LoadX509KeyPair - load an X509 key pair (private key , certificate) -// from the provided paths. The private key may be encrypted and is -// decrypted using the ENV_VAR: OPERATOR_CERT_PASSWD. -func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) { - certPEMBlock, err := os.ReadFile(certFile) - if err != nil { - return tls.Certificate{}, err - } - keyPEMBlock, err := os.ReadFile(keyFile) - if err != nil { - return tls.Certificate{}, err - } - key, rest := pem.Decode(keyPEMBlock) - if len(rest) > 0 { - return tls.Certificate{}, errors.New("the private key contains additional data") - } - if x509.IsEncryptedPEMBlock(key) { - password := env.Get(EnvCertPassword, "") - if len(password) == 0 { - return tls.Certificate{}, errors.New("no password") - } - decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password)) - if decErr != nil { - return tls.Certificate{}, decErr - } - keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey}) - } - return tls.X509KeyPair(certPEMBlock, keyPEMBlock) +// operatorCSRName Internal func returns the given operator CSR name +func operatorCSRName() string { + return getCSRName("operator") } diff --git a/pkg/controller/cluster/tls.go b/pkg/controller/cluster/tls.go new file mode 100644 index 00000000000..51d707740b0 --- /dev/null +++ b/pkg/controller/cluster/tls.go @@ -0,0 +1,302 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cluster + +import ( + "context" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "fmt" + "os" + "time" + + miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" + "github.com/minio/operator/pkg/controller/cluster/certificates" + xcerts "github.com/minio/pkg/certs" + "github.com/minio/pkg/env" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" +) + +// generateTLSCert Generic method to generate TLS Certificartes for different services +func (c *Controller) generateTLSCert(serviceName string, secretName string, deploymentName string) (*string, *string) { + ctx := context.Background() + namespace := miniov2.GetNSFromFile() + csrName := getCSRName(serviceName) + // operator deployment for owner reference + operatorDeployment, err := c.kubeClientSet.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{}) + if err != nil { + panic(err) + } + + publicCertPath := fmt.Sprintf("/tmp/%s/public.crt", serviceName) + publicKeyPath := fmt.Sprintf("/tmp/%s/private.key", serviceName) + + for { + // TLS certificates + tlsCertSecret, err := c.kubeClientSet.CoreV1().Secrets(namespace).Get(ctx, secretName, metav1.GetOptions{}) + if err != nil { + if k8serrors.IsNotFound(err) { + klog.Infof("%s TLS secret not found: %v", serviceName, err) + if err = c.checkAndCreateCSR(ctx, operatorDeployment, serviceName, csrName, secretName); err != nil { + klog.Infof("Waiting for the %s certificates to be issued %v", serviceName, err.Error()) + time.Sleep(time.Second * 10) + } else { + err = c.deleteCSR(ctx, csrName) + if err != nil { + klog.Infof(err.Error()) + } + } + } + } else { + publicCertKey, privateKeyKey := c.getKeyNames(tlsCertSecret) + if val, ok := tlsCertSecret.Data[publicCertKey]; ok { + err := os.WriteFile(publicCertPath, val, 0o644) + if err != nil { + panic(err) + } + } else { + panic(fmt.Errorf("missing '%s' in %s/%s", publicCertKey, tlsCertSecret.Namespace, tlsCertSecret.Name)) + } + if val, ok := tlsCertSecret.Data[privateKeyKey]; ok { + err := os.WriteFile(publicKeyPath, val, 0o644) + if err != nil { + panic(err) + } + } else { + panic(fmt.Errorf("missing '%s' in %s/%s", publicCertKey, tlsCertSecret.Namespace, tlsCertSecret.Name)) + } + break + } + } + + // validate certificates if they are valid, if not panic right here. + if _, err = tls.LoadX509KeyPair(publicCertPath, publicKeyPath); err != nil { + panic(err) + } + + return &publicCertPath, &publicKeyPath +} + +// checkAndCreateCSR Queries for the Certificate signing request +func (c *Controller) checkAndCreateCSR(ctx context.Context, deployment metav1.Object, serviceName string, csrName string, secretName string) error { + var err error + if certificates.GetCertificatesAPIVersion(c.kubeClientSet) == certificates.CSRV1 { + _, err = c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{}) + } else { + _, err = c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Get(ctx, csrName, metav1.GetOptions{}) + } + if err != nil { + if k8serrors.IsNotFound(err) { + klog.V(2).Infof("Creating a new Certificate Signing Request for %s Server Certs, cluster %q", serviceName) + if err = c.createAndStoreCSR(ctx, deployment, serviceName, csrName, secretName); err != nil { + return err + } + return fmt.Errorf("waiting for %s cert", serviceName) + } + return err + } + return nil +} + +// getKeyNames Identify the K8s secret keys containing the public and private TLS certificate keys +func (c *Controller) getKeyNames(tlsCertificateSecret *corev1.Secret) (string, string) { + // default secret keys for Opaque k8s secret + publicCertKey := "public.crt" + privateKeyKey := "private.key" + // if secret type is k8s tls or cert-manager use the right secret keys + if tlsCertificateSecret.Type == "kubernetes.io/tls" || tlsCertificateSecret.Type == "cert-manager.io/v1alpha2" || tlsCertificateSecret.Type == "cert-manager.io/v1" { + publicCertKey = "tls.crt" + privateKeyKey = "tls.key" + } + return publicCertKey, privateKeyKey +} + +// createCertificateSecret Stores the private and public keys in a Secret +func (c *Controller) createCertificateSecret(ctx context.Context, deployment metav1.Object, labels map[string]string, secretName string, pkBytes, certBytes []byte) error { + secret := &corev1.Secret{ + Type: "Opaque", + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: miniov2.GetNSFromFile(), + Labels: labels, + OwnerReferences: []metav1.OwnerReference{ + *metav1.NewControllerRef(deployment, schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }), + }, + }, + Data: map[string][]byte{ + "private.key": pkBytes, + "public.crt": certBytes, + }, + } + _, err := c.kubeClientSet.CoreV1().Secrets(miniov2.GetNSFromFile()).Create(ctx, secret, metav1.CreateOptions{}) + return err +} + +// createAndStoreCSR handles all the steps required to create the CSR: from creation of keys, submitting CSR and +// finally creating a secret storing private key and certificate for TLS +// This Method Blocks till the CSR Request is approved via kubectl approve +func (c *Controller) createAndStoreCSR(ctx context.Context, deployment metav1.Object, serviceName string, csrName string, secretName string) error { + privKeysBytes, csrBytes, err := generateCSRCryptoData(serviceName) + if err != nil { + klog.Errorf("Private Key and CSR generation failed with error: %v", err) + return err + } + namespace := miniov2.GetNSFromFile() + err = c.createCertificateSigningRequest(ctx, map[string]string{}, csrName, namespace, csrBytes) + if err != nil { + klog.Errorf("Unexpected error during the creation of the csr/%s: %v", csrName, err) + return err + } + + // fetch certificate from CSR + certBytes, err := c.fetchCertificate(ctx, csrName) + if err != nil { + klog.Errorf("Unexpected error during the creation of the csr/%s: %v", csrName, err) + return err + } + + // PEM encode private ECDSA key + encodedPrivateKey := pem.EncodeToMemory(&pem.Block{Type: privateKeyType, Bytes: privKeysBytes}) + + // Create secret + err = c.createCertificateSecret(ctx, deployment, map[string]string{}, secretName, encodedPrivateKey, certBytes) + if err != nil { + klog.Errorf("Unexpected error during the creation of the %s/%s secret: %v", deployment.GetNamespace(), secretName, err) + return err + } + return nil +} + +// createTLSConfig defines the tls.Config for the http servers. +// Defines the http Protocols, TLS versions, Cipher suites between other TLS settings +func (c *Controller) createTLSConfig(certsManager *xcerts.Manager) *tls.Config { + config := &tls.Config{ + PreferServerCipherSuites: true, + CurvePreferences: []tls.CurveID{tls.CurveP256}, + NextProtos: []string{"h2", "http/1.1"}, + MinVersion: tls.VersionTLS12, + CipherSuites: []uint16{ + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + }, + GetCertificate: certsManager.GetCertificate, + } + return config +} + +// LoadX509KeyPair Internal func load an X509 key pair (private key , certificate) +// from the provided paths. The private key may be encrypted and is +// decrypted using the ENV_VAR: OPERATOR_CERT_PASSWD. +func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) { + certPEMBlock, err := os.ReadFile(certFile) + if err != nil { + return tls.Certificate{}, err + } + keyPEMBlock, err := os.ReadFile(keyFile) + if err != nil { + return tls.Certificate{}, err + } + key, rest := pem.Decode(keyPEMBlock) + if len(rest) > 0 { + return tls.Certificate{}, errors.New("the private key contains additional data") + } + if x509.IsEncryptedPEMBlock(key) { + password := env.Get(CertPasswordEnv, "") + if len(password) == 0 { + return tls.Certificate{}, errors.New("no password") + } + decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password)) + if decErr != nil { + return tls.Certificate{}, decErr + } + keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey}) + } + return tls.X509KeyPair(certPEMBlock, keyPEMBlock) +} + +// getCSRName Internal func to return the CSR name +func getCSRName(serviceName string) string { + namespace := miniov2.GetNSFromFile() + return fmt.Sprintf("%s-%s-csr", serviceName, namespace) +} + +// generateCSRCryptoData Internal func Creates the private Key material +func generateCSRCryptoData(serviceName string) ([]byte, []byte, error) { + privateKey, err := newPrivateKey(miniov2.DefaultEllipticCurve) + if err != nil { + klog.Errorf("Unexpected error during the ECDSA Key generation: %v", err) + return nil, nil, err + } + + privKeyBytes, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + klog.Errorf("Unexpected error during encoding the ECDSA Private Key: %v", err) + return nil, nil, err + } + + opCommon := fmt.Sprintf("%s.%s.svc.%s", serviceName, miniov2.GetNSFromFile(), miniov2.GetClusterDomain()) + opCommonNoDomain := fmt.Sprintf("%s.%s.svc", serviceName, miniov2.GetNSFromFile()) + + csrTemplate := x509.CertificateRequest{ + Subject: pkix.Name{ + CommonName: fmt.Sprintf("system:node:%s", opCommonNoDomain), + Organization: []string{"system:nodes"}, + }, + Extensions: []pkix.Extension{ + { + Id: nil, + Critical: false, + Value: []byte(serviceName), + }, + { + Id: nil, + Critical: false, + Value: []byte(opCommonNoDomain), + }, + { + Id: nil, + Critical: false, + Value: []byte(opCommon), + }, + }, + SignatureAlgorithm: x509.ECDSAWithSHA512, + DNSNames: []string{serviceName, opCommonNoDomain, opCommon}, + } + + csrBytes, err := x509.CreateCertificateRequest(rand.Reader, &csrTemplate, privateKey) + if err != nil { + klog.Errorf("Unexpected error during creating the CSR: %v", err) + return nil, nil, err + } + return privKeyBytes, csrBytes, nil +} From 3d927022c66fe0ba3e6b3bbee2b1f2b72d42be9e Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Thu, 17 Nov 2022 00:22:13 -0800 Subject: [PATCH 02/11] create dir --- pkg/controller/cluster/tls.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/controller/cluster/tls.go b/pkg/controller/cluster/tls.go index 51d707740b0..daa5534aeb2 100644 --- a/pkg/controller/cluster/tls.go +++ b/pkg/controller/cluster/tls.go @@ -50,6 +50,8 @@ func (c *Controller) generateTLSCert(serviceName string, secretName string, depl panic(err) } + _ = os.Mkdir(fmt.Sprintf("/tmp/%s", serviceName), os.ModePerm) + publicCertPath := fmt.Sprintf("/tmp/%s/public.crt", serviceName) publicKeyPath := fmt.Sprintf("/tmp/%s/private.key", serviceName) From 0f53205bcad6932f9749d5bfe123ad4651b3acd4 Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Thu, 17 Nov 2022 14:20:01 -0800 Subject: [PATCH 03/11] be safe creating dir --- pkg/controller/cluster/tls.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/controller/cluster/tls.go b/pkg/controller/cluster/tls.go index daa5534aeb2..7b145031a67 100644 --- a/pkg/controller/cluster/tls.go +++ b/pkg/controller/cluster/tls.go @@ -50,7 +50,10 @@ func (c *Controller) generateTLSCert(serviceName string, secretName string, depl panic(err) } - _ = os.Mkdir(fmt.Sprintf("/tmp/%s", serviceName), os.ModePerm) + mkdirerr := os.MkdirAll(fmt.Sprintf("/tmp/%s", serviceName), 0o777) + if mkdirerr != nil { + panic(mkdirerr) + } publicCertPath := fmt.Sprintf("/tmp/%s/public.crt", serviceName) publicKeyPath := fmt.Sprintf("/tmp/%s/private.key", serviceName) From ea6bf238c9794deb833f38c9240a4a6c54d961bc Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Mon, 21 Nov 2022 12:31:17 -0800 Subject: [PATCH 04/11] CSR should not error when trying to delete any no longer existing --- pkg/controller/cluster/minio.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/controller/cluster/minio.go b/pkg/controller/cluster/minio.go index 5ed4601e35a..ea436d78b28 100644 --- a/pkg/controller/cluster/minio.go +++ b/pkg/controller/cluster/minio.go @@ -70,10 +70,18 @@ func (c *Controller) checkAndCreateMinIOCSR(ctx context.Context, nsName types.Na func (c *Controller) deleteCSR(ctx context.Context, csrName string) error { if certificates.GetCertificatesAPIVersion(c.kubeClientSet) == certificates.CSRV1 { if err := c.kubeClientSet.CertificatesV1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}); err != nil { + // CSR have a short time live, we should not return error when a NotFound is thrown + // https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/#request-signing-process + if k8serrors.IsNotFound(err) { + return nil + } return err } } else { if err := c.kubeClientSet.CertificatesV1beta1().CertificateSigningRequests().Delete(ctx, csrName, metav1.DeleteOptions{}); err != nil { + if k8serrors.IsNotFound(err) { + return nil + } return err } } From fa6db423a4042baa1d7e83c05c8024a41dc248fd Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Tue, 6 Dec 2022 03:21:47 -0600 Subject: [PATCH 05/11] console cert --- pkg/controller/cluster/console.go | 76 ++++++++++++++++++----- pkg/controller/cluster/main-controller.go | 25 ++++++++ pkg/controller/cluster/operator.go | 16 +++++ resources/base/console-ui.yaml | 29 +++++++++ resources/base/deployment.yaml | 3 + 5 files changed, 135 insertions(+), 14 deletions(-) diff --git a/pkg/controller/cluster/console.go b/pkg/controller/cluster/console.go index d9dc87f0803..a5c9871fd58 100644 --- a/pkg/controller/cluster/console.go +++ b/pkg/controller/cluster/console.go @@ -18,9 +18,11 @@ package cluster import ( "context" + "os" miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" "github.com/minio/operator/pkg/resources/services" + "github.com/minio/pkg/env" corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" @@ -97,18 +99,64 @@ func (c *Controller) checkConsoleSvc(ctx context.Context, tenant *miniov2.Tenant return err } -// isConsoleTLS Internal func, reads MINIO_OPERATOR_TLS_ENABLE ENV to identify if Operator TLS is enabled, default "off" +// generateConsoleTLSCert Issues the Operator Console TLS Certificate +func (c *Controller) generateConsoleTLSCert() (*string, *string) { + return c.generateTLSCert("console", OperatorConsoleTLSSecretName, getConsoleDeploymentName()) +} + +func (c *Controller) recreateOperatorConsoleCertsIfRequired(ctx context.Context) error { + namespace := miniov2.GetNSFromFile() + operatorConsoleTLSSecret, err := c.getTLSSecret(ctx, namespace, OperatorConsoleTLSSecretName) + if err != nil { + if k8serrors.IsNotFound(err) { + klog.V(2).Info("TLS certificate not found. Generating one.") + // Generate new certificate KeyPair for Operator Console + c.generateConsoleTLSCert() + return nil + } + return err + } + + needsRenewal, err := c.certNeedsRenewal(operatorConsoleTLSSecret) + if err != nil { + return err + } + + if !needsRenewal { + return nil + } + + // Expired cert. Delete the secret + CSR and re-create the cert + err = c.deleteCSR(ctx, consoleCSRName()) + if err != nil { + return err + } + klog.V(2).Info("Deleting the TLS secret of expired console cert") + err = c.kubeClientSet.CoreV1().Secrets(namespace).Delete(ctx, OperatorConsoleTLSSecretName, metav1.DeleteOptions{}) + if err != nil { + return err + } + + klog.V(2).Info("Generating a fresh TLS certificate for Console") + // Generate new certificate KeyPair for Operator Console + c.generateConsoleTLSCert() + + return nil +} + +// isOperatorConsoleTLS Internal func, reads MINIO_OPERATOR_TLS_ENABLE ENV to identify if Operator Console TLS is enabled, default "off" // **WARNING** This will change and will be default to "on" in operator v5 -// func isConsoleTLS() bool { -// value, set := os.LookupEnv(ConsoleTLSEnv) -// // By default, Console TLS is NOT used. -// return (set && value == "on") || set -// } - -// func getConsoleDeploymentName() string { -// return env.Get("MINIO_CONSOLE_DEPLOYMENT_NAME", DefaultConsoleDeploymentName) -// } - -// func (c *Controller) generateConsoleTLSCert() (*string, *string) { -// return c.generateTLSCert("console", OperatorConsoleTLSSecretName, getConsoleDeploymentName()) -// } +func isOperatorConsoleTLS() bool { + value, set := os.LookupEnv(ConsoleTLSEnv) + // By default, Console TLS is NOT used. + return (set && value == "on") +} + +func getConsoleDeploymentName() string { + return env.Get("MINIO_CONSOLE_DEPLOYMENT_NAME", DefaultConsoleDeploymentName) +} + +// consoleCSRName Internal func returns the given console CSR name +func consoleCSRName() string { + return getCSRName("console") +} diff --git a/pkg/controller/cluster/main-controller.go b/pkg/controller/cluster/main-controller.go index af2f6ac8d93..65206c1ba97 100644 --- a/pkg/controller/cluster/main-controller.go +++ b/pkg/controller/cluster/main-controller.go @@ -343,6 +343,8 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error { apiServerWillStart := make(chan interface{}) // we need to make sure the HTTP Upgrade server is ready before starting operator upgradeServerWillStart := make(chan interface{}) + // pausing the process until console has it's TLS certificate (if enabled) + consoleTLS := make(chan interface{}) go func() { // Request kubernetes version from Kube ApiServer @@ -384,12 +386,35 @@ func (c *Controller) Start(threadiness int, stopCh <-chan struct{}) error { } }() + go func() { + klog.Infof("Starting console TLS certificate setup") + if isOperatorConsoleTLS() { + klog.Infof("Console TLS enabled") + err := c.recreateOperatorConsoleCertsIfRequired(ctx) + close(consoleTLS) + if err != nil { + panic(err) + } + klog.Infof("Restarting Console pods") + err = c.rolloutRestartDeployment(getConsoleDeploymentName()) + if err != nil { + klog.Errorf("Console deployment didn't restart: %s", err) + } + } else { + klog.Infof("Console TLS is not enabled") + close(consoleTLS) + } + }() + klog.Info("Waiting for API to start") <-apiServerWillStart klog.Info("Waiting for Upgrade Server to start") <-upgradeServerWillStart + klog.Info("Waiting for Console TLS") + <-consoleTLS + // Start the informer factories to begin populating the informer caches klog.Info("Starting Tenant controller") diff --git a/pkg/controller/cluster/operator.go b/pkg/controller/cluster/operator.go index ded09d0ac7d..134203a46a8 100644 --- a/pkg/controller/cluster/operator.go +++ b/pkg/controller/cluster/operator.go @@ -51,6 +51,22 @@ const ( var serverCertsManager *xcerts.Manager +// rolloutRestartDeployment - executes the equivalent to kubectl rollout restart deployment +func (c *Controller) rolloutRestartDeployment(deployName string) error { + ctx := context.Background() + namespace := miniov2.GetNSFromFile() + deployment, err := c.kubeClientSet.AppsV1().Deployments(namespace).Get(ctx, deployName, metav1.GetOptions{}) + if err != nil { + return err + } + + _, errUpdt := c.kubeClientSet.AppsV1().Deployments(namespace).Update(ctx, deployment, metav1.UpdateOptions{FieldManager: "kubectl-rollout"}) + if errUpdt != nil { + return errUpdt + } + return nil +} + // generateOperatorTLSCert Issues the Operator TLS Certificate func (c *Controller) generateOperatorTLSCert() (*string, *string) { return c.generateTLSCert("operator", OperatorTLSSecretName, getOperatorDeploymentName()) diff --git a/resources/base/console-ui.yaml b/resources/base/console-ui.yaml index 2a3b540b03b..d0c4a9b6864 100644 --- a/resources/base/console-ui.yaml +++ b/resources/base/console-ui.yaml @@ -296,6 +296,11 @@ metadata: app.kubernetes.io/instance: minio-operator app.kubernetes.io/name: operator spec: + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate replicas: 1 selector: matchLabels: @@ -310,6 +315,7 @@ spec: containers: - args: - server + - --certs-dir=/tmp/certs env: - name: CONSOLE_OPERATOR_MODE value: "on" @@ -325,4 +331,27 @@ spec: name: http - containerPort: 9433 name: https + volumeMounts: + - mountPath: /tmp/certs + name: tls-certificates + volumes: + - name: tls-certificates + projected: + sources: + - secret: + items: + - key: public.crt + path: public.crt + - key: public.crt + path: CAs/public.crt + - key: private.key + path: private.key + - key: tls.crt + path: tls.crt + - key: tls.crt + path: CAs/tls.crt + - key: tls.key + path: tls.key + name: operator-console-tls + optional: true serviceAccountName: console-sa diff --git a/resources/base/deployment.yaml b/resources/base/deployment.yaml index 33af9bb37a8..ba07f8850cb 100644 --- a/resources/base/deployment.yaml +++ b/resources/base/deployment.yaml @@ -32,6 +32,9 @@ spec: runAsUser: 1000 runAsGroup: 1000 runAsNonRoot: true + env: + - name: MINIO_CONSOLE_TLS_ENABLE + value: "off" affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: From f8756f0c427be881d470140750fd64d5c76b421c Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Tue, 6 Dec 2022 12:04:03 -0600 Subject: [PATCH 06/11] Fix restart Console pods with TLS certificate --- go.mod | 7 ++++--- go.sum | 14 ++++++++------ kubectl-minio/go.mod | 6 +++--- kubectl-minio/go.sum | 12 ++++++------ pkg/controller/cluster/operator.go | 19 +++++++++++++++---- 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/go.mod b/go.mod index 037d1e9c3b2..8c42337268a 100644 --- a/go.mod +++ b/go.mod @@ -21,12 +21,13 @@ require ( golang.org/x/crypto v0.0.0-20221012134737-56aed061732a golang.org/x/time v0.1.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.25.3 + k8s.io/api v0.25.4 k8s.io/apiextensions-apiserver v0.25.3 - k8s.io/apimachinery v0.25.3 - k8s.io/client-go v0.25.3 + k8s.io/apimachinery v0.25.4 + k8s.io/client-go v0.25.4 k8s.io/code-generator v0.25.3 k8s.io/klog/v2 v2.80.1 + k8s.io/kubectl v0.25.4 sigs.k8s.io/controller-runtime v0.13.0 ) diff --git a/go.sum b/go.sum index 822b035a84a..5172940d670 100644 --- a/go.sum +++ b/go.sum @@ -1819,14 +1819,14 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.2.1/go.mod h1:lPVVZ2BS5TfnjLyizF7o7hv7j9/L+8cZY2hLyjP9cGY= -k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ= -k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= +k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs= +k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ= k8s.io/apiextensions-apiserver v0.25.3 h1:bfI4KS31w2f9WM1KLGwnwuVlW3RSRPuIsfNF/3HzR0k= k8s.io/apiextensions-apiserver v0.25.3/go.mod h1:ZJqwpCkxIx9itilmZek7JgfUAM0dnTsA48I4krPqRmo= -k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc= -k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= -k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0= -k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA= +k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc= +k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= +k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= +k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= k8s.io/code-generator v0.25.3 h1:BEH+wDi90bGyrYcY4abGtUqaOX7G94RRrEu8l+SvIeo= k8s.io/code-generator v0.25.3/go.mod h1:9F5fuVZOMWRme7MYj2YT3L9ropPWPokd9VRhVyD3+0w= k8s.io/gengo v0.0.0-20211129171323-c02415ce4185 h1:TT1WdmqqXareKxZ/oNXEUSwKlLiHzPMyB0t8BaFeBYI= @@ -1837,6 +1837,8 @@ k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea h1:3QOH5+2fGsY8e1qf+GIFpg+zw/JGNrgyZRQR7/m6uWg= k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea/go.mod h1:C/N6wCaBHeBHkHUesQOQy2/MZqGgMAFPqGsGQLdbZBU= +k8s.io/kubectl v0.25.4 h1:O3OA1z4V1ZyvxCvScjq0pxAP7ABgznr8UvnVObgI6Dc= +k8s.io/kubectl v0.25.4/go.mod h1:CKMrQ67Bn2YCP26tZStPQGq62zr9pvzEf65A0navm8k= k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73 h1:H9TCJUUx+2VA0ZiD9lvtaX8fthFsMoD+Izn93E/hm8U= k8s.io/utils v0.0.0-20220823124924-e9cbc92d1a73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= mvdan.cc/gofumpt v0.1.1/go.mod h1:yXG1r1WqZVKWbVRtBWKWX9+CxGYfA51nSomhM0woR48= diff --git a/kubectl-minio/go.mod b/kubectl-minio/go.mod index 74877b3e5d7..b912c79e211 100644 --- a/kubectl-minio/go.mod +++ b/kubectl-minio/go.mod @@ -12,11 +12,11 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.6.0 - k8s.io/api v0.25.3 + k8s.io/api v0.25.4 k8s.io/apiextensions-apiserver v0.25.3 - k8s.io/apimachinery v0.25.3 + k8s.io/apimachinery v0.25.4 k8s.io/cli-runtime v0.25.3 - k8s.io/client-go v0.25.3 + k8s.io/client-go v0.25.4 k8s.io/klog/v2 v2.80.1 sigs.k8s.io/kustomize/api v0.12.1 sigs.k8s.io/kustomize/kyaml v0.13.9 diff --git a/kubectl-minio/go.sum b/kubectl-minio/go.sum index b1d3d7026a7..abfb33e6005 100644 --- a/kubectl-minio/go.sum +++ b/kubectl-minio/go.sum @@ -408,16 +408,16 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.25.3 h1:Q1v5UFfYe87vi5H7NU0p4RXC26PPMT8KOpr1TLQbCMQ= -k8s.io/api v0.25.3/go.mod h1:o42gKscFrEVjHdQnyRenACrMtbuJsVdP+WVjqejfzmI= +k8s.io/api v0.25.4 h1:3YO8J4RtmG7elEgaWMb4HgmpS2CfY1QlaOz9nwB+ZSs= +k8s.io/api v0.25.4/go.mod h1:IG2+RzyPQLllQxnhzD8KQNEu4c4YvyDTpSMztf4A0OQ= k8s.io/apiextensions-apiserver v0.25.3 h1:bfI4KS31w2f9WM1KLGwnwuVlW3RSRPuIsfNF/3HzR0k= k8s.io/apiextensions-apiserver v0.25.3/go.mod h1:ZJqwpCkxIx9itilmZek7JgfUAM0dnTsA48I4krPqRmo= -k8s.io/apimachinery v0.25.3 h1:7o9ium4uyUOM76t6aunP0nZuex7gDf8VGwkR5RcJnQc= -k8s.io/apimachinery v0.25.3/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= +k8s.io/apimachinery v0.25.4 h1:CtXsuaitMESSu339tfhVXhQrPET+EiWnIY1rcurKnAc= +k8s.io/apimachinery v0.25.4/go.mod h1:jaF9C/iPNM1FuLl7Zuy5b9v+n35HGSh6AQ4HYRkCqwo= k8s.io/cli-runtime v0.25.3 h1:Zs7P7l7db/5J+KDePOVtDlArAa9pZXaDinGWGZl0aM8= k8s.io/cli-runtime v0.25.3/go.mod h1:InHHsjkyW5hQsILJGpGjeruiDZT/R0OkROQgD6GzxO4= -k8s.io/client-go v0.25.3 h1:oB4Dyl8d6UbfDHD8Bv8evKylzs3BXzzufLiO27xuPs0= -k8s.io/client-go v0.25.3/go.mod h1:t39LPczAIMwycjcXkVc+CB+PZV69jQuNx4um5ORDjQA= +k8s.io/client-go v0.25.4 h1:3RNRDffAkNU56M/a7gUfXaEzdhZlYhoW8dgViGy5fn8= +k8s.io/client-go v0.25.4/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.80.1 h1:atnLQ121W371wYYFawwYx1aEY2eUfs4l3J72wtgAwV4= k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= diff --git a/pkg/controller/cluster/operator.go b/pkg/controller/cluster/operator.go index 134203a46a8..47ca36283a3 100644 --- a/pkg/controller/cluster/operator.go +++ b/pkg/controller/cluster/operator.go @@ -24,14 +24,18 @@ import ( "os" "time" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" miniov2 "github.com/minio/operator/pkg/apis/minio.min.io/v2" xcerts "github.com/minio/pkg/certs" "github.com/minio/pkg/env" "k8s.io/klog/v2" + k8sscheme "k8s.io/kubectl/pkg/scheme" ) const ( @@ -59,10 +63,17 @@ func (c *Controller) rolloutRestartDeployment(deployName string) error { if err != nil { return err } - - _, errUpdt := c.kubeClientSet.AppsV1().Deployments(namespace).Update(ctx, deployment, metav1.UpdateOptions{FieldManager: "kubectl-rollout"}) - if errUpdt != nil { - return errUpdt + if deployment.Spec.Template.ObjectMeta.Annotations == nil { + deployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + deployment.Spec.Template.ObjectMeta.Annotations["kubectl.kubernetes.io/restartedAt"] = time.Now().Format(time.RFC3339) + data, err := runtime.Encode(k8sscheme.Codecs.LegacyCodec(appsv1.SchemeGroupVersion), deployment) + if err != nil { + return err + } + _, err2 := c.kubeClientSet.AppsV1().Deployments(namespace).Patch(ctx, deployName, types.StrategicMergePatchType, data, metav1.PatchOptions{FieldManager: "kubectl-rollout"}) + if err2 != nil { + return err2 } return nil } From 68238f81c8c2ed68978aacb19240692465b41aee Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Tue, 6 Dec 2022 18:08:02 -0600 Subject: [PATCH 07/11] Add option to enable TLS for operator console in the kubectl-minio init command --- kubectl-minio/cmd/init.go | 7 ++++--- kubectl-minio/cmd/resources/deployment.go | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/kubectl-minio/cmd/init.go b/kubectl-minio/cmd/init.go index e6e227aa6da..3a30d8d758c 100644 --- a/kubectl-minio/cmd/init.go +++ b/kubectl-minio/cmd/init.go @@ -80,6 +80,7 @@ func newInitCmd(out io.Writer, errOut io.Writer) *cobra.Command { f.StringVar(&o.operatorOpts.NSToWatch, "namespace-to-watch", "", "namespace where operator looks for MinIO tenants, leave empty for all namespaces") f.StringVar(&o.operatorOpts.ImagePullSecret, "image-pull-secret", "", "image pull secret to be used for pulling MinIO Operator") f.StringVar(&o.operatorOpts.ConsoleImage, "console-image", "", "console image") + f.BoolVar(&o.operatorOpts.ConsoleTLS, "console-tls", false, "enable tls for Operator console") f.StringVar(&o.operatorOpts.TenantMinIOImage, "default-minio-image", "", "default tenant MinIO image") f.StringVar(&o.operatorOpts.TenantConsoleImage, "default-console-image", "", "default tenant Console image") f.StringVar(&o.operatorOpts.TenantKesImage, "default-kes-image", "", "default tenant KES image") @@ -148,13 +149,13 @@ func (o *operatorInitCmd) run(writer io.Writer) error { }, }) } - if o.operatorOpts.NSToWatch != "" { + if o.operatorOpts.ConsoleTLS { operatorDepPatches = append(operatorDepPatches, opInterface{ Op: "add", Path: "/spec/template/spec/containers/0/env/0", Value: corev1.EnvVar{ - Name: "WATCHED_NAMESPACE", - Value: o.operatorOpts.NSToWatch, + Name: "MINIO_CONSOLE_TLS_ENABLE", + Value: "on", }, }) } diff --git a/kubectl-minio/cmd/resources/deployment.go b/kubectl-minio/cmd/resources/deployment.go index 1ac06b0d820..7c2ecfa0ec8 100644 --- a/kubectl-minio/cmd/resources/deployment.go +++ b/kubectl-minio/cmd/resources/deployment.go @@ -24,6 +24,7 @@ type OperatorOptions struct { ClusterDomain string ImagePullSecret string ConsoleImage string + ConsoleTLS bool TenantMinIOImage string TenantConsoleImage string TenantKesImage string From 8d18c15a9d6197c5500cbdfb26abe3113a3c0b2b Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Fri, 16 Dec 2022 12:49:58 -0600 Subject: [PATCH 08/11] overwrited NSToWatch by mistake, reverting it --- kubectl-minio/cmd/init.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/kubectl-minio/cmd/init.go b/kubectl-minio/cmd/init.go index d6f98ea6ced..e2474ce8bef 100644 --- a/kubectl-minio/cmd/init.go +++ b/kubectl-minio/cmd/init.go @@ -149,6 +149,16 @@ func (o *operatorInitCmd) run(writer io.Writer) error { }, }) } + if o.operatorOpts.NSToWatch != "" { + operatorDepPatches = append(operatorDepPatches, opInterface{ + Op: "add", + Path: "/spec/template/spec/containers/0/env/0", + Value: corev1.EnvVar{ + Name: "WATCHED_NAMESPACE", + Value: o.operatorOpts.NSToWatch, + }, + }) + } if o.operatorOpts.ConsoleTLS { operatorDepPatches = append(operatorDepPatches, opInterface{ Op: "add", From f2c009c44f7d83d64795c370a9c238e8eb9f002b Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Fri, 16 Dec 2022 12:50:18 -0600 Subject: [PATCH 09/11] deploy strategy not needed for now --- resources/base/console-ui.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/resources/base/console-ui.yaml b/resources/base/console-ui.yaml index 7857dfba578..e3eaad74b6d 100644 --- a/resources/base/console-ui.yaml +++ b/resources/base/console-ui.yaml @@ -296,11 +296,6 @@ metadata: app.kubernetes.io/instance: minio-operator app.kubernetes.io/name: operator spec: - strategy: - rollingUpdate: - maxSurge: 25% - maxUnavailable: 25% - type: RollingUpdate replicas: 1 selector: matchLabels: From eeb2fe9671d8b4b65d551938971810361a2ff36e Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Fri, 16 Dec 2022 12:51:13 -0600 Subject: [PATCH 10/11] add MINIO_CONSOLE_TLS_ENABLE variable to the helm template for reference --- helm/operator/values.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/helm/operator/values.yaml b/helm/operator/values.yaml index b7573c84294..affa550895e 100644 --- a/helm/operator/values.yaml +++ b/helm/operator/values.yaml @@ -5,6 +5,8 @@ operator: # env: # - name: MINIO_OPERATOR_TLS_ENABLE # value: "off" + # - name: MINIO_CONSOLE_TLS_ENABLE + # value: "off" # - name: CLUSTER_DOMAIN # value: "cluster.domain" # - name: WATCHED_NAMESPACE From 3bb9dda1a260b9709a09fea6418648ebd7cff450 Mon Sep 17 00:00:00 2001 From: pjuarezd Date: Fri, 16 Dec 2022 13:03:54 -0600 Subject: [PATCH 11/11] gofumpt complaining in formating --- kubectl-minio/cmd/helpers/constants.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kubectl-minio/cmd/helpers/constants.go b/kubectl-minio/cmd/helpers/constants.go index 3f2101ecac1..3a74a19a20f 100644 --- a/kubectl-minio/cmd/helpers/constants.go +++ b/kubectl-minio/cmd/helpers/constants.go @@ -45,7 +45,5 @@ const ( DefaultKESImage = "minio/kes:v0.18.0" ) -var ( - // KESReplicas is the number of replicas for MinIO KES - KESReplicas int32 = 2 -) +// KESReplicas is the number of replicas for MinIO KES +var KESReplicas int32 = 2