diff --git a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index 59c58e33..6623c123 100644 --- a/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/api/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -50,9 +50,15 @@ spec: description: OctaviaAmphoraControllerSpec defines common state for all Octavia Amphora Controllers properties: + certspassphrasesecret: + default: octavia-ca-passphrase + description: Name of secret containing passphrase for the CA private + keys + type: string certssecret: - description: '*kubebuilder:validation:Required Secret containing certs - for securing communication with amphora based Load Balancers' + default: octavia-certs-secret + description: LoadBalancerCerts - Secret containing certs for securing + communication with amphora based Load Balancers type: string containerImage: description: ContainerImage - Amphora Controller Container Image URL diff --git a/api/bases/octavia.openstack.org_octavias.yaml b/api/bases/octavia.openstack.org_octavias.yaml index fa729a1b..9df71518 100644 --- a/api/bases/octavia.openstack.org_octavias.yaml +++ b/api/bases/octavia.openstack.org_octavias.yaml @@ -456,9 +456,15 @@ spec: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment properties: + certspassphrasesecret: + default: octavia-ca-passphrase + description: Name of secret containing passphrase for the CA private + keys + type: string certssecret: - description: '*kubebuilder:validation:Required Secret containing - certs for securing communication with amphora based Load Balancers' + default: octavia-certs-secret + description: LoadBalancerCerts - Secret containing certs for securing + communication with amphora based Load Balancers type: string containerImage: description: ContainerImage - Amphora Controller Container Image @@ -632,9 +638,15 @@ spec: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment properties: + certspassphrasesecret: + default: octavia-ca-passphrase + description: Name of secret containing passphrase for the CA private + keys + type: string certssecret: - description: '*kubebuilder:validation:Required Secret containing - certs for securing communication with amphora based Load Balancers' + default: octavia-certs-secret + description: LoadBalancerCerts - Secret containing certs for securing + communication with amphora based Load Balancers type: string containerImage: description: ContainerImage - Amphora Controller Container Image @@ -808,9 +820,15 @@ spec: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment properties: + certspassphrasesecret: + default: octavia-ca-passphrase + description: Name of secret containing passphrase for the CA private + keys + type: string certssecret: - description: '*kubebuilder:validation:Required Secret containing - certs for securing communication with amphora based Load Balancers' + default: octavia-certs-secret + description: LoadBalancerCerts - Secret containing certs for securing + communication with amphora based Load Balancers type: string containerImage: description: ContainerImage - Amphora Controller Container Image diff --git a/api/v1beta1/amphoracontroller_types.go b/api/v1beta1/amphoracontroller_types.go index efb112eb..dad57810 100644 --- a/api/v1beta1/amphoracontroller_types.go +++ b/api/v1beta1/amphoracontroller_types.go @@ -77,10 +77,16 @@ type OctaviaAmphoraControllerSpec struct { // Secret containing OpenStack password information for octavia OctaviaDatabasePassword, AdminPassword Secret string `json:"secret"` - // *kubebuilder:validation:Required - // Secret containing certs for securing communication with amphora based Load Balancers + // +kubebuilder:validation:Required + // +kubebuilder:default=octavia-certs-secret + // LoadBalancerCerts - Secret containing certs for securing communication with amphora based Load Balancers LoadBalancerCerts string `json:"certssecret"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=octavia-ca-passphrase + // Name of secret containing passphrase for the CA private keys + CAKeyPassphraseSecret string `json:"certspassphrasesecret"` + // +kubebuilder:validation:Optional // +kubebuilder:default={database: OctaviaDatabasePassword, service: OctaviaPassword} // PasswordSelectors - Selectors to identify the DB and AdminUser password from the Secret diff --git a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml index 59c58e33..6623c123 100644 --- a/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml +++ b/config/crd/bases/octavia.openstack.org_octaviaamphoracontrollers.yaml @@ -50,9 +50,15 @@ spec: description: OctaviaAmphoraControllerSpec defines common state for all Octavia Amphora Controllers properties: + certspassphrasesecret: + default: octavia-ca-passphrase + description: Name of secret containing passphrase for the CA private + keys + type: string certssecret: - description: '*kubebuilder:validation:Required Secret containing certs - for securing communication with amphora based Load Balancers' + default: octavia-certs-secret + description: LoadBalancerCerts - Secret containing certs for securing + communication with amphora based Load Balancers type: string containerImage: description: ContainerImage - Amphora Controller Container Image URL diff --git a/config/crd/bases/octavia.openstack.org_octavias.yaml b/config/crd/bases/octavia.openstack.org_octavias.yaml index fa729a1b..9df71518 100644 --- a/config/crd/bases/octavia.openstack.org_octavias.yaml +++ b/config/crd/bases/octavia.openstack.org_octavias.yaml @@ -456,9 +456,15 @@ spec: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment properties: + certspassphrasesecret: + default: octavia-ca-passphrase + description: Name of secret containing passphrase for the CA private + keys + type: string certssecret: - description: '*kubebuilder:validation:Required Secret containing - certs for securing communication with amphora based Load Balancers' + default: octavia-certs-secret + description: LoadBalancerCerts - Secret containing certs for securing + communication with amphora based Load Balancers type: string containerImage: description: ContainerImage - Amphora Controller Container Image @@ -632,9 +638,15 @@ spec: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment properties: + certspassphrasesecret: + default: octavia-ca-passphrase + description: Name of secret containing passphrase for the CA private + keys + type: string certssecret: - description: '*kubebuilder:validation:Required Secret containing - certs for securing communication with amphora based Load Balancers' + default: octavia-certs-secret + description: LoadBalancerCerts - Secret containing certs for securing + communication with amphora based Load Balancers type: string containerImage: description: ContainerImage - Amphora Controller Container Image @@ -808,9 +820,15 @@ spec: description: OctaviaHousekeeping - Spec definition for the Octavia Housekeeping agent for the Octavia deployment properties: + certspassphrasesecret: + default: octavia-ca-passphrase + description: Name of secret containing passphrase for the CA private + keys + type: string certssecret: - description: '*kubebuilder:validation:Required Secret containing - certs for securing communication with amphora based Load Balancers' + default: octavia-certs-secret + description: LoadBalancerCerts - Secret containing certs for securing + communication with amphora based Load Balancers type: string containerImage: description: ContainerImage - Amphora Controller Container Image diff --git a/config/samples/octavia_v1beta1_octavia.yaml b/config/samples/octavia_v1beta1_octavia.yaml index 93125cd3..3ac72e32 100644 --- a/config/samples/octavia_v1beta1_octavia.yaml +++ b/config/samples/octavia_v1beta1_octavia.yaml @@ -21,7 +21,7 @@ spec: serviceUser: octavia serviceAccount: octavia role: housekeeping - certssecret: todo + certssecret: octavia-amp-cert-data secret: osp-secret preserveJobs: false customServiceConfig: | @@ -33,7 +33,7 @@ spec: serviceUser: octavia serviceAccount: octavia role: healthmanager - certssecret: todo + certssecret: octavia-amp-cert-data secret: osp-secret preserveJobs: false customServiceConfig: | @@ -45,7 +45,7 @@ spec: serviceUser: octavia serviceAccount: octavia role: worker - certssecret: todo + certssecret: octavia-amp-cert-data secret: osp-secret preserveJobs: false customServiceConfig: | diff --git a/controllers/amphoracontroller_controller.go b/controllers/amphoracontroller_controller.go index 9f2adbba..59501c72 100644 --- a/controllers/amphoracontroller_controller.go +++ b/controllers/amphoracontroller_controller.go @@ -31,6 +31,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" nad "github.com/openstack-k8s-operators/lib-common/modules/common/networkattachment" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" "github.com/openstack-k8s-operators/lib-common/modules/common/util" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" @@ -253,6 +254,17 @@ func (r *OctaviaAmphoraControllerReconciler) reconcileNormal(ctx context.Context return ctrl.Result{}, err } + err = amphoracontrollers.EnsureAmphoraCerts(ctx, instance, helper, &Log) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ServiceConfigReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ServiceConfigReadyErrorMessage, + err.Error())) + return ctrl.Result{}, err + } + instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.InputReadyMessage) // @@ -417,12 +429,25 @@ func (r *OctaviaAmphoraControllerReconciler) generateServiceConfigMaps( if err != nil { return err } - templateParameters["ServiceUser"] = instance.Spec.ServiceUser + caPassSecret, _, err := secret.GetSecret( + ctx, helper, instance.Spec.CAKeyPassphraseSecret, instance.Namespace) + if err != nil { + return err + } + spec := instance.Spec + templateParameters["ServiceUser"] = spec.ServiceUser templateParameters["KeystoneInternalURL"] = keystoneInternalURL templateParameters["KeystonePublicURL"] = keystonePublicURL - templateParameters["ServiceRoleName"] = instance.Spec.Role + templateParameters["ServiceRoleName"] = spec.Role templateParameters["LbMgmtNetworkId"] = templateVars.LbMgmtNetworkID templateParameters["AmpFlavorId"] = templateVars.AmphoraDefaultFlavorID + serverCAPassphrase := caPassSecret.Data["server-ca-passphrase"] + if serverCAPassphrase != nil { + templateParameters["ServerCAKeyPassphrase"] = string(serverCAPassphrase) + } else { + // Can't do string(nil) + templateParameters["ServerCAKeyPassphrase"] = "" + } // TODO(beagles): populate the template parameters cms := []util.Template{ diff --git a/pkg/amphoracontrollers/amphora_certs.go b/pkg/amphoracontrollers/amphora_certs.go new file mode 100644 index 00000000..6c10acb1 --- /dev/null +++ b/pkg/amphoracontrollers/amphora_certs.go @@ -0,0 +1,215 @@ +package amphoracontrollers + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" + + "github.com/go-logr/logr" + "github.com/openstack-k8s-operators/lib-common/modules/common/helper" + "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + octaviav1 "github.com/openstack-k8s-operators/octavia-operator/api/v1beta1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +var ( + subjectDefault = pkix.Name{ + Organization: []string{"Dis"}, + Country: []string{"US"}, + Province: []string{"Oregon"}, + Locality: []string{"Springfield"}, + StreetAddress: []string{"Denial"}, + PostalCode: []string{""}, + CommonName: "www.example.com", + } +) + +// generateKey generates a PEM encoded private RSA key and applies PEM +// encryption if given passphrase is not an empty string. +func generateKey(passphrase []byte) (*rsa.PrivateKey, []byte, error) { + priv, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return nil, nil, err + } + + var pemBlock *pem.Block + if passphrase != nil { + pemBlock, err = x509.EncryptPEMBlock( //nolint:staticcheck + rand.Reader, + "RSA PRIVATE KEY", + x509.MarshalPKCS1PrivateKey(priv), + passphrase, + x509.PEMCipherAES128) + if err != nil { + fmt.Println("Error encrypting private CA key:", err) + return priv, nil, err + } + } else { + privBytes := x509.MarshalPKCS1PrivateKey(priv) + pemBlock = &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes} + } + + privPEM := new(bytes.Buffer) + err = pem.Encode(privPEM, pemBlock) + if err != nil { + return priv, nil, err + } + + return priv, privPEM.Bytes(), nil +} + +func generateCACert(caPrivKey *rsa.PrivateKey, commonName string) ([]byte, error) { + caTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: subjectDefault, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + BasicConstraintsValid: true, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCRLSign | x509.KeyUsageCertSign, + } + caTemplate.Subject.CommonName = commonName + + caBytes, err := x509.CreateCertificate( + rand.Reader, caTemplate, caTemplate, &caPrivKey.PublicKey, caPrivKey) + if err != nil { + return nil, err + } + caCertPEM := new(bytes.Buffer) + err = pem.Encode(caCertPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: caBytes, + }) + if err != nil { + return nil, err + } + return caCertPEM.Bytes(), nil +} + +// Create a certificate and key for the client and sign it with the CA +func generateClientCert(caCertPEM []byte, caPrivKey *rsa.PrivateKey) ([]byte, error) { + + certTemplate := &x509.Certificate{ + SerialNumber: big.NewInt(2019), + Subject: subjectDefault, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(0, 1, 0), + IsCA: false, + BasicConstraintsValid: false, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageEmailProtection}, + } + + certBytes, err := x509.CreateCertificate( + rand.Reader, certTemplate, certTemplate, &caPrivKey.PublicKey, caPrivKey) + if err != nil { + return nil, err + } + + certPEM := new(bytes.Buffer) + err = pem.Encode(certPEM, &pem.Block{ + Type: "CERTIFICATE", + Bytes: certBytes, + }) + if err != nil { + return nil, err + } + + return certPEM.Bytes(), nil +} + +// EnsureAmphoraCerts ensures Amphora certificates exist in the secret store +func EnsureAmphoraCerts(ctx context.Context, instance *octaviav1.OctaviaAmphoraController, h *helper.Helper, log *logr.Logger) error { + var oAmpSecret *corev1.Secret + var serverCAPass []byte = nil + + _, _, err := secret.GetSecret(ctx, h, instance.Spec.LoadBalancerCerts, instance.Namespace) + if err != nil { + if !k8serrors.IsNotFound(err) { + err = fmt.Errorf("Error retrieving secret %s - %w", instance.Spec.LoadBalancerCerts, err) + return err + } + + cAPassSecret, _, err := secret.GetSecret( + ctx, h, instance.Spec.CAKeyPassphraseSecret, instance.Namespace) + if err != nil { + log.Info("Could not read server CA passphrase. No encryption will be applied to the generated key.") + } else { + serverCAPass = cAPassSecret.Data["server-ca-passphrase"] + } + + serverCAKey, serverCAKeyPEM, err := generateKey(serverCAPass) + if err != nil { + err = fmt.Errorf("Error while generating server CA key: %w", err) + return err + } + serverCACert, err := generateCACert(serverCAKey, "Octavia server CA") + if err != nil { + err = fmt.Errorf("Error while generating server CA certificate: %w", err) + return err + } + + clientCAKey, _, err := generateKey(nil) + if err != nil { + err = fmt.Errorf("Error while generating client CA key: %w", err) + return err + } + clientCACert, err := generateCACert(clientCAKey, "Octavia client CA") + if err != nil { + err = fmt.Errorf("Error while generating amphora client CA certificate: %w", err) + return err + } + + clientKey, clientKeyPEM, err := generateKey(nil) + if err != nil { + err = fmt.Errorf("Error while generating amphora client key: %w", err) + return err + } + clientCert, err := generateClientCert(clientCACert, clientKey) + if err != nil { + err = fmt.Errorf("Error while generating amphora client certificate: %w", err) + return err + } + clientKeyAndCert := append(clientKeyPEM, clientCert...) + + oAmpSecret = &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: instance.Spec.LoadBalancerCerts, + Namespace: instance.Namespace, + }, + + // note: the client CA key seem to be needed only for generating the + // client CA cert and should not get mounted to the pods + Data: map[string][]byte{ + "server_ca.key.pem": serverCAKeyPEM, + "server_ca.cert.pem": serverCACert, + "client_ca.cert.pem": clientCACert, + // Unencrypted client key and cert + "client.cert-and-key.pem": clientKeyAndCert, + }, + } + + // err = h.GetClient().Create(ctx, oAmpSecret) + _, result, err := secret.CreateOrPatchSecret(ctx, h, instance, oAmpSecret) + + if err != nil { + err = fmt.Errorf("Error creating certs secret %s - %w", + instance.Spec.LoadBalancerCerts, err) + return err + } else if result != controllerutil.OperationResultNone { + return nil + } + } + + return nil +} diff --git a/pkg/amphoracontrollers/deployment.go b/pkg/amphoracontrollers/deployment.go index 8b060c45..06d7d544 100644 --- a/pkg/amphoracontrollers/deployment.go +++ b/pkg/amphoracontrollers/deployment.go @@ -41,6 +41,10 @@ func Deployment( // The API pod has an extra volume so the API and the provider agent can // communicate with each other. volumes := octavia.GetVolumes(instance.Name) + volumes = append(volumes, GetCertVolume(instance.Spec.LoadBalancerCerts)...) + + volumeMounts := octavia.GetVolumeMounts(serviceName) + volumeMounts = append(volumeMounts, GetCertVolumeMount()...) // TODO(beagles): service debugging @@ -100,7 +104,7 @@ func Deployment( Name: serviceName, Image: instance.Spec.ContainerImage, Env: env.MergeEnvs([]corev1.EnvVar{}, envVars), - VolumeMounts: octavia.GetVolumeMounts(serviceName), + VolumeMounts: volumeMounts, Resources: instance.Spec.Resources, ReadinessProbe: readinessProbe, LivenessProbe: livenessProbe, diff --git a/pkg/amphoracontrollers/volumes.go b/pkg/amphoracontrollers/volumes.go new file mode 100644 index 00000000..1f72f42f --- /dev/null +++ b/pkg/amphoracontrollers/volumes.go @@ -0,0 +1,55 @@ +/* + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amphoracontrollers + +import ( + corev1 "k8s.io/api/core/v1" +) + +const ( + configVolume = "amphora-certs" +) + +var ( + // Files get mounted as root:root, but process is running as octavia + configMode int32 = 0644 +) + +// GetCertVolume - service volumes +func GetCertVolume(certSecretName string) []corev1.Volume { + return []corev1.Volume{ + { + Name: configVolume, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + DefaultMode: &configMode, + SecretName: certSecretName, + }, + }, + }, + } +} + +// GetCertVolumeMount - certificate VolumeMount +func GetCertVolumeMount() []corev1.VolumeMount { + return []corev1.VolumeMount{ + { + Name: configVolume, + MountPath: "/etc/octavia/certs", + ReadOnly: true, + }, + } +} diff --git a/templates/octaviaamphoracontroller/config/octavia.conf b/templates/octaviaamphoracontroller/config/octavia.conf index 915fc0cc..5ef02ae8 100644 --- a/templates/octaviaamphoracontroller/config/octavia.conf +++ b/templates/octaviaamphoracontroller/config/octavia.conf @@ -22,13 +22,20 @@ auth_type=password # region_name=regionOne interface=internal [certificates] +cert_generator = local_cert_generator +ca_certificate = /etc/octavia/certs/server_ca.cert.pem +ca_private_key = /etc/octavia/certs/server_ca.key.pem +ca_private_key_passphrase = {{ .ServerCAKeyPassphrase }} [compute] [networking] port_detach_timeout=300 [haproxy_amphora] +client_cert = /etc/octavia/certs/client.cert-and-key.pem +server_ca = /etc/octavia/certs/server_ca.cert.pem [controller_worker] amp_boot_network_list={{ .LbMgmtNetworkId }} amp_flavor_id={{ .AmpFlavorId }} +client_ca = /etc/octavia/certs/client_ca.cert.pem [task_flow] [oslo_messaging] # topic=octavia-rpc