Skip to content
This repository has been archived by the owner on Jun 25, 2024. It is now read-only.

Commit

Permalink
WIP - Initial commit to generate EDPM certs
Browse files Browse the repository at this point in the history
This is lightly tested so far (and so is still WIP), but it gives an idea of how EDPM certs
could be generated and mounted for each of the services that need them.

While I will test and refine, I'm hoping that this PR starts
conversations and questions so that we can decide on an implementation.

The basic idea is as follows:

Each service has a boolean parameter HasTLSCerts which determines
whether or not certs are expected for this service.  If so, then we
expect that a secret with the name <nodeSet.Name>-<service.Name>-certs will
be created.

This secret contains the TLS certs, keys and cacerts for every node in
the nodeset.  They are referenced as <nodeName>-tls.key, nodeName-tls.crt, nodeName-ca.crt

This secret will be mounted in the openstackAnsibleEE pod at
/var/lib/openstack/certs.  It will be up to the ansible playbook to
move the appropriate certs, keys etc. for each node to the node, and do
any reconfiguration of the service.

More details are below:
1. After the DNS data is created in the openstackdataplanenodeset_controller,
   a call is made to GenerateTLSCerts for each service that has HasTLSCerts: True.
2. GenerateTLSCerts right now only generates a single cert for each node in the
   nodeset using the DNSNames for that node.  As the cert contains the
   DNSNames for all the node's interfaces, it should be usable on all
   interfaces.  This cert is stored in the secret "nodeName-cert".  This
   is the default cert - and it should only be created once - ie. the
   first time this code is run.
3.  Its likely that most services will be able to use this cert.  If a
    service needs a different kind of cert (maybe using ips - for ovn
    for instance, or a different issuer), then code needs to be added to
    the switch statement for that particular service.
4. The secret containing the cert will need to have an ownership
   reference updated so that we can trigger a reconcile when the cert
   changes or is renewed.
  • Loading branch information
vakwetu committed Oct 10, 2023
1 parent 819a117 commit bffa898
Show file tree
Hide file tree
Showing 13 changed files with 208 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ spec:
items:
type: string
type: array
hasTLSCerts:
type: boolean
label:
type: string
openStackAnsibleEERunnerImage:
Expand Down
5 changes: 5 additions & 0 deletions api/v1beta1/openstackdataplaneservice_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,11 @@ type OpenStackDataPlaneServiceSpec struct {
// OpenStackAnsibleEERunnerImage image to use as the ansibleEE runner image
// +kubebuilder:validation:Optional
OpenStackAnsibleEERunnerImage string `json:"openStackAnsibleEERunnerImage,omitempty" yaml:"openStackAnsibleEERunnerImage,omitempty"`

// HasTLSCerts - Whether the nodes have TLS certs
// +kubebuilder:validation:Optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"}
HasTLSCerts bool `json:"hasTLSCerts,omitempty"`
}

// OpenStackDataPlaneServiceStatus defines the observed state of OpenStackDataPlaneService
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ spec:
items:
type: string
type: array
hasTLSCerts:
type: boolean
label:
type: string
openStackAnsibleEERunnerImage:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ metadata:
spec:
label: dataplane-deployment-libvirt
playbook: osp.edpm.libvirt
hasTLSCerts: True
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ spec:
secrets:
- nova-cell1-compute-config
playbook: osp.edpm.nova
hasTLSCerts: True
16 changes: 16 additions & 0 deletions controllers/openstackdataplanenodeset_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,22 @@ func (r *OpenStackDataPlaneNodeSetReconciler) Reconcile(ctx context.Context, req
instance.Status.DNSClusterAddresses = dnsClusterAddresses
instance.Status.CtlplaneSearchDomain = ctlplaneSearchDomain

// Issue certs for TLS for services that need them
for _, serviceName := range instance.Spec.Services {
service, err := deployment.GetService(ctx, helper, serviceName)
if err != nil {
return ctrl.Result{}, err
}
if service.Spec.HasTLSCerts {
result, err = deployment.EnsureTLSCerts(ctx, helper, instance, allIPSets, serviceName)
if err != nil {
return ctrl.Result{}, err
} else if (result != ctrl.Result{}) {
return result, nil
}
}
}

ansibleSSHPrivateKeySecret := instance.Spec.NodeTemplate.AnsibleSSHPrivateKeySecret

var secretKeys = []string{}
Expand Down
1 change: 1 addition & 0 deletions docs/openstack_dataplaneservice.md
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ OpenStackDataPlaneServiceSpec defines the desired state of OpenStackDataPlaneSer
| configMaps | ConfigMaps list of ConfigMap names to mount as ExtraMounts for the OpenStackAnsibleEE | []string | false |
| secrets | Secrets list of Secret names to mount as ExtraMounts for the OpenStackAnsibleEE | []string | false |
| openStackAnsibleEERunnerImage | OpenStackAnsibleEERunnerImage image to use as the ansibleEE runner image | string | false |
| hasTLSCerts | HasTLSCerts - Whether the nodes have TLS certs | bool | false |

[Back to Custom Resources](#custom-resources)

Expand Down
5 changes: 3 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.19
replace github.com/openstack-k8s-operators/dataplane-operator/api => ./api

require (
github.com/cert-manager/cert-manager v1.11.5
github.com/go-logr/logr v1.2.4
github.com/google/uuid v1.3.1
github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
Expand All @@ -13,6 +14,7 @@ require (
github.com/openstack-k8s-operators/dataplane-operator/api v0.0.0-20230724101130-2d6fe1f4706b
github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20231001103054-f74a88ed4971
github.com/openstack-k8s-operators/lib-common/modules/ansible v0.3.1-0.20231006072650-7fe7fe16bcd1
github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231006072650-7fe7fe16bcd1
github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231006072650-7fe7fe16bcd1
github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20231006072650-7fe7fe16bcd1
github.com/openstack-k8s-operators/lib-common/modules/test v0.3.1-0.20231006072650-7fe7fe16bcd1
Expand All @@ -23,12 +25,12 @@ require (
k8s.io/api v0.26.9
k8s.io/apimachinery v0.26.9
k8s.io/client-go v0.26.9
k8s.io/utils v0.0.0-20230726121419-3b25d923346b
sigs.k8s.io/controller-runtime v0.14.6
)

require (
github.com/beorn7/perks v1.0.1 // indirect
github.com/cert-manager/cert-manager v1.11.5 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.10.2 // indirect
Expand Down Expand Up @@ -82,7 +84,6 @@ require (
k8s.io/component-base v0.26.9 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230515203736-54b630e78af5 // indirect
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
sigs.k8s.io/gateway-api v0.6.0 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,8 @@ github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20231001103054-f
github.com/openstack-k8s-operators/infra-operator/apis v0.1.1-0.20231001103054-f74a88ed4971/go.mod h1:zqFs5MrBKeaE4HQroUgMWwIkBwmmcygg6sghcidSdCA=
github.com/openstack-k8s-operators/lib-common/modules/ansible v0.3.1-0.20231006072650-7fe7fe16bcd1 h1:+IMX5kc/uo5MM/wbmzXOpZg581KLJHNedkYOKDygC7s=
github.com/openstack-k8s-operators/lib-common/modules/ansible v0.3.1-0.20231006072650-7fe7fe16bcd1/go.mod h1:A9sWNibvjr1a9B/mpy4k6J9xkH11fnn0Dx/X1EZ3On8=
github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231006072650-7fe7fe16bcd1 h1:sE/qio/WNUEng0VBmefSr46e/cq4R83payEzge/Y48U=
github.com/openstack-k8s-operators/lib-common/modules/certmanager v0.0.0-20231006072650-7fe7fe16bcd1/go.mod h1:u1pqzqGNLcof95aqhLfU6xHVTD6ZTc5gWy2FE03UrZQ=
github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231006072650-7fe7fe16bcd1 h1:ALZWU2GFDSoOKoBsGbsdgAzlJzGFFsBVFyLvrJIZ+ss=
github.com/openstack-k8s-operators/lib-common/modules/common v0.3.1-0.20231006072650-7fe7fe16bcd1/go.mod h1:Ozg6SxfwOtMkiH553c0XQBWuygZQq4jDQCpR4hZqlxM=
github.com/openstack-k8s-operators/lib-common/modules/storage v0.3.1-0.20231006072650-7fe7fe16bcd1 h1:+vRt690N+He4uJM0Cvk7Fguw0zs395A8qfV5Uq8B7kw=
Expand Down
4 changes: 4 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ import (
baremetalv1 "github.com/openstack-k8s-operators/openstack-baremetal-operator/api/v1beta1"
"sigs.k8s.io/controller-runtime/pkg/client/config"

certmgrv1 "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1"
certmgrmetav1 "github.com/cert-manager/cert-manager/pkg/apis/meta/v1"
dataplanev1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1"
dataplanev1beta1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1"
"github.com/openstack-k8s-operators/dataplane-operator/controllers"
Expand All @@ -60,6 +62,8 @@ func init() {
utilruntime.Must(baremetalv1.AddToScheme(scheme))
utilruntime.Must(infranetworkv1.AddToScheme(scheme))
utilruntime.Must(dataplanev1beta1.AddToScheme(scheme))
utilruntime.Must(certmgrv1.AddToScheme(scheme))
utilruntime.Must(certmgrmetav1.AddToScheme(scheme))
//+kubebuilder:scaffold:scheme
}

Expand Down
134 changes: 134 additions & 0 deletions pkg/deployment/cert.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
/*
Copyright 2023.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deployment

import (
"context"
"fmt"
"strings"
"time"

corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"

dataplanev1 "github.com/openstack-k8s-operators/dataplane-operator/api/v1beta1"
infranetworkv1 "github.com/openstack-k8s-operators/infra-operator/apis/network/v1beta1"
"github.com/openstack-k8s-operators/lib-common/modules/certmanager"
"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
"github.com/openstack-k8s-operators/lib-common/modules/common/secret"
)

// EnsureTLSCerts generates a secret containing all the certificates for the relevant service
// This secret will be mounted by the ansibleEE pod as an extra mount when the service is deployed.
func EnsureTLSCerts(ctx context.Context, helper *helper.Helper,
instance *dataplanev1.OpenStackDataPlaneNodeSet,
allIPSets map[string]infranetworkv1.IPSet,
serviceName string) (ctrl.Result, error) {

certsData := map[string][]byte{}

// for each node in the nodeset, issue all the TLS certs needed based on the
// ips or DNS Names
for nodeName := range instance.Spec.Nodes {
var dnsNames []string
var secretName string
var certName string
var certSecret *corev1.Secret = nil
var err error
var result ctrl.Result

// TODO(alee) decide if we want to use other labels
// For now we just add the hostname so we can select all the certs on one node
labels := map[string]string{
"hostname": nodeName,
}

ipSet, ok := allIPSets[nodeName]
if ok {
for _, res := range ipSet.Status.Reservation {
fqdnName := strings.Join([]string{nodeName, res.DNSDomain}, ".")
dnsNames = append(dnsNames, fqdnName)
}
}

switch serviceName {
default:
// The default case provides a cert with all the dns names for the host.
// This will probably be sufficient for most services. If a service needs
// a different kind of cert (for example, containing ips, or using a different
// issuer) then add a case for the service in this switch statement

secretName = "cert-" + nodeName
certSecret, _, err = secret.GetSecret(ctx, helper, secretName, instance.Namespace)
if err != nil {
if !k8serrors.IsNotFound(err) {
err = fmt.Errorf("Error retrieving secret %s - %w", secretName, err)
return ctrl.Result{}, err
}

certName = secretName
duration := ptr.To(time.Hour * 24 * 365)
certSecret, result, err = certmanager.EnsureCert(ctx, helper, RootCAIssuerInternalLabel,
certName, duration, dnsNames, nil, labels)
if err != nil {
return ctrl.Result{}, err
} else if (result != ctrl.Result{}) {
return result, nil
}
}
}

// TODO(alee) Add an owner reference to the secret so it can be monitored
// We'll do this once stuggi adds a function to do this in libcommon

// To use this cert, add it to the relevant service data
// TODO(alee) We only need the cert and key. The cacert will come from another label
for key, value := range certSecret.Data {
certsData[nodeName+"-"+key] = value
}
}

// create a secret to hold the certs for the service
serviceCertsSecret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: GetServiceCertsSecretName(instance, serviceName),
Namespace: instance.Namespace,
},
Data: certsData,
}
_, result, err := secret.CreateOrPatchSecret(ctx, helper, instance, serviceCertsSecret)
if err != nil {
err = fmt.Errorf("Error creating certs secret for %s - %w", serviceName, err)
return ctrl.Result{}, err
} else if result != controllerutil.OperationResultNone {
return ctrl.Result{RequeueAfter: time.Second * 5}, nil
}

return ctrl.Result{}, nil
}

// GetServiceCertsSecretName - return name of secret to be mounted in ansibleEE which contains
// all the TLS certs for the relevant service
// The convention we use here is "<nodeset.name>-<service>-certs", so for example,
// openstack-epdm-nova-certs.
func GetServiceCertsSecretName(instance *dataplanev1.OpenStackDataPlaneNodeSet, serviceName string) string {
return fmt.Sprintf("%s-%s-certs", instance.Name, serviceName)
}
6 changes: 6 additions & 0 deletions pkg/deployment/const.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,4 +47,10 @@ const (

// ConfigPaths base path for volume mounts in OpenStackAnsibleEE pod
ConfigPaths = "/var/lib/openstack/configs"

// RootCAIssuerInternalLabel for internal RootCA to issue internal TLS Certs
RootCAIssuerInternalLabel = "osp-rootca-issuer"

// CertPaths base path for cert volume mount in OpenStackAnsibleEE pod
CertPaths = "/var/lib/openstack/certs"
)
32 changes: 31 additions & 1 deletion pkg/deployment/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func Deploy(
// specific mounts.
aeeSpec.ExtraMounts = make([]storage.VolMounts, len(aeeSpecMounts))
copy(aeeSpec.ExtraMounts, aeeSpecMounts)
aeeSpec, err = addServiceExtraMounts(ctx, helper, aeeSpec, foundService)
aeeSpec, err = addServiceExtraMounts(ctx, helper, aeeSpec, foundService, nodeSet)

if err != nil {
return &ctrl.Result{}, err
Expand Down Expand Up @@ -228,6 +228,7 @@ func addServiceExtraMounts(
helper *helper.Helper,
aeeSpec dataplanev1.AnsibleEESpec,
service dataplanev1.OpenStackDataPlaneService,
nodeSet *dataplanev1.OpenStackDataPlaneNodeSet,
) (dataplanev1.AnsibleEESpec, error) {

client := helper.GetClient()
Expand Down Expand Up @@ -326,6 +327,35 @@ func addServiceExtraMounts(

aeeSpec.ExtraMounts = append(aeeSpec.ExtraMounts, volMounts)
}

// Add mount for TLS certs
if service.Spec.HasTLSCerts {
volMounts := storage.VolMounts{}
secretName := GetServiceCertsSecretName(nodeSet, service.Name)
sec := &corev1.Secret{}
err := client.Get(ctx, types.NamespacedName{Name: secretName, Namespace: service.Namespace}, sec)
if err != nil {
return aeeSpec, err
}
volume := corev1.Volume{
Name: secretName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: secretName,
},
},
}

volumeMount := corev1.VolumeMount{
Name: secretName,
MountPath: path.Join(CertPaths, service.Name),
}

volMounts.Volumes = append(volMounts.Volumes, volume)
volMounts.Mounts = append(volMounts.Mounts, volumeMount)
aeeSpec.ExtraMounts = append(aeeSpec.ExtraMounts, volMounts)

}
return aeeSpec, nil

}

0 comments on commit bffa898

Please sign in to comment.