Adding option to deploy service across nodesets

Dataplane operator can now override the playbook target to 'all'. diverse scenarios across hosts. Services can be marked as global and deployed on all node sets Services can hold multiple inventory secrets. Kuttle and functional tests were expanded to new functionality. Signed-off-by: Jiri Podivin <[email protected]>
openstack-k8s-operators · Feb 21, 2024 · 00ff6ce · 00ff6ce
1 parent 75fab92
commit 00ff6ce
Show file tree

Hide file tree

Showing 17 changed files with 1,607 additions and 62 deletions.
diff --git a/api/bases/dataplane.openstack.org_openstackdataplaneservices.yaml b/api/bases/dataplane.openstack.org_openstackdataplaneservices.yaml
@@ -40,6 +40,8 @@ spec:
                 items:
                   type: string
                 type: array
+              deployOnAllNodeSets:
+                type: boolean
               openStackAnsibleEERunnerImage:
                 type: string
               play:

diff --git a/api/v1beta1/openstackdataplaneservice_types.go b/api/v1beta1/openstackdataplaneservice_types.go
@@ -73,6 +73,11 @@ type OpenStackDataPlaneServiceSpec struct {
 	// +kubebuilder:default=false
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"}
 	AddCertMounts bool `json:"addCertMounts" yaml:"addCertMounts"`
+
+	// DeployOnAllNodeSets - should the service be deploy across all nodesets
+	// This will override default target of a service play, setting it to 'all'.
+	// +kubebuilder:validation:Optional
+	DeployOnAllNodeSets *bool `json:"deployOnAllNodeSets,omitempty" yaml:"deployOnAllNodeSets,omitempty"`
 }
 
 // OpenStackDataPlaneServiceStatus defines the observed state of OpenStackDataPlaneService

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/dataplane.openstack.org_openstackdataplaneservices.yaml b/config/crd/bases/dataplane.openstack.org_openstackdataplaneservices.yaml
@@ -40,6 +40,8 @@ spec:
                 items:
                   type: string
                 type: array
+              deployOnAllNodeSets:
+                type: boolean
               openStackAnsibleEERunnerImage:
                 type: string
               play:

diff --git a/controllers/openstackdataplanedeployment_controller.go b/controllers/openstackdataplanedeployment_controller.go
@@ -213,13 +213,23 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 	// All nodeSets successfully fetched.
 	// Mark InputReadyCondition=True
 	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.ReadyMessage)
+	shouldRequeue := false
+	haveError := false
+
+	globalInventorySecrets := []string{}
+	globalSSHKeySecrets := map[string]string{}
+
+	// Gathering individual inventory and ssh secrets for later use
+	for _, nodeSet := range nodeSets.Items {
+		// Add inventory secret to list of inventories for global services
+		globalInventorySecrets = append(globalInventorySecrets, fmt.Sprintf("dataplanenodeset-%s", nodeSet.Name))
+		globalSSHKeySecrets[nodeSet.Name] = nodeSet.Spec.NodeTemplate.AnsibleSSHPrivateKeySecret
+	}
 
 	// Deploy each nodeSet
 	// The loop starts and checks NodeSet deployments sequentially. However, after they
 	// are started, they are running in parallel, since the loop does not wait
 	// for the first started NodeSet to finish before starting the next.
-	shouldRequeue := false
-	haveError := false
 	for _, nodeSet := range nodeSets.Items {
 
 		Log.Info(fmt.Sprintf("Deploying NodeSet: %s", nodeSet.Name))
@@ -235,8 +245,6 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 		ansibleEESpec.AnsibleLimit = instance.Spec.AnsibleLimit
 		ansibleEESpec.ExtraVars = instance.Spec.AnsibleExtraVars
 
-		nodeSetSecretInv := fmt.Sprintf("dataplanenodeset-%s", nodeSet.Name)
-
 		if nodeSet.Status.DNSClusterAddresses != nil && nodeSet.Status.CtlplaneSearchDomain != "" {
 			ansibleEESpec.DNSConfig = &corev1.PodDNSConfig{
 				Nameservers: nodeSet.Status.DNSClusterAddresses,
@@ -245,13 +253,14 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 		}
 
 		deployer := deployment.Deployer{
-			Ctx:             ctx,
-			Helper:          helper,
-			NodeSet:         &nodeSet,
-			Deployment:      instance,
-			Status:          &instance.Status,
-			AeeSpec:         &ansibleEESpec,
-			InventorySecret: nodeSetSecretInv,
+			Ctx:                         ctx,
+			Helper:                      helper,
+			NodeSet:                     &nodeSet,
+			Deployment:                  instance,
+			Status:                      &instance.Status,
+			AeeSpec:                     &ansibleEESpec,
+			InventorySecrets:            globalInventorySecrets,
+			AnsibleSSHPrivateKeySecrets: globalSSHKeySecrets,
 		}
 
 		// When ServicesOverride is set on the OpenStackDataPlaneDeployment,

diff --git a/docs/assemblies/custom_resources.adoc b/docs/assemblies/custom_resources.adoc
@@ -297,6 +297,11 @@ OpenStackDataPlaneServiceSpec defines the desired state of OpenStackDataPlaneSer
 | AddCertMounts - Whether to add cert mounts
 | bool
 | true
+
+| deployOnAllNodeSets
+| DeployOnAllNodeSets - should the service be deploy across all nodesets This will override default target of a service play, setting it to 'all'.
+| *bool
+| false
 |===
 
 <<custom-resources,Back to Custom Resources>>

diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go
@@ -39,13 +39,14 @@ import (
 
 // Deployer defines a data structure with all of the relevant objects required for a full deployment.
 type Deployer struct {
-	Ctx             context.Context
-	Helper          *helper.Helper
-	NodeSet         *dataplanev1.OpenStackDataPlaneNodeSet
-	Deployment      *dataplanev1.OpenStackDataPlaneDeployment
-	Status          *dataplanev1.OpenStackDataPlaneDeploymentStatus
-	AeeSpec         *dataplanev1.AnsibleEESpec
-	InventorySecret string
+	Ctx                         context.Context
+	Helper                      *helper.Helper
+	NodeSet                     *dataplanev1.OpenStackDataPlaneNodeSet
+	Deployment                  *dataplanev1.OpenStackDataPlaneDeployment
+	Status                      *dataplanev1.OpenStackDataPlaneDeploymentStatus
+	AeeSpec                     *dataplanev1.AnsibleEESpec
+	InventorySecrets            []string
+	AnsibleSSHPrivateKeySecrets map[string]string
 }
 
 // Deploy function encapsulating primary deloyment handling

diff --git a/pkg/deployment/inventory.go b/pkg/deployment/inventory.go
@@ -54,6 +54,8 @@ func GenerateNodeSetInventory(ctx context.Context, helper *helper.Helper,
 		nodeSetGroup.Vars["role_tags"] = instance.Spec.Tags
 	}
 
+	nodeSetGroup.Vars["ansible_ssh_private_key_file"] = fmt.Sprintf("/runner/env/ssh_key/ssh_key_%s", instance.Name)
+
 	for _, node := range instance.Spec.Nodes {
 		host := nodeSetGroup.AddHost(strings.Split(node.HostName, ".")[0])
 		// Use ansible_host if provided else use hostname. Fall back to

diff --git a/pkg/deployment/service.go b/pkg/deployment/service.go
@@ -48,9 +48,10 @@ func (d *Deployer) DeployService(foundService dataplanev1.OpenStackDataPlaneServ
 		d.Helper,
 		d.Deployment,
 		&foundService,
-		d.NodeSet.Spec.NodeTemplate.AnsibleSSHPrivateKeySecret,
-		d.InventorySecret,
-		d.AeeSpec)
+		d.AnsibleSSHPrivateKeySecrets,
+		d.InventorySecrets,
+		d.AeeSpec,
+		d.NodeSet.Name)
 
 	if err != nil {
 		d.Helper.GetLogger().Error(err, fmt.Sprintf("Unable to execute Ansible for %s", foundService.Name))

diff --git a/pkg/util/ansible_execution.go b/pkg/util/ansible_execution.go
@@ -18,6 +18,7 @@ package util
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -41,12 +42,21 @@ func AnsibleExecution(
 	helper *helper.Helper,
 	obj client.Object,
 	service *dataplanev1.OpenStackDataPlaneService,
-	sshKeySecret string,
-	inventorySecret string,
+	sshKeySecrets map[string]string,
+	inventorySecrets []string,
 	aeeSpec *dataplanev1.AnsibleEESpec,
+	targetNodeset string,
 ) error {
 	var err error
 	var cmdLineArguments strings.Builder
+	var inventoryVolume corev1.Volume
+	var inventoryName string
+	var inventoryMountPath string
+	var sshKeyName string
+	var sshKeyMountPath string
+	var sshKeyMountSubPath string
+
+	ansibleEEMounts := storage.VolMounts{}
 
 	ansibleEE, err := GetAnsibleExecution(ctx, helper, obj, service.Name)
 	if err != nil && !k8serrors.IsNotFound(err) {
@@ -98,51 +108,87 @@ func AnsibleExecution(
 			ansibleEE.Spec.Playbook = service.Spec.Playbook
 		}
 
-		ansibleEEMounts := storage.VolMounts{}
-		sshKeyVolume := corev1.Volume{
-			Name: "ssh-key",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: sshKeySecret,
-					Items: []corev1.KeyToPath{
-						{
-							Key:  "ssh-privatekey",
-							Path: "ssh_key",
+		// If we have a service that ought to be deployed everywhere
+		// substitute the existing play target with 'all'
+		// Check if we have ExtraVars before accessing it
+		if ansibleEE.Spec.ExtraVars == nil {
+			ansibleEE.Spec.ExtraVars = make(map[string]json.RawMessage)
+		}
+		if service.Spec.DeployOnAllNodeSets != nil && *service.Spec.DeployOnAllNodeSets {
+			ansibleEE.Spec.ExtraVars["edpm_override_hosts"] = json.RawMessage([]byte("\"all\""))
+			util.LogForObject(helper, fmt.Sprintf("for service %s, substituting existing ansible play host with 'all'.", service.Name), ansibleEE)
+		} else {
+			ansibleEE.Spec.ExtraVars["edpm_override_hosts"] = json.RawMessage([]byte(fmt.Sprintf("\"%s\"", targetNodeset)))
+			util.LogForObject(helper, fmt.Sprintf("for service %s, substituting existing ansible play host with 'all'.", service.Name), ansibleEE)
+		}
+
+		for sshKeyNodeName, sshKeySecret := range sshKeySecrets {
+			if service.Spec.DeployOnAllNodeSets != nil && *service.Spec.DeployOnAllNodeSets {
+				sshKeyName = fmt.Sprintf("ssh-key-%s", sshKeyNodeName)
+				sshKeyMountSubPath = fmt.Sprintf("ssh_key_%s", targetNodeset)
+				sshKeyMountPath = fmt.Sprintf("/runner/env/ssh_key/%s", sshKeyMountSubPath)
+			} else {
+				sshKeyName = "ssh-key"
+				sshKeyMountSubPath = "ssh_key"
+				sshKeyMountPath = "/runner/env/ssh_key"
+			}
+			sshKeyVolume := corev1.Volume{
+				Name: sshKeyName,
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: sshKeySecret,
+						Items: []corev1.KeyToPath{
+							{
+								Key:  "ssh-privatekey",
+								Path: sshKeyMountSubPath,
+							},
 						},
 					},
 				},
-			},
-		}
-		sshKeyMount := corev1.VolumeMount{
-			Name:      "ssh-key",
-			MountPath: "/runner/env/ssh_key",
-			SubPath:   "ssh_key",
+			}
+			sshKeyMount := corev1.VolumeMount{
+				Name:      sshKeyName,
+				MountPath: sshKeyMountPath,
+				SubPath:   sshKeyMountSubPath,
+			}
+			// Mount ssh secrets
+			ansibleEEMounts.Mounts = append(ansibleEEMounts.Mounts, sshKeyMount)
+			ansibleEEMounts.Volumes = append(ansibleEEMounts.Volumes, sshKeyVolume)
 		}
 
-		inventoryVolume := corev1.Volume{
-			Name: "inventory",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: inventorySecret,
-					Items: []corev1.KeyToPath{
-						{
-							Key:  "inventory",
-							Path: "inventory",
+		// Mounting inventory and secrets
+		for inventoryIndex, inventorySecret := range inventorySecrets {
+			if service.Spec.DeployOnAllNodeSets != nil && *service.Spec.DeployOnAllNodeSets {
+				inventoryName = fmt.Sprintf("inventory-%d", inventoryIndex)
+				inventoryMountPath = fmt.Sprintf("/runner/inventory/%s", inventoryName)
+			} else {
+				inventoryName = "inventory"
+				inventoryMountPath = "/runner/inventory/hosts"
+			}
+
+			inventoryVolume = corev1.Volume{
+				Name: inventoryName,
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: inventorySecret,
+						Items: []corev1.KeyToPath{
+							{
+								Key:  inventoryName,
+								Path: inventoryName,
+							},
 						},
 					},
 				},
-			},
+			}
+			inventoryMount := corev1.VolumeMount{
+				Name:      inventoryName,
+				MountPath: inventoryMountPath,
+				SubPath:   inventoryName,
+			}
+			// Inventory mount
+			ansibleEEMounts.Mounts = append(ansibleEEMounts.Mounts, inventoryMount)
+			ansibleEEMounts.Volumes = append(ansibleEEMounts.Volumes, inventoryVolume)
 		}
-		inventoryMount := corev1.VolumeMount{
-			Name:      "inventory",
-			MountPath: "/runner/inventory/hosts",
-			SubPath:   "inventory",
-		}
-
-		ansibleEEMounts.Volumes = append(ansibleEEMounts.Volumes, sshKeyVolume)
-		ansibleEEMounts.Volumes = append(ansibleEEMounts.Volumes, inventoryVolume)
-		ansibleEEMounts.Mounts = append(ansibleEEMounts.Mounts, sshKeyMount)
-		ansibleEEMounts.Mounts = append(ansibleEEMounts.Mounts, inventoryMount)
 
 		ansibleEE.Spec.ExtraMounts = append(aeeSpec.ExtraMounts, []storage.VolMounts{ansibleEEMounts}...)
 		ansibleEE.Spec.Env = aeeSpec.Env

diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go
@@ -41,8 +41,13 @@ func CreateDataplaneDeployment(name types.NamespacedName, spec map[string]interf
 }
 
 // Create an OpenStackDataPlaneService with a given NamespacedName, assert on success
-func CreateDataplaneService(name types.NamespacedName) *unstructured.Unstructured {
-	raw := DefaultDataplaneService(name)
+func CreateDataplaneService(name types.NamespacedName, globalService bool) *unstructured.Unstructured {
+	var raw map[string]interface{}
+	if globalService {
+		raw = DefaultDataplaneGlobalService(name)
+	} else {
+		raw = DefaultDataplaneService(name)
+	}
 	return th.CreateUnstructured(raw)
 }
 
@@ -241,6 +246,24 @@ func DefaultDataplaneService(name types.NamespacedName) map[string]interface{} {
 		}}
 }
 
+// Create an empty OpenStackDataPlaneService struct
+// containing only given NamespacedName as metadata
+func DefaultDataplaneGlobalService(name types.NamespacedName) map[string]interface{} {
+
+	return map[string]interface{}{
+
+		"apiVersion": "dataplane.openstack.org/v1beta1",
+		"kind":       "OpenStackDataPlaneService",
+		"metadata": map[string]interface{}{
+			"name":      name.Name,
+			"namespace": name.Namespace,
+		},
+		"spec": map[string]interface{}{
+			"deployOnAllNodeSets": true,
+		},
+	}
+}
+
 // Get resources
 
 // Retrieve OpenStackDataPlaneDeployment and check for errors