Adding option to deploy service across nodesets

Dataplane operator can now override the playbook target to 'all'. diverse scenarios across hosts. Services can be marked as global and deployed on all node sets Services can hold multiple inventory secrets. Kuttle tests were expanded to new functionality. Signed-off-by: Jiri Podivin <[email protected]>
openstack-k8s-operators · Feb 14, 2024 · cab8fbf · cab8fbf
1 parent 74ff3ca
commit cab8fbf
Show file tree

Hide file tree

Showing 16 changed files with 1,531 additions and 39 deletions.
diff --git a/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml b/api/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml
@@ -44,6 +44,10 @@ spec:
             type: object
           spec:
             properties:
+              allNodeSetsServices:
+                items:
+                  type: string
+                type: array
               ansibleLimit:
                 type: string
               ansibleSkipTags:

diff --git a/api/bases/dataplane.openstack.org_openstackdataplaneservices.yaml b/api/bases/dataplane.openstack.org_openstackdataplaneservices.yaml
@@ -40,6 +40,8 @@ spec:
                 items:
                   type: string
                 type: array
+              deployOnAllNodeSets:
+                type: boolean
               label:
                 maxLength: 53
                 pattern: '[a-z]([-a-z0-9]*[a-z0-9])'

diff --git a/api/v1beta1/openstackdataplanedeployment_types.go b/api/v1beta1/openstackdataplanedeployment_types.go
@@ -48,6 +48,10 @@ type OpenStackDataPlaneDeploymentSpec struct {
 	// +kubebuilder:validation:Minimum:=1
 	// +kubebuilder:default:=15
 	DeploymentRequeueTime int `json:"deploymentRequeueTime"`
+
+	// Services which should be deployed on all nodesets
+	// +kubebuilder:validation:Optional
+	AllNodeSetsServices []string `json:"allNodeSetsServices"`
 }
 
 // OpenStackDataPlaneDeploymentStatus defines the observed state of OpenStackDataPlaneDeployment

diff --git a/api/v1beta1/openstackdataplaneservice_types.go b/api/v1beta1/openstackdataplaneservice_types.go
@@ -82,6 +82,11 @@ type OpenStackDataPlaneServiceSpec struct {
 	// +kubebuilder:default=false
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,xDescriptors={"urn:alm:descriptor:com.tectonic.ui:booleanSwitch"}
 	AddCertMounts bool `json:"addCertMounts" yaml:"addCertMounts"`
+
+	// DeployOnAllNodeSets - should the service be deploy across all nodesets
+	// This will override default target of a service play, setting it to 'all'.
+	// +kubebuilder:validation:Optional
+	DeployOnAllNodeSets *bool `json:"deployOnAllNodeSets,omitempty" yaml:"deployOnAllNodeSets,omitempty"`
 }
 
 // OpenStackDataPlaneServiceStatus defines the observed state of OpenStackDataPlaneService

diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go
diff --git a/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml b/config/crd/bases/dataplane.openstack.org_openstackdataplanedeployments.yaml
@@ -44,6 +44,10 @@ spec:
             type: object
           spec:
             properties:
+              allNodeSetsServices:
+                items:
+                  type: string
+                type: array
               ansibleLimit:
                 type: string
               ansibleSkipTags:

diff --git a/config/crd/bases/dataplane.openstack.org_openstackdataplaneservices.yaml b/config/crd/bases/dataplane.openstack.org_openstackdataplaneservices.yaml
@@ -40,6 +40,8 @@ spec:
                 items:
                   type: string
                 type: array
+              deployOnAllNodeSets:
+                type: boolean
               label:
                 maxLength: 53
                 pattern: '[a-z]([-a-z0-9]*[a-z0-9])'

diff --git a/controllers/openstackdataplanedeployment_controller.go b/controllers/openstackdataplanedeployment_controller.go
@@ -195,13 +195,29 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 	// All nodeSets successfully fetched.
 	// Mark InputReadyCondition=True
 	instance.Status.Conditions.MarkTrue(condition.InputReadyCondition, condition.ReadyMessage)
+	shouldRequeue := false
+	haveError := false
+
+	// Declare nodeSet for global services
+	globalNodeset := dataplanev1.OpenStackDataPlaneNodeSet{}
+	globalAnsibleEESpec := globalNodeset.GetAnsibleEESpec()
+	globalAnsibleEESpec.AnsibleTags = instance.Spec.AnsibleTags
+	globalAnsibleEESpec.AnsibleSkipTags = instance.Spec.AnsibleSkipTags
+	globalAnsibleEESpec.AnsibleLimit = instance.Spec.AnsibleLimit
+
+	globalInventorySecrets := []string{}
+
+	// Gathering individual inventory secrets for later use
+	for _, nodeSet := range nodeSets.Items {
+		nodeSetSecretInv := fmt.Sprintf("dataplanenodeset-%s", nodeSet.Name)
+		// Add inventory secret to list of inventories for global services
+		globalInventorySecrets = append(globalInventorySecrets, nodeSetSecretInv)
+	}
 
 	// Deploy each nodeSet
 	// The loop starts and checks NodeSet deployments sequentially. However, after they
 	// are started, they are running in parallel, since the loop does not wait
 	// for the first started NodeSet to finish before starting the next.
-	shouldRequeue := false
-	haveError := false
 	for _, nodeSet := range nodeSets.Items {
 
 		Log.Info(fmt.Sprintf("Deploying NodeSet: %s", nodeSet.Name))
@@ -226,13 +242,13 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 		}
 
 		deployer := deployment.Deployer{
-			Ctx:             ctx,
-			Helper:          helper,
-			NodeSet:         &nodeSet,
-			Deployment:      instance,
-			Status:          &instance.Status,
-			AeeSpec:         &ansibleEESpec,
-			InventorySecret: nodeSetSecretInv,
+			Ctx:              ctx,
+			Helper:           helper,
+			NodeSet:          &nodeSet,
+			Deployment:       instance,
+			Status:           &instance.Status,
+			AeeSpec:          &ansibleEESpec,
+			InventorySecrets: []string{nodeSetSecretInv},
 		}
 
 		// When ServicesOverride is set on the OpenStackDataPlaneDeployment,
@@ -266,6 +282,47 @@ func (r *OpenStackDataPlaneDeploymentReconciler) Reconcile(ctx context.Context,
 				condition.Type(dataplanev1.NodeSetDeploymentReadyCondition),
 				condition.DeploymentReadyMessage)
 		}
+
+	}
+
+	if haveError {
+		return ctrl.Result{}, err
+	}
+
+	if shouldRequeue {
+		logger.Info("Not all NodeSets done for OpenStackDeployment")
+		return ctrl.Result{}, nil
+	}
+
+	// If we have any services we want to deploy everywhere, deploy them now
+	if len(instance.Spec.AllNodeSetsServices) != 0 {
+
+		globalDeployer := deployment.Deployer{
+			Ctx:              ctx,
+			Helper:           helper,
+			NodeSet:          &globalNodeset,
+			Deployment:       instance,
+			Status:           &instance.Status,
+			AeeSpec:          &globalAnsibleEESpec,
+			InventorySecrets: globalInventorySecrets,
+		}
+
+		deployResult, err := globalDeployer.Deploy(instance.Spec.AllNodeSetsServices)
+		if err != nil {
+			util.LogErrorForObject(helper, err, "OpenStackDeployment error for all nodesets", instance)
+			haveError = true
+			instance.Status.Conditions.MarkFalse(
+				condition.ReadyCondition,
+				condition.ErrorReason,
+				condition.SeverityWarning,
+				dataplanev1.DataPlaneNodeSetErrorMessage,
+				err.Error())
+		}
+		if deployResult != nil {
+			shouldRequeue = true
+		} else {
+			logger.Info("Global OpenStackDeployment succeeded")
+		}
 	}
 
 	if haveError {
@@ -327,6 +384,20 @@ func (r *OpenStackDataPlaneDeploymentReconciler) setHashes(
 		}
 	}
 
+	// Now do the same for global services
+	for _, serviceName := range instance.Spec.AllNodeSetsServices {
+		err = deployment.GetDeploymentHashesForService(
+			ctx,
+			helper,
+			instance.Namespace,
+			serviceName,
+			instance.Status.ConfigMapHashes,
+			instance.Status.SecretHashes)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 

diff --git a/docs/assemblies/custom_resources.adoc b/docs/assemblies/custom_resources.adoc
@@ -297,6 +297,11 @@ OpenStackDataPlaneServiceSpec defines the desired state of OpenStackDataPlaneSer
 | AddCertMounts - Whether to add cert mounts
 | bool
 | true
+
+| deployOnAllNodeSets
+| DeployOnAllNodeSets - should the service be deploy across all nodesets This will override default target of a service play, setting it to 'all'.
+| *bool
+| false
 |===
 
 <<custom-resources,Back to Custom Resources>>
@@ -642,6 +647,11 @@ OpenStackDataPlaneDeploymentSpec defines the desired state of OpenStackDataPlane
 | Time before the deployment is requeued in seconds
 | int
 | true
+
+| allNodeSetsServices
+| Services which should be deployed on all nodesets
+| []string
+| true
 |===
 
 <<custom-resources,Back to Custom Resources>>

diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go
@@ -39,13 +39,13 @@ import (
 
 // Deployer defines a data structure with all of the relevant objects required for a full deployment.
 type Deployer struct {
-	Ctx             context.Context
-	Helper          *helper.Helper
-	NodeSet         *dataplanev1.OpenStackDataPlaneNodeSet
-	Deployment      *dataplanev1.OpenStackDataPlaneDeployment
-	Status          *dataplanev1.OpenStackDataPlaneDeploymentStatus
-	AeeSpec         *dataplanev1.AnsibleEESpec
-	InventorySecret string
+	Ctx              context.Context
+	Helper           *helper.Helper
+	NodeSet          *dataplanev1.OpenStackDataPlaneNodeSet
+	Deployment       *dataplanev1.OpenStackDataPlaneDeployment
+	Status           *dataplanev1.OpenStackDataPlaneDeploymentStatus
+	AeeSpec          *dataplanev1.AnsibleEESpec
+	InventorySecrets []string
 }
 
 // Deploy function encapsulating primary deloyment handling

diff --git a/pkg/deployment/service.go b/pkg/deployment/service.go
@@ -49,7 +49,7 @@ func (d *Deployer) DeployService(foundService dataplanev1.OpenStackDataPlaneServ
 		d.Deployment,
 		&foundService,
 		d.NodeSet.Spec.NodeTemplate.AnsibleSSHPrivateKeySecret,
-		d.InventorySecret,
+		d.InventorySecrets,
 		d.AeeSpec)
 
 	if err != nil {

diff --git a/pkg/util/ansible_execution.go b/pkg/util/ansible_execution.go
@@ -18,6 +18,7 @@ package util
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -42,11 +43,16 @@ func AnsibleExecution(
 	obj client.Object,
 	service *dataplanev1.OpenStackDataPlaneService,
 	sshKeySecret string,
-	inventorySecret string,
+	inventorySecrets []string,
 	aeeSpec *dataplanev1.AnsibleEESpec,
 ) error {
 	var err error
 	var cmdLineArguments strings.Builder
+	var inventoryVolume corev1.Volume
+	var inventoryName string
+	var inventoryMountPath string
+	log := helper.GetLogger()
+	ansibleEEMounts := storage.VolMounts{}
 
 	ansibleEE, err := GetAnsibleExecution(ctx, helper, obj, service.Spec.Label)
 	if err != nil && !k8serrors.IsNotFound(err) {
@@ -99,7 +105,15 @@ func AnsibleExecution(
 			ansibleEE.Spec.Playbook = service.Spec.Playbook
 		}
 
-		ansibleEEMounts := storage.VolMounts{}
+		// If we have a service that ought to be deployed everywhere
+		// substitute the existing play target with 'all'
+		if service.Spec.DeployOnAllNodeSets != nil && *service.Spec.DeployOnAllNodeSets {
+			ansibleEE.Spec.ExtraVars = map[string]json.RawMessage{
+				"edpm_override_hosts": json.RawMessage([]byte("\"all\"")),
+			}
+			log.Info(fmt.Sprintf("for service %s, substituting existing ansible play host with 'all'.", service.Name))
+		}
+
 		sshKeyVolume := corev1.Volume{
 			Name: "ssh-key",
 			VolumeSource: corev1.VolumeSource{
@@ -119,32 +133,44 @@ func AnsibleExecution(
 			MountPath: "/runner/env/ssh_key",
 			SubPath:   "ssh_key",
 		}
+		// Mount ssh secrets
+		ansibleEEMounts.Volumes = append(ansibleEEMounts.Volumes, sshKeyVolume)
+		ansibleEEMounts.Mounts = append(ansibleEEMounts.Mounts, sshKeyMount)
 
-		inventoryVolume := corev1.Volume{
-			Name: "inventory",
-			VolumeSource: corev1.VolumeSource{
-				Secret: &corev1.SecretVolumeSource{
-					SecretName: inventorySecret,
-					Items: []corev1.KeyToPath{
-						{
-							Key:  "inventory",
-							Path: "inventory",
+		// Mounting inventory secrets
+		for inventoryIndex, inventorySecret := range inventorySecrets {
+			if service.Spec.DeployOnAllNodeSets != nil && *service.Spec.DeployOnAllNodeSets {
+				inventoryName = fmt.Sprintf("inventory-%d", inventoryIndex)
+				inventoryMountPath = fmt.Sprintf("/runner/inventory/%s", inventoryName)
+			} else {
+				inventoryName = "inventory"
+				inventoryMountPath = "/runner/inventory/hosts"
+			}
+
+			inventoryVolume = corev1.Volume{
+				Name: inventoryName,
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName: inventorySecret,
+						Items: []corev1.KeyToPath{
+							{
+								Key:  inventoryName,
+								Path: inventoryName,
+							},
 						},
 					},
 				},
-			},
-		}
-		inventoryMount := corev1.VolumeMount{
-			Name:      "inventory",
-			MountPath: "/runner/inventory/hosts",
-			SubPath:   "inventory",
+			}
+			inventoryMount := corev1.VolumeMount{
+				Name:      inventoryName,
+				MountPath: inventoryMountPath,
+				SubPath:   inventoryName,
+			}
+			// Inventory mount
+			ansibleEEMounts.Mounts = append(ansibleEEMounts.Mounts, inventoryMount)
+			ansibleEEMounts.Volumes = append(ansibleEEMounts.Volumes, inventoryVolume)
 		}
 
-		ansibleEEMounts.Volumes = append(ansibleEEMounts.Volumes, sshKeyVolume)
-		ansibleEEMounts.Volumes = append(ansibleEEMounts.Volumes, inventoryVolume)
-		ansibleEEMounts.Mounts = append(ansibleEEMounts.Mounts, sshKeyMount)
-		ansibleEEMounts.Mounts = append(ansibleEEMounts.Mounts, inventoryMount)
-
 		ansibleEE.Spec.ExtraMounts = append(aeeSpec.ExtraMounts, []storage.VolMounts{ansibleEEMounts}...)
 		ansibleEE.Spec.Env = aeeSpec.Env