Skip to content

Commit

Permalink
Refactor infra creation to improve the overall infra setup time
Browse files Browse the repository at this point in the history
  • Loading branch information
@dharaneeshvrd
dharaneeshvrd committed Jul 3, 2024
1 parent 392efd0 commit 21ca9b0
Showing 1 changed file with 150 additions and 62 deletions.
212 changes: 150 additions & 62 deletions controllers/ibmpowervscluster_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,12 @@ import (
"context"
"fmt"
"strings"
"sync"
"time"

"github.com/pkg/errors"

corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
Expand Down Expand Up @@ -111,142 +113,228 @@ func (r *IBMPowerVSClusterReconciler) Reconcile(ctx context.Context, req ctrl.Re
return r.reconcile(clusterScope)
}

func (r *IBMPowerVSClusterReconciler) reconcile(clusterScope *scope.PowerVSClusterScope) (ctrl.Result, error) { //nolint:gocyclo
if controllerutil.AddFinalizer(clusterScope.IBMPowerVSCluster, infrav1beta2.IBMPowerVSClusterFinalizer) {
return ctrl.Result{}, nil
}
type updatePowerVSCluster struct {
cluster *infrav1beta2.IBMPowerVSCluster
mu sync.Mutex
}

// check for annotation set for cluster resource and decide on proceeding with infra creation.
// do not proceed further if "powervs.cluster.x-k8s.io/create-infra=true" annotation is not set.
if !scope.CheckCreateInfraAnnotation(*clusterScope.IBMPowerVSCluster) {
clusterScope.IBMPowerVSCluster.Status.Ready = true
return ctrl.Result{}, nil
}
type reconcileResult struct {
reconcile.Result
error
}

// validate PER availability for the PowerVS zone, proceed further only if PowerVS zone support PER.
// more information about PER can be found here: https://cloud.ibm.com/docs/power-iaas?topic=power-iaas-per
if err := clusterScope.IsPowerVSZoneSupportsPER(); err != nil {
clusterScope.Error(err, "error checking PER capability for PowerVS zone")
return reconcile.Result{}, err
func (update *updatePowerVSCluster) condition(condition bool, conditionArgs ...interface{}) {
update.mu.Lock()
defer update.mu.Unlock()
if condition {
conditions.MarkTrue(update.cluster, conditionArgs[0].(capiv1beta1.ConditionType))
return
}

// reconcile service resource group
clusterScope.Info("Reconciling resource group")
if err := clusterScope.ReconcileResourceGroup(); err != nil {
clusterScope.Error(err, "failed to reconcile resource group")
return reconcile.Result{}, err
}
conditions.MarkFalse(update.cluster, conditionArgs[0].(capiv1beta1.ConditionType), conditionArgs[1].(string), conditionArgs[2].(capiv1beta1.ConditionSeverity), conditionArgs[3].(string), conditionArgs[4:]...)
}

powerVSCluster := clusterScope.IBMPowerVSCluster
func (r *IBMPowerVSClusterReconciler) reconcilePowerVSResources(clusterScope *scope.PowerVSClusterScope, updatePowerVSCluster *updatePowerVSCluster, ch chan reconcileResult, wg *sync.WaitGroup) {
defer wg.Done()
// reconcile PowerVS service instance
clusterScope.Info("Reconciling PowerVS service instance")
if requeue, err := clusterScope.ReconcilePowerVSServiceInstance(); err != nil {
clusterScope.Error(err, "failed to reconcile PowerVS service instance")
conditions.MarkFalse(powerVSCluster, infrav1beta2.ServiceInstanceReadyCondition, infrav1beta2.ServiceInstanceReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
return reconcile.Result{}, err
updatePowerVSCluster.condition(false, infrav1beta2.ServiceInstanceReadyCondition, infrav1beta2.ServiceInstanceReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
ch <- reconcileResult{reconcile.Result{}, err}
return
} else if requeue {
clusterScope.Info("PowerVS service instance creation is pending, requeuing")
return reconcile.Result{RequeueAfter: 1 * time.Minute}, nil
ch <- reconcileResult{reconcile.Result{Requeue: true}, nil}
return
}
conditions.MarkTrue(powerVSCluster, infrav1beta2.ServiceInstanceReadyCondition)
updatePowerVSCluster.condition(true, infrav1beta2.ServiceInstanceReadyCondition)

clusterScope.IBMPowerVSClient.WithClients(powervs.ServiceOptions{CloudInstanceID: clusterScope.GetServiceInstanceID()})

// reconcile network
clusterScope.Info("Reconciling network")
if requeue, err := clusterScope.ReconcileNetwork(); err != nil {
clusterScope.Error(err, "failed to reconcile PowerVS network")
conditions.MarkFalse(powerVSCluster, infrav1beta2.NetworkReadyCondition, infrav1beta2.NetworkReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
return reconcile.Result{}, err
updatePowerVSCluster.condition(false, infrav1beta2.NetworkReadyCondition, infrav1beta2.NetworkReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
ch <- reconcileResult{reconcile.Result{}, err}
return
} else if requeue {
clusterScope.Info("PowerVS network creation is pending, requeuing")
return reconcile.Result{RequeueAfter: 1 * time.Minute}, nil
clusterScope.Info("PowerVS network creation is pending")
return
}
conditions.MarkTrue(powerVSCluster, infrav1beta2.NetworkReadyCondition)
updatePowerVSCluster.condition(true, infrav1beta2.NetworkReadyCondition)
}

// reconcile VPC
func (r *IBMPowerVSClusterReconciler) reconcileVPCResources(clusterScope *scope.PowerVSClusterScope, updatePowerVSCluster *updatePowerVSCluster, ch chan reconcileResult, wg *sync.WaitGroup) {
defer wg.Done()
clusterScope.Info("Reconciling VPC")
if requeue, err := clusterScope.ReconcileVPC(); err != nil {
clusterScope.Error(err, "failed to reconcile VPC")
conditions.MarkFalse(powerVSCluster, infrav1beta2.VPCReadyCondition, infrav1beta2.VPCReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
return reconcile.Result{}, err
updatePowerVSCluster.condition(false, infrav1beta2.VPCReadyCondition, infrav1beta2.VPCReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
ch <- reconcileResult{reconcile.Result{}, err}
return
} else if requeue {
clusterScope.Info("VPC creation is pending, requeuing")
return reconcile.Result{RequeueAfter: 15 * time.Second}, nil
ch <- reconcileResult{reconcile.Result{Requeue: true}, nil}
return
}
conditions.MarkTrue(powerVSCluster, infrav1beta2.VPCReadyCondition)
updatePowerVSCluster.condition(true, infrav1beta2.VPCReadyCondition)

// reconcile VPC Subnet
clusterScope.Info("Reconciling VPC subnets")
if requeue, err := clusterScope.ReconcileVPCSubnets(); err != nil {
clusterScope.Error(err, "failed to reconcile VPC subnets")
conditions.MarkFalse(powerVSCluster, infrav1beta2.VPCSubnetReadyCondition, infrav1beta2.VPCSubnetReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
return reconcile.Result{}, err
updatePowerVSCluster.condition(false, infrav1beta2.VPCSubnetReadyCondition, infrav1beta2.VPCSubnetReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
ch <- reconcileResult{reconcile.Result{}, err}
return
} else if requeue {
clusterScope.Info("VPC subnet creation is pending, requeuing")
return reconcile.Result{RequeueAfter: 15 * time.Second}, nil
ch <- reconcileResult{reconcile.Result{Requeue: true}, nil}
return
}
conditions.MarkTrue(powerVSCluster, infrav1beta2.VPCSubnetReadyCondition)
updatePowerVSCluster.condition(true, infrav1beta2.VPCSubnetReadyCondition)

// reconcile VPC security group
clusterScope.Info("Reconciling VPC security group")
if err := clusterScope.ReconcileVPCSecurityGroups(); err != nil {
clusterScope.Error(err, "failed to reconcile VPC security groups")
conditions.MarkFalse(powerVSCluster, infrav1beta2.VPCSecurityGroupReadyCondition, infrav1beta2.VPCSecurityGroupReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
updatePowerVSCluster.condition(false, infrav1beta2.VPCSecurityGroupReadyCondition, infrav1beta2.VPCSecurityGroupReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
ch <- reconcileResult{reconcile.Result{}, err}
return
}
updatePowerVSCluster.condition(true, infrav1beta2.VPCSecurityGroupReadyCondition)

// reconcile LoadBalancer
clusterScope.Info("Reconciling VPC load balancers")
if requeue, err := clusterScope.ReconcileLoadBalancers(); err != nil {
clusterScope.Error(err, "failed to reconcile VPC load balancers")
updatePowerVSCluster.condition(false, infrav1beta2.LoadBalancerReadyCondition, infrav1beta2.LoadBalancerReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
ch <- reconcileResult{reconcile.Result{}, err}
return
} else if requeue {
clusterScope.Info("VPC load balancer creation is pending")
return
}
updatePowerVSCluster.condition(true, infrav1beta2.LoadBalancerReadyCondition)
}

func (r *IBMPowerVSClusterReconciler) reconcile(clusterScope *scope.PowerVSClusterScope) (ctrl.Result, error) { //nolint:gocyclo
if controllerutil.AddFinalizer(clusterScope.IBMPowerVSCluster, infrav1beta2.IBMPowerVSClusterFinalizer) {
return ctrl.Result{}, nil
}

// check for annotation set for cluster resource and decide on proceeding with infra creation.
// do not proceed further if "powervs.cluster.x-k8s.io/create-infra=true" annotation is not set.
if !scope.CheckCreateInfraAnnotation(*clusterScope.IBMPowerVSCluster) {
clusterScope.IBMPowerVSCluster.Status.Ready = true
return ctrl.Result{}, nil
}

// validate PER availability for the PowerVS zone, proceed further only if PowerVS zone support PER.
// more information about PER can be found here: https://cloud.ibm.com/docs/power-iaas?topic=power-iaas-per
if err := clusterScope.IsPowerVSZoneSupportsPER(); err != nil {
clusterScope.Error(err, "error checking PER capability for PowerVS zone")
return reconcile.Result{}, err
}
conditions.MarkTrue(powerVSCluster, infrav1beta2.VPCSecurityGroupReadyCondition)

// reconcile service resource group
clusterScope.Info("Reconciling resource group")
if err := clusterScope.ReconcileResourceGroup(); err != nil {
clusterScope.Error(err, "failed to reconcile resource group")
return reconcile.Result{}, err
}

updatePowerVSCluster := &updatePowerVSCluster{
cluster: clusterScope.IBMPowerVSCluster,
}

var wg sync.WaitGroup
ch := make(chan reconcileResult)

// reconcile PowerVS resources
wg.Add(1)
go r.reconcilePowerVSResources(clusterScope, updatePowerVSCluster, ch, &wg)

// reconcile VPC
wg.Add(1)
go r.reconcileVPCResources(clusterScope, updatePowerVSCluster, ch, &wg)

// wait for above reconcile to complete and close the channel
go func() {
wg.Wait()
close(ch)
}()

var requeue bool
var errList []error
// receive return values from the channel and decide the requeue
for val := range ch {
if val.Requeue {
requeue = true
}
if val.error != nil {
errList = append(errList, val.error)
}
}

if requeue && len(errList) > 1 {
return ctrl.Result{RequeueAfter: 30 * time.Second}, kerrors.NewAggregate(errList)
} else if requeue {
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
} else if len(errList) > 1 {
return ctrl.Result{}, kerrors.NewAggregate(errList)
}

// reconcile Transit Gateway
clusterScope.Info("Reconciling Transit Gateway")
if requeue, err := clusterScope.ReconcileTransitGateway(); err != nil {
clusterScope.Error(err, "failed to reconcile transit gateway")
conditions.MarkFalse(powerVSCluster, infrav1beta2.TransitGatewayReadyCondition, infrav1beta2.TransitGatewayReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
conditions.MarkFalse(updatePowerVSCluster.cluster, infrav1beta2.TransitGatewayReadyCondition, infrav1beta2.TransitGatewayReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
return reconcile.Result{}, err
} else if requeue {
clusterScope.Info("Transit gateway creation is pending, requeuing")
return reconcile.Result{RequeueAfter: 1 * time.Minute}, nil
}
conditions.MarkTrue(powerVSCluster, infrav1beta2.TransitGatewayReadyCondition)

// reconcile LoadBalancer
clusterScope.Info("Reconciling VPC load balancers")
if requeue, err := clusterScope.ReconcileLoadBalancers(); err != nil {
clusterScope.Error(err, "failed to reconcile VPC load balancers")
conditions.MarkFalse(powerVSCluster, infrav1beta2.LoadBalancerReadyCondition, infrav1beta2.LoadBalancerReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
return reconcile.Result{}, err
} else if requeue {
clusterScope.Info("VPC load balancer creation is pending, requeuing")
return reconcile.Result{RequeueAfter: 1 * time.Minute}, nil
}
conditions.MarkTrue(updatePowerVSCluster.cluster, infrav1beta2.TransitGatewayReadyCondition)

// reconcile COSInstance
if clusterScope.IBMPowerVSCluster.Spec.Ignition != nil {
clusterScope.Info("Reconciling COS service instance")
if err := clusterScope.ReconcileCOSInstance(); err != nil {
conditions.MarkFalse(powerVSCluster, infrav1beta2.COSInstanceReadyCondition, infrav1beta2.COSInstanceReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
conditions.MarkFalse(updatePowerVSCluster.cluster, infrav1beta2.COSInstanceReadyCondition, infrav1beta2.COSInstanceReconciliationFailedReason, capiv1beta1.ConditionSeverityError, err.Error())
return reconcile.Result{}, err
}
conditions.MarkTrue(powerVSCluster, infrav1beta2.COSInstanceReadyCondition)
conditions.MarkTrue(updatePowerVSCluster.cluster, infrav1beta2.COSInstanceReadyCondition)
}

var networkReady, loadBalancerReady bool
for _, cond := range clusterScope.IBMPowerVSCluster.Status.Conditions {
if cond.Type == infrav1beta2.NetworkReadyCondition && cond.Status == corev1.ConditionTrue {
networkReady = true
}
if cond.Type == infrav1beta2.LoadBalancerReadyCondition && cond.Status == corev1.ConditionTrue {
loadBalancerReady = true
}
}

if !networkReady || !loadBalancerReady {
clusterScope.Info("Network or LoadBalancer still not ready, requeuing")
return ctrl.Result{RequeueAfter: 30 * time.Second}, nil
}

// update cluster object with loadbalancer host
loadBalancer := clusterScope.PublicLoadBalancer()
if loadBalancer == nil {
return reconcile.Result{}, fmt.Errorf("failed to fetch public loadbalancer")
}
if clusterScope.GetLoadBalancerState(loadBalancer.Name) == nil || *clusterScope.GetLoadBalancerState(loadBalancer.Name) != infrav1beta2.VPCLoadBalancerStateActive {
clusterScope.Info("LoadBalancer state is not active")
return reconcile.Result{RequeueAfter: time.Minute}, nil
}

clusterScope.Info("Getting load balancer host")
hostName := clusterScope.GetLoadBalancerHostName(loadBalancer.Name)
if hostName == nil || *hostName == "" {
clusterScope.Info("LoadBalancer hostname is not yet available, requeuing")
return reconcile.Result{RequeueAfter: time.Minute}, nil
}
conditions.MarkTrue(powerVSCluster, infrav1beta2.LoadBalancerReadyCondition)

clusterScope.IBMPowerVSCluster.Spec.ControlPlaneEndpoint.Host = *clusterScope.GetLoadBalancerHostName(loadBalancer.Name)
clusterScope.IBMPowerVSCluster.Spec.ControlPlaneEndpoint.Port = clusterScope.APIServerPort()
Expand Down

0 comments on commit 21ca9b0

Please sign in to comment.