diff --git a/service/controller/azure_machine_pool.go b/service/controller/azure_machine_pool.go index 09bfc0442f..0b4fa72d81 100644 --- a/service/controller/azure_machine_pool.go +++ b/service/controller/azure_machine_pool.go @@ -27,6 +27,7 @@ import ( "github.com/giantswarm/azure-operator/v5/service/controller/debugger" "github.com/giantswarm/azure-operator/v5/service/controller/internal/vmsku" "github.com/giantswarm/azure-operator/v5/service/controller/resource/azureconfig" + "github.com/giantswarm/azure-operator/v5/service/controller/resource/azuremachinepoolconditions" "github.com/giantswarm/azure-operator/v5/service/controller/resource/cloudconfigblob" "github.com/giantswarm/azure-operator/v5/service/controller/resource/ipam" "github.com/giantswarm/azure-operator/v5/service/controller/resource/nodepool" @@ -137,6 +138,20 @@ func NewAzureMachinePoolResourceSet(config AzureMachinePoolConfig) ([]resource.I organizationClientFactory = client.NewOrganizationFactory(c) } + var azureMachinePoolConditionsResource resource.Interface + { + c := azuremachinepoolconditions.Config{ + AzureClientsFactory: &organizationClientFactory, + CtrlClient: config.K8sClient.CtrlClient(), + Logger: config.Logger, + } + + azureMachinePoolConditionsResource, err = azuremachinepoolconditions.New(c) + if err != nil { + return nil, microerror.Mask(err) + } + } + var newDebugger *debugger.Debugger { c := debugger.Config{ @@ -322,6 +337,7 @@ func NewAzureMachinePoolResourceSet(config AzureMachinePoolConfig) ([]resource.I } resources := []resource.Interface{ + azureMachinePoolConditionsResource, sparkResource, cloudconfigblobResource, ipamResource, diff --git a/service/controller/key/key.go b/service/controller/key/key.go index bafdcdd186..26da368eea 100644 --- a/service/controller/key/key.go +++ b/service/controller/key/key.go @@ -46,6 +46,7 @@ const ( masterNatGatewayName = "masters-nat-gw" prefixMaster = "master" prefixWorker = "worker" + subnetDeploymentPrefix = "subnet" virtualNetworkSuffix = "VirtualNetwork" vpnGatewaySubnet = "GatewaySubnet" vpnGatewaySuffix = "VPNGateway" @@ -629,6 +630,10 @@ func NodePoolDeploymentName(azureMachinePool *expcapzv1alpha3.AzureMachinePool) return NodePoolVMSSName(azureMachinePool) } +func SubnetDeploymentName(subnetName string) string { + return fmt.Sprintf("%s-%s", subnetDeploymentPrefix, subnetName) +} + func MachinePoolID(getter LabelsGetter) (string, error) { machinePoolID, exists := getter.GetLabels()[apiextensionslabels.MachinePool] if !exists { diff --git a/service/controller/resource/azuremachinepoolconditions/conditionready.go b/service/controller/resource/azuremachinepoolconditions/conditionready.go new file mode 100644 index 0000000000..29f53fa454 --- /dev/null +++ b/service/controller/resource/azuremachinepoolconditions/conditionready.go @@ -0,0 +1,66 @@ +package azuremachinepoolconditions + +import ( + "context" + + azureconditions "github.com/giantswarm/apiextensions/v3/pkg/conditions/azure" + "github.com/giantswarm/microerror" + corev1 "k8s.io/api/core/v1" + capzexp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + capi "sigs.k8s.io/cluster-api/api/v1alpha3" + capiconditions "sigs.k8s.io/cluster-api/util/conditions" +) + +func (r *Resource) ensureReadyCondition(ctx context.Context, azureMachinePool *capzexp.AzureMachinePool) error { + r.logDebug(ctx, "ensuring condition Ready") + var err error + + // Ensure VMMSReady condition + err = r.ensureSubnetReadyCondition(ctx, azureMachinePool) + if err != nil { + return microerror.Mask(err) + } + + // Ensure VMMSReady condition + err = r.ensureVmssReadyCondition(ctx, azureMachinePool) + if err != nil { + return microerror.Mask(err) + } + + // List of conditions that all need to be True for the Ready condition to + // be True: + // - VMSSReady: node pool VMSS is ready + // - SubnetReady: node pool subnet is ready + conditionsToSummarize := capiconditions.WithConditions( + azureconditions.SubnetReadyCondition, + azureconditions.VMSSReadyCondition) + + // Update Ready condition + capiconditions.SetSummary( + azureMachinePool, + conditionsToSummarize, + capiconditions.AddSourceRef()) + + // Now check current Ready condition so we can log the value + r.logConditionStatus(ctx, azureMachinePool, capi.ReadyCondition) + r.logDebug(ctx, "ensured condition Ready") + return nil +} + +func (r *Resource) logConditionStatus(ctx context.Context, azureMachinePool *capzexp.AzureMachinePool, conditionType capi.ConditionType) { + condition := capiconditions.Get(azureMachinePool, conditionType) + + if condition == nil { + r.logWarning(ctx, "condition %s not set", conditionType) + } else { + messageFormat := "condition %s set to %s" + messageArgs := []interface{}{conditionType, condition.Status} + if condition.Status != corev1.ConditionTrue { + messageFormat += ", Reason=%s, Severity=%s, Message=%s" + messageArgs = append(messageArgs, condition.Reason) + messageArgs = append(messageArgs, condition.Severity) + messageArgs = append(messageArgs, condition.Message) + } + r.logDebug(ctx, messageFormat, messageArgs...) + } +} diff --git a/service/controller/resource/azuremachinepoolconditions/conditionsubnetready.go b/service/controller/resource/azuremachinepoolconditions/conditionsubnetready.go new file mode 100644 index 0000000000..11833c493c --- /dev/null +++ b/service/controller/resource/azuremachinepoolconditions/conditionsubnetready.go @@ -0,0 +1,105 @@ +package azuremachinepoolconditions + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/network/mgmt/2019-11-01/network" + azureconditions "github.com/giantswarm/apiextensions/v3/pkg/conditions/azure" + "github.com/giantswarm/microerror" + capzexp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + capi "sigs.k8s.io/cluster-api/api/v1alpha3" + capiconditions "sigs.k8s.io/cluster-api/util/conditions" + + "github.com/giantswarm/azure-operator/v5/pkg/helpers" + "github.com/giantswarm/azure-operator/v5/service/controller/key" +) + +const ( + SubnetNotFoundReason = "SubnetNotFound" + SubnetProvisioningStatePrefix = "SubnetProvisioningState" +) + +func (r *Resource) ensureSubnetReadyCondition(ctx context.Context, azureMachinePool *capzexp.AzureMachinePool) error { + r.logDebug(ctx, "ensuring condition %s", azureconditions.SubnetReadyCondition) + + deploymentsClient, err := r.azureClientsFactory.GetDeploymentsClient(ctx, azureMachinePool.ObjectMeta) + if err != nil { + return microerror.Mask(err) + } + + // Now let's first check ARM deployment state + subnetDeploymentName := key.SubnetDeploymentName(azureMachinePool.Name) + isSubnetDeploymentSuccessful, err := r.checkIfDeploymentIsSuccessful(ctx, deploymentsClient, azureMachinePool, subnetDeploymentName, azureconditions.SubnetReadyCondition) + if err != nil { + return microerror.Mask(err) + } else if !isSubnetDeploymentSuccessful { + // Function checkIfDeploymentIsSuccessful that is called above, if it + // sees that the deployment is not succeeded, for whatever reason, it + // will also set appropriate condition value, so our job here is done. + return nil + } + + // Deployment is successful, we proceed with checking the actual Azure + // subnet. + subnetsClient, err := r.azureClientsFactory.GetSubnetsClient(ctx, azureMachinePool.ObjectMeta) + if err != nil { + return microerror.Mask(err) + } + + azureCluster, err := helpers.GetAzureClusterFromMetadata(ctx, r.ctrlClient, azureMachinePool.ObjectMeta) + if err != nil { + return microerror.Mask(err) + } + + subnetName := azureMachinePool.Name + subnet, err := subnetsClient.Get(ctx, azureCluster.Name, azureCluster.Spec.NetworkSpec.Vnet.Name, subnetName, "") + if IsNotFound(err) { + r.setSubnetNotFound(ctx, azureMachinePool, subnetName, azureconditions.SubnetReadyCondition) + return nil + } else if err != nil { + return microerror.Mask(err) + } + + // Note: Here we check if the subnet exists and that its provisioning state + // is succeeded. It would be good to also check network security group, + // routing table and service endpoints. + if subnet.ProvisioningState == network.Succeeded { + capiconditions.MarkTrue(azureMachinePool, azureconditions.SubnetReadyCondition) + } else { + r.setSubnetProvisioningStateNotSuccessful(ctx, azureMachinePool, subnetName, subnet.ProvisioningState, azureconditions.SubnetReadyCondition) + } + + r.logConditionStatus(ctx, azureMachinePool, azureconditions.SubnetReadyCondition) + r.logDebug(ctx, "ensured condition %s", azureconditions.SubnetReadyCondition) + return nil +} + +func (r *Resource) setSubnetNotFound(ctx context.Context, cr capiconditions.Setter, subnetName string, condition capi.ConditionType) { + message := "Subnet %s is not found" + messageArgs := subnetName + capiconditions.MarkFalse( + cr, + condition, + SubnetNotFoundReason, + capi.ConditionSeverityError, + message, + messageArgs) + + r.logWarning(ctx, message, messageArgs) +} + +func (r *Resource) setSubnetProvisioningStateNotSuccessful(ctx context.Context, cr capiconditions.Setter, subnetName string, provisioningState network.ProvisioningState, condition capi.ConditionType) { + message := "Subnet %s provisioning state is %s" + messageArgs := []interface{}{subnetName, provisioningState} + reason := SubnetProvisioningStatePrefix + string(provisioningState) + + capiconditions.MarkFalse( + cr, + condition, + reason, + capi.ConditionSeverityWarning, + message, + messageArgs...) + + r.logWarning(ctx, message, messageArgs...) +} diff --git a/service/controller/resource/azuremachinepoolconditions/conditionvmssready.go b/service/controller/resource/azuremachinepoolconditions/conditionvmssready.go new file mode 100644 index 0000000000..540f6c3bcc --- /dev/null +++ b/service/controller/resource/azuremachinepoolconditions/conditionvmssready.go @@ -0,0 +1,153 @@ +package azuremachinepoolconditions + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/compute/mgmt/2019-07-01/compute" + azureconditions "github.com/giantswarm/apiextensions/v3/pkg/conditions/azure" + "github.com/giantswarm/microerror" + capzexp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" + capi "sigs.k8s.io/cluster-api/api/v1alpha3" + capiconditions "sigs.k8s.io/cluster-api/util/conditions" + + "github.com/giantswarm/azure-operator/v5/service/controller/key" +) + +const ( + VMSSNotFoundReason = "VMSSNotFound" + VMSSProvisioningStatePrefix = "VMSSProvisioningState" + VMSSProvisioningStateUnknownReason = "VMSSProvisioningStateUnknown" + VmssProvisioningStateSucceeded = string(compute.ProvisioningStateSucceeded) + VmssProvisioningStateFailed = string(compute.ProvisioningStateFailed) +) + +func (r *Resource) ensureVmssReadyCondition(ctx context.Context, azureMachinePool *capzexp.AzureMachinePool) error { + r.logDebug(ctx, "ensuring condition %s", azureconditions.VMSSReadyCondition) + + deploymentsClient, err := r.azureClientsFactory.GetDeploymentsClient(ctx, azureMachinePool.ObjectMeta) + if err != nil { + return microerror.Mask(err) + } + + // Now let's first check ARM deployment state + deploymentName := key.NodePoolDeploymentName(azureMachinePool) + isDeploymentSuccessful, err := r.checkIfDeploymentIsSuccessful(ctx, deploymentsClient, azureMachinePool, deploymentName, azureconditions.VMSSReadyCondition) + if err != nil { + return microerror.Mask(err) + } else if !isDeploymentSuccessful { + // Function checkIfDeploymentIsSuccessful that is called above, if it + // sees that the deployment is not succeeded, for whatever reason, it + // will also set appropriate condition value, so our job here is done. + return nil + } + + // Deployment is successful, we proceed with checking the actual Azure + // VMSS. + vmssClient, err := r.azureClientsFactory.GetVirtualMachineScaleSetsClient(ctx, azureMachinePool.ObjectMeta) + if err != nil { + return microerror.Mask(err) + } + + resourceGroupName := key.ClusterName(azureMachinePool) + vmssName := key.NodePoolVMSSName(azureMachinePool) + + vmss, err := vmssClient.Get(ctx, resourceGroupName, vmssName) + if IsNotFound(err) { + r.setVMSSNotFound(ctx, azureMachinePool, vmssName, azureconditions.VMSSReadyCondition) + return nil + } else if err != nil { + return microerror.Mask(err) + } + + // Note: Here we are only checking the provisioning state of VMSS. Ideally + // we would check the provisioning and power state of all instances, but + // that would require more VMSS instance API calls that have very low + // throttling limits, so we will add that later, once throttling situation + // is better. + + // Check if VMSS provisioning state is set. We expect that it is, since we + // already checked the deployment, but it's not impossible that the VMSS + // resource got changed for some reason. + if vmss.ProvisioningState == nil { + r.setVMSSProvisioningStateUnknown(ctx, azureMachinePool, deploymentName, azureconditions.VMSSReadyCondition) + return nil + } + + switch *vmss.ProvisioningState { + // VMSS provisioning state is Succeeded, all good. + case VmssProvisioningStateSucceeded: + capiconditions.MarkTrue(azureMachinePool, azureconditions.VMSSReadyCondition) + // VMSS provisioning state is Failed, VMSS has some issues. + case VmssProvisioningStateFailed: + r.setVMSSProvisioningStateFailed(ctx, azureMachinePool, vmssName, azureconditions.VMSSReadyCondition) + default: + // VMSS provisioning state not Succeeded, set current state to VMSSReady condition. + r.setVMSSProvisioningStateWarning(ctx, azureMachinePool, vmssName, *vmss.ProvisioningState, azureconditions.VMSSReadyCondition) + } + + // Log current VMSSReady condition + r.logConditionStatus(ctx, azureMachinePool, azureconditions.VMSSReadyCondition) + r.logDebug(ctx, "ensured condition %s", azureconditions.VMSSReadyCondition) + return nil +} + +func (r *Resource) setVMSSNotFound(ctx context.Context, cr capiconditions.Setter, vmssName string, condition capi.ConditionType) { + message := "VMSS %s is not found, which should not happen when the deployment is successful" + messageArgs := vmssName + capiconditions.MarkFalse( + cr, + condition, + VMSSNotFoundReason, + capi.ConditionSeverityError, + message, + messageArgs) + + r.logWarning(ctx, message, messageArgs) +} + +func (r *Resource) setVMSSProvisioningStateUnknown(ctx context.Context, cr capiconditions.Setter, deploymentName string, condition capi.ConditionType) { + message := "VMSS %s provisioning state not returned by Azure API, check back in few minutes" + messageArgs := deploymentName + capiconditions.MarkFalse( + cr, + condition, + VMSSProvisioningStateUnknownReason, + capi.ConditionSeverityWarning, + message, + messageArgs) + + r.logWarning(ctx, message, messageArgs) +} + +func (r *Resource) setVMSSProvisioningStateFailed(ctx context.Context, cr capiconditions.Setter, vmssName string, condition capi.ConditionType) { + message := "VMSS %s failed, it might succeed after retrying, see Azure portal for more details" + messageArgs := vmssName + reason := VMSSProvisioningStatePrefix + VmssProvisioningStateFailed + + capiconditions.MarkFalse( + cr, + condition, + reason, + capi.ConditionSeverityError, + message, + messageArgs) + + r.logWarning(ctx, message, messageArgs) +} + +func (r *Resource) setVMSSProvisioningStateWarning(ctx context.Context, cr capiconditions.Setter, vmssName string, currentProvisioningState string, condition capi.ConditionType) { + message := "Deployment %s has not succeeded yet, current state is %s, " + + "check back in few minutes, see Azure portal for more details" + messageArgs := []interface{}{vmssName, currentProvisioningState} + reason := VMSSProvisioningStatePrefix + currentProvisioningState + + capiconditions.MarkFalse( + cr, + condition, + reason, + capi.ConditionSeverityWarning, + message, + messageArgs...) + + r.logWarning(ctx, message, messageArgs...) +} diff --git a/service/controller/resource/azuremachinepoolconditions/create.go b/service/controller/resource/azuremachinepoolconditions/create.go new file mode 100644 index 0000000000..4c860d8402 --- /dev/null +++ b/service/controller/resource/azuremachinepoolconditions/create.go @@ -0,0 +1,30 @@ +package azuremachinepoolconditions + +import ( + "context" + + "github.com/giantswarm/microerror" + + "github.com/giantswarm/azure-operator/v5/service/controller/key" +) + +func (r *Resource) EnsureCreated(ctx context.Context, cr interface{}) error { + var err error + azureMachinePool, err := key.ToAzureMachinePool(cr) + if err != nil { + return microerror.Mask(err) + } + + // ensure Ready condition + err = r.ensureReadyCondition(ctx, &azureMachinePool) + if err != nil { + return microerror.Mask(err) + } + + err = r.ctrlClient.Status().Update(ctx, &azureMachinePool) + if err != nil { + return microerror.Mask(err) + } + + return nil +} diff --git a/service/controller/resource/azuremachinepoolconditions/delete.go b/service/controller/resource/azuremachinepoolconditions/delete.go new file mode 100644 index 0000000000..a8cdb2bd03 --- /dev/null +++ b/service/controller/resource/azuremachinepoolconditions/delete.go @@ -0,0 +1,9 @@ +package azuremachinepoolconditions + +import ( + "context" +) + +func (r *Resource) EnsureDeleted(ctx context.Context, obj interface{}) error { + return nil +} diff --git a/service/controller/resource/azuremachinepoolconditions/deploymentchecks.go b/service/controller/resource/azuremachinepoolconditions/deploymentchecks.go new file mode 100644 index 0000000000..cd3b52d2d9 --- /dev/null +++ b/service/controller/resource/azuremachinepoolconditions/deploymentchecks.go @@ -0,0 +1,140 @@ +package azuremachinepoolconditions + +import ( + "context" + + "github.com/Azure/azure-sdk-for-go/services/resources/mgmt/2019-05-01/resources" + "github.com/giantswarm/microerror" + capi "sigs.k8s.io/cluster-api/api/v1alpha3" + capiconditions "sigs.k8s.io/cluster-api/util/conditions" + + "github.com/giantswarm/azure-operator/v5/service/controller/key" +) + +const ( + DeploymentNotFoundReason = "DeploymentNotFound" + DeploymentProvisioningStateUnknownReason = "DeploymentProvisioningStateUnknown" + DeploymentProvisioningStatePrefix = "DeploymentProvisioningState" + DeploymentProvisioningStateSucceeded = "Succeeded" + DeploymentProvisioningStateFailed = "Failed" +) + +type CRWithConditions interface { + key.LabelsGetter + capiconditions.Setter +} + +func (r *Resource) checkIfDeploymentIsSuccessful(ctx context.Context, deploymentsClient *resources.DeploymentsClient, cr CRWithConditions, deploymentName string, conditionType capi.ConditionType) (bool, error) { + deployment, err := deploymentsClient.Get(ctx, key.ClusterName(cr), deploymentName) + if IsNotFound(err) { + // Deployment has not been found, which means that we still + // didn't start deploying it. + r.setDeploymentNotFound(ctx, cr, deploymentName, conditionType) + return false, nil + } else if err != nil { + // Error while getting Subnet deployment, let's check if + // deployment provisioning state is set. + if !isProvisioningStateSet(&deployment) { + return false, microerror.Mask(err) + } + + currentProvisioningState := *deployment.Properties.ProvisioningState + r.setProvisioningStateWarning(ctx, cr, deploymentName, currentProvisioningState, conditionType) + return false, nil + } + + // We got the Subnet deployment without errors, but for some reason the provisioning state is + // not set. + if !isProvisioningStateSet(&deployment) { + r.setProvisioningStateUnknown(ctx, cr, deploymentName, conditionType) + return false, nil + } + + // Now let's finally check what's the current subnet deployment + // provisioning state. + currentProvisioningState := *deployment.Properties.ProvisioningState + + switch currentProvisioningState { + case DeploymentProvisioningStateSucceeded: + return true, nil + case DeploymentProvisioningStateFailed: + // Subnet deployment has failed. + r.setProvisioningStateWarningFailed(ctx, cr, deploymentName, conditionType) + default: + // Subnet deployment is probably still running. + r.setProvisioningStateWarning(ctx, cr, deploymentName, currentProvisioningState, conditionType) + } + + return false, nil +} + +func isProvisioningStateSet(deployment *resources.DeploymentExtended) bool { + if deployment.Properties != nil && + deployment.Properties.ProvisioningState != nil && + *deployment.Properties.ProvisioningState != "" { + return true + } + + return false +} + +func (r *Resource) setProvisioningStateWarningFailed(ctx context.Context, cr capiconditions.Setter, deploymentName string, condition capi.ConditionType) { + message := "Deployment %s failed, it might succeed after retrying, see Azure portal for more details" + messageArgs := deploymentName + reason := DeploymentProvisioningStatePrefix + DeploymentProvisioningStateFailed + + capiconditions.MarkFalse( + cr, + condition, + reason, + capi.ConditionSeverityError, + message, + messageArgs) + + r.logWarning(ctx, message, messageArgs) +} + +func (r *Resource) setProvisioningStateWarning(ctx context.Context, cr capiconditions.Setter, deploymentName string, currentProvisioningState string, condition capi.ConditionType) { + message := "Deployment %s has not succeeded yet, current state is %s, " + + "check back in few minutes, see Azure portal for more details" + messageArgs := []interface{}{deploymentName, currentProvisioningState} + reason := DeploymentProvisioningStatePrefix + currentProvisioningState + + capiconditions.MarkFalse( + cr, + condition, + reason, + capi.ConditionSeverityWarning, + message, + messageArgs...) + + r.logWarning(ctx, message, messageArgs...) +} + +func (r *Resource) setProvisioningStateUnknown(ctx context.Context, cr capiconditions.Setter, deploymentName string, condition capi.ConditionType) { + message := "Deployment %s provisioning state not returned by Azure API, check back in few minutes" + messageArgs := deploymentName + capiconditions.MarkFalse( + cr, + condition, + DeploymentProvisioningStateUnknownReason, + capi.ConditionSeverityWarning, + message, + messageArgs) + + r.logWarning(ctx, message, messageArgs) +} + +func (r *Resource) setDeploymentNotFound(ctx context.Context, cr capiconditions.Setter, deploymentName string, condition capi.ConditionType) { + message := "Deployment %s is not found, check back in few minutes" + messageArgs := deploymentName + capiconditions.MarkFalse( + cr, + condition, + DeploymentNotFoundReason, + capi.ConditionSeverityWarning, + message, + messageArgs) + + r.logWarning(ctx, message, messageArgs) +} diff --git a/service/controller/resource/azuremachinepoolconditions/error.go b/service/controller/resource/azuremachinepoolconditions/error.go new file mode 100644 index 0000000000..c2df475e29 --- /dev/null +++ b/service/controller/resource/azuremachinepoolconditions/error.go @@ -0,0 +1,43 @@ +package azuremachinepoolconditions + +import ( + "github.com/Azure/go-autorest/autorest" + "github.com/giantswarm/microerror" +) + +var invalidConfigError = µerror.Error{ + Kind: "invalidConfigError", +} + +// IsInvalidConfig asserts invalidConfigError. +func IsInvalidConfig(err error) bool { + return microerror.Cause(err) == invalidConfigError +} + +var notFoundError = µerror.Error{ + Kind: "notFoundError", +} + +// IsNotFound asserts notFoundError. +func IsNotFound(err error) bool { + if err == nil { + return false + } + + c := microerror.Cause(err) + + if c == notFoundError { + return true + } + + { + dErr, ok := c.(autorest.DetailedError) + if ok { + if dErr.StatusCode == 404 { + return true + } + } + } + + return false +} diff --git a/service/controller/resource/azuremachinepoolconditions/resource.go b/service/controller/resource/azuremachinepoolconditions/resource.go new file mode 100644 index 0000000000..bb50ca02b2 --- /dev/null +++ b/service/controller/resource/azuremachinepoolconditions/resource.go @@ -0,0 +1,63 @@ +package azuremachinepoolconditions + +import ( + "context" + "fmt" + + "github.com/giantswarm/microerror" + "github.com/giantswarm/micrologger" + "sigs.k8s.io/controller-runtime/pkg/client" + + azureclient "github.com/giantswarm/azure-operator/v5/client" +) + +const ( + // Name is the identifier of the resource. + Name = "azuremachinepoolconditions" +) + +type Config struct { + AzureClientsFactory *azureclient.OrganizationFactory + CtrlClient client.Client + Logger micrologger.Logger +} + +// Resource ensures that AzureMachinePool Status Conditions are set. +type Resource struct { + azureClientsFactory *azureclient.OrganizationFactory + ctrlClient client.Client + logger micrologger.Logger +} + +func New(config Config) (*Resource, error) { + if config.AzureClientsFactory == nil { + return nil, microerror.Maskf(invalidConfigError, "%T.AzureClientsFactory must not be empty", config) + } + if config.CtrlClient == nil { + return nil, microerror.Maskf(invalidConfigError, "%T.CtrlClient must not be empty", config) + } + if config.Logger == nil { + return nil, microerror.Maskf(invalidConfigError, "%T.Logger must not be empty", config) + } + + r := &Resource{ + azureClientsFactory: config.AzureClientsFactory, + ctrlClient: config.CtrlClient, + logger: config.Logger, + } + + return r, nil +} + +// Name returns the resource name. +func (r *Resource) Name() string { + return Name +} + +func (r *Resource) logDebug(ctx context.Context, message string, messageArgs ...interface{}) { + r.logger.LogCtx(ctx, "level", "debug", "message", fmt.Sprintf(message, messageArgs...)) +} + +func (r *Resource) logWarning(ctx context.Context, message string, messageArgs ...interface{}) { + r.logger.LogCtx(ctx, "level", "warning", "message", fmt.Sprintf(message, messageArgs...)) +} diff --git a/service/controller/resource/machinepoolconditions/resource.go b/service/controller/resource/machinepoolconditions/resource.go index 28fdfdf622..523d190d31 100644 --- a/service/controller/resource/machinepoolconditions/resource.go +++ b/service/controller/resource/machinepoolconditions/resource.go @@ -20,7 +20,7 @@ type Config struct { Logger micrologger.Logger } -// Resource ensures that AzureCluster Status Conditions are set. +// Resource ensures that MachinePool Status Conditions are set. type Resource struct { ctrlClient client.Client logger micrologger.Logger diff --git a/service/controller/resource/nodepool/conditions.go b/service/controller/resource/nodepool/conditions.go deleted file mode 100644 index a0f452048f..0000000000 --- a/service/controller/resource/nodepool/conditions.go +++ /dev/null @@ -1,90 +0,0 @@ -package nodepool - -import ( - "context" - "fmt" - - azureconditions "github.com/giantswarm/apiextensions/v3/pkg/conditions/azure" - "github.com/giantswarm/microerror" - capzexp "sigs.k8s.io/cluster-api-provider-azure/exp/api/v1alpha3" - capi "sigs.k8s.io/cluster-api/api/v1alpha3" - conditions "sigs.k8s.io/cluster-api/util/conditions" -) - -const ( - ProvisioningStateSucceeded = "Succeeded" - ProvisioningStateFailed = "Failed" -) - -func (r *Resource) UpdateDeploymentSucceededCondition(ctx context.Context, azureMachinePool *capzexp.AzureMachinePool, provisioningState *string) error { - conditionType := azureconditions.DeploymentSucceededCondition - var conditionReason string - var conditionSeverity capi.ConditionSeverity - logger := r.Logger.With("level", "debug", "type", "AzureMachinePool", "message", "setting Status.Condition", "conditionType", conditionType) - - if provisioningState == nil { - conditionReason = "DeploymentNotFound" - conditionSeverity = capi.ConditionSeverityWarning - conditions.MarkFalse( - azureMachinePool, - conditionType, - conditionReason, - conditionSeverity, - "Deployment has not been found.") - logger.LogCtx(ctx, "conditionStatus", false, "conditionReason", conditionReason, "conditionSeverity", conditionSeverity) - } else { - switch *provisioningState { - case ProvisioningStateSucceeded: - conditions.MarkTrue(azureMachinePool, conditionType) - logger.LogCtx(ctx, "conditionStatus", true) - case ProvisioningStateFailed: - conditionSeverity = capi.ConditionSeverityError - conditionReason = "ProvisioningStateFailed" - conditions.MarkFalse( - azureMachinePool, - conditionType, - conditionReason, - conditionSeverity, - "Deployment has failed.") - logger.LogCtx(ctx, "conditionStatus", false, "conditionReason", conditionReason, "conditionSeverity", conditionSeverity) - default: - conditionSeverity = capi.ConditionSeverityWarning - conditionReason = fmt.Sprintf("ProvisioningState%s", *provisioningState) - conditions.MarkFalse( - azureMachinePool, - conditionType, - conditionReason, - conditionSeverity, - "Current deployment provisioning status is %s.", - *provisioningState) - logger.LogCtx(ctx, "conditionStatus", false, "conditionReason", conditionReason, "conditionSeverity", conditionSeverity) - } - } - - // Preview implementation only: DeploymentSucceeded -> Ready - // In the final version it will include more detailed and more accurate conditions, e.g. checking the power state of VMSS instances. - if conditions.IsTrue(azureMachinePool, azureconditions.DeploymentSucceededCondition) { - conditions.MarkTrue(azureMachinePool, capi.ReadyCondition) - } else { - conditionReason = "Deploying" - conditionSeverity = capi.ConditionSeverityWarning - conditions.MarkFalse( - azureMachinePool, - capi.ReadyCondition, - conditionReason, - conditionSeverity, - "Node pool deployment is in progress.") - } - - err := r.CtrlClient.Status().Update(ctx, azureMachinePool) - if err != nil { - r.Logger.LogCtx(ctx, - "level", "error", - "type", "AzureMachinePool", - "conditionType", conditionType, - "message", "error while setting Status.Condition") - return microerror.Mask(err) - } - - return nil -} diff --git a/service/controller/resource/nodepool/create_deployment_uninitialized.go b/service/controller/resource/nodepool/create_deployment_uninitialized.go index 172972a00c..f4821339fc 100644 --- a/service/controller/resource/nodepool/create_deployment_uninitialized.go +++ b/service/controller/resource/nodepool/create_deployment_uninitialized.go @@ -113,15 +113,6 @@ func (r *Resource) deploymentUninitializedTransition(ctx context.Context, obj in return currentState, microerror.Mask(err) } - defer func() { - var currentProvisioningState *string - if currentDeployment.Properties != nil && currentDeployment.Properties.ProvisioningState != nil { - currentProvisioningState = currentDeployment.Properties.ProvisioningState - } - // Update DeploymentSucceeded Condition for this AzureMachinePool - _ = r.UpdateDeploymentSucceededCondition(ctx, &azureMachinePool, currentProvisioningState) - }() - // Figure out if we need to submit the ARM Deployment. deploymentNeedsToBeSubmitted := currentDeployment.IsHTTPStatus(http.StatusNotFound) nodesNeedToBeRolled := false diff --git a/service/controller/resource/nodepool/create_scale_workers.go b/service/controller/resource/nodepool/create_scale_workers.go index 260efbe8c0..9b825c8421 100644 --- a/service/controller/resource/nodepool/create_scale_workers.go +++ b/service/controller/resource/nodepool/create_scale_workers.go @@ -66,24 +66,12 @@ func (r *Resource) scaleUpWorkerVMSSTransition(ctx context.Context, obj interfac // Ensure the deployment is successful before we move on with scaling. currentDeployment, err := deploymentsClient.Get(ctx, key.ClusterID(&azureMachinePool), key.NodePoolDeploymentName(&azureMachinePool)) if IsDeploymentNotFound(err) { - // Update DeploymentSucceeded Condition for this AzureMachinePool - _ = r.UpdateDeploymentSucceededCondition(ctx, &azureMachinePool, nil) - // Deployment not found, we need to apply it again. return DeploymentUninitialized, microerror.Mask(err) } else if err != nil { return currentState, microerror.Mask(err) } - defer func() { - var currentProvisioningState *string - if currentDeployment.Properties != nil && currentDeployment.Properties.ProvisioningState != nil { - currentProvisioningState = currentDeployment.Properties.ProvisioningState - } - // Update DeploymentSucceeded Condition for this AzureMachinePool - _ = r.UpdateDeploymentSucceededCondition(ctx, &azureMachinePool, currentProvisioningState) - }() - switch *currentDeployment.Properties.ProvisioningState { case "Failed", "Canceled": // Deployment is failed or canceled, I need to go back and re-apply it. diff --git a/service/controller/resource/subnet/resource.go b/service/controller/resource/subnet/resource.go index d2b9bd48ae..b1916de783 100644 --- a/service/controller/resource/subnet/resource.go +++ b/service/controller/resource/subnet/resource.go @@ -24,7 +24,6 @@ import ( ) const ( - mainDeploymentName = "subnet" // Name is the identifier of the resource. Name = "subnet" ) @@ -119,10 +118,6 @@ func (r *Resource) EnsureCreated(ctx context.Context, obj interface{}) error { return nil } -func getSubnetARMDeploymentName(subnetName string) string { - return fmt.Sprintf("%s-%s", mainDeploymentName, subnetName) -} - func (r *Resource) ensureSubnets(ctx context.Context, deploymentsClient *azureresource.DeploymentsClient, storageAccountsClient *storage.AccountsClient, natGatewaysClient *network.NatGatewaysClient, azureCluster capzv1alpha3.AzureCluster) error { armTemplate, err := subnet.GetARMTemplate() if err != nil { @@ -141,7 +136,7 @@ func (r *Resource) ensureSubnets(ctx context.Context, deploymentsClient *azurere } for i := 0; i < len(azureCluster.Spec.NetworkSpec.Subnets); i++ { - deploymentName := getSubnetARMDeploymentName(azureCluster.Spec.NetworkSpec.Subnets[i].Name) + deploymentName := key.SubnetDeploymentName(azureCluster.Spec.NetworkSpec.Subnets[i].Name) currentDeployment, err := deploymentsClient.Get(ctx, key.ClusterID(&azureCluster), deploymentName) if IsNotFound(err) { // fallthrough @@ -266,7 +261,7 @@ func (r *Resource) garbageCollectSubnets(ctx context.Context, deploymentsClient return microerror.Mask(err) } - err = r.deleteARMDeployment(ctx, deploymentsClient, key.ClusterID(&azureCluster), getSubnetARMDeploymentName(*subnetInAzure.Name)) + err = r.deleteARMDeployment(ctx, deploymentsClient, key.ClusterID(&azureCluster), key.SubnetDeploymentName(*subnetInAzure.Name)) if err != nil { return microerror.Mask(err) }