Skip to content

Commit

Permalink
refactor privatedns reconciliation to use async pattern
Browse files Browse the repository at this point in the history
  • Loading branch information
@shysank
shysank committed Jan 26, 2022
1 parent 3f46575 commit 77dbc7b
Show file tree
Hide file tree
Showing 2 changed files with 433 additions and 948 deletions.
325 changes: 186 additions & 139 deletions azure/services/privatedns/privatedns.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,205 +19,252 @@ package privatedns
import (
"context"

"github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
"sigs.k8s.io/cluster-api-provider-azure/azure/services/async"
"sigs.k8s.io/cluster-api-provider-azure/util/reconciler"

infrav1 "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"

"github.com/Azure/azure-sdk-for-go/services/privatedns/mgmt/2018-09-01/privatedns"
"github.com/Azure/go-autorest/autorest/to"
"github.com/pkg/errors"

"sigs.k8s.io/cluster-api-provider-azure/azure"
"sigs.k8s.io/cluster-api-provider-azure/azure/converters"
"sigs.k8s.io/cluster-api-provider-azure/util/tele"
)

const serviceName = "privatedns"

// Scope defines the scope interface for a private dns service.
type Scope interface {
azure.ClusterDescriber
PrivateDNSSpec() *azure.PrivateDNSSpec
azure.Authorizer
azure.AsyncStatusUpdater
PrivateDNSSpec() (zoneSpec azure.ResourceSpecGetter, linksSpec, recordsSpec []azure.ResourceSpecGetter)
}

// Service provides operations on Azure resources.
type Service struct {
Scope Scope
client
Scope Scope
zoneClient async.Creator
vnetLinkClient async.Creator
zoneReconciler async.Reconciler
vnetLinkReconciler async.Reconciler
recordReconciler async.Reconciler
}

// New creates a new private dns service.
func New(scope Scope) *Service {
zoneClient := newPrivateZonesClient(scope)
vnetLinkClient := newVirtualNetworkLinksClient(scope)
recordSetsClient := newRecordSetsClient(scope)
return &Service{
Scope: scope,
client: newClient(scope),
Scope: scope,
zoneClient: zoneClient,
vnetLinkClient: vnetLinkClient,
zoneReconciler: async.New(scope, zoneClient, zoneClient),
vnetLinkReconciler: async.New(scope, vnetLinkClient, vnetLinkClient),
recordReconciler: async.New(scope, recordSetsClient, recordSetsClient),
}
}

// Reconcile creates or updates the private zone, links it to the vnet, and creates DNS records.
func (s *Service) Reconcile(ctx context.Context) error {
ctx, log, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.Reconcile")
ctx, _, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.Reconcile")
defer done()

zoneSpec := s.Scope.PrivateDNSSpec()
if zoneSpec != nil {
// Skip the reconciliation of private DNS zone which is not managed by capz.
isManaged, err := s.isPrivateDNSManaged(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName)
if err != nil && !azure.ResourceNotFound(err) {
return errors.Wrapf(err, "could not get private DNS zone state of %s in resource group %s", zoneSpec.ZoneName, s.Scope.ResourceGroup())
}
// If resource is not found, it means it should be created and hence setting isVnetLinkManaged to true
// will allow the reconciliation to continue
if err != nil && azure.ResourceNotFound(err) {
isManaged = true
}
if !isManaged {
log.V(1).Info("Skipping reconciliation of unmanaged private DNS zone", "private DNS", zoneSpec.ZoneName)
log.V(1).Info("Tag the DNS manually from azure to manage it with capz."+
"Please see https://capz.sigs.k8s.io/topics/custom-dns.html#manage-dns-via-capz-tool", "private DNS", zoneSpec.ZoneName)
return nil
}
// Create the private DNS zone.
log.V(2).Info("creating private DNS zone", "private dns zone", zoneSpec.ZoneName)
pDNS := privatedns.PrivateZone{
Location: to.StringPtr(azure.Global),
Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{
ClusterName: s.Scope.ClusterName(),
Lifecycle: infrav1.ResourceLifecycleOwned,
Additional: s.Scope.AdditionalTags(),
})),
}
err = s.client.CreateOrUpdateZone(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName, pDNS)
if err != nil {
return errors.Wrapf(err, "failed to create private DNS zone %s", zoneSpec.ZoneName)
}
log.V(2).Info("successfully created private DNS zone", "private dns zone", zoneSpec.ZoneName)
for _, linkSpec := range zoneSpec.Links {
// If the virtual network link is not managed by capz, skip its reconciliation
isVnetLinkManaged, err := s.isVnetLinkManaged(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName, linkSpec.LinkName)
if err != nil && !azure.ResourceNotFound(err) {
return errors.Wrapf(err, "could not get vnet link state of %s in resource group %s", zoneSpec.ZoneName, s.Scope.ResourceGroup())
}
// If resource is not found, it means it should be created and hence setting isVnetLinkManaged to true
// will allow the reconciliation to continue
if err != nil && azure.ResourceNotFound(err) {
isVnetLinkManaged = true
}
if !isVnetLinkManaged {
log.V(2).Info("Skipping vnet link reconciliation for unmanaged vnet link", "vnet link", linkSpec.LinkName, "private dns zone", zoneSpec.ZoneName)
continue
}
// Link each virtual network.
log.V(2).Info("creating a virtual network link", "virtual network", linkSpec.VNetName, "private dns zone", zoneSpec.ZoneName)
link := privatedns.VirtualNetworkLink{
VirtualNetworkLinkProperties: &privatedns.VirtualNetworkLinkProperties{
VirtualNetwork: &privatedns.SubResource{
ID: to.StringPtr(azure.VNetID(s.Scope.SubscriptionID(), linkSpec.VNetResourceGroup, linkSpec.VNetName)),
},
RegistrationEnabled: to.BoolPtr(false),
},
Location: to.StringPtr(azure.Global),
Tags: converters.TagsToMap(infrav1.Build(infrav1.BuildParams{
ClusterName: s.Scope.ClusterName(),
Lifecycle: infrav1.ResourceLifecycleOwned,
Additional: s.Scope.AdditionalTags(),
})),
}
err = s.client.CreateOrUpdateLink(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName, linkSpec.LinkName, link)
if err != nil {
return errors.Wrapf(err, "failed to create virtual network link %s", linkSpec.LinkName)
ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout)
defer cancel()

zoneSpec, links, records := s.Scope.PrivateDNSSpec()
if zoneSpec == nil {
return nil
}

if err := s.reconcileZone(ctx, zoneSpec); err != nil {
s.Scope.UpdatePutStatus(infrav1.PrivateDNSReadyCondition, serviceName, err)
return err
}

if err := s.reconcileLinks(ctx, links); err != nil {
s.Scope.UpdatePutStatus(infrav1.PrivateDNSReadyCondition, serviceName, err)
return err
}

if err := s.reconcileRecords(ctx, records); err != nil {
s.Scope.UpdatePutStatus(infrav1.PrivateDNSReadyCondition, serviceName, err)
return err
}

s.Scope.UpdatePutStatus(infrav1.PrivateDNSReadyCondition, serviceName, nil)
return nil
}

func (s *Service) reconcileZone(ctx context.Context, zoneSpec azure.ResourceSpecGetter) error {
ctx, _, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.reconcileZone")
defer done()

_, err := s.zoneReconciler.CreateResource(ctx, zoneSpec, serviceName)
return err
}

func (s *Service) reconcileLinks(ctx context.Context, links []azure.ResourceSpecGetter) error {
ctx, _, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.reconcileLinks")
defer done()

var resErr error

// We go through the list of links to reconcile each one, independently of the result of the previous one.
// If multiple errors occur, we return the most pressing one.
// Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (ie. error creating) -> operationNotDoneError (ie. creating in progress) -> no error (ie. created)
for _, linkSpec := range links {
if _, err := s.vnetLinkReconciler.CreateResource(ctx, linkSpec, serviceName); err != nil {
if !azure.IsOperationNotDoneError(err) || resErr == nil {
resErr = err
}
log.V(2).Info("successfully created virtual network link", "virtual network", linkSpec.VNetName, "private dns zone", zoneSpec.ZoneName)
}
// Create the record(s).
for _, record := range zoneSpec.Records {
log.V(2).Info("creating record set", "private dns zone", zoneSpec.ZoneName, "record", record.Hostname)
set := privatedns.RecordSet{
RecordSetProperties: &privatedns.RecordSetProperties{
TTL: to.Int64Ptr(300),
},
}
recordType := converters.GetRecordType(record.IP)
if recordType == privatedns.A {
set.RecordSetProperties.ARecords = &[]privatedns.ARecord{{
Ipv4Address: &record.IP,
}}
} else if recordType == privatedns.AAAA {
set.RecordSetProperties.AaaaRecords = &[]privatedns.AaaaRecord{{
Ipv6Address: &record.IP,
}}
}
err := s.client.CreateOrUpdateRecordSet(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName, recordType, record.Hostname, set)
if err != nil {
return errors.Wrapf(err, "failed to create record %s in private DNS zone %s", record.Hostname, zoneSpec.ZoneName)
}

return resErr
}

func (s *Service) reconcileRecords(ctx context.Context, records []azure.ResourceSpecGetter) error {
ctx, _, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.reconcileRecords")
defer done()

var resErr error

// We go through the list of links to reconcile each one, independently of the result of the previous one.
// If multiple errors occur, we return the most pressing one.
// Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (ie. error creating) -> operationNotDoneError (ie. creating in progress) -> no error (ie. created)
for _, recordSpec := range records {
if _, err := s.recordReconciler.CreateResource(ctx, recordSpec, serviceName); err != nil {
if !azure.IsOperationNotDoneError(err) || resErr == nil {
resErr = err
}
log.V(2).Info("successfully created record set", "private dns zone", zoneSpec.ZoneName, "record", record.Hostname)
}
}
return nil

return resErr
}

// Delete deletes the private zone and vnet links.
func (s *Service) Delete(ctx context.Context) error {
ctx, log, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.Delete")
ctx, _, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.Delete")
defer done()

zoneSpec := s.Scope.PrivateDNSSpec()
if zoneSpec != nil {
for _, linkSpec := range zoneSpec.Links {
// If the virtual network link is not managed by capz, skip its removal
isVnetLinkManaged, err := s.isVnetLinkManaged(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName, linkSpec.LinkName)
if err != nil && !azure.ResourceNotFound(err) {
return errors.Wrapf(err, "could not get vnet link state of %s in resource group %s", zoneSpec.ZoneName, s.Scope.ResourceGroup())
}
if !isVnetLinkManaged {
log.V(2).Info("Skipping vnet link deletion for unmanaged vnet link", "vnet link", linkSpec.LinkName, "private dns zone", zoneSpec.ZoneName)
continue
}
log.V(2).Info("removing virtual network link", "virtual network", linkSpec.VNetName, "private dns zone", zoneSpec.ZoneName)
err = s.client.DeleteLink(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName, linkSpec.LinkName)
if err != nil && !azure.ResourceNotFound(err) {
return errors.Wrapf(err, "failed to delete virtual network link %s with zone %s in resource group %s", linkSpec.VNetName, zoneSpec.ZoneName, s.Scope.ResourceGroup())
}
}
// Skip the deletion of private DNS zone which is not managed by capz.
isManaged, err := s.isPrivateDNSManaged(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName)
ctx, cancel := context.WithTimeout(ctx, reconciler.DefaultAzureServiceReconcileTimeout)
defer cancel()

zoneSpec, links, _ := s.Scope.PrivateDNSSpec()
if zoneSpec == nil {
return nil
}

if err := s.deleteLinks(ctx, links); err != nil {
s.Scope.UpdateDeleteStatus(infrav1.PrivateDNSReadyCondition, serviceName, err)
return err
}

if err := s.deleteZone(ctx, zoneSpec); err != nil {
s.Scope.UpdateDeleteStatus(infrav1.PrivateDNSReadyCondition, serviceName, err)
return err
}

s.Scope.UpdateDeleteStatus(infrav1.PrivateDNSReadyCondition, serviceName, nil)

return nil
}

func (s *Service) deleteLinks(ctx context.Context, links []azure.ResourceSpecGetter) error {
ctx, log, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.deleteLinks")
defer done()

var resErr error

// We go through the list of links to delete each one, independently of the result of the previous one.
// If multiple errors occur, we return the most pressing one.
// Order of precedence (highest -> lowest) is: error that is not an operationNotDoneError (ie. error creating) -> operationNotDoneError (ie. creating in progress) -> no error (ie. created)
for _, linkSpec := range links {
// If the virtual network link is not managed by capz, skip its reconciliation
isVnetLinkManaged, err := s.isVnetLinkManaged(ctx, linkSpec)
if err != nil && !azure.ResourceNotFound(err) {
return errors.Wrapf(err, "could not get private DNS zone state of %s in resource group %s", zoneSpec.ZoneName, s.Scope.ResourceGroup())
return errors.Wrapf(err, "could not get vnet link state of %s in resource group %s",
linkSpec.OwnerResourceName(), linkSpec.ResourceGroupName())
}
if !isManaged {
log.V(1).Info("Skipping private DNS zone deletion for unmanaged private DNS zone", "private DNS", zoneSpec.ZoneName)
return nil
}
// Delete the private DNS zone, which also deletes all records.
log.V(2).Info("deleting private dns zone", "private dns zone", zoneSpec.ZoneName)
err = s.client.DeleteZone(ctx, s.Scope.ResourceGroup(), zoneSpec.ZoneName)
// If resource is not found, it means it should be created and hence setting isVnetLinkManaged to true
// will allow the reconciliation to continue
if err != nil && azure.ResourceNotFound(err) {
// already deleted
return nil
isVnetLinkManaged = true
}
if err != nil && !azure.ResourceNotFound(err) {
return errors.Wrapf(err, "failed to delete private dns zone %s in resource group %s", zoneSpec.ZoneName, s.Scope.ResourceGroup())
if !isVnetLinkManaged {
log.V(2).Info("Skipping vnet link deletion for unmanaged vnet link", "vnet link",
linkSpec.ResourceName(), "private dns zone", linkSpec.OwnerResourceName())
continue
}

if err := s.vnetLinkReconciler.DeleteResource(ctx, linkSpec, serviceName); err != nil {
if !azure.IsOperationNotDoneError(err) || resErr == nil {
resErr = err
}
}
log.V(2).Info("successfully deleted private dns zone", "private dns zone", zoneSpec.ZoneName)
}
return nil

return resErr
}

func (s *Service) deleteZone(ctx context.Context, zoneSpec azure.ResourceSpecGetter) error {
ctx, log, done := tele.StartSpanWithLogger(ctx, "privatedns.Service.deleteZone")
defer done()

// Skip the reconciliation of private DNS zone which is not managed by capz.
isManaged, err := s.isPrivateDNSManaged(ctx, zoneSpec)
if err != nil && !azure.ResourceNotFound(err) {
return errors.Wrapf(err, "could not get private DNS zone state of %s in resource group %s", zoneSpec.ResourceName(), zoneSpec.ResourceGroupName())
}

// If resource is not found, it means it should be created and hence setting isVnetLinkManaged to true
// will allow the reconciliation to continue
if err != nil && azure.ResourceNotFound(err) {
isManaged = true
}
if !isManaged {
log.V(1).Info("Skipping deletion of unmanaged private DNS zone", "private DNS", zoneSpec.ResourceName())
return nil
}

err = s.zoneReconciler.DeleteResource(ctx, zoneSpec, serviceName)
return err
}

// isPrivateDNSManaged returns true if the private DNS has an owned tag with the cluster name as value,
// meaning that the DNS lifecycle is managed.
func (s *Service) isPrivateDNSManaged(ctx context.Context, resourceGroup, zoneName string) (bool, error) {
zone, err := s.client.GetZone(ctx, resourceGroup, zoneName)
func (s *Service) isPrivateDNSManaged(ctx context.Context, spec azure.ResourceSpecGetter) (bool, error) {
result, err := s.zoneClient.Get(ctx, spec)
if err != nil {
return false, err
}
zone, ok := result.(privatedns.PrivateZone)
if !ok {
return false, errors.Errorf("%T is not a privatedns.PrivateZone", zone)
}

tags := converters.MapToTags(zone.Tags)
return tags.HasOwned(s.Scope.ClusterName()), nil
}

// isVnetLinkManaged returns true if the vnet link has an owned tag with the cluster name as value,
// meaning that the vnet link lifecycle is managed.
func (s *Service) isVnetLinkManaged(ctx context.Context, resourceGroupName, zoneName, vnetLinkName string) (bool, error) {
zone, err := s.client.GetLink(ctx, resourceGroupName, zoneName, vnetLinkName)
func (s *Service) isVnetLinkManaged(ctx context.Context, spec azure.ResourceSpecGetter) (bool, error) {
result, err := s.vnetLinkClient.Get(ctx, spec)
if err != nil {
return false, err
}
tags := converters.MapToTags(zone.Tags)

link, ok := result.(privatedns.VirtualNetworkLink)
if !ok {
return false, errors.Errorf("%T is not a privatedns.VirtualNetworkLink", link)
}

tags := converters.MapToTags(link.Tags)
return tags.HasOwned(s.Scope.ClusterName()), nil
}
Loading

0 comments on commit 77dbc7b

Please sign in to comment.