Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ensure record for DNSEntries are left untouched during migration #408

Merged
merged 3 commits into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.23.0

require (
github.com/ahmetb/gen-crd-api-reference-docs v0.3.1-0.20241014194617-ffc4efda75d4
github.com/gardener/external-dns-management v0.22.1
github.com/gardener/external-dns-management v0.22.2
github.com/gardener/gardener v1.110.0
github.com/go-logr/logr v1.4.2
github.com/hashicorp/go-multierror v1.1.1
Expand Down Expand Up @@ -43,6 +43,7 @@ require (
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/gardener/cert-management v0.17.1 // indirect
github.com/gardener/controller-manager-library v0.2.1-0.20241206090116-9fadce45689c // indirect
github.com/gardener/etcd-druid v0.25.0 // indirect
github.com/gardener/machine-controller-manager v0.55.1 // indirect
github.com/go-logr/zapr v1.3.0 // indirect
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -95,10 +95,12 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/gardener/cert-management v0.17.1 h1:vawZGN+rsCRMviacnnMSWELbuIJsNXHaqaLbZ4hYADw=
github.com/gardener/cert-management v0.17.1/go.mod h1:cwSsyN935017HojKVuWqw2TBhiaxSisX132D9Tn+n9I=
github.com/gardener/controller-manager-library v0.2.1-0.20241206090116-9fadce45689c h1:im/eYurY6+CzlRtxYzes6IfqMPIuzQ7Cy3sA8D/t528=
github.com/gardener/controller-manager-library v0.2.1-0.20241206090116-9fadce45689c/go.mod h1:fyLOrcaKtGno4McZKW21b6QtwNghCF0IemTLKcwKZlM=
github.com/gardener/etcd-druid v0.25.0 h1:mR9/x5r27pO+I+XzpNcN2DDenam+7ITrhc7qKt9rbsI=
github.com/gardener/etcd-druid v0.25.0/go.mod h1:6C0eyfdlw6CowLm/l4ZiKwrvkc+5NHrnc/rY2wCUwys=
github.com/gardener/external-dns-management v0.22.1 h1:WEwCDOersJ7ezeDJelbGVac1BTmEveJuds3JlJc84Xg=
github.com/gardener/external-dns-management v0.22.1/go.mod h1:2P7PamBPMKIOZMYRhl/VFhxZEBn4VUTdjESjKPxvOXA=
github.com/gardener/external-dns-management v0.22.2 h1:caSPJBLFHv9Y95IAwk1HvarIUCjDccLcyuyjW1qqwhM=
github.com/gardener/external-dns-management v0.22.2/go.mod h1:adBY3qQ39Fvc2PvihP4xzEE5Y2//GuurMXQpKylMOJ4=
github.com/gardener/gardener v1.110.0 h1:Ix/NeYJyYIIDRHqO0126JYPGNVKy2kDEco7RyXuCYwo=
github.com/gardener/gardener v1.110.0/go.mod h1:Ge2wQMWm0NmQZP3L/WMejpfXsnGbfTFBEZud819P3vU=
github.com/gardener/machine-controller-manager v0.55.1 h1:d6mTnuYko+jWeIi7tAFWgWnL1nR5hGcI6pRCDcH0TGY=
Expand Down
14 changes: 11 additions & 3 deletions pkg/controller/common/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,10 @@ import (
)

const (
ANNOTATION_OPERATION = constants.GardenerOperation
ANNOTATION_OPERATION_MIGRATE = constants.GardenerOperationMigrate
ANNOTATION_OPERATION_RESTORE = constants.GardenerOperationRestore
ANNOTATION_OPERATION = constants.GardenerOperation
ANNOTATION_OPERATION_MIGRATE = constants.GardenerOperationMigrate
ANNOTATION_OPERATION_RESTORE = constants.GardenerOperationRestore
ANNOTATION_OPERATION_RESTORE_STEP1 = "restore/step1"
marc1404 marked this conversation as resolved.
Show resolved Hide resolved
)

func CopyMap(m map[string]string) map[string]string {
Expand Down Expand Up @@ -67,6 +68,13 @@ func IsRestoring(ex *extensionsv1alpha1.Extension) bool {
return ex.Annotations[ANNOTATION_OPERATION] == ANNOTATION_OPERATION_RESTORE
}

func IsRestoringStep1(ex *extensionsv1alpha1.Extension) bool {
if ex.Annotations == nil {
return false
}
return ex.Annotations[ANNOTATION_OPERATION] == ANNOTATION_OPERATION_RESTORE_STEP1
}

// ShortenID shortens an identifier longer than maxlen characters by cutting the string
// and adding a hash suffix so that total length is maxlen. Identifiers are preserved
// if length < maxlen.
Expand Down
82 changes: 81 additions & 1 deletion pkg/controller/lifecycle/actuator.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"time"

dnsv1alpha1 "github.com/gardener/external-dns-management/pkg/apis/dns/v1alpha1"
"github.com/gardener/external-dns-management/pkg/dns"
"github.com/gardener/gardener/extensions/pkg/controller"
"github.com/gardener/gardener/extensions/pkg/controller/extension"
"github.com/gardener/gardener/extensions/pkg/util"
Expand Down Expand Up @@ -258,6 +259,21 @@ func (a *actuator) delete(ctx context.Context, log logr.Logger, ex *extensionsv1

// Restore the Extension resource.
func (a *actuator) Restore(ctx context.Context, log logr.Logger, ex *extensionsv1alpha1.Extension) error {
// First run extension reconciliation with deactivated DNSOwner to avoid
// zone reconciliation before all entries are reconciled.
// Premature zone reconciliation can lead to DNS entries being deleted temporarily.
exCopy := ex.DeepCopy()
if exCopy.Annotations == nil {
exCopy.Annotations = map[string]string{}
}
exCopy.Annotations[common.ANNOTATION_OPERATION] = common.ANNOTATION_OPERATION_RESTORE_STEP1
if err := a.Reconcile(ctx, log, exCopy); err != nil {
return err
}

if err := a.waitForEntryReconciliation(ctx, log, ex); err != nil {
return err
}
return a.Reconcile(ctx, log, ex)
}

Expand All @@ -268,9 +284,73 @@ func (a *actuator) Migrate(ctx context.Context, log logr.Logger, ex *extensionsv
return err
}

if err := a.ignoreDNSEntriesForMigration(ctx, ex); err != nil {
return err
}
marc1404 marked this conversation as resolved.
Show resolved Hide resolved

return a.delete(ctx, log, ex, true)
}

func (a *actuator) ignoreDNSEntriesForMigration(ctx context.Context, ex *extensionsv1alpha1.Extension) error {
entriesHelper := common.NewShootDNSEntriesHelper(ctx, a.Client(), ex)
list, err := entriesHelper.List()
if err != nil {
return err
}
for _, entry := range list {
patch := client.MergeFrom(entry.DeepCopy())
if entry.Annotations == nil {
entry.Annotations = map[string]string{}
}
entry.Annotations[dns.AnnotationHardIgnore] = "true"
if err := client.IgnoreNotFound(a.Client().Patch(ctx, &entry, patch)); err != nil {
return fmt.Errorf("failed to ignore DNS entry %q: %w", entry.Name, err)
}
}
return nil
}

func (a *actuator) waitForEntryReconciliation(ctx context.Context, log logr.Logger, ex *extensionsv1alpha1.Extension) error {
entriesHelper := common.NewShootDNSEntriesHelper(ctx, a.Client(), ex)
list, err := entriesHelper.List()
if err != nil {
return err
}

// annotate all entries with gardener.cloud/operation=reconcile
for _, entry := range list {
patch := client.MergeFrom(entry.DeepCopy())
if entry.Annotations == nil {
entry.Annotations = map[string]string{}
}
entry.Annotations[v1beta1constants.GardenerOperation] = v1beta1constants.GardenerOperationReconcile
delete(entry.Annotations, dns.AnnotationHardIgnore) // should not be needed as the DNSEntries have been recreated, but just to be sure
if err := client.IgnoreNotFound(a.Client().Patch(ctx, &entry, patch)); err != nil {
return fmt.Errorf("failed to revert ignore DNS entry %q: %w", entry.Name, err)
}
}

// wait for all entries to be reconciled, i.e. gardener.cloud/operation annotation is removed
start := time.Now()
for _, entry := range list {
for {
if err := a.Client().Get(ctx, client.ObjectKeyFromObject(&entry), &entry); err != nil {
return err
}
if _, ok := entry.Annotations[v1beta1constants.GardenerOperation]; !ok {
log.Info("DNS entry reconciled", "entry", entry.Name)
break
}
if time.Since(start) > 3*time.Minute {
return fmt.Errorf("timeout waiting for DNS entry %q to be reconciled", entry.Name)
}
time.Sleep(1 * time.Second)
}
}

return nil
}

func (a *actuator) isManagingDNSProviders(dns *gardencorev1beta1.DNS) bool {
return a.Config().ManageDNSProviders && dns != nil && dns.Domain != nil
}
Expand Down Expand Up @@ -312,7 +392,7 @@ func (a *actuator) createOrUpdateSeedResources(ctx context.Context, dnsconfig *a
if !deploymentEnabled || a.isHibernated(cluster) {
replicas = 0
}
shootActive := !common.IsMigrating(ex)
shootActive := !common.IsMigrating(ex) && !common.IsRestoringStep1(ex)

chartValues := map[string]interface{}{
"serviceName": service.ServiceName,
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/replication/reconciler.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ func (r *reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reco
if err != nil {
return result, err
}
if common.IsMigrating(ext) {
if ext == nil || common.IsMigrating(ext) {
return result, nil
}
statehandler, err := common.NewStateHandler(ctx, r.Env, ext, false)
Expand Down
Loading