From 731a48427567d897aa3a814bea13796f52789505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Wenkai=20Yin=28=E5=B0=B9=E6=96=87=E5=BC=80=29?= Date: Tue, 7 Feb 2023 07:48:23 +0800 Subject: [PATCH] Add secret restore item action to handle service account token secret MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add secret restore item action to handle service account token secret: 1. Skip the restoration for the auto-created service account token secret 2. Remove several fields for non-auto-created service account token secret to make sure the secret can be restored Signed-off-by: Wenkai Yin(尹文开) --- changelogs/unreleased/5843-ywk253100 | 1 + pkg/cmd/server/plugin/plugin.go | 13 ++- pkg/cmd/server/server.go | 4 +- pkg/restore/secret_action.go | 107 ++++++++++++++++++++ pkg/restore/secret_action_test.go | 142 +++++++++++++++++++++++++++ 5 files changed, 264 insertions(+), 3 deletions(-) create mode 100644 changelogs/unreleased/5843-ywk253100 create mode 100644 pkg/restore/secret_action.go create mode 100644 pkg/restore/secret_action_test.go diff --git a/changelogs/unreleased/5843-ywk253100 b/changelogs/unreleased/5843-ywk253100 new file mode 100644 index 0000000000..f1d970f24c --- /dev/null +++ b/changelogs/unreleased/5843-ywk253100 @@ -0,0 +1 @@ +Add secret restore item action to handle service account token secret \ No newline at end of file diff --git a/pkg/cmd/server/plugin/plugin.go b/pkg/cmd/server/plugin/plugin.go index 010db9c5fc..a42f88730b 100644 --- a/pkg/cmd/server/plugin/plugin.go +++ b/pkg/cmd/server/plugin/plugin.go @@ -58,7 +58,8 @@ func NewCommand(f client.Factory) *cobra.Command { RegisterRestoreItemAction("velero.io/crd-preserve-fields", newCRDV1PreserveUnknownFieldsItemAction). RegisterRestoreItemAction("velero.io/change-pvc-node-selector", newChangePVCNodeSelectorItemAction(f)). RegisterRestoreItemAction("velero.io/apiservice", newAPIServiceRestoreItemAction). - RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction) + RegisterRestoreItemAction("velero.io/admission-webhook-configuration", newAdmissionWebhookConfigurationAction). + RegisterRestoreItemAction("velero.io/secret", newSecretRestoreItemAction(f)) if !features.IsEnabled(velerov1api.APIGroupVersionsFeatureFlag) { // Do not register crd-remap-version BIA if the API Group feature flag is enabled, so that the v1 CRD can be backed up pluginServer = pluginServer.RegisterBackupItemAction("velero.io/crd-remap-version", newRemapCRDVersionAction(f)) @@ -234,3 +235,13 @@ func newAPIServiceRestoreItemAction(logger logrus.FieldLogger) (interface{}, err func newAdmissionWebhookConfigurationAction(logger logrus.FieldLogger) (interface{}, error) { return restore.NewAdmissionWebhookConfigurationAction(logger), nil } + +func newSecretRestoreItemAction(f client.Factory) plugincommon.HandlerInitializer { + return func(logger logrus.FieldLogger) (interface{}, error) { + client, err := f.KubebuilderClient() + if err != nil { + return nil, err + } + return restore.NewSecretAction(logger, client), nil + } +} diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go index ea998ba106..200121dca6 100644 --- a/pkg/cmd/server/server.go +++ b/pkg/cmd/server/server.go @@ -499,9 +499,9 @@ func (s *server) veleroResourcesExist() error { // - VolumeSnapshots are needed to create PVCs using the VolumeSnapshot as their data source. // - PVs go before PVCs because PVCs depend on them. // - PVCs go before pods or controllers so they can be mounted as volumes. +// - Service accounts go before secrets so service account token secrets can be filled automatically. // - Secrets and config maps go before pods or controllers so they can be mounted // as volumes. -// - Service accounts go before pods or controllers so pods can use them. // - Limit ranges go before pods or controllers so pods can use them. // - Pods go before controllers so they can be explicitly restored and potentially // have pod volume restores run before controllers adopt the pods. @@ -525,9 +525,9 @@ var defaultRestorePriorities = restore.Priorities{ "volumesnapshots.snapshot.storage.k8s.io", "persistentvolumes", "persistentvolumeclaims", + "serviceaccounts", "secrets", "configmaps", - "serviceaccounts", "limitranges", "pods", // we fully qualify replicasets.apps because prior to Kubernetes 1.16, replicasets also diff --git a/pkg/restore/secret_action.go b/pkg/restore/secret_action.go new file mode 100644 index 0000000000..1e63372b65 --- /dev/null +++ b/pkg/restore/secret_action.go @@ -0,0 +1,107 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "context" + "fmt" + "strings" + + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/util/kube" +) + +// SecretAction is a restore item action for secrets +type SecretAction struct { + logger logrus.FieldLogger + client client.Client +} + +// NewSecretAction creates a new SecretAction instance +func NewSecretAction(logger logrus.FieldLogger, client client.Client) *SecretAction { + return &SecretAction{ + logger: logger, + client: client, + } +} + +// AppliesTo indicates which resources this action applies +func (s *SecretAction) AppliesTo() (velero.ResourceSelector, error) { + return velero.ResourceSelector{ + IncludedResources: []string{"secrets"}, + }, nil +} + +// Execute the action +func (s *SecretAction) Execute(input *velero.RestoreItemActionExecuteInput) (*velero.RestoreItemActionExecuteOutput, error) { + s.logger.Info("Executing SecretAction") + defer s.logger.Info("Done executing SecretAction") + + var secret corev1.Secret + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(input.Item.UnstructuredContent(), &secret); err != nil { + return nil, errors.Wrap(err, "unable to convert secret from runtime.Unstructured") + } + + log := s.logger.WithField("secret", kube.NamespaceAndName(&secret)) + if secret.Type != corev1.SecretTypeServiceAccountToken { + log.Debug("No match found - including this secret") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + }, nil + } + + // The auto created service account token secret will be created by kube controller automatically again(before Kubernetes v1.22), no need to restore. + // This will cause the patch operation of managedFields failed if we restore it as the secret is removed immediately + // after restoration and the patch operation reports not found error. + list := &corev1.ServiceAccountList{} + if err := s.client.List(context.Background(), list, &client.ListOptions{Namespace: secret.Namespace}); err != nil { + return nil, errors.Wrap(err, "unable to list the service accounts") + } + for _, sa := range list.Items { + if strings.HasPrefix(secret.Name, fmt.Sprintf("%s-token-", sa.Name)) { + log.Debug("auto created service account token secret found - excluding this secret") + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: input.Item, + SkipRestore: true, + }, nil + } + } + + log.Debug("service account token secret(not auto created) found - remove some fields from this secret") + // If the annotation and data are not removed, the secret cannot be restored successfully. + // The kube controller will fill the annotation and data with new value automatically: + // https://kubernetes.io/docs/concepts/configuration/secret/#service-account-token-secrets + delete(secret.Annotations, "kubernetes.io/service-account.uid") + delete(secret.Data, "token") + delete(secret.Data, "ca.crt") + + res, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&secret) + if err != nil { + return nil, errors.Wrap(err, "unable to convert secret to runtime.Unstructured") + } + + return &velero.RestoreItemActionExecuteOutput{ + UpdatedItem: &unstructured.Unstructured{Object: res}, + }, nil +} diff --git a/pkg/restore/secret_action_test.go b/pkg/restore/secret_action_test.go new file mode 100644 index 0000000000..a627e5296f --- /dev/null +++ b/pkg/restore/secret_action_test.go @@ -0,0 +1,142 @@ +/* +Copyright The Velero Contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package restore + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/vmware-tanzu/velero/pkg/plugin/velero" + "github.com/vmware-tanzu/velero/pkg/test" +) + +func TestSecretActionAppliesTo(t *testing.T) { + action := NewSecretAction(test.NewLogger(), nil) + actual, err := action.AppliesTo() + require.NoError(t, err) + assert.Equal(t, velero.ResourceSelector{IncludedResources: []string{"secrets"}}, actual) +} + +func TestSecretActionExecute(t *testing.T) { + tests := []struct { + name string + input *corev1.Secret + serviceAccount *corev1.ServiceAccount + skipped bool + output *corev1.Secret + }{ + { + name: "not service account token secret", + input: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + Name: "default-token-sfafa", + }, + Type: corev1.SecretTypeOpaque, + }, + skipped: false, + output: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + Name: "default-token-sfafa", + }, + Type: corev1.SecretTypeOpaque, + }, + }, + { + name: "auto created service account token", + input: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + Name: "default-token-sfafa", + }, + Type: corev1.SecretTypeServiceAccountToken, + }, + serviceAccount: &corev1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + Name: "default", + }, + }, + skipped: true, + }, + { + name: "not auto created service account token", + input: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + Name: "my-token", + Annotations: map[string]string{ + "kubernetes.io/service-account.uid": "uid", + "key": "value", + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{ + "token": []byte("token"), + "ca.crt": []byte("ca"), + "key": []byte("value"), + }, + }, + skipped: false, + output: &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "foo", + Name: "my-token", + Annotations: map[string]string{ + "key": "value", + }, + }, + Type: corev1.SecretTypeServiceAccountToken, + Data: map[string][]byte{ + "key": []byte("value"), + }, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + secretUnstructured, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.input) + require.NoError(t, err) + var serviceAccounts []client.Object + if tc.serviceAccount != nil { + serviceAccounts = append(serviceAccounts, tc.serviceAccount) + } + client := fake.NewClientBuilder().WithObjects(serviceAccounts...).Build() + action := NewSecretAction(test.NewLogger(), client) + res, err := action.Execute(&velero.RestoreItemActionExecuteInput{ + Item: &unstructured.Unstructured{Object: secretUnstructured}, + }) + require.NoError(t, err) + assert.Equal(t, tc.skipped, res.SkipRestore) + if !tc.skipped { + r, err := runtime.DefaultUnstructuredConverter.ToUnstructured(tc.output) + require.NoError(t, err) + assert.EqualValues(t, &unstructured.Unstructured{Object: r}, res.UpdatedItem) + } + }) + } +}