Compare backup and cluster objects before logging

When restoring resources that raise an already exists error, check their
equality before logging a message on the restore. If they're the same
except for some metadata, don't generate a message.

The restore process was modified so that if an object had an empty
namespace string, no namespace key is created on the object. This was to
avoid manipulating the copy of the current cluster's object by adding
the target namespace.

There are some cases right now that are known to not be equal via this
method:

- The `default` ServiceAccount in a namespace will not match, primarily
because of differing default tokens. These will be handled in their own
patch
- IP addresses for Services are recorded in the backup object, but are
either not present on the cluster object, or different. An issue for
this already exists at vmware-tanzu#354
- Endpoints have differing values for `renewTime`. This may be
insubstantial, but isn't currently handled by the resetMetadataAndStatus
function.
- PersistentVolume objects do not match on spec fields, such as
claimRef and cloud provider persistent disk info

Signed-off-by: Nolan Brubaker <[email protected]>

pkg/restore/restore.go

@@ -31,6 +31,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
@@ -634,20 +635,36 @@ func (ctx *context) restoreResource(resource, namespace, resourcePath string) (a
 		}
 
 		// necessary because we may have remapped the namespace
-		obj.SetNamespace(namespace)
+		// if the namespace is blank, don't create the key
+		if namespace != "" {
+			obj.SetNamespace(namespace)
+		}
 
 		// add an ark-restore label to each resource for easy ID
 		addLabel(obj, api.RestoreLabelKey, ctx.restore.Name)
 
 		ctx.infof("Restoring %s: %v", obj.GroupVersionKind().Kind, obj.GetName())
-		_, err = resourceClient.Create(obj)
-		if apierrors.IsAlreadyExists(err) {
-			addToResult(&warnings, namespace, err)
+		_, restoreErr := resourceClient.Create(obj)
+		if apierrors.IsAlreadyExists(restoreErr) {
+			fromCluster, err := resourceClient.Get(obj.GetName(), metav1.GetOptions{})
+			if err != nil {
+				addToResult(&warnings, namespace, errors.WithStack(err))
+			}
+			equal, err := objectsAreEqual(fromCluster, obj)
+			// Log any errors trying to check equality
+			if err != nil {
+				ctx.infof(err.Error())
+			}
+			if !equal {
+				e := fmt.Errorf("%s and backup does not match.", restoreErr)
+				addToResult(&warnings, namespace, e)
+			}
 			continue
 		}
-		if err != nil {
+		// Error was something other than an AlreadyExists
+		if restoreErr != nil {
 			ctx.infof("error restoring %s: %v", obj.GetName(), err)
-			addToResult(&errs, namespace, fmt.Errorf("error restoring %s: %v", fullPath, err))
+			addToResult(&errs, namespace, fmt.Errorf("error restoring %s: %v", fullPath, restoreErr))
 			continue
 		}
 
@@ -722,6 +739,23 @@ func (ctx *context) executePVAction(obj *unstructured.Unstructured) (*unstructur
 	return updated2, nil
 }
 
+// objectsAreEqual takes two unstructured objects and checks for equality.
+func objectsAreEqual(fromCluster, fromBackup *unstructured.Unstructured) (bool, error) {
+	// Remove insubstantial metadata
+	fromCluster, err := resetMetadataAndStatus(fromCluster)
+	if err != nil {
+		return false, err
+	}
+
+	// We know the cluster won't have the restore name label, so
+	// copy it over from the backup
+	restoreName := fromBackup.GetLabels()[api.RestoreLabelKey]
+	addLabel(fromCluster, api.RestoreLabelKey, restoreName)
+
+	// If there are no specific actions needed based on the type, simply check for equality.
+	return equality.Semantic.DeepEqual(fromBackup, fromCluster), nil
+}
+
 func isPVReady(obj runtime.Unstructured) bool {
 	phase, err := collections.GetString(obj.UnstructuredContent(), "status.phase")
 	if err != nil {

pkg/restore/restore_test.go

@@ -942,10 +942,6 @@ func toUnstructured(objs ...runtime.Object) []unstructured.Unstructured {
 
 		delete(metadata, "creationTimestamp")
 
-		if _, exists := metadata["namespace"]; !exists {
-			metadata["namespace"] = ""
-		}
-
 		delete(unstructuredObj.Object, "status")
 
 		res = append(res, unstructuredObj)