diff --git a/controllers/external/util.go b/controllers/external/util.go
index 299c8fe5bbb4..42b488fe427c 100644
--- a/controllers/external/util.go
+++ b/controllers/external/util.go
@@ -73,6 +73,10 @@ type CloneTemplateInput struct {
 	// Labels is an optional map of labels to be added to the object.
 	// +optional
 	Labels map[string]string
+
+	// Annotations is an optional map of annotations to be added to the object.
+	// +optional
+	Annotations map[string]string
 }
 
 // CloneTemplate uses the client and the reference to create a new object from the template.
@@ -88,6 +92,7 @@ func CloneTemplate(ctx context.Context, in *CloneTemplateInput) (*corev1.ObjectR
 		ClusterName: in.ClusterName,
 		OwnerRef:    in.OwnerRef,
 		Labels:      in.Labels,
+		Annotations: in.Annotations,
 	}
 	to, err := GenerateTemplate(generateTemplateInput)
 	if err != nil {
@@ -127,6 +132,10 @@ type GenerateTemplateInput struct {
 	// Labels is an optional map of labels to be added to the object.
 	// +optional
 	Labels map[string]string
+
+	// Annotations is an optional map of annotations to be added to the object.
+	// +optional
+	Annotations map[string]string
 }
 
 func GenerateTemplate(in *GenerateTemplateInput) (*unstructured.Unstructured, error) {
@@ -146,10 +155,14 @@ func GenerateTemplate(in *GenerateTemplateInput) (*unstructured.Unstructured, er
 	to.SetName(names.SimpleNameGenerator.GenerateName(in.Template.GetName() + "-"))
 	to.SetNamespace(in.Namespace)
 
-	if to.GetAnnotations() == nil {
-		to.SetAnnotations(map[string]string{})
-	}
+	// Set annotations.
 	annotations := to.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	for key, value := range in.Annotations {
+		annotations[key] = value
+	}
 	annotations[clusterv1.TemplateClonedFromNameAnnotation] = in.TemplateRef.Name
 	annotations[clusterv1.TemplateClonedFromGroupKindAnnotation] = in.TemplateRef.GroupVersionKind().GroupKind().String()
 	to.SetAnnotations(annotations)
diff --git a/controllers/external/util_test.go b/controllers/external/util_test.go
index 671bbe8a9c6b..ab1bca0f969b 100644
--- a/controllers/external/util_test.go
+++ b/controllers/external/util_test.go
@@ -130,7 +130,12 @@ func TestCloneTemplateResourceFound(t *testing.T) {
 				"template": map[string]interface{}{
 					"metadata": map[string]interface{}{
 						"annotations": map[string]interface{}{
-							"test": "annotations",
+							"test-template": "annotations",
+							"precedence":    "template",
+						},
+						"labels": map[string]interface{}{
+							"test-template": "label",
+							"precedence":    "template",
 						},
 					},
 					"spec": map[string]interface{}{
@@ -175,7 +180,12 @@ func TestCloneTemplateResourceFound(t *testing.T) {
 		ClusterName: testClusterName,
 		OwnerRef:    owner.DeepCopy(),
 		Labels: map[string]string{
-			"test-label-1": "value-1",
+			"precedence":               "input",
+			clusterv1.ClusterLabelName: "should-be-overwritten",
+		},
+		Annotations: map[string]string{
+			"precedence": "input",
+			clusterv1.TemplateClonedFromNameAnnotation: "should-be-overwritten",
 		},
 	})
 	g.Expect(err).NotTo(HaveOccurred())
@@ -201,10 +211,12 @@ func TestCloneTemplateResourceFound(t *testing.T) {
 
 	cloneLabels := clone.GetLabels()
 	g.Expect(cloneLabels).To(HaveKeyWithValue(clusterv1.ClusterLabelName, testClusterName))
-	g.Expect(cloneLabels).To(HaveKeyWithValue("test-label-1", "value-1"))
+	g.Expect(cloneLabels).To(HaveKeyWithValue("test-template", "label"))
+	g.Expect(cloneLabels).To(HaveKeyWithValue("precedence", "input"))
 
 	cloneAnnotations := clone.GetAnnotations()
-	g.Expect(cloneAnnotations).To(HaveKeyWithValue("test", "annotations"))
+	g.Expect(cloneAnnotations).To(HaveKeyWithValue("test-template", "annotations"))
+	g.Expect(cloneAnnotations).To(HaveKeyWithValue("precedence", "input"))
 
 	g.Expect(cloneAnnotations).To(HaveKeyWithValue(clusterv1.TemplateClonedFromNameAnnotation, templateRef.Name))
 	g.Expect(cloneAnnotations).To(HaveKeyWithValue(clusterv1.TemplateClonedFromGroupKindAnnotation, templateRef.GroupVersionKind().GroupKind().String()))
diff --git a/controllers/machineset_controller.go b/controllers/machineset_controller.go
index 073d42137c80..290281bb335a 100644
--- a/controllers/machineset_controller.go
+++ b/controllers/machineset_controller.go
@@ -361,6 +361,7 @@ func (r *MachineSetReconciler) syncReplicas(ctx context.Context, ms *clusterv1.M
 				Namespace:   machine.Namespace,
 				ClusterName: machine.Spec.ClusterName,
 				Labels:      machine.Labels,
+				Annotations: machine.Annotations,
 			})
 			if err != nil {
 				return errors.Wrapf(err, "failed to clone infrastructure configuration for MachineSet %q in namespace %q", ms.Name, ms.Namespace)
diff --git a/controllers/machineset_controller_test.go b/controllers/machineset_controller_test.go
index 44b58e907bf7..096526a19f16 100644
--- a/controllers/machineset_controller_test.go
+++ b/controllers/machineset_controller_test.go
@@ -83,6 +83,10 @@ var _ = Describe("MachineSet Reconciler", func() {
 						Labels: map[string]string{
 							"label-1": "true",
 						},
+						Annotations: map[string]string{
+							"annotation-1": "true",
+							"precedence":   "MachineSet",
+						},
 					},
 					Spec: clusterv1.MachineSpec{
 						ClusterName: testCluster.Name,
@@ -127,7 +131,11 @@ var _ = Describe("MachineSet Reconciler", func() {
 		infraResource := map[string]interface{}{
 			"kind":       "InfrastructureMachine",
 			"apiVersion": "infrastructure.cluster.x-k8s.io/v1alpha4",
-			"metadata":   map[string]interface{}{},
+			"metadata": map[string]interface{}{
+				"annotations": map[string]interface{}{
+					"precedence": "InfrastructureMachineTemplate",
+				},
+			},
 			"spec": map[string]interface{}{
 				"size": "3xlarge",
 			},
@@ -191,6 +199,22 @@ var _ = Describe("MachineSet Reconciler", func() {
 			return len(machines.Items)
 		}, timeout).Should(BeEquivalentTo(replicas))
 
+		By("Creating a InfrastructureMachine for each Machine")
+		infraMachines := &unstructured.UnstructuredList{}
+		infraMachines.SetAPIVersion("infrastructure.cluster.x-k8s.io/v1alpha4")
+		infraMachines.SetKind("InfrastructureMachine")
+		Eventually(func() int {
+			if err := testEnv.List(ctx, infraMachines, client.InNamespace(namespace.Name)); err != nil {
+				return -1
+			}
+			return len(machines.Items)
+		}, timeout).Should(BeEquivalentTo(replicas))
+		for _, im := range infraMachines.Items {
+			Expect(im.GetAnnotations()).To(HaveKeyWithValue("annotation-1", "true"), "have annotations of MachineTemplate applied")
+			Expect(im.GetAnnotations()).To(HaveKeyWithValue("precedence", "MachineSet"), "the annotations from the MachineSpec template to overwrite the infrastructure template ones")
+			Expect(im.GetLabels()).To(HaveKeyWithValue("label-1", "true"), "have labels of MachineTemplate applied")
+		}
+
 		// Set the infrastructure reference as ready.
 		for _, m := range machines.Items {
 			fakeBootstrapRefReady(*m.Spec.Bootstrap.ConfigRef, bootstrapResource)