Fix upgrade deadlock in Pipeline and Triggers

Add a mechanism to disable webhook validation if the configMaps
are not already there in the cluster.

The dead lock is caused due to a cluster level cyclic dependency in
tektoncd/pipelines and in tektoncd/triggers. More details will be captured in
issues in the both repository.

Note: infact this deadlock can occur in any tektoncd project which follows tektoncd/pipeline
file list structure in `config/`.

Signed-off-by: Nikhil Thomas <[email protected]>

pkg/reconciler/common/deadlockbreaker.go

@@ -0,0 +1,73 @@
+package common
+
+import (
+	"context"
+
+	"github.com/manifestival/manifestival"
+	"github.com/tektoncd/operator/pkg/apis/operator/v1alpha1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+var webhookNames = map[string]string{
+	v1alpha1.PipelineResourceName: "config.webhook.pipeline.tekton.dev",
+	v1alpha1.TriggerResourceName: "config.webhook.triggers.tekton.dev",
+}
+
+
+func PreemptDeadlock(ctx context.Context, m *manifestival.Manifest, component string) error {
+	ok, err := configMapsExistOnCluster(m)
+	if err != nil {
+		return err
+	}
+	if ok {
+		return nil
+	}
+	webhookName := webhookNames[component]
+	err = removeValidatingWebhookRules(m, webhookName)
+	if err != nil {
+		return err
+	}
+
+	return deleteDeployments(m)
+}
+
+func configMapsExistOnCluster(m *manifestival.Manifest) (bool, error) {
+	client := m.Client
+	configMaps := m.Filter(manifestival.ByKind("ConfigMap"))
+	for _, cm := range configMaps.Resources() {
+		ok, err := configMapExistOnCluster(&cm, client)
+		if err != nil {
+			return false, err
+		}
+		if !ok {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func configMapExistOnCluster(cm *unstructured.Unstructured, client manifestival.Client) (bool, error) {
+	cm, err := client.Get(cm)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
+func removeValidatingWebhookRules(m *manifestival.Manifest, webhookName string) error {
+	cmValidationWebHook := m.Filter(manifestival.ByName(webhookName))
+	err := unstructured.SetNestedField(cmValidationWebHook.Resources()[0].Object, nil, "webhooks", "rules")
+	if err != nil {
+		return err
+	}
+	return cmValidationWebHook.Apply()
+}
+
+func deleteDeployments(m *manifestival.Manifest) error {
+	deployments := m.Filter(manifestival.ByKind("Deployemnt"))
+	return deployments.Delete()
+}

pkg/reconciler/kubernetes/tektonpipeline/tektonpipeline.go

@@ -250,8 +250,9 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, tp *v1alpha1.TektonPipel
 		return nil
 	} else if ready.Status == corev1.ConditionFalse {
 		tp.Status.MarkInstallerSetNotReady(ready.Message)
+		err = common.PreemptDeadlock(ctx, &r.manifest, v1alpha1.PipelineResourceName)
 		r.enqueueAfter(tp, 10*time.Second)
-		return nil
+		return err
 	}
 
 	// Mark InstallerSet Ready

pkg/reconciler/kubernetes/tektontrigger/tektontrigger.go

@@ -264,8 +264,9 @@ func (r *Reconciler) ReconcileKind(ctx context.Context, tt *v1alpha1.TektonTrigg
 		return nil
 	} else if ready.Status == corev1.ConditionFalse {
 		tt.Status.MarkInstallerSetNotReady(ready.Message)
+		err = common.PreemptDeadlock(ctx, &r.manifest, v1alpha1.PipelineResourceName)
 		r.enqueueAfter(tt, 10*time.Second)
-		return nil
+		return err
 	}
 
 	// Mark InstallerSet Ready