Run CI YAML Example tests using go test

Update to accurately report specific example failures by name. E2E YAML
tests now use same harness and output format as other tests. Fixes tektoncd#1251

Add a new `yaml` build tag so that existing integration tests aren't
affected. Future updates to SubEnv may allow e2e tests to be run
entirely locally by avoiding external deps, see tektoncd#1372. NOTE: If an
example creates more than one clustertask, they will not all be cleaned
up.

test/README.md

@@ -206,9 +206,12 @@ go test -v -tags=e2e -count=1 ./test -run ^TestTaskRun
 To run the YAML e2e tests, run the following command:
 
 ```bash
-./test/e2e-tests-yaml.sh
+go test -v -count=1 -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/
 ```
 
+To limit parallelism of tests, use `-parallel=n` where `n` is the number of
+tests to run in parallel.
+
 ### Running upgrade tests
 
 There are two scenarios in upgrade tests. One is to install the previous release, upgrade to the current release, and

test/clients.go

@@ -46,21 +46,74 @@ import (
 	"github.com/tektoncd/pipeline/pkg/client/clientset/versioned/typed/pipeline/v1beta1"
 	resourceversioned "github.com/tektoncd/pipeline/pkg/client/resource/clientset/versioned"
 	resourcev1alpha1 "github.com/tektoncd/pipeline/pkg/client/resource/clientset/versioned/typed/resource/v1alpha1"
+	rbacv1 "k8s.io/client-go/kubernetes/typed/rbac/v1"
+	"k8s.io/client-go/tools/clientcmd"
 	knativetest "knative.dev/pkg/test"
 )
 
 // clients holds instances of interfaces for making requests to the Pipeline controllers.
 type clients struct {
 	KubeClient *knativetest.KubeClient
 
+	RbacV1Client *rbacv1.RbacV1Client
+
 	PipelineClient         v1beta1.PipelineInterface
+	ClusterTaskClient      v1beta1.ClusterTaskInterface
 	TaskClient             v1beta1.TaskInterface
 	TaskRunClient          v1beta1.TaskRunInterface
 	PipelineRunClient      v1beta1.PipelineRunInterface
 	PipelineResourceClient resourcev1alpha1.PipelineResourceInterface
 	ConditionClient        v1alpha1.ConditionInterface
 }
 
+// newClients instantiates and returns several clientsets required for making requests to the
+// Pipeline cluster specified by the combination of clusterName and configPath. Clients can
+// make requests within namespace.
+func newClientsWithOverrides(t *testing.T, configPath, clusterName, namespace string, overrides *clientcmd.ConfigOverrides) *clients {
+	t.Helper()
+	var err error
+	c := &clients{}
+
+	c.KubeClient, err = knativetest.NewKubeClient(configPath, clusterName)
+	if err != nil {
+		t.Fatalf("failed to create kubeclient from config file at %s: %s", configPath, err)
+	}
+
+	// From BuildClientConfig in knative.dev/pkg/test/clients.go
+	// Override the cluster name if provided.
+	if clusterName != "" {
+		overrides.Context.Cluster = clusterName
+	}
+	cfg, err := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(
+		&clientcmd.ClientConfigLoadingRules{ExplicitPath: configPath},
+		overrides).ClientConfig()
+	if err != nil {
+		t.Fatalf("failed to create configuration obj from %s for cluster %s: %s", configPath, clusterName, err)
+	}
+
+	c.RbacV1Client, err = rbacv1.NewForConfig(cfg)
+	if err != nil {
+		t.Fatalf("failed to create rbac client from config file at %s: %s", configPath, err)
+	}
+
+	cs, err := versioned.NewForConfig(cfg)
+	if err != nil {
+		t.Fatalf("failed to create pipeline clientset from config file at %s: %s", configPath, err)
+	}
+	rcs, err := resourceversioned.NewForConfig(cfg)
+	if err != nil {
+		t.Fatalf("failed to create pipeline clientset from config file at %s: %s", configPath, err)
+	}
+	c.PipelineClient = cs.TektonV1beta1().Pipelines(namespace)
+	c.ClusterTaskClient = cs.TektonV1beta1().ClusterTasks()
+	c.TaskClient = cs.TektonV1beta1().Tasks(namespace)
+	c.TaskRunClient = cs.TektonV1beta1().TaskRuns(namespace)
+	c.PipelineRunClient = cs.TektonV1beta1().PipelineRuns(namespace)
+	c.PipelineResourceClient = rcs.TektonV1alpha1().PipelineResources(namespace)
+	c.ConditionClient = cs.TektonV1alpha1().Conditions(namespace)
+	return c
+}
+
 // newClients instantiates and returns several clientsets required for making requests to the
 // Pipeline cluster specified by the combination of clusterName and configPath. Clients can
 // make requests within namespace.
@@ -79,6 +132,11 @@ func newClients(t *testing.T, configPath, clusterName, namespace string) *client
 		t.Fatalf("failed to create configuration obj from %s for cluster %s: %s", configPath, clusterName, err)
 	}
 
+	c.RbacV1Client, err = rbacv1.NewForConfig(cfg)
+	if err != nil {
+		t.Fatalf("failed to create rbac client from config file at %s: %s", configPath, err)
+	}
+
 	cs, err := versioned.NewForConfig(cfg)
 	if err != nil {
 		t.Fatalf("failed to create pipeline clientset from config file at %s: %s", configPath, err)
@@ -88,6 +146,7 @@ func newClients(t *testing.T, configPath, clusterName, namespace string) *client
 		t.Fatalf("failed to create pipeline clientset from config file at %s: %s", configPath, err)
 	}
 	c.PipelineClient = cs.TektonV1beta1().Pipelines(namespace)
+	c.ClusterTaskClient = cs.TektonV1beta1().ClusterTasks()
 	c.TaskClient = cs.TektonV1beta1().Tasks(namespace)
 	c.TaskRunClient = cs.TektonV1beta1().TaskRuns(namespace)
 	c.PipelineRunClient = cs.TektonV1beta1().PipelineRuns(namespace)

test/e2e-common.sh

@@ -18,115 +18,6 @@
 
 source $(git rev-parse --show-toplevel)/vendor/github.com/tektoncd/plumbing/scripts/e2e-tests.sh
 
-function teardown() {
-    subheader "Tearing down Tekton Pipelines"
-    ko delete --ignore-not-found=true -f config/
-    # teardown will be called when run against an existing cluster to cleanup before
-    # continuing, so we must wait for the cleanup to complete or the subsequent attempt
-    # to deploy to the same namespace will fail
-    wait_until_object_does_not_exist namespace tekton-pipelines
-}
-
-function output_yaml_test_results() {
-  # If formatting fails for any reason, use yaml as a fall back.
-  kubectl get $1.tekton.dev -o=custom-columns-file=${REPO_ROOT_DIR}/test/columns.txt || \
-    kubectl get $1.tekton.dev -oyaml
-}
-
-function output_pods_logs() {
-    echo ">>> $1"
-    kubectl get $1.tekton.dev -o yaml
-    local runs=$(kubectl get $1.tekton.dev --output=jsonpath="{.items[*].metadata.name}")
-    set +e
-    for run in ${runs}; do
-	echo ">>>> $1 ${run}"
-	case "$1" in
-	    "taskrun")
-		tkn taskrun logs --nocolour ${run}
-		;;
-	    "pipelinerun")
-		tkn pipelinerun logs --nocolour ${run}
-		;;
-	esac
-    done
-    set -e
-    echo ">>>> Pods"
-    kubectl get pods -o yaml
-}
-
-# Called by `fail_test` (provided by `e2e-tests.sh`) to dump info on test failure
-function dump_extra_cluster_state() {
-  echo ">>> Pipeline controller log:"
-  kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-controller tekton-pipelines)
-  echo ">>> Pipeline webhook log:"
-  kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-webhook tekton-pipelines)
-}
-
-function validate_run() {
-  local tests_finished=0
-  for i in {1..90}; do
-    local finished="$(kubectl get $1.tekton.dev --output=jsonpath='{.items[*].status.conditions[*].status}')"
-    if [[ ! "$finished" == *"Unknown"* ]]; then
-      tests_finished=1
-      break
-    fi
-    sleep 10
-  done
-
-  return ${tests_finished}
-}
-
-function check_results() {
-  local failed=0
-  results="$(kubectl get $1.tekton.dev --output=jsonpath='{range .items[*]}{.metadata.name}={.status.conditions[*].type}{.status.conditions[*].status}{" "}{end}')"
-  for result in ${results}; do
-    if [[ ! "${result,,}" == *"=succeededtrue" ]]; then
-      echo "ERROR: test ${result} but should be succeededtrue"
-      failed=1
-    fi
-  done
-
-  return ${failed}
-}
-
-function create_resources() {
-  local resource=$1
-  echo ">> Creating resources ${resource}"
-
-  # Applying the resources, either *taskruns or * *pipelineruns except those
-  # in the no-ci directory
-  for file in $(find ${REPO_ROOT_DIR}/examples/${resource}s/ -name '*.yaml' -not -path '*/no-ci/*' | sort); do
-    perl -p -e 's/gcr.io\/christiewilson-catfactory/$ENV{KO_DOCKER_REPO}/g' ${file} | ko create -f - || return 1
-  done
-}
-
-function run_tests() {
-  local resource=$1
-
-  # Wait for tests to finish.
-  echo ">> Waiting for tests to finish for ${resource}"
-  if validate_run $resource; then
-    echo "ERROR: tests timed out"
-  fi
-
-  # Check that tests passed.
-  echo ">> Checking test results for ${resource}"
-  if check_results $resource; then
-    echo ">> All YAML tests passed"
-    return 0
-  fi
-  return 1
-}
-
-function run_yaml_tests() {
-  echo ">> Starting tests for the resource ${1}/${2}"
-  create_resources ${1}/${2} || fail_test "Could not create ${2}/${1} from the examples"
-  if ! run_tests ${2}; then
-    return 1
-  fi
-  return 0
-}
-
 function install_pipeline_crd() {
   echo ">> Deploying Tekton Pipelines"
   ko resolve -f config/ \

test/e2e-tests-upgrade.sh

@@ -51,15 +51,7 @@ failed=0
 go_test_e2e -timeout=20m ./test || failed=1
 
 # Run the post-integration tests.
-for test in taskrun pipelinerun; do
-  header "Running YAML e2e tests for ${test}s"
-  if ! run_yaml_tests ${test}; then
-    echo "ERROR: one or more YAML tests failed"
-    output_yaml_test_results ${test}
-    output_pods_logs ${test}
-    failed=1
-  fi
-done
+go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1
 
 # Remove all the pipeline CRDs, and clean up the environment for next Scenario.
 uninstall_pipeline_crd
@@ -87,15 +79,7 @@ go_test_e2e -timeout=20m ./test || failed=1
 # Run the post-integration tests. We do not need to install the resources again, since
 # they are installed before the upgrade. We verify if they still work, after going through
 # the upgrade.
-for test in taskrun pipelinerun; do
-  header "Running YAML e2e tests for ${test}s"
-  if ! run_tests ${test}; then
-    echo "ERROR: one or more YAML tests failed"
-    output_yaml_test_results ${test}
-    output_pods_logs ${test}
-    failed=1
-  fi
-done
+go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1
 
 (( failed )) && fail_test
 

test/e2e-tests.sh

@@ -36,7 +36,7 @@ go_test_e2e -timeout=20m ./test/... || failed=1
 # Run these _after_ the integration tests b/c they don't quite work all the way
 # and they cause a lot of noise in the logs, making it harder to debug integration
 # test failures.
-${REPO_ROOT_DIR}/test/e2e-tests-yaml.sh --run-tests || failed=1
+go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1
 
 (( failed )) && fail_test
 success