Run CI YAML Example tests using go test

Update to accurately report specific example failures by name. E2E YAML
tests now use same harness and output format as other tests. Fixes #1251

Add a new `yaml` build tag so that existing integration tests aren't
affected. Future updates to SubEnv may allow e2e tests to be run entirely
locally by avoiding external deps, see #1372

test/README.md

@@ -206,9 +206,12 @@ go test -v -tags=e2e -count=1 ./test -run ^TestTaskRun
 To run the YAML e2e tests, run the following command:
 
 ```bash
-./test/e2e-tests-yaml.sh
+go test -v -count=1 -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/
 ```
 
+To limit parallelism of tests, use `-parallel=n` where `n` is the number of
+tests to run in parallel.
+
 ### Running upgrade tests
 
 There are two scenarios in upgrade tests. One is to install the previous release, upgrade to the current release, and

test/e2e-common.sh

@@ -18,115 +18,6 @@
 
 source $(git rev-parse --show-toplevel)/vendor/github.com/tektoncd/plumbing/scripts/e2e-tests.sh
 
-function teardown() {
-    subheader "Tearing down Tekton Pipelines"
-    ko delete --ignore-not-found=true -f config/
-    # teardown will be called when run against an existing cluster to cleanup before
-    # continuing, so we must wait for the cleanup to complete or the subsequent attempt
-    # to deploy to the same namespace will fail
-    wait_until_object_does_not_exist namespace tekton-pipelines
-}
-
-function output_yaml_test_results() {
-  # If formatting fails for any reason, use yaml as a fall back.
-  kubectl get $1.tekton.dev -o=custom-columns-file=${REPO_ROOT_DIR}/test/columns.txt || \
-    kubectl get $1.tekton.dev -oyaml
-}
-
-function output_pods_logs() {
-    echo ">>> $1"
-    kubectl get $1.tekton.dev -o yaml
-    local runs=$(kubectl get $1.tekton.dev --output=jsonpath="{.items[*].metadata.name}")
-    set +e
-    for run in ${runs}; do
-	echo ">>>> $1 ${run}"
-	case "$1" in
-	    "taskrun")
-		tkn taskrun logs --nocolour ${run}
-		;;
-	    "pipelinerun")
-		tkn pipelinerun logs --nocolour ${run}
-		;;
-	esac
-    done
-    set -e
-    echo ">>>> Pods"
-    kubectl get pods -o yaml
-}
-
-# Called by `fail_test` (provided by `e2e-tests.sh`) to dump info on test failure
-function dump_extra_cluster_state() {
-  echo ">>> Pipeline controller log:"
-  kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-controller tekton-pipelines)
-  echo ">>> Pipeline webhook log:"
-  kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-webhook tekton-pipelines)
-}
-
-function validate_run() {
-  local tests_finished=0
-  for i in {1..90}; do
-    local finished="$(kubectl get $1.tekton.dev --output=jsonpath='{.items[*].status.conditions[*].status}')"
-    if [[ ! "$finished" == *"Unknown"* ]]; then
-      tests_finished=1
-      break
-    fi
-    sleep 10
-  done
-
-  return ${tests_finished}
-}
-
-function check_results() {
-  local failed=0
-  results="$(kubectl get $1.tekton.dev --output=jsonpath='{range .items[*]}{.metadata.name}={.status.conditions[*].type}{.status.conditions[*].status}{" "}{end}')"
-  for result in ${results}; do
-    if [[ ! "${result,,}" == *"=succeededtrue" ]]; then
-      echo "ERROR: test ${result} but should be succeededtrue"
-      failed=1
-    fi
-  done
-
-  return ${failed}
-}
-
-function create_resources() {
-  local resource=$1
-  echo ">> Creating resources ${resource}"
-
-  # Applying the resources, either *taskruns or * *pipelineruns except those
-  # in the no-ci directory
-  for file in $(find ${REPO_ROOT_DIR}/examples/${resource}s/ -name '*.yaml' -not -path '*/no-ci/*' | sort); do
-    perl -p -e 's/gcr.io\/christiewilson-catfactory/$ENV{KO_DOCKER_REPO}/g' ${file} | ko create -f - || return 1
-  done
-}
-
-function run_tests() {
-  local resource=$1
-
-  # Wait for tests to finish.
-  echo ">> Waiting for tests to finish for ${resource}"
-  if validate_run $resource; then
-    echo "ERROR: tests timed out"
-  fi
-
-  # Check that tests passed.
-  echo ">> Checking test results for ${resource}"
-  if check_results $resource; then
-    echo ">> All YAML tests passed"
-    return 0
-  fi
-  return 1
-}
-
-function run_yaml_tests() {
-  echo ">> Starting tests for the resource ${1}/${2}"
-  create_resources ${1}/${2} || fail_test "Could not create ${2}/${1} from the examples"
-  if ! run_tests ${2}; then
-    return 1
-  fi
-  return 0
-}
-
 function install_pipeline_crd() {
   echo ">> Deploying Tekton Pipelines"
   ko resolve -f config/ \

test/e2e-tests-upgrade.sh

@@ -51,15 +51,7 @@ failed=0
 go_test_e2e -timeout=20m ./test || failed=1
 
 # Run the post-integration tests.
-for test in taskrun pipelinerun; do
-  header "Running YAML e2e tests for ${test}s"
-  if ! run_yaml_tests ${test}; then
-    echo "ERROR: one or more YAML tests failed"
-    output_yaml_test_results ${test}
-    output_pods_logs ${test}
-    failed=1
-  fi
-done
+go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1
 
 # Remove all the pipeline CRDs, and clean up the environment for next Scenario.
 uninstall_pipeline_crd
@@ -87,15 +79,7 @@ go_test_e2e -timeout=20m ./test || failed=1
 # Run the post-integration tests. We do not need to install the resources again, since
 # they are installed before the upgrade. We verify if they still work, after going through
 # the upgrade.
-for test in taskrun pipelinerun; do
-  header "Running YAML e2e tests for ${test}s"
-  if ! run_tests ${test}; then
-    echo "ERROR: one or more YAML tests failed"
-    output_yaml_test_results ${test}
-    output_pods_logs ${test}
-    failed=1
-  fi
-done
+go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1
 
 (( failed )) && fail_test