diff --git a/test/README.md b/test/README.md index 5b135f929e8..12184d120a8 100644 --- a/test/README.md +++ b/test/README.md @@ -206,9 +206,12 @@ go test -v -tags=e2e -count=1 ./test -run ^TestTaskRun To run the YAML e2e tests, run the following command: ```bash -./test/e2e-tests-yaml.sh +go test -v -count=1 -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ ``` +To limit parallelism of tests, use `-parallel=n` where `n` is the number of +tests to run in parallel. + ### Running upgrade tests There are two scenarios in upgrade tests. One is to install the previous release, upgrade to the current release, and diff --git a/test/e2e-common.sh b/test/e2e-common.sh index e0a6328bc33..1fba0249576 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -18,115 +18,6 @@ source $(git rev-parse --show-toplevel)/vendor/github.com/tektoncd/plumbing/scripts/e2e-tests.sh -function teardown() { - subheader "Tearing down Tekton Pipelines" - ko delete --ignore-not-found=true -f config/ - # teardown will be called when run against an existing cluster to cleanup before - # continuing, so we must wait for the cleanup to complete or the subsequent attempt - # to deploy to the same namespace will fail - wait_until_object_does_not_exist namespace tekton-pipelines -} - -function output_yaml_test_results() { - # If formatting fails for any reason, use yaml as a fall back. - kubectl get $1.tekton.dev -o=custom-columns-file=${REPO_ROOT_DIR}/test/columns.txt || \ - kubectl get $1.tekton.dev -oyaml -} - -function output_pods_logs() { - echo ">>> $1" - kubectl get $1.tekton.dev -o yaml - local runs=$(kubectl get $1.tekton.dev --output=jsonpath="{.items[*].metadata.name}") - set +e - for run in ${runs}; do - echo ">>>> $1 ${run}" - case "$1" in - "taskrun") - tkn taskrun logs --nocolour ${run} - ;; - "pipelinerun") - tkn pipelinerun logs --nocolour ${run} - ;; - esac - done - set -e - echo ">>>> Pods" - kubectl get pods -o yaml -} - -# Called by `fail_test` (provided by `e2e-tests.sh`) to dump info on test failure -function dump_extra_cluster_state() { - echo ">>> Pipeline controller log:" - kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-controller tekton-pipelines) - echo ">>> Pipeline webhook log:" - kubectl -n tekton-pipelines logs $(get_app_pod tekton-pipelines-webhook tekton-pipelines) -} - -function validate_run() { - local tests_finished=0 - for i in {1..90}; do - local finished="$(kubectl get $1.tekton.dev --output=jsonpath='{.items[*].status.conditions[*].status}')" - if [[ ! "$finished" == *"Unknown"* ]]; then - tests_finished=1 - break - fi - sleep 10 - done - - return ${tests_finished} -} - -function check_results() { - local failed=0 - results="$(kubectl get $1.tekton.dev --output=jsonpath='{range .items[*]}{.metadata.name}={.status.conditions[*].type}{.status.conditions[*].status}{" "}{end}')" - for result in ${results}; do - if [[ ! "${result,,}" == *"=succeededtrue" ]]; then - echo "ERROR: test ${result} but should be succeededtrue" - failed=1 - fi - done - - return ${failed} -} - -function create_resources() { - local resource=$1 - echo ">> Creating resources ${resource}" - - # Applying the resources, either *taskruns or * *pipelineruns except those - # in the no-ci directory - for file in $(find ${REPO_ROOT_DIR}/examples/${resource}s/ -name '*.yaml' -not -path '*/no-ci/*' | sort); do - perl -p -e 's/gcr.io\/christiewilson-catfactory/$ENV{KO_DOCKER_REPO}/g' ${file} | ko create -f - || return 1 - done -} - -function run_tests() { - local resource=$1 - - # Wait for tests to finish. - echo ">> Waiting for tests to finish for ${resource}" - if validate_run $resource; then - echo "ERROR: tests timed out" - fi - - # Check that tests passed. - echo ">> Checking test results for ${resource}" - if check_results $resource; then - echo ">> All YAML tests passed" - return 0 - fi - return 1 -} - -function run_yaml_tests() { - echo ">> Starting tests for the resource ${1}/${2}" - create_resources ${1}/${2} || fail_test "Could not create ${2}/${1} from the examples" - if ! run_tests ${2}; then - return 1 - fi - return 0 -} - function install_pipeline_crd() { echo ">> Deploying Tekton Pipelines" ko resolve -f config/ \ diff --git a/test/e2e-tests-upgrade.sh b/test/e2e-tests-upgrade.sh index cce0c60bd89..c793d36cdee 100755 --- a/test/e2e-tests-upgrade.sh +++ b/test/e2e-tests-upgrade.sh @@ -51,15 +51,7 @@ failed=0 go_test_e2e -timeout=20m ./test || failed=1 # Run the post-integration tests. -for test in taskrun pipelinerun; do - header "Running YAML e2e tests for ${test}s" - if ! run_yaml_tests ${test}; then - echo "ERROR: one or more YAML tests failed" - output_yaml_test_results ${test} - output_pods_logs ${test} - failed=1 - fi -done +go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1 # Remove all the pipeline CRDs, and clean up the environment for next Scenario. uninstall_pipeline_crd @@ -87,15 +79,7 @@ go_test_e2e -timeout=20m ./test || failed=1 # Run the post-integration tests. We do not need to install the resources again, since # they are installed before the upgrade. We verify if they still work, after going through # the upgrade. -for test in taskrun pipelinerun; do - header "Running YAML e2e tests for ${test}s" - if ! run_tests ${test}; then - echo "ERROR: one or more YAML tests failed" - output_yaml_test_results ${test} - output_pods_logs ${test} - failed=1 - fi -done +go_test_e2e -tags='e2e,yaml' -timeout=20m -test.run '^TestYaml' ./test/ || failed=1 (( failed )) && fail_test diff --git a/test/e2e-tests-yaml.sh b/test/e2e-tests-yaml.sh deleted file mode 100755 index 6d2061dca9f..00000000000 --- a/test/e2e-tests-yaml.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2019 The Tekton Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script calls out to scripts in tektoncd/plumbing to setup a cluster -# and deploy Tekton Pipelines to it for running integration tests. - -source $(git rev-parse --show-toplevel)/test/e2e-common.sh - -# Script entry point. - -initialize $@ - -header "Setting up environment" - -# Handle failures ourselves, so we can dump useful info. -set +o errexit -set +o pipefail - -install_pipeline_crd - -# Run the tests -failed=0 -for version in v1alpha1 v1beta1; do - for test in taskrun pipelinerun; do - header "Running YAML e2e tests for ${version} ${test}s" - if ! run_yaml_tests ${version} ${test}; then - echo "ERROR: one or more YAML tests failed" - output_yaml_test_results ${test} - output_pods_logs ${test} - failed=1 - fi - done - # Clean resources - delete_pipeline_resources - for res in services pods configmaps secrets serviceaccounts persistentvolumeclaims; do - kubectl delete --ignore-not-found=true ${res} --all - done -done - -(( failed )) && fail_test - -success diff --git a/test/e2e_yaml_test.go b/test/e2e_yaml_test.go new file mode 100644 index 00000000000..fecb8807644 --- /dev/null +++ b/test/e2e_yaml_test.go @@ -0,0 +1,182 @@ +// +build e2e,yaml + +/* +Copyright 2020 The Tekton Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "testing" + "path/filepath" + "io/ioutil" + "os" + "os/exec" + "regexp" + "io" + "strings" + knativetest "knative.dev/pkg/test" +) + +// Generalized for tekton CRDs +func GetCreatedTektonCrd(input []byte, kind string) (string, error) { + re := regexp.MustCompile(kind + `.tekton.dev\/(.+) created`) + submatch := re.FindSubmatch(input) + if submatch == nil { + return "", nil + } + return string(submatch[1]), nil +} + +func ValidatePipelineRunDone(t *testing.T, c *clients, pipelineRunName string) { + err := WaitForPipelineRunState(c, pipelineRunName, pipelineRunTimeout, Succeed(pipelineRunName), pipelineRunName) + + if err != nil { + t.Fatal(err) + } + return +} + +func ValidateTaskRunDone(t *testing.T, c *clients, taskRunName string) { + // Per test basis + err := WaitForTaskRunState(c, taskRunName, Succeed(taskRunName), taskRunName) + + if err != nil { + t.Fatal(err) + } + return +} + +// Substitute docker repos/bucket paths from environment to allow tests on +// local clusters +func SubEnv(input []byte) ([]byte, error) { + val, ok := os.LookupEnv("KO_DOCKER_REPO") + if ok { + re := regexp.MustCompile(`gcr.io\/christiewilson-catfactory`) + return re.ReplaceAll(input, []byte(val)), nil + } else { + return input, nil + } +} + +func KoCreate(input []byte, namespace string) ([]byte, error) { + cmd := exec.Command("ko", "create", "-n", namespace, "-f", "-") + stdin, err := cmd.StdinPipe() + if err != nil { + return []byte("Cannot create Stdin Pipe"), err + } + + go func() { + defer stdin.Close() + io.WriteString(stdin, string(input)) + }() + + out, err := cmd.CombinedOutput() + return out, err +} + +func YamlSubtest(path string, validateFunc func(t *testing.T, c *clients, name string), kind string) func(subt *testing.T) { + return func(subt *testing.T) { + subt.Parallel() + + // Setup unique namespaces for each test so they can run in complete + // isolation + c, namespace := setup(subt) + knativetest.CleanupOnInterrupt(func() { tearDown(subt, c, namespace) }, subt.Logf) + defer tearDown(subt, c, namespace) + + inputExample, err := ioutil.ReadFile(path) + + if err != nil { + subt.Fatal(err) + } + + subbedInput, err := SubEnv(inputExample) + if err != nil { + subt.Fatal(err) + } + + out, err := KoCreate(subbedInput, namespace) + if err != nil { + subt.Fatalf("%s Output: %s", err, out) + } + + // Parse from KoCreate for now + name, err := GetCreatedTektonCrd(out, kind) + if name == "" { + // Nothing to check from ko create, this is not a taskrun or pipeline + // run + subt.Skipf("pipelinerun or taskrun not created") + } else if err != nil { + subt.Fatal(err) + } + + validateFunc(subt, c, name) + } +} + +func GetExamplePaths(t *testing.T, dir string) ([]string) { + var examplePaths[]string + + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + t.Fatal(err) + } + // Do not append root and any other folders named "examples" + if info.Name() == "examples" && info.IsDir() { + return nil + } + if info.Name() == "no-ci" && info.IsDir() { + return filepath.SkipDir + } + if info.IsDir() == false && filepath.Ext(info.Name()) == ".yaml" { + examplePaths = append(examplePaths, path) + return nil + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + return examplePaths +} + +func ExtractTestName(baseDir string, path string) (string) { + re := regexp.MustCompile(baseDir + "/(.+).yaml") + submatch := re.FindSubmatch([]byte(path)) + if submatch == nil { + return path + } + return string(submatch[1]) +} + +func TestYaml(t *testing.T) { + baseDir := "../examples" + + for _, path := range GetExamplePaths(t, baseDir) { + // Extract a test name + testName := ExtractTestName(baseDir, path) + validateFunc := ValidatePipelineRunDone + kind := "pipelinerun" + + if strings.Contains(path, "/taskruns/") { + validateFunc = ValidateTaskRunDone + kind = "taskrun" + } + + t.Run(testName, YamlSubtest(path, validateFunc, kind)) + } +}