From 44c751426e6e9c056f0fa6cd87f9506f292d11b3 Mon Sep 17 00:00:00 2001 From: Siddhesh Ghadi Date: Mon, 22 Feb 2021 10:43:22 +0530 Subject: [PATCH] Add ppc64le nightly tests for pipelines - Add secret & resource details to docs. - Update nightly test EventListener to include ppc64le architecture. - Add TriggerTemplate for nightly pipeline tests on ppc64le. - Add cronjob to trigger nightly ppc64le pipeline tests. - Add tasks to deploy, test & clean pipeline components on remote ppc64le cluster. - Add service account for triggering nightly tests. Signed-off-by: Siddhesh Ghadi --- docs/dogfooding.md | 5 + .../cleanup/bastion-p-nightly/README.md | 1 + .../cleanup/bastion-p-nightly/cronjob.yaml | 20 +++ .../bastion-p-nightly/kustomization.yaml | 5 + .../dogfooding/cleanup/kustomization.yaml | 1 + .../nightly-tests/kustomization.yaml | 1 + .../nightly-tests/ppc64le/kustomization.yaml | 3 + .../ppc64le/pipeline-nightly-test/README.md | 1 + .../pipeline-nightly-test/cronjob.yaml | 26 +++ .../pipeline-nightly-test/kustomization.yaml | 5 + .../bastion-p/cleanup_tekton.yaml | 49 ++++++ .../bastion-p/deploy_tekton_component.yaml | 48 ++++++ .../bastion-p/kustomization.yaml | 8 + .../bastion-p/test_tekton_component.yaml | 57 +++++++ .../nightly-tests/eventlistener.yaml | 12 ++ .../nightly-tests/kustomization.yaml | 2 + ...pipeline-deploy-test-ppc64le-template.yaml | 154 ++++++++++++++++++ .../nightly-tests/serviceaccount.yaml | 14 ++ 18 files changed, 412 insertions(+) create mode 100644 tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/README.md create mode 100644 tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/cronjob.yaml create mode 100644 tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/kustomization.yaml create mode 100644 tekton/cronjobs/dogfooding/nightly-tests/ppc64le/kustomization.yaml create mode 100644 tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/README.md create mode 100644 tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/cronjob.yaml create mode 100644 tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/kustomization.yaml create mode 100644 tekton/resources/nightly-tests/bastion-p/cleanup_tekton.yaml create mode 100644 tekton/resources/nightly-tests/bastion-p/deploy_tekton_component.yaml create mode 100644 tekton/resources/nightly-tests/bastion-p/kustomization.yaml create mode 100644 tekton/resources/nightly-tests/bastion-p/test_tekton_component.yaml create mode 100644 tekton/resources/nightly-tests/pipeline-deploy-test-ppc64le-template.yaml diff --git a/docs/dogfooding.md b/docs/dogfooding.md index a419929c5..1b2cf45ac 100644 --- a/docs/dogfooding.md +++ b/docs/dogfooding.md @@ -24,6 +24,11 @@ Secrets which have been applied to the dogfooding cluster but are not committed - `mario-github-token` used for updating PRs - In the bastion-z namespace: - `s390x-k8s-ssh` used to ssh access s390x remote machine + - In the bastion-p namespace: + - `ppc64le-kubeconfig` used to access ppc64le remote k8 cluster + - `registry-credentials` used to access registry on remote machine + - `registry-certificate` self-signed certificate for registry on remote machine + - `ppc64le-cluster` headless service & endpoint to resolve remote machine address - `GCP` secrets: - `nightly-account` is used by nightly releases to push releases to the nightly bucket. It's a token for service account diff --git a/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/README.md b/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/README.md new file mode 100644 index 000000000..a2a33ffb5 --- /dev/null +++ b/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/README.md @@ -0,0 +1 @@ +Cron Job to daily cleanup pr/tr from the default namespace in the dogfooding cluster diff --git a/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/cronjob.yaml b/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/cronjob.yaml new file mode 100644 index 000000000..232dfcbe5 --- /dev/null +++ b/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/cronjob.yaml @@ -0,0 +1,20 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: cleanup-trigger +spec: + schedule: "0 11 * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: trigger + env: + - name: NAMESPACE + value: "bastion-p" + - name: CLUSTER_RESOURCE + value: "dogfooding-tektoncd-cleaner" + - name: CLEANUP_KEEP + value: "20" diff --git a/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/kustomization.yaml b/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/kustomization.yaml new file mode 100644 index 000000000..684646b93 --- /dev/null +++ b/tekton/cronjobs/dogfooding/cleanup/bastion-p-nightly/kustomization.yaml @@ -0,0 +1,5 @@ +bases: +- ../../../bases/cleanup +patchesStrategicMerge: +- cronjob.yaml +nameSuffix: "-dogfooding-bastion-p" diff --git a/tekton/cronjobs/dogfooding/cleanup/kustomization.yaml b/tekton/cronjobs/dogfooding/cleanup/kustomization.yaml index 7e10eb0da..6746f813c 100644 --- a/tekton/cronjobs/dogfooding/cleanup/kustomization.yaml +++ b/tekton/cronjobs/dogfooding/cleanup/kustomization.yaml @@ -4,3 +4,4 @@ resources: - bastion-z-nightly - tekton-ci-nightly - tekton-nightly-nightly +- bastion-p-nightly diff --git a/tekton/cronjobs/dogfooding/nightly-tests/kustomization.yaml b/tekton/cronjobs/dogfooding/nightly-tests/kustomization.yaml index 7b63aeeb0..831f5d89a 100644 --- a/tekton/cronjobs/dogfooding/nightly-tests/kustomization.yaml +++ b/tekton/cronjobs/dogfooding/nightly-tests/kustomization.yaml @@ -1,3 +1,4 @@ namespace: default resources: - s390x +- ppc64le diff --git a/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/kustomization.yaml b/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/kustomization.yaml new file mode 100644 index 000000000..beafed2b5 --- /dev/null +++ b/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/kustomization.yaml @@ -0,0 +1,3 @@ +namespace: default +resources: +- pipeline-nightly-test diff --git a/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/README.md b/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/README.md new file mode 100644 index 000000000..0bad08004 --- /dev/null +++ b/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/README.md @@ -0,0 +1 @@ +Cron Job to run nightly pipeline e2e tests. diff --git a/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/cronjob.yaml b/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/cronjob.yaml new file mode 100644 index 000000000..ba3bd9185 --- /dev/null +++ b/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/cronjob.yaml @@ -0,0 +1,26 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: nightly-test-trigger +spec: + schedule: "0 1 * * *" + concurrencyPolicy: Forbid + jobTemplate: + spec: + template: + spec: + containers: + - name: trigger + env: + - name: SINK_URL + value: "http://el-test-nightly.default.svc.cluster.local:8080" + - name: TARGET_PROJECT + value: "pipeline" + - name: NAMESPACE + value: "bastion-p" + - name: REGISTRY + value: "ppc64le-cluster.bastion-p.svc.cluster.local:443" + - name: TARGET_ARCH + value: "ppc64le" + - name: REMOTE_SECRET_NAME + value: "ppc64le-kubeconfig" diff --git a/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/kustomization.yaml b/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/kustomization.yaml new file mode 100644 index 000000000..ee6dc81c4 --- /dev/null +++ b/tekton/cronjobs/dogfooding/nightly-tests/ppc64le/pipeline-nightly-test/kustomization.yaml @@ -0,0 +1,5 @@ +bases: +- ../../../../bases/nightly-tests +patchesStrategicMerge: +- cronjob.yaml +nameSuffix: "-pipeline-ppc64le" diff --git a/tekton/resources/nightly-tests/bastion-p/cleanup_tekton.yaml b/tekton/resources/nightly-tests/bastion-p/cleanup_tekton.yaml new file mode 100644 index 000000000..8899c9828 --- /dev/null +++ b/tekton/resources/nightly-tests/bastion-p/cleanup_tekton.yaml @@ -0,0 +1,49 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: cleanup-tekton-nightly +spec: + workspaces: + - name: k8s-config + description: workspace to get k8s config file + mountPath: /root/.kube + - name: source-code + description: workspace with source code for tekton component + mountPath: /workspace + params: + - name: package + - name: resources + description: space separated list of resources to be deleted + default: "conditions pipelineresources tasks pipelines taskruns pipelineruns" + - name: plumbing-path + description: path in the workspace for plumbing source code + default: src/github.com/tektoncd/plumbing + steps: + - name: cleanup-resources + image: gcr.io/tekton-releases/dogfooding/kubectl:latest + env: + - name: KUBECONFIG + value: /root/.kube/config + command: + - /bin/sh + args: + - -ce + - | + kubectl delete ns -l tekton.dev/test-e2e=true + for res in $(params.resources); do + kubectl delete --ignore-not-found=true ${res}.tekton.dev --all || return true + done + - name: uninstall-tekton-project + image: gcr.io/tekton-releases/dogfooding/test-runner:latest + workingdir: /workspace/src/$(params.package) + env: + - name: KUBECONFIG + value: /root/.kube/config + command: + - /bin/bash + args: + - -ce + - | + source $(workspaces.source-code.path)/$(params.plumbing-path)/scripts/library.sh + ko delete --ignore-not-found=true -f config/ + wait_until_object_does_not_exist namespace tekton-pipelines diff --git a/tekton/resources/nightly-tests/bastion-p/deploy_tekton_component.yaml b/tekton/resources/nightly-tests/bastion-p/deploy_tekton_component.yaml new file mode 100644 index 000000000..a6be2c7aa --- /dev/null +++ b/tekton/resources/nightly-tests/bastion-p/deploy_tekton_component.yaml @@ -0,0 +1,48 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: deploy-tekton-project-nightly +spec: + workspaces: + - name: k8s-config + description: workspace to get k8s config file + mountPath: /root/.kube + - name: registry-credentials + description: workspace to get registry credentials + mountPath: /tekton/home/.docker + - name: registry-certificate + description: workspace to get registry self-signed certificate + mountPath: /opt/ssl/certs + - name: source-code + description: workspace with source code for tekton component + mountPath: /workspace + params: + - name: package + description: package to install + - name: container-registry + description: container registry used to publish build images + - name: target-arch + description: target architecture for tests (s390x, ppc64le, arm64) + steps: + - name: deploy + workingdir: /workspace/src/$(params.package) + image: gcr.io/tekton-releases/dogfooding/test-runner:latest + env: + - name: GOPATH + value: /workspace + - name: KO_DOCKER_REPO + value: $(params.container-registry) + - name: KUBECONFIG + value: /root/.kube/config + - name: SSL_CERT_FILE + value: /opt/ssl/certs/registry.crt + command: + - /bin/bash + args: + - -ce + - | + # busybox binary in distroless image doesn't work on ppc64le + # https://bugs.busybox.net/show_bug.cgi?id=13491 + sed -i 's/gcr.io\/distroless\/base@sha256:[a-zA-Z0-9]*/registry.access.redhat.com\/ubi8\/ubi-minimal:latest/g' config/controller.yaml + ko apply --platform=linux/$(params.target-arch) -f config/ + kubectl wait -n tekton-pipelines --for=condition=ready pods --all --timeout=120s \ No newline at end of file diff --git a/tekton/resources/nightly-tests/bastion-p/kustomization.yaml b/tekton/resources/nightly-tests/bastion-p/kustomization.yaml new file mode 100644 index 000000000..694da1622 --- /dev/null +++ b/tekton/resources/nightly-tests/bastion-p/kustomization.yaml @@ -0,0 +1,8 @@ +namespace: bastion-p +commonAnnotations: + managed-by: Tekton + +resources: +- deploy_tekton_component.yaml +- test_tekton_component.yaml +- cleanup_tekton.yaml diff --git a/tekton/resources/nightly-tests/bastion-p/test_tekton_component.yaml b/tekton/resources/nightly-tests/bastion-p/test_tekton_component.yaml new file mode 100644 index 000000000..b35e5efa8 --- /dev/null +++ b/tekton/resources/nightly-tests/bastion-p/test_tekton_component.yaml @@ -0,0 +1,57 @@ +apiVersion: tekton.dev/v1beta1 +kind: Task +metadata: + name: test-e2e-tekton-component +spec: + workspaces: + - name: k8s-config + description: workspace to get k8s config file + mountPath: /root/.kube + - name: source-code + description: workspace with source code for tekton component + mountPath: /workspace + params: + - name: package + description: package (and its children) under test + - name: tests-path + description: path to the tests within "tests" git resource + default: ./test + - name: plumbing-path + description: path in the workspace for plumbing source code + default: src/github.com/tektoncd/plumbing + - name: timeout + description: timeout for the go test runner + default: 45m + - name: container-registry + description: container registry used to push images during tests e.g. gcr.io/tekton-e2e-tests or icr.io/tekton-e2e-tests + - name: tags + default: e2e + - name: target-arch + description: target architecture for tests (s390x, ppc64le, arm64) + steps: + - name: run-e2e-tests + image: gcr.io/tekton-releases/dogfooding/test-runner:latest + workingdir: $(workspaces.source-code.path)/src/$(params.package) + env: + - name: REPO_ROOT_DIR + value: $(workspaces.source-code.path)/src/$(params.package) + - name: GOPATH + value: /workspace + - name: KO_DOCKER_REPO + value: $(params.container-registry) + - name: TEST_RUNTIME_ARCH + value: $(params.target-arch) + - name: SYSTEM_NAMESPACE + value: tekton-pipelines + - name: KUBECONFIG + value: /root/.kube/config + command: + - /bin/bash + args: + - -ce + - | + source $(workspaces.source-code.path)/$(params.plumbing-path)/scripts/library.sh + # extend test timeout (from 10 minutes to 20 minutes) to resolve https://github.com/tektoncd/pipeline/issues/3627 + sed -i 's/timeout = 10/timeout = 20/g' test/wait.go + header "Running Go $(params.tags) tests" + report_go_test -v -count=1 -tags=$(params.tags) -timeout=$(params.timeout) $(params.tests-path) -kubeconfig /root/.kube/config \ No newline at end of file diff --git a/tekton/resources/nightly-tests/eventlistener.yaml b/tekton/resources/nightly-tests/eventlistener.yaml index 24afb2cfd..b0d049ada 100644 --- a/tekton/resources/nightly-tests/eventlistener.yaml +++ b/tekton/resources/nightly-tests/eventlistener.yaml @@ -53,6 +53,18 @@ spec: - ref: trigger-to-deploy-test-tekton-project template: ref: tekton-operator-nightly-test-s390x + - name: pipeline-nightly-test-trigger-ppc64le + interceptors: + - cel: + filter: >- + 'trigger-template' in body && + body['trigger-template'] == 'pipeline' && + 'arch' in body.params.target && + body.params.target.arch == 'ppc64le' + bindings: + - ref: trigger-to-deploy-test-tekton-project + template: + name: tekton-pipeline-nightly-test-ppc64le --- apiVersion: triggers.tekton.dev/v1alpha1 kind: TriggerBinding diff --git a/tekton/resources/nightly-tests/kustomization.yaml b/tekton/resources/nightly-tests/kustomization.yaml index 492445fc9..8528adf7f 100644 --- a/tekton/resources/nightly-tests/kustomization.yaml +++ b/tekton/resources/nightly-tests/kustomization.yaml @@ -3,9 +3,11 @@ commonAnnotations: resources: - bastion-z +- bastion-p - eventlistener.yaml - pipeline-deploy-test-s390x-template.yaml - triggers-deploy-test-s390x-template.yaml - cli-deploy-test-s390x-template.yaml - operator-deploy-test-s390x-template.yaml +- pipeline-deploy-test-ppc64le-template.yaml - serviceaccount.yaml diff --git a/tekton/resources/nightly-tests/pipeline-deploy-test-ppc64le-template.yaml b/tekton/resources/nightly-tests/pipeline-deploy-test-ppc64le-template.yaml new file mode 100644 index 000000000..0f614183e --- /dev/null +++ b/tekton/resources/nightly-tests/pipeline-deploy-test-ppc64le-template.yaml @@ -0,0 +1,154 @@ +# Copyright 2021 The Tekton Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: triggers.tekton.dev/v1alpha1 +kind: TriggerTemplate +metadata: + name: tekton-pipeline-nightly-test-ppc64le +spec: + params: + - name: containerRegistry + - name: targetArch + - name: namespace + - name: remoteSecret + resourcetemplates: + - apiVersion: tekton.dev/v1beta1 + kind: PipelineRun + metadata: + generateName: tekton-pipeline-$(tt.params.targetArch)-nightly-run- + namespace: $(tt.params.namespace) + spec: + timeout: 2h + workspaces: + # this workspace will be used to share info between tasks + - name: shared-workspace + volumeClaimTemplate: + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi + # this workspace will be used to store k8s config + - name: k8s-config + secret: + secretName: $(tt.params.remoteSecret) + # this workspace will be used to store registry credentials + - name: registry-credentials + secret: + secretName: registry-credentials + # this workspace will be used to store registry self-signed certificate + - name: registry-certificate + secret: + secretName: registry-certificate + pipelineSpec: + workspaces: + - name: shared-workspace + - name: k8s-config + - name: registry-credentials + - name: registry-certificate + params: + - name: package + - name: container-registry + - name: target-arch + tasks: + - name: git-clone-plumbing + taskRef: + name: git-clone + params: + - name: url + value: https://github.com/tektoncd/plumbing + - name: revision + value: main + - name: subdirectory + value: src/github.com/tektoncd/plumbing + workspaces: + - name: output + workspace: shared-workspace + subPath: source-code + - name: git-clone-pipeline + runAfter: [git-clone-plumbing] + taskRef: + name: git-clone + params: + - name: url + value: https://github.com/tektoncd/pipeline + - name: revision + value: master + - name: subdirectory + value: src/github.com/tektoncd/pipeline + workspaces: + - name: output + workspace: shared-workspace + subPath: source-code + - name: deploy-pipeline + runAfter: [git-clone-pipeline] + taskRef: + name: deploy-tekton-project-nightly + workspaces: + - name: k8s-config + workspace: k8s-config + - name: registry-credentials + workspace: registry-credentials + - name: registry-certificate + workspace: registry-certificate + - name: source-code + workspace: shared-workspace + subPath: source-code + retries: 2 + params: + - name: package + value: $(params.package) + - name: container-registry + value: $(params.container-registry) + - name: target-arch + value: $(params.target-arch) + - name: e2e-test-pipeline + runAfter: [deploy-pipeline] + taskRef: + name: test-e2e-tekton-component + workspaces: + - name: k8s-config + workspace: k8s-config + - name: source-code + workspace: shared-workspace + subPath: source-code + retries: 2 + params: + - name: package + value: $(params.package) + - name: container-registry + value: $(params.container-registry) + - name: target-arch + value: $(params.target-arch) + finally: + - name: cleanup + taskRef: + name: cleanup-tekton-nightly + workspaces: + - name: k8s-config + workspace: k8s-config + - name: source-code + workspace: shared-workspace + subPath: source-code + retries: 2 + params: + - name: package + value: $(params.package) + params: + - name: package + value: github.com/tektoncd/pipeline + - name: container-registry + value: $(tt.params.containerRegistry) + - name: target-arch + value: $(tt.params.targetArch) diff --git a/tekton/resources/nightly-tests/serviceaccount.yaml b/tekton/resources/nightly-tests/serviceaccount.yaml index c16dfc540..20130fe32 100644 --- a/tekton/resources/nightly-tests/serviceaccount.yaml +++ b/tekton/resources/nightly-tests/serviceaccount.yaml @@ -31,6 +31,20 @@ roleRef: kind: ClusterRole name: tekton-test-triggers-nightly --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tekton-test-nightly-trigger-bastion-p + namespace: bastion-p +subjects: +- kind: ServiceAccount + name: tekton-test-nightly + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tekton-test-triggers-nightly +--- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: