From 766aa6d3b1c0a4b8536ec1e13bbf258e9ba0ab43 Mon Sep 17 00:00:00 2001 From: "Ricardo M. Oliveira" Date: Tue, 16 Apr 2024 18:10:17 -0300 Subject: [PATCH] Add kfp-tekton integration tests and manifests Signed-off-by: Ricardo M. Oliveira --- .github/workflows/backend.yml | 83 +++++++++++++++++++ .github/workflows/codeql.yml | 72 ++++++++++++++++ .../kustomization.yaml | 27 ++++++ .../namespace.yaml | 4 + .../params.yaml | 4 + .../env/plain-multi-user/kustomization.yaml | 22 +++++ .../kustomize/env/plain/kustomization.yaml | 19 +++++ .../kustomization.yaml | 81 ++++++++++++++++++ .../tekton-config.yaml | 10 +++ .../kustomization.yaml | 81 ++++++++++++++++++ .../kustomization.yaml | 78 +++++++++++++++++ 11 files changed, 481 insertions(+) create mode 100644 .github/workflows/backend.yml create mode 100644 .github/workflows/codeql.yml create mode 100644 manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml create mode 100644 manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml create mode 100644 manifests/kustomize/cluster-scoped-resources-tekton/params.yaml create mode 100644 manifests/kustomize/env/plain-multi-user/kustomization.yaml create mode 100644 manifests/kustomize/env/plain/kustomization.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml create mode 100644 manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml create mode 100644 manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml new file mode 100644 index 000000000000..759769b65652 --- /dev/null +++ b/.github/workflows/backend.yml @@ -0,0 +1,83 @@ +name: KFP Tekton backend unit tests + +on: + push: + branches: [master] + + # Run tests for any PRs which change the backend code + pull_request: + paths: + - 'go.mod' + - 'backend/**' + - 'scripts/deploy/github/**' + - 'manifests/kustomize/**' + +env: + GITHUB_ACTION: "true" + SETUPTOOLS_USE_DISTUTILS: "stdlib" + +jobs: + run-go-unittests: + runs-on: ubuntu-latest + steps: + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: 1.20.x + - name: Checkout code + uses: actions/checkout@v4 + - name: "run go unit tests" + run: go test -v -cover ./backend/... + backend-integration: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Create k8s Kind Cluster + uses: container-tools/kind-action@v2 + with: + cluster_name: kfp-tekton + kubectl_version: v1.26.3 + version: v0.17.0 + node_image: kindest/node:v1.26.3 + - name: build images + run: ./scripts/deploy/github/build-images.sh + - name: Set up Python 3.10 + uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: "deploy kfp-tekton" + run: ./scripts/deploy/github/deploy-kfp.sh + - name: Install sdk + run: | + python3 -m venv .venv + . .venv/bin/activate + pip install -e sdk/python + - name: "flip coin test" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-flip-coin.sh" ./scripts/deploy/github/e2e-test.sh + - name: "static loop test" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-static-loop.sh" ./scripts/deploy/github/e2e-test.sh + - name: "dynamic loop test" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-dynamic-loop.sh" ./scripts/deploy/github/e2e-test.sh + - name: "secret as env" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-secret-as-env.sh" ./scripts/deploy/github/e2e-test.sh + - name: "secret as volume" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-secret-as-volume.sh" ./scripts/deploy/github/e2e-test.sh + - name: "use env" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-env.sh" ./scripts/deploy/github/e2e-test.sh + - name: "use volume" + run: | + . .venv/bin/activate + TEST_SCRIPT="test-volume.sh" ./scripts/deploy/github/e2e-test.sh diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000000..2f0b0b472e01 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,72 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "CodeQL" + +on: + schedule: + # Every Friday at 19:39 + - cron: '39 19 * * 5' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'go', 'javascript', 'python' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml new file mode 100644 index 000000000000..ea1a04a385cf --- /dev/null +++ b/manifests/kustomize/cluster-scoped-resources-tekton/kustomization.yaml @@ -0,0 +1,27 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: kubeflow + +resources: +- namespace.yaml +bases: +- ../third-party/application/cluster-scoped +# - ../third-party/argo/installs/namespace/cluster-scoped +- ../base/pipeline/cluster-scoped +- ../base/cache-deployer/cluster-scoped +vars: +# NOTE: var name must be unique globally to allow composition of multiple kustomize +# packages. Therefore, we added prefix `kfp-cluster-scoped-` to distinguish it from +# others. +- name: kfp-cluster-scoped-namespace + objref: + # cache deployer sa's metadata.namespace will be first transformed by namespace field in kustomization.yaml + # so that we only need to change kustomization.yaml's namespace field for namespace customization. + kind: ServiceAccount + name: kubeflow-pipelines-cache-deployer-sa + apiVersion: v1 + fieldref: + fieldpath: metadata.namespace +configurations: +- params.yaml diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml new file mode 100644 index 000000000000..3c65856e7b73 --- /dev/null +++ b/manifests/kustomize/cluster-scoped-resources-tekton/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: '$(kfp-cluster-scoped-namespace)' diff --git a/manifests/kustomize/cluster-scoped-resources-tekton/params.yaml b/manifests/kustomize/cluster-scoped-resources-tekton/params.yaml new file mode 100644 index 000000000000..cc253fe26603 --- /dev/null +++ b/manifests/kustomize/cluster-scoped-resources-tekton/params.yaml @@ -0,0 +1,4 @@ +# Allow Kustomize var to replace following fields. +varReference: +- path: metadata/name + kind: Namespace diff --git a/manifests/kustomize/env/plain-multi-user/kustomization.yaml b/manifests/kustomize/env/plain-multi-user/kustomization.yaml new file mode 100644 index 000000000000..47c8daea9cf7 --- /dev/null +++ b/manifests/kustomize/env/plain-multi-user/kustomization.yaml @@ -0,0 +1,22 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../base/installs/multi-user + - ../../base/metadata/base + - ../../base/metadata/options/istio + - ../../third-party/mysql/base + - ../../third-party/mysql/options/istio + - ../../third-party/minio/base + - ../../third-party/minio/options/istio + - ../../third-party/metacontroller/base + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow diff --git a/manifests/kustomize/env/plain/kustomization.yaml b/manifests/kustomize/env/plain/kustomization.yaml new file mode 100644 index 000000000000..7bf943cab335 --- /dev/null +++ b/manifests/kustomize/env/plain/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../cluster-scoped-resources-tekton + - ../../base/installs/generic + - ../../base/metadata/base + - ../../third-party/minio/base + - ../../third-party/mysql/base + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +# !!! If you want to customize the namespace, +# please also update base/cache-deployer/cluster-scoped/cache-deployer-clusterrolebinding.yaml +namespace: kubeflow diff --git a/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml new file mode 100644 index 000000000000..807b318cf14d --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/kustomization.yaml @@ -0,0 +1,81 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../third-party/tekton/installs/cluster + - ../../third-party/tekton-custom-task + - ../plain-multi-user + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. +commonLabels: + application-crd-id: kubeflow-pipelines + +images: +- name: gcr.io/ml-pipeline/api-server + newName: quay.io/aipipeline/apiserver + newTag: 2.0.5 +- name: gcr.io/ml-pipeline/persistenceagent + newName: quay.io/aipipeline/persistenceagent + newTag: 2.0.5 +- name: gcr.io/ml-pipeline/scheduledworkflow + newName: quay.io/aipipeline/scheduledworkflow + newTag: 2.0.5 + +patchesStrategicMerge: +- tekton-config.yaml + +patches: +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline + spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-persistenceagent + spec: + template: + spec: + containers: + - name: ml-pipeline-persistenceagent + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-scheduledworkflow + spec: + template: + spec: + containers: + - name: ml-pipeline-scheduledworkflow + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-ui + spec: + template: + spec: + containers: + - name: ml-pipeline-ui + env: + - name: POD_LOG_CONTAINER_NAME + value: step-user-main \ No newline at end of file diff --git a/manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml new file mode 100644 index 000000000000..5707255acdd9 --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-tekton-multi-user/tekton-config.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: feature-flags + namespace: tekton-pipelines + labels: + app.kubernetes.io/instance: default + app.kubernetes.io/part-of: tekton-pipelines +data: + running-in-environment-with-injected-sidecars: "true" diff --git a/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml new file mode 100644 index 000000000000..cb9814e88d3e --- /dev/null +++ b/manifests/kustomize/env/platform-agnostic-tekton/kustomization.yaml @@ -0,0 +1,81 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../third-party/tekton/installs/cluster +- ../../third-party/tekton-custom-task +- ../plain + +# Identifier for application manager to apply ownerReference. +# The ownerReference ensures the resources get garbage collected +# when application is deleted. + +images: +- name: gcr.io/ml-pipeline/api-server + newName: quay.io/aipipeline/apiserver + newTag: 2.0.5 +- name: gcr.io/ml-pipeline/persistenceagent + newName: quay.io/aipipeline/persistenceagent + newTag: 2.0.5 +- name: gcr.io/ml-pipeline/scheduledworkflow + newName: quay.io/aipipeline/scheduledworkflow + newTag: 2.0.5 + +labels: +- includeSelectors: true + pairs: + application-crd-id: kubeflow-pipelines + +patches: +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline + spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-persistenceagent + spec: + template: + spec: + containers: + - name: ml-pipeline-persistenceagent + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-scheduledworkflow + spec: + template: + spec: + containers: + - name: ml-pipeline-scheduledworkflow + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-ui + spec: + template: + spec: + containers: + - name: ml-pipeline-ui + env: + - name: POD_LOG_CONTAINER_NAME + value: step-user-main diff --git a/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml b/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml new file mode 100644 index 000000000000..342bbfe848df --- /dev/null +++ b/manifests/kustomize/env/platform-openshift-pipelines/kustomization.yaml @@ -0,0 +1,78 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../plain +- ../../third-party/openshift-pipelines-custom-task +- ../../third-party/openshift/standalone + +patches: +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline + spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-persistenceagent + spec: + template: + spec: + containers: + - name: ml-pipeline-persistenceagent + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-scheduledworkflow + spec: + template: + spec: + containers: + - name: ml-pipeline-scheduledworkflow + env: + - name: EXECUTIONTYPE + value: PipelineRun +- patch: |- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: ml-pipeline-ui + spec: + template: + spec: + containers: + - name: ml-pipeline-ui + env: + - name: POD_LOG_CONTAINER_NAME + value: step-user-main + + +images: +- name: gcr.io/ml-pipeline/api-server + newName: quay.io/aipipeline/apiserver + newTag: 2.0.0 +- name: gcr.io/ml-pipeline/persistenceagent + newName: quay.io/aipipeline/persistenceagent + newTag: 2.0.0 +- name: gcr.io/ml-pipeline/scheduledworkflow + newName: quay.io/aipipeline/scheduledworkflow + newTag: 2.0.0 + +labels: +- includeSelectors: true + pairs: + application-crd-id: kubeflow-pipelines