diff --git a/, b/, deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/.github/workflows/generate_flyte_manifest.yml b/.github/workflows/generate_flyte_manifest.yml index e0c9c589ab9..b2cdb2f2d6d 100644 --- a/.github/workflows/generate_flyte_manifest.yml +++ b/.github/workflows/generate_flyte_manifest.yml @@ -23,7 +23,6 @@ jobs: VERSION: ${{ github.event.inputs.next-version }} run: | make release_automation - make kustomize make helm - name: Create Pull Request id: cpr @@ -39,17 +38,12 @@ jobs: title: 'Update Flyte components' body: | Updated flyte deployment - - Updated GCP Flyte kustomize generated manifest file - - Updated EKS Flyte kustomize generated manifest file - - Updated Sandbox Flyte kustomize generated manifest file - - Updated TEST Flyte kustomize generated manifest file - Updated GCP Flyte helm generated manifest file - Updated EKS Flyte helm generated manifest file - Updated Sandbox Flyte helm generated manifest file - Updated TEST Flyte helm generated manifest file - Auto-generated by [flyte-bot] labels: | - kustomize helm team-reviewers: | flyte-maintainers diff --git a/.github/workflows/lite-image-manual.yml b/.github/workflows/lite-image-manual.yml deleted file mode 100644 index 8b5831b0e24..00000000000 --- a/.github/workflows/lite-image-manual.yml +++ /dev/null @@ -1,70 +0,0 @@ -name: Manually push - -on: - workflow_dispatch: - inputs: - flyte_version: - description: 'flyte version' - required: true - default: 'latest' - type: string - -jobs: - sandbox-lite-build-dind: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Set flyte version to release - id: set_version - run: | - echo ::set-output name=flyte_version::$(echo ${{ github.event.inputs.flyte_version }}) - - name: Prepare DIND Image Names - id: dind-names - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/${{ github.repository_owner }}/flyte-sandbox-lite - tags: | - ${{ steps.set_version.outputs.flyte_version }} - type=sha,format=long, prefix=dind- - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-single-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-single-buildx - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - - name: Build and push DIND Image - uses: docker/build-push-action@v2 - with: - context: . - platforms: linux/arm64, linux/amd64 - push: true - target: dind - tags: ${{ steps.dind-names.outputs.tags }} - build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" - file: Dockerfile.sandbox-lite - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index dc78d4b5591..5b9a6056a44 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -62,60 +62,3 @@ jobs: tags: ${{ steps.dind-names.outputs.tags }} build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" file: docker/sandbox/Dockerfile - - trigger-sandbox-lite-build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - with: - fetch-depth: "0" - - name: Setup Golang caches - uses: actions/cache@v3 - with: - path: | - /root/.cache/go-build - /root/go/pkg/mod - key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }} - restore-keys: | - ${{ runner.os }}-golang- - - name: Set flyte version to release - id: set_version - run: | - if [ ${{ github.event_name}} = "release" ]; then - echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }}) - else - echo ::set-output name=flyte_version::latest - fi - - name: Prepare DIND Image Names - id: dind-names - uses: docker/metadata-action@v3 - with: - # list of Docker images to use as base name for tags - images: | - ghcr.io/${{ github.repository_owner }}/flyte-sandbox-lite - tags: | - ${{ steps.set_version.outputs.flyte_version }} - type=sha,format=long - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Login to GitHub Container Registry - if: ${{ github.event_name == 'release' }} - uses: docker/login-action@v1 - with: - registry: ghcr.io - username: "${{ secrets.FLYTE_BOT_USERNAME }}" - password: "${{ secrets.FLYTE_BOT_PAT }}" - - name: Build and push DIND Image - uses: docker/build-push-action@v2 - with: - context: . - platforms: linux/arm64, linux/amd64 - push: ${{ github.event_name == 'release' }} - target: dind - tags: ${{ steps.dind-names.outputs.tags }} - build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}" - file: Dockerfile.sandbox-lite diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 831beda79d9..28813360deb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -70,16 +70,6 @@ jobs: conda activate monodocs-env make docs - generate_kustomize: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 - with: - go-version: "1.21" - - name: Kustomize and diff - run: DELTA_CHECK=true make kustomize - generate_helm: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/validate-helm-charts.yaml b/.github/workflows/validate-helm-charts.yaml new file mode 100644 index 00000000000..dc55ad980c8 --- /dev/null +++ b/.github/workflows/validate-helm-charts.yaml @@ -0,0 +1,84 @@ +name: Validate helm charts & manifests + +on: + pull_request: + branches: + - master + paths: + - deployment + - docker/sandbox-bundled/manifests + +jobs: + lint-and-test-charts: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + sparse-checkout: charts + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + check-latest: true + + - name: Set up chart-testing + uses: helm/chart-testing-action@v2 + + - name: Detect charts changed (list-changed) + id: charts-changed + run: | + changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }}) + if [[ -n "$changed" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + fi + + - name: Run chart-testing (lint) + if: steps.charts-changed.outputs.changed == 'true' + run: ct lint --target-branch ${{ github.event.repository.default_branch }} + + - name: Create KinD cluster + if: steps.charts-changed.outputs.changed == 'true' + uses: helm/kind-action@v1 + + - name: Run chart-testing (install) + if: steps.charts-changed.outputs.changed == 'true' + run: ct install --target-branch ${{ github.event.repository.default_branch }} + + validate-manifests: + needs: + - lint-and-test-charts + runs-on: ubuntu-latest + defaults: + run: + shell: bash + strategy: + matrix: + k8s_versions: [ "1.29.2", "1.28.7", "1.27.11" ] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + sparse-checkout: | + deployment + docker/sandbox-bundled/manifests + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Install kubeconform + run: | + curl -L -o kubeconform.tar.gz https://github.com/yannh/kubeconform/releases/download/v0.6.4/kubeconform-linux-amd64.tar.gz + tar -zvxf kubeconform.tar.gz + chmod +x kubeconform + sudo mv kubeconform /usr/local/bin/kubeconform + + - name: Validate manifests + run: | + kubeconform -strict -summary -skip CustomResourceDefinition -ignore-filename-pattern "deployment/stats/prometheus/*" -kubernetes-version ${{ matrix.k8s_versions }} ./deployment ./docker/sandbox-bundled/manifests diff --git a/.gitignore b/.gitignore index a8e78b52d68..301bf266a85 100644 --- a/.gitignore +++ b/.gitignore @@ -20,8 +20,6 @@ __pycache__/ /charts/flyteagent/Chart.lock /charts/flyte-deps/Chart.lock /charts/flyte-core/Chart.lock -/docker/sandbox-lite/storage/charts -/docker/sandbox-lite/storage/Chart.lock **/*.bak- .python-version dist/ diff --git a/CHANGELOG/CHANGELOG-v1.11.0.md b/CHANGELOG/CHANGELOG-v1.11.0.md new file mode 100644 index 00000000000..2c576654897 --- /dev/null +++ b/CHANGELOG/CHANGELOG-v1.11.0.md @@ -0,0 +1,32 @@ +# Flyte v1.11.0 Release Notes + +We're excited to announce the release of Flyte v1.11.0! This version brings a host of improvements, bug fixes, and new features designed to enhance your experience with Flyte. From operational enhancements to documentation updates, this release aims to make Flyte more robust, user-friendly, and feature-rich. + +## Highlights + +- **Agents hit General Availability (GA):** Agents, now in General Availability, are long-running, stateless services that facilitate asynchronous job launches on platforms like Databricks or Snowflake and enable external service calls. They are versatile, supporting implementations in any language through a protobuf interface, enhancing Flyte's flexibility and operational efficiency. +- **Improved Caching:** Support for loading cached sublists with multiple data types has been introduced, eliminating issues related to cache retrieval across varied data formats. +- **Tracing and Observability:** The introduction of opentelemetry BlobstoreClientTracer in flyteadmin enhances observability, allowing for better monitoring and troubleshooting. +- **Security Enhancements:** Added securityContext configuration to Flyte-core charts, strengthening the security posture of Flyte deployments. +- **Documentation Overhaul:** Continuous improvements and updates have been made to the documentation, fixing broken links and updating content for better clarity and usability. +- **Operational Improvements:** This release introduces enhancements such as adding a service account for V1 Ray Jobs, caching console assets in a single binary, and conditional mounting of secrets to improve the operational efficiency of Flyte. Additionally, we are removing `kustomize` from our deployment process to simplify the configuration and management of Flyte instances, making it easier for users to maintain and streamline their deployment workflows. + + +## Bug Fixes + +- **Fixed Literal in Launchplan:** Added fixed_literal to the launchplan template, addressing issues with hardcoded values in workflows. +- **Corrected Metadata and Resources:** Fixes have been applied to correct IsParent metadata in ArrayNode eventing and to address invalid "resources" scope issues in deployment configurations. +- **Enhanced Stability and Performance:** Numerous bug fixes have been implemented to address stability and performance issues, including fixes for data catalog errors, yaml comment errors in pod template examples, and more. + +## Documentation and Guides + +- **Comprehensive Guides:** New guides and documentation updates have been added, including a ChatGPT Agent Setup guide and an Airflow migration guide. Improvements in documentation for developing agents have been integrated into the broader enhancements for this release. +- **Updated Troubleshooting and Configuration Docs:** New troubleshooting guides for spark task execution and updates to deployment configuration documents enhance the knowledge base for Flyte users. + +## Contributors + +We extend our deepest gratitude to all the contributors who made this release possible. Special shoutouts to @neilisaur, @lowc1012, @MortalHappiness, @novahow, and @pryce-turner for making their first contributions! + +**For a full list of changes, enhancements, and bug fixes, visit our [changelog](https://github.com/flyteorg/flyte/compare/v1.10.7...v1.11.0).** + +Thank you for your continued support of Flyte. We look forward to hearing your feedback on this release! diff --git a/Dockerfile.sandbox-lite b/Dockerfile.sandbox-lite deleted file mode 100644 index 095c83b6e1e..00000000000 --- a/Dockerfile.sandbox-lite +++ /dev/null @@ -1,115 +0,0 @@ -# syntax=docker/dockerfile:1.3 - - -ARG FLYTE_VERSION="latest" -FROM ghcr.io/flyteorg/flyteconsole-release:${FLYTE_VERSION} AS flyteconsole - -FROM --platform=${BUILDPLATFORM} golang:1.19.0-alpine3.16 AS go_builder - -ARG TARGETARCH -ENV GOARCH=${TARGETARCH} -ENV GOOS=linux - -# Install dependencies -RUN apk add --no-cache build-base - -COPY go.mod go.sum /app/flyte/ -WORKDIR /app/flyte - -COPY datacatalog datacatalog -COPY flyteadmin flyteadmin -COPY flytecopilot flytecopilot -COPY flyteidl flyteidl -COPY flyteplugins flyteplugins -COPY flytepropeller flytepropeller -COPY flytestdlib flytestdlib - -RUN go mod download - -COPY --from=flyteconsole /app/ cmd/single/dist - -COPY cmd/ /app/flyte/cmd/ -RUN --mount=type=cache,target=/root/.cache/go-build --mount=type=cache,target=/root/go/pkg/mod go build -tags console -v -o /flyte cmd/main.go - -FROM alpine:3.15 AS base - -# Install dependencies -RUN apk add --no-cache openssl - -# Make directory to store artifacts -RUN mkdir -p /flyteorg/bin /flyteorg/share - -# Install k3s -ARG K3S_VERSION="v1.21.1%2Bk3s1" -ARG TARGETARCH - -RUN case $TARGETARCH in \ - amd64) export SUFFIX=;; \ - arm64) export SUFFIX=-arm64;; \ - aarch64) export SUFFIX=-arm64;; \ - # TODO: Check if we need to add case fail - esac; \ - wget -q -O /flyteorg/bin/k3s https://github.com/k3s-io/k3s/releases/download/${K3S_VERSION}/k3s${SUFFIX} \ - && chmod +x /flyteorg/bin/k3s - -# Install Helm -ARG HELM_VERSION="v3.6.3" - -RUN wget -q -O /flyteorg/bin/get_helm.sh https://raw.githubusercontent.com/helm/helm/${HELM_VERSION}/scripts/get-helm-3 && \ - chmod 700 /flyteorg/bin/get_helm.sh && \ - sh /flyteorg/bin/get_helm.sh --version ${HELM_VERSION} && \ - mv /usr/local/bin/helm /flyteorg/bin/helm && \ - rm /flyteorg/bin/get_helm.sh - -# Install flytectl -RUN wget -q -O - https://raw.githubusercontent.com/flyteorg/flytectl/master/install.sh | BINDIR=/flyteorg/bin sh -s - -# Install buildkit-cli-for-kubectl -COPY --from=go_builder /flyte /flyteorg/bin/ - -# Copy flyte chart -COPY charts/flyte-deps/ /flyteorg/share/flyte-deps - -# Copy scripts -COPY docker/sandbox/kubectl docker/sandbox/cgroup-v2-hack.sh /flyteorg/bin/ - -# Copy Flyte config -COPY flyte.yaml /flyteorg/share/flyte.yaml - -FROM docker:20.10.14-dind-alpine3.15 AS dind - -# Install dependencies -RUN apk add --no-cache bash git make tini curl jq - -# Copy artifacts from base -COPY --from=base /flyteorg/ /flyteorg/ - -# Copy entrypoints -COPY docker/sandbox-lite/flyte-entrypoint-dind.sh /flyteorg/bin/flyte-entrypoint.sh - -# Copy cluster resource templates -COPY docker/sandbox-lite/templates/ /etc/flyte/clusterresource/templates/ - -ENV FLYTE_VERSION "${FLYTE_VERSION}" - -ARG FLYTE_TEST="release" -ENV FLYTE_TEST "${FLYTE_TEST}" -ENV FLYTE_DEV "False" - -RUN addgroup -S docker - -# Update PATH variable -ENV PATH "/flyteorg/bin:${PATH}" -ENV POD_NAMESPACE "flyte" - -# Declare volumes for k3s -VOLUME /var/lib/kubelet -VOLUME /var/lib/rancher/k3s -VOLUME /var/lib/cni -VOLUME /var/log - -# Expose Flyte ports -# 30080 for console, 30081 for gRPC, 30082 for k8s dashboard, 30084 for minio api, 30088 for minio console -EXPOSE 30080 30081 30082 30084 30088 30089 - -ENTRYPOINT ["tini", "flyte-entrypoint.sh"] diff --git a/Makefile b/Makefile index 64af8207879..3f351392894 100644 --- a/Makefile +++ b/Makefile @@ -30,10 +30,6 @@ linux_compile: cmd/single/dist update_boilerplate: @boilerplate/update.sh -.PHONY: kustomize -kustomize: - KUSTOMIZE_VERSION=3.9.2 bash script/generate_kustomize.sh - .PHONY: helm helm: ## Generate K8s Manifest from Helm Charts. bash script/generate_helm.sh diff --git a/charts/flyte-binary/README.md b/charts/flyte-binary/README.md index 78c301f2b36..fcd66f3186b 100644 --- a/charts/flyte-binary/README.md +++ b/charts/flyte-binary/README.md @@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment | configuration.auth.oidc.clientId | string | `""` | | | configuration.auth.oidc.clientSecret | string | `""` | | | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` | | -| configuration.co-pilot.image.tag | string | `"v1.11.0-b1"` | | +| configuration.co-pilot.image.tag | string | `"v1.11.0"` | | | configuration.database.dbname | string | `"flyte"` | | | configuration.database.host | string | `"127.0.0.1"` | | | configuration.database.options | string | `"sslmode=disable"` | | diff --git a/charts/flyte-binary/eks-production.yaml b/charts/flyte-binary/eks-production.yaml index b73da0278a6..987269a5aa5 100644 --- a/charts/flyte-binary/eks-production.yaml +++ b/charts/flyte-binary/eks-production.yaml @@ -68,6 +68,7 @@ configuration: - container - sidecar - K8S-ARRAY #used for MapTasks + - agent-service default-for-task-types: - container: container - container_array: K8S-ARRAY diff --git a/charts/flyte-binary/eks-starter.yaml b/charts/flyte-binary/eks-starter.yaml index 51fd15f112b..20257bf0723 100644 --- a/charts/flyte-binary/eks-starter.yaml +++ b/charts/flyte-binary/eks-starter.yaml @@ -68,6 +68,7 @@ configuration: - container - sidecar - K8S-ARRAY #used for MapTasks + - agent-service default-for-task-types: - container: container - container_array: K8S-ARRAY diff --git a/charts/flyte-binary/values.yaml b/charts/flyte-binary/values.yaml index 61bb73e72d5..e84b1c54f49 100644 --- a/charts/flyte-binary/values.yaml +++ b/charts/flyte-binary/values.yaml @@ -159,7 +159,7 @@ configuration: # repository CoPilot sidecar image repository repository: cr.flyte.org/flyteorg/flytecopilot # FLYTECOPILOT_IMAGE # tag CoPilot sidecar image tag - tag: v1.11.0-b1 # FLYTECOPILOT_TAG + tag: v1.11.0 # FLYTECOPILOT_TAG # agentService Flyte Agent configuration agentService: defaultAgent: diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index 5f0e3dcbb52..5a18f902cba 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -95,16 +95,16 @@ helm install gateway bitnami/contour -n flyte | configmap.clusters.clusterConfigs | list | `[]` | | | configmap.clusters.labelClusterMap | object | `{}` | | | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). | | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"heartbeat-grace-period-multiplier":3,"max-reservation-heartbeat":"30s","metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | | configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. | -| configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | -| configmap.enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) | -| configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins | +| configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | +| configmap.enabled_plugins.tasks.task-plugins | object | `{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}` | Plugins configuration, [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#TaskPluginConfig) | +| configmap.enabled_plugins.tasks.task-plugins.enabled-plugins | list | `["container","sidecar","k8s-array","agent-service"]` | [Enabled Plugins](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/config#Config). Enable sagemaker*, athena if you install the backend plugins | | configmap.k8s | object | `{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}}}` | Kubernetes specific Flyte configuration | | configmap.k8s.plugins.k8s | object | `{"default-cpus":"100m","default-env-vars":[],"default-memory":"100Mi"}` | Configuration section for all K8s specific plugins [Configuration structure](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config) | | configmap.remoteData.remoteData.region | string | `"us-east-1"` | | @@ -130,7 +130,7 @@ helm install gateway bitnami/contour -n flyte | datacatalog.extraArgs | object | `{}` | Appends extra command line arguments to the main command | | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| datacatalog.image.tag | string | `"v1.11.0-b1"` | Docker image tag | +| datacatalog.image.tag | string | `"v1.11.0"` | Docker image tag | | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | datacatalog.podEnv | object | `{}` | Additional Datacatalog container environment variables | @@ -165,7 +165,7 @@ helm install gateway bitnami/contour -n flyte | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command | | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | | | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyteadmin.image.tag | string | `"v1.11.0-b1"` | | +| flyteadmin.image.tag | string | `"v1.11.0"` | | | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create | | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | @@ -202,7 +202,7 @@ helm install gateway bitnami/contour -n flyte | flyteconsole.ga.tracking_id | string | `"G-0QW4DJWJ20"` | | | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | | | flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyteconsole.image.tag | string | `"v1.11.0"` | | +| flyteconsole.image.tag | string | `"v1.10.3"` | | | flyteconsole.imagePullSecrets | list | `[]` | ImagePullSecrets to assign to the Flyteconsole deployment | | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | @@ -226,7 +226,7 @@ helm install gateway bitnami/contour -n flyte | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command | | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | | | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flytepropeller.image.tag | string | `"v1.11.0-b1"` | | +| flytepropeller.image.tag | string | `"v1.11.0"` | | | flytepropeller.manager | bool | `false` | | | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | @@ -256,7 +256,7 @@ helm install gateway bitnami/contour -n flyte | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | -| flytescheduler.image.tag | string | `"v1.11.0-b1"` | Docker image tag | +| flytescheduler.image.tag | string | `"v1.11.0"` | Docker image tag | | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | | flytescheduler.podEnv | object | `{}` | Additional Flytescheduler container environment variables | diff --git a/charts/flyte-core/templates/common/ingress.yaml b/charts/flyte-core/templates/common/ingress.yaml index 69f7c4ae070..d812f2abe3a 100644 --- a/charts/flyte-core/templates/common/ingress.yaml +++ b/charts/flyte-core/templates/common/ingress.yaml @@ -7,154 +7,90 @@ - path: /flyteidl.service.SignalService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.SignalService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.AdminService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.AdminService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.DataProxyService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.DataProxyService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.AuthMetadataService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.AuthMetadataService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.IdentityService pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /flyteidl.service.IdentityService/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /grpc.health.v1.Health pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} - path: /grpc.health.v1.Health/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: {{ $grpcPort }} -{{- else }} - serviceName: flyteadmin - servicePort: {{ $grpcPort }} -{{- end }} {{- end }} {{- if .Values.common.ingress.enabled }} -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} apiVersion: networking.k8s.io/v1 -{{- else }} -apiVersion: networking.k8s.io/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ template "flyte.name" . }} @@ -171,288 +107,173 @@ spec: - path: /* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: ssl-redirect port: name: use-annotation -{{- else }} - serviceName: ssl-redirect - servicePort: use-annotation -{{- end }} {{- end }} # This is useful only for frontend development {{- if .Values.common.ingress.webpackHMR }} - path: /__webpack_hmr pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteconsole port: number: 80 -{{- else }} - serviceName: flyteconsole - servicePort: 80 -{{- end }} {{- end }} # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml - path: /console pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteconsole port: number: 80 -{{- else }} - serviceName: flyteconsole - servicePort: 80 -{{- end }} - path: /console/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteconsole port: number: 80 -{{- else }} - serviceName: flyteconsole - servicePort: 80 -{{- end }} - path: /api pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /api/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /healthcheck pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /v1/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} {{- if .Values.deployRedoc }} # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: redoc port: number: 87 -{{- else }} - serviceName: redoc - servicePort: 87 -{{- end }} # Port 87 in FlyteAdmin maps to the redoc container. - path: /openapi/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: redoc port: number: 87 -{{- else }} - serviceName: redoc - servicePort: 87 -{{- end }} {{- end }} - path: /.well-known pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /.well-known/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /login pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /login/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /logout pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /logout/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /callback pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /callback/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /me pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /config pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /config/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /oauth2 pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} - path: /oauth2/* pathType: ImplementationSpecific backend: -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} service: name: flyteadmin port: number: 80 -{{- else }} - serviceName: flyteadmin - servicePort: 80 -{{- end }} {{- if not .Values.common.ingress.separateGrpcIngress }} {{- include "grpcRoutes" . | nindent 10 -}} {{- end }} @@ -469,11 +290,7 @@ spec: # Certain ingress controllers like nginx cannot serve HTTP 1 and GRPC with a single ingress because GRPC can only # enabled on the ingress object, not on backend services (GRPC annotation is set on the ingress, not on the services). --- -{{- if $.Capabilities.APIVersions.Has "networking.k8s.io/v1/Ingress" }} apiVersion: networking.k8s.io/v1 -{{- else }} -apiVersion: networking.k8s.io/v1beta1 -{{- end }} kind: Ingress metadata: name: {{ template "flyte.name" . }}-grpc diff --git a/charts/flyte-core/values-eks.yaml b/charts/flyte-core/values-eks.yaml index b05cd1f61cc..1dbaded1c7e 100644 --- a/charts/flyte-core/values-eks.yaml +++ b/charts/flyte-core/values-eks.yaml @@ -277,6 +277,7 @@ configmap: - container - sidecar - k8s-array + - agent-service # - sagemaker_hyperparameter_tuning # - sagemaker_custom_training # - sagemaker_training diff --git a/charts/flyte-core/values-gcp.yaml b/charts/flyte-core/values-gcp.yaml index 4051fb23b92..181772c0c6e 100644 --- a/charts/flyte-core/values-gcp.yaml +++ b/charts/flyte-core/values-gcp.yaml @@ -292,6 +292,7 @@ configmap: - container - sidecar - k8s-array + - agent-service default-for-task-types: container: container sidecar: sidecar diff --git a/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml b/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml index 380db08be71..edfd9478bcd 100644 --- a/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml +++ b/charts/flyte-core/values-keycloak-idp-flyteclients-without-browser.yaml @@ -669,6 +669,7 @@ configmap: - container - sidecar - k8s-array + - agent-service default-for-task-types: container: container sidecar: sidecar diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 32fb5c58278..0319b3bacae 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -16,7 +16,7 @@ flyteadmin: image: # -- Docker image for Flyteadmin deployment repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE - tag: v1.11.0-b1 # FLYTEADMIN_TAG + tag: v1.11.0 # FLYTEADMIN_TAG pullPolicy: IfNotPresent # -- Additional flyteadmin container environment variables # @@ -142,7 +142,7 @@ flytescheduler: # -- Docker image for Flytescheduler deployment repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE # -- Docker image tag - tag: v1.11.0-b1 # FLYTESCHEDULER_TAG + tag: v1.11.0 # FLYTESCHEDULER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytescheduler deployment @@ -208,7 +208,7 @@ datacatalog: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE # -- Docker image tag - tag: v1.11.0-b1 # DATACATALOG_TAG + tag: v1.11.0 # DATACATALOG_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment @@ -296,7 +296,7 @@ flytepropeller: image: # -- Docker image for Flytepropeller deployment repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE - tag: v1.11.0-b1 # FLYTEPROPELLER_TAG + tag: v1.11.0 # FLYTEPROPELLER_TAG pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment resources: @@ -379,7 +379,7 @@ flyteconsole: image: # -- Docker image for Flyteconsole deployment repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE - tag: v1.11.0 # FLYTECONSOLE_TAG + tag: v1.10.3 # FLYTECONSOLE_TAG pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment resources: @@ -731,7 +731,7 @@ configmap: # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 # FLYTECOPILOT_IMAGE + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0 # FLYTECOPILOT_IMAGE start-timeout: 30s # -- Core propeller configuration @@ -792,8 +792,7 @@ configmap: - container - sidecar - k8s-array - # -- Uncomment to enable agent service - # - agent-service + - agent-service default-for-task-types: container: container sidecar: sidecar diff --git a/charts/flyte-deps/Chart.yaml b/charts/flyte-deps/Chart.yaml index fbd72a0bc23..ea84c798e77 100644 --- a/charts/flyte-deps/Chart.yaml +++ b/charts/flyte-deps/Chart.yaml @@ -11,7 +11,7 @@ dependencies: - name: spark-operator alias: sparkoperator version: 1.1.15 - repository: https://googlecloudplatform.github.io/spark-on-k8s-operator + repository: https://kubeflow.github.io/spark-operator condition: sparkoperator.enabled - name: dask-kubernetes-operator alias: daskoperator diff --git a/charts/flyte-deps/README.md b/charts/flyte-deps/README.md index 6a5eb13e4f6..014e19c2513 100644 --- a/charts/flyte-deps/README.md +++ b/charts/flyte-deps/README.md @@ -9,8 +9,8 @@ A Helm chart for Flyte dependency | Repository | Name | Version | |------------|------|---------| | https://charts.bitnami.com/bitnami | contour | 7.10.1 | -| https://googlecloudplatform.github.io/spark-on-k8s-operator | sparkoperator(spark-operator) | 1.1.15 | | https://helm.dask.org | daskoperator(dask-kubernetes-operator) | 2022.12.0 | +| https://kubeflow.github.io/spark-operator | sparkoperator(spark-operator) | 1.1.15 | | https://kubernetes.github.io/dashboard/ | kubernetes-dashboard | 4.0.2 | ### SANDBOX INSTALLATION: diff --git a/charts/flyte/Chart.yaml b/charts/flyte/Chart.yaml index 0fcfb6679be..daed727185a 100644 --- a/charts/flyte/Chart.yaml +++ b/charts/flyte/Chart.yaml @@ -15,7 +15,7 @@ dependencies: - name: spark-operator alias: sparkoperator version: 1.1.15 - repository: https://googlecloudplatform.github.io/spark-on-k8s-operator + repository: https://kubeflow.github.io/spark-operator condition: sparkoperator.enabled - name: dask-kubernetes-operator alias: daskoperator diff --git a/charts/flyte/README.md b/charts/flyte/README.md index f2ac67bdc21..5ead2db7308 100644 --- a/charts/flyte/README.md +++ b/charts/flyte/README.md @@ -10,8 +10,8 @@ A Helm chart for Flyte Sandbox |------------|------|---------| | file://../flyte-core | flyte(flyte-core) | v0.1.10 | | https://charts.bitnami.com/bitnami | contour | 7.10.1 | -| https://googlecloudplatform.github.io/spark-on-k8s-operator | sparkoperator(spark-operator) | 1.1.15 | | https://helm.dask.org | daskoperator(dask-kubernetes-operator) | 2022.12.0 | +| https://kubeflow.github.io/spark-operator | sparkoperator(spark-operator) | 1.1.15 | | https://kubernetes.github.io/dashboard/ | kubernetes-dashboard | 4.0.2 | **NOTE:** Flyte sandbox helm chart is deprecated, From now follow the sandbox [docs](https://docs.flyte.org/en/latest/deployment/sandbox.html) for installing it on cloud @@ -71,7 +71,7 @@ helm upgrade -f values-sandbox.yaml flyte . | contour.tolerations | list | `[]` | tolerations for Contour deployment | | daskoperator | object | `{"enabled":false}` | Optional: Dask Plugin using the Dask Operator | | daskoperator.enabled | bool | `false` | - enable or disable the dask operator deployment installation | -| flyte | object | `{"cluster_resource_manager":{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]},"common":{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"host":"","separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":true}},"configmap":{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}},"datacatalog":{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v1.11.0-b1"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"db":{"admin":{"database":{"dbname":"flyteadmin","host":"postgres","port":5432,"username":"postgres"}},"datacatalog":{"database":{"dbname":"datacatalog","host":"postgres","port":5432,"username":"postgres"}}},"deployRedoc":true,"flyteadmin":{"additionalVolumeMounts":[],"additionalVolumes":[],"affinity":{},"configPath":"/etc/flyte/config/*.yaml","env":[],"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteadmin","tag":"v1.11.0-b1"},"initialProjects":["flytesnacks","flytetester","flyteexamples"],"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flyteconsole":{"affinity":{},"ga":{"enabled":true,"tracking_id":"G-0QW4DJWJ20"},"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteconsole","tag":"v1.11.0"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"275Mi"},"requests":{"cpu":"10m","memory":"250Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]},"flytepropeller":{"affinity":{},"cacheSizeMbs":0,"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytepropeller","tag":"v1.11.0-b1"},"manager":false,"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flytescheduler":{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v1.11.0-b1"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"storage":{"bucketName":"my-s3-bucket","custom":{},"gcs":null,"s3":{"region":"us-east-1"},"type":"sandbox"},"webhook":{"enabled":true,"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]}},"workflow_notifications":{"config":{},"enabled":false},"workflow_scheduler":{"enabled":true,"type":"native"}}` | ------------------------------------------------------------------- Core System settings This section consists of Core components of Flyte and their deployment settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and Flyteconsole | +| flyte | object | `{"cluster_resource_manager":{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]},"common":{"databaseSecret":{"name":"","secretManifest":{}},"flyteNamespaceTemplate":{"enabled":false},"ingress":{"albSSLRedirect":false,"annotations":{"nginx.ingress.kubernetes.io/app-root":"/console"},"enabled":true,"host":"","separateGrpcIngress":false,"separateGrpcIngressAnnotations":{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"},"tls":{"enabled":false},"webpackHMR":true}},"configmap":{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}},"datacatalog":{"affinity":{},"configPath":"/etc/datacatalog/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/datacatalog","tag":"v1.11.0"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"NodePort"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"db":{"admin":{"database":{"dbname":"flyteadmin","host":"postgres","port":5432,"username":"postgres"}},"datacatalog":{"database":{"dbname":"datacatalog","host":"postgres","port":5432,"username":"postgres"}}},"deployRedoc":true,"flyteadmin":{"additionalVolumeMounts":[],"additionalVolumes":[],"affinity":{},"configPath":"/etc/flyte/config/*.yaml","env":[],"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteadmin","tag":"v1.11.0"},"initialProjects":["flytesnacks","flytetester","flyteexamples"],"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"loadBalancerSourceRanges":[],"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flyteconsole":{"affinity":{},"ga":{"enabled":true,"tracking_id":"G-0QW4DJWJ20"},"image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flyteconsole","tag":"v1.10.3"},"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"500m","memory":"275Mi"},"requests":{"cpu":"10m","memory":"250Mi"}},"service":{"annotations":{},"type":"ClusterIP"},"tolerations":[]},"flytepropeller":{"affinity":{},"cacheSizeMbs":0,"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytepropeller","tag":"v1.11.0"},"manager":false,"nodeSelector":{},"podAnnotations":{},"replicaCount":1,"resources":{"limits":{"cpu":"200m","ephemeral-storage":"100Mi","memory":"200Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"flytescheduler":{"affinity":{},"configPath":"/etc/flyte/config/*.yaml","image":{"pullPolicy":"IfNotPresent","repository":"cr.flyte.org/flyteorg/flytescheduler","tag":"v1.11.0"},"nodeSelector":{},"podAnnotations":{},"resources":{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}},"secrets":{},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]},"tolerations":[]},"storage":{"bucketName":"my-s3-bucket","custom":{},"gcs":null,"s3":{"region":"us-east-1"},"type":"sandbox"},"webhook":{"enabled":true,"service":{"annotations":{"projectcontour.io/upstream-protocol.h2c":"grpc"},"type":"ClusterIP"},"serviceAccount":{"annotations":{},"create":true,"imagePullSecrets":[]}},"workflow_notifications":{"config":{},"enabled":false},"workflow_scheduler":{"enabled":true,"type":"native"}}` | ------------------------------------------------------------------- Core System settings This section consists of Core components of Flyte and their deployment settings. This includes FlyteAdmin service, Datacatalog, FlytePropeller and Flyteconsole | | flyte.cluster_resource_manager | object | `{"config":{"cluster_resources":{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}},"enabled":true,"service_account_name":"flyteadmin","templates":[{"key":"aa_namespace","value":"apiVersion: v1\nkind: Namespace\nmetadata:\n name: {{ namespace }}\nspec:\n finalizers:\n - kubernetes\n"},{"key":"ab_project_resource_quota","value":"apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }}\nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }}\n limits.memory: {{ projectQuotaMemory }}\n"}]}` | Configuration for the Cluster resource manager component. This is an optional component, that enables automatic cluster configuration. This is useful to set default quotas, manage namespaces etc that map to a project/domain | | flyte.cluster_resource_manager.config.cluster_resources | object | `{"customData":[{"production":[{"projectQuotaCpu":{"value":"5"}},{"projectQuotaMemory":{"value":"4000Mi"}}]},{"staging":[{"projectQuotaCpu":{"value":"2"}},{"projectQuotaMemory":{"value":"3000Mi"}}]},{"development":[{"projectQuotaCpu":{"value":"4"}},{"projectQuotaMemory":{"value":"3000Mi"}}]}],"refresh":"5m","refreshInterval":"5m","standaloneDeployment":false,"templatePath":"/etc/flyte/clusterresource/templates"}` | ClusterResource parameters Refer to the [structure](https://pkg.go.dev/github.com/lyft/flyteadmin@v0.3.37/pkg/runtime/interfaces#ClusterResourceConfig) to customize. | | flyte.cluster_resource_manager.config.cluster_resources.standaloneDeployment | bool | `false` | Starts the cluster resource manager in standalone mode with requisite auth credentials to call flyteadmin service endpoints | @@ -91,15 +91,15 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.common.ingress.separateGrpcIngressAnnotations | object | `{"nginx.ingress.kubernetes.io/backend-protocol":"GRPC"}` | - Extra Ingress annotations applied only to the GRPC ingress. Only makes sense if `separateGrpcIngress` is enabled. | | flyte.common.ingress.tls | object | `{"enabled":false}` | - TLS Settings | | flyte.common.ingress.webpackHMR | bool | `true` | - Enable or disable HMR route to flyteconsole. This is useful only for frontend development. | -| flyte.configmap | object | `{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ----------------------------------------------------------------- CONFIGMAPS SETTINGS | +| flyte.configmap | object | `{"adminServer":{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}},"catalog":{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}},"console":{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"},"copilot":{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0","name":"flyte-copilot-","start-timeout":"30s"}}}},"core":{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}},"datacatalogServer":{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}},"domain":{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]},"enabled_plugins":{"tasks":{"task-plugins":{"default-for-task-types":{"bigquery_query_job_task":"agent-service","container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array","agent-service"]}}},"k8s":{"plugins":{"k8s":{"default-cpus":"100m","default-env-vars":[{"FLYTE_AWS_ENDPOINT":"http://minio.flyte:9000"},{"FLYTE_AWS_ACCESS_KEY_ID":"minio"},{"FLYTE_AWS_SECRET_ACCESS_KEY":"miniostorage"}],"default-memory":"200Mi"}}},"logger":{"logger":{"level":5,"show-source":true}},"remoteData":{"remoteData":{"region":"us-east-1","scheme":"local","signedUrls":{"durationMinutes":3}}},"resource_manager":{"propeller":{"resourcemanager":{"redis":null,"type":"noop"}}},"task_logs":{"plugins":{"logs":{"cloudwatch-enabled":false,"kubernetes-enabled":true,"kubernetes-template-uri":"http://localhost:30082/#/log/{{ \"{{\" }} .namespace {{ \"}}\" }}/{{ \"{{\" }} .podName {{ \"}}\" }}/pod?namespace={{ \"{{\" }} .namespace {{ \"}}\" }}"}}},"task_resource_defaults":{"task_resources":{"defaults":{"cpu":"100m","memory":"200Mi","storage":"5Mi"},"limits":{"cpu":2,"gpu":1,"memory":"1Gi","storage":"20Mi"}}}}` | ----------------------------------------------------------------- CONFIGMAPS SETTINGS | | flyte.configmap.adminServer | object | `{"auth":{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}},"flyteadmin":{"eventVersion":2,"metadataStoragePrefix":["metadata","admin"],"metricsScope":"flyte:","profilerPort":10254,"roleNameKey":"iam.amazonaws.com/role","testing":{"host":"http://flyteadmin"}},"server":{"grpcPort":8089,"httpPort":8088,"security":{"allowCors":true,"allowedHeaders":["Content-Type","flyte-authorization"],"allowedOrigins":["*"],"secure":false,"useAuth":false}}}` | FlyteAdmin server configuration | | flyte.configmap.adminServer.auth | object | `{"appAuth":{"thirdPartyConfig":{"flyteClient":{"clientId":"flytectl","redirectUri":"http://localhost:53593/callback","scopes":["offline","all"]}}},"authorizedUris":["https://localhost:30081","http://flyteadmin:80","http://flyteadmin.flyte.svc.cluster.local:80"],"userAuth":{"openId":{"baseUrl":"https://accounts.google.com","clientId":"657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com","scopes":["profile","openid"]}}}` | Authentication configuration | | flyte.configmap.adminServer.server.security.secure | bool | `false` | Controls whether to serve requests over SSL/TLS. | | flyte.configmap.adminServer.server.security.useAuth | bool | `false` | Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. | | flyte.configmap.catalog | object | `{"catalog-cache":{"endpoint":"datacatalog:89","insecure":true,"type":"datacatalog"}}` | Catalog Client configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/catalog#Config) Additional advanced Catalog configuration [here](https://pkg.go.dev/github.com/lyft/flyteplugins/go/tasks/pluginmachinery/catalog#Config) | | flyte.configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | -| flyte.configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | -| flyte.configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | +| flyte.configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | +| flyte.configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | | flyte.configmap.core | object | `{"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | flyte.configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | flyte.configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | @@ -120,7 +120,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.datacatalog.configPath | string | `"/etc/datacatalog/config/*.yaml"` | Default regex string for searching configuration files | | flyte.datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment | -| flyte.datacatalog.image.tag | string | `"v1.11.0-b1"` | Docker image tag | +| flyte.datacatalog.image.tag | string | `"v1.11.0"` | Docker image tag | | flyte.datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment | | flyte.datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods | | flyte.datacatalog.replicaCount | int | `1` | Replicas count for Datacatalog deployment | @@ -136,7 +136,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flyteadmin.env | list | `[]` | Additional flyteadmin container environment variables e.g. SendGrid's API key - name: SENDGRID_API_KEY value: "" e.g. secret environment variable (you can combine it with .additionalVolumes): - name: SENDGRID_API_KEY valueFrom: secretKeyRef: name: sendgrid-secret key: api_key | | flyte.flyteadmin.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment | -| flyte.flyteadmin.image.tag | string | `"v1.11.0-b1"` | Docker image tag | +| flyte.flyteadmin.image.tag | string | `"v1.11.0"` | Docker image tag | | flyte.flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create | | flyte.flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment | | flyte.flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods | @@ -151,7 +151,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flyteconsole.affinity | object | `{}` | affinity for Flyteconsole deployment | | flyte.flyteconsole.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment | -| flyte.flyteconsole.image.tag | string | `"v1.11.0"` | Docker image tag | +| flyte.flyteconsole.image.tag | string | `"v1.10.3"` | Docker image tag | | flyte.flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment | | flyte.flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods | | flyte.flyteconsole.replicaCount | int | `1` | Replicas count for Flyteconsole deployment | @@ -162,7 +162,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flytepropeller.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyte.flytepropeller.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment | -| flyte.flytepropeller.image.tag | string | `"v1.11.0-b1"` | Docker image tag | +| flyte.flytepropeller.image.tag | string | `"v1.11.0"` | Docker image tag | | flyte.flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment | | flyte.flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods | | flyte.flytepropeller.replicaCount | int | `1` | Replicas count for Flytepropeller deployment | @@ -176,7 +176,7 @@ helm upgrade -f values-sandbox.yaml flyte . | flyte.flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files | | flyte.flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | flyte.flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment | -| flyte.flytescheduler.image.tag | string | `"v1.11.0-b1"` | Docker image tag | +| flyte.flytescheduler.image.tag | string | `"v1.11.0"` | Docker image tag | | flyte.flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment | | flyte.flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods | | flyte.flytescheduler.resources | object | `{"limits":{"cpu":"250m","ephemeral-storage":"100Mi","memory":"500Mi"},"requests":{"cpu":"10m","ephemeral-storage":"50Mi","memory":"50Mi"}}` | Default resources requests and limits for Flytescheduler deployment | diff --git a/charts/flyte/values.yaml b/charts/flyte/values.yaml index 50a1fc54025..f0d2f7c75a5 100755 --- a/charts/flyte/values.yaml +++ b/charts/flyte/values.yaml @@ -16,7 +16,7 @@ flyte: # -- Docker image for Flyteadmin deployment repository: cr.flyte.org/flyteorg/flyteadmin # FLYTEADMIN_IMAGE # -- Docker image tag - tag: v1.11.0-b1 # FLYTEADMIN_TAG + tag: v1.11.0 # FLYTEADMIN_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Additional flyteadmin container environment variables @@ -84,7 +84,7 @@ flyte: # -- Docker image for Flytescheduler deployment repository: cr.flyte.org/flyteorg/flytescheduler # FLYTESCHEDULER_IMAGE # -- Docker image tag - tag: v1.11.0-b1 # FLYTESCHEDULER_TAG + tag: v1.11.0 # FLYTESCHEDULER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytescheduler deployment @@ -129,7 +129,7 @@ flyte: # -- Docker image for Datacatalog deployment repository: cr.flyte.org/flyteorg/datacatalog # DATACATALOG_IMAGE # -- Docker image tag - tag: v1.11.0-b1 # DATACATALOG_TAG + tag: v1.11.0 # DATACATALOG_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Datacatalog deployment @@ -178,7 +178,7 @@ flyte: # -- Docker image for Flytepropeller deployment repository: cr.flyte.org/flyteorg/flytepropeller # FLYTEPROPELLER_IMAGE # -- Docker image tag - tag: v1.11.0-b1 # FLYTEPROPELLER_TAG + tag: v1.11.0 # FLYTEPROPELLER_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flytepropeller deployment @@ -223,7 +223,7 @@ flyte: # -- Docker image for Flyteconsole deployment repository: cr.flyte.org/flyteorg/flyteconsole # FLYTECONSOLE_IMAGE # -- Docker image tag - tag: v1.11.0 # FLYTECONSOLE_TAG + tag: v1.10.3 # FLYTECONSOLE_TAG # -- Docker image pull policy pullPolicy: IfNotPresent # -- Default resources requests and limits for Flyteconsole deployment @@ -471,7 +471,7 @@ flyte: # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) co-pilot: name: flyte-copilot- - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 # FLYTECOPILOT_IMAGE + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0 # FLYTECOPILOT_IMAGE start-timeout: 30s # -- Core propeller configuration diff --git a/charts/flyteagent/README.md b/charts/flyteagent/README.md index f889c095bd4..d8627166733 100644 --- a/charts/flyteagent/README.md +++ b/charts/flyteagent/README.md @@ -20,7 +20,7 @@ A Helm chart for Flyte agent | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy | | image.repository | string | `"ghcr.io/flyteorg/flyteagent"` | Docker image for flyteagent deployment | -| image.tag | string | `"1.10.7"` | Docker image tag | +| image.tag | string | `"1.10.8b4"` | Docker image tag | | nameOverride | string | `""` | | | nodeSelector | object | `{}` | nodeSelector for flyteagent deployment | | podAnnotations | object | `{}` | Annotations for flyteagent pods | @@ -30,6 +30,7 @@ A Helm chart for Flyte agent | ports.containerPort | int | `8000` | | | ports.name | string | `"agent-grpc"` | | | priorityClassName | string | `""` | Sets priorityClassName for datacatalog pod(s). | +| readinessProbe | object | `{"grpc":{"port":8000},"initialDelaySeconds":1,"periodSeconds":3}` | https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out | | replicaCount | int | `1` | Replicas count for flyteagent deployment | | resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"200Mi"},"requests":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"200Mi"}}` | Default resources requests and limits for flyteagent deployment | | securityContext | object | `{"allowPrivilegeEscalation":false}` | Security context for container | diff --git a/charts/flyteagent/templates/agent/deployment.yaml b/charts/flyteagent/templates/agent/deployment.yaml index 2c6a903f8fa..caad9ca44a0 100644 --- a/charts/flyteagent/templates/agent/deployment.yaml +++ b/charts/flyteagent/templates/agent/deployment.yaml @@ -39,6 +39,10 @@ spec: ports: - containerPort: {{ .Values.ports.containerPort }} name: {{ .Values.ports.name }} + readinessProbe: + {{- with .Values.readinessProbe -}} + {{ tpl (toYaml .) $ | nindent 10 }} + {{- end }} securityContext: {{- toYaml .Values.securityContext | nindent 12 }} resources: {{- toYaml .Values.resources | nindent 10 }} diff --git a/charts/flyteagent/values.yaml b/charts/flyteagent/values.yaml index aee84dc2b28..266ec9c0eb7 100755 --- a/charts/flyteagent/values.yaml +++ b/charts/flyteagent/values.yaml @@ -23,7 +23,7 @@ image: # -- Docker image for flyteagent deployment repository: ghcr.io/flyteorg/flyteagent # -- Docker image tag - tag: 1.10.7 # FLYTEAGENT_TAG + tag: 1.10.8b4 # FLYTEAGENT_TAG # -- Docker image pull policy pullPolicy: IfNotPresent ports: @@ -56,6 +56,13 @@ serviceAccount: imagePullSecrets: [] # -- Security context for pod podSecurityContext: {} +# -- Readiness probe for flyteagent. Use readinessProbe: {} if agent doesn't implement grpc-health-checking service. +# -- https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out +readinessProbe: + grpc: + port: 8000 + initialDelaySeconds: 1 + periodSeconds: 3 # -- Security context for container securityContext: allowPrivilegeEscalation: false diff --git a/datacatalog/go.mod b/datacatalog/go.mod index 4116c4cc7f5..7c04fd7832b 100644 --- a/datacatalog/go.mod +++ b/datacatalog/go.mod @@ -43,7 +43,7 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/flyteorg/stow v0.3.8 // indirect + github.com/flyteorg/stow v0.3.10 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-gormigrate/gormigrate/v2 v2.1.1 // indirect diff --git a/datacatalog/go.sum b/datacatalog/go.sum index 786de45ede6..b1642f638fe 100644 --- a/datacatalog/go.sum +++ b/datacatalog/go.sum @@ -113,8 +113,8 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flyteorg/stow v0.3.8 h1:4a6BtfgDR86fUwa48DkkZTcp6WK4oQXSfewPd/kN0Z4= -github.com/flyteorg/stow v0.3.8/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= +github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8= +github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= diff --git a/deployment/agent/flyte_agent_helm_generated.yaml b/deployment/agent/flyte_agent_helm_generated.yaml index 46762b4cff2..4e78b991ac3 100644 --- a/deployment/agent/flyte_agent_helm_generated.yaml +++ b/deployment/agent/flyte_agent_helm_generated.yaml @@ -78,7 +78,7 @@ spec: - pyflyte - serve - agent - image: "ghcr.io/flyteorg/flyteagent:1.10.7" + image: "ghcr.io/flyteorg/flyteagent:1.10.8b4" imagePullPolicy: "IfNotPresent" name: flyteagent volumeMounts: @@ -87,6 +87,11 @@ spec: ports: - containerPort: 8000 name: agent-grpc + readinessProbe: + grpc: + port: 8000 + initialDelaySeconds: 1 + periodSeconds: 3 securityContext: allowPrivilegeEscalation: false resources: diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml index 83c0d249c53..a5b9e0081fc 100644 --- a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml +++ b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml @@ -429,7 +429,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -489,6 +489,7 @@ data: - container - sidecar - k8s-array + - agent-service k8s.yaml: | plugins: k8s: @@ -868,7 +869,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -889,7 +890,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -907,7 +908,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -924,7 +925,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -951,7 +952,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1056,7 +1057,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1112,7 +1113,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1186,7 +1187,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1204,7 +1205,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1267,7 +1268,7 @@ spec: template: metadata: annotations: - configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" + configChecksum: "64a0a51a13929685cf9e74444b216f25bfc9dbdb542d6d491db345519111197" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1293,7 +1294,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1347,9 +1348,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b1 + app.kubernetes.io/version: v1.11.0 annotations: - configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" + configChecksum: "64a0a51a13929685cf9e74444b216f25bfc9dbdb542d6d491db345519111197" spec: securityContext: fsGroup: 65534 @@ -1361,7 +1362,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1388,7 +1389,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml deleted file mode 100644 index b4ee31f106a..00000000000 --- a/deployment/eks/flyte_generated.yaml +++ /dev/null @@ -1,9526 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: flyte ---- -apiVersion: v1 -kind: Namespace -metadata: - name: kubeflow ---- -apiVersion: v1 -kind: Namespace -metadata: - name: sparkoperator ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: flyteworkflows.flyte.lyft.com -spec: - group: flyte.lyft.com - names: - kind: FlyteWorkflow - plural: flyteworkflows - shortNames: - - fly - singular: flyteworkflow - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - type: object - x-kubernetes-preserve-unknown-fields: true - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (unknown) - creationTimestamp: null - name: scheduledsparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: ScheduledSparkApplication - listKind: ScheduledSparkApplicationList - plural: scheduledsparkapplications - shortNames: - - scheduledsparkapp - singular: scheduledsparkapplication - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - concurrencyPolicy: - type: string - failedRunHistoryLimit: - format: int32 - type: integer - schedule: - type: string - successfulRunHistoryLimit: - format: int32 - type: integer - suspend: - type: boolean - template: - properties: - arguments: - items: - type: string - type: array - batchScheduler: - type: string - batchSchedulerOptions: - properties: - priorityClassName: - type: string - queue: - type: string - type: object - deps: - properties: - files: - items: - type: string - type: array - jars: - items: - type: string - type: array - pyFiles: - items: - type: string - type: array - type: object - driver: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - type: string - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - serviceAccount: - type: string - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - executor: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - deleteOnTermination: - type: boolean - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - instances: - format: int32 - minimum: 1 - type: integer - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - failureRetries: - format: int32 - type: integer - hadoopConf: - additionalProperties: - type: string - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - type: string - imagePullSecrets: - items: - type: string - type: array - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - type: string - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - metricsPropertiesFile: - type: string - prometheus: - properties: - configFile: - type: string - configuration: - type: string - jmxExporterJar: - type: string - port: - format: int32 - maximum: 49151 - minimum: 1024 - type: integer - required: - - jmxExporterJar - type: object - required: - - exposeDriverMetrics - - exposeExecutorMetrics - type: object - nodeSelector: - additionalProperties: - type: string - type: object - pythonVersion: - enum: - - "2" - - "3" - type: string - restartPolicy: - properties: - onFailureRetries: - format: int32 - minimum: 0 - type: integer - onFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - onSubmissionFailureRetries: - format: int32 - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - type: - enum: - - Never - - Always - - OnFailure - type: string - type: object - retryInterval: - format: int64 - type: integer - serviceAccount: - type: string - sparkConf: - additionalProperties: - type: string - type: object - sparkConfigMap: - type: string - sparkVersion: - type: string - timeToLiveSeconds: - format: int64 - type: integer - type: - enum: - - Java - - Python - - Scala - - R - type: string - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - type: string - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - required: - - sources - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - driver - - executor - - sparkVersion - - type - type: object - required: - - schedule - - template - type: object - status: - properties: - lastRun: - format: date-time - nullable: true - type: string - lastRunName: - type: string - nextRun: - format: date-time - nullable: true - type: string - pastFailedRunNames: - items: - type: string - type: array - pastSuccessfulRunNames: - items: - type: string - type: array - reason: - type: string - scheduleState: - type: string - type: object - required: - - metadata - - spec - type: object - version: v1beta2 - versions: - - name: v1beta2 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (unknown) - creationTimestamp: null - name: sparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: SparkApplication - listKind: SparkApplicationList - plural: sparkapplications - shortNames: - - sparkapp - singular: sparkapplication - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - arguments: - items: - type: string - type: array - batchScheduler: - type: string - batchSchedulerOptions: - properties: - priorityClassName: - type: string - queue: - type: string - type: object - deps: - properties: - files: - items: - type: string - type: array - jars: - items: - type: string - type: array - pyFiles: - items: - type: string - type: array - type: object - driver: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - type: string - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - serviceAccount: - type: string - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - executor: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - deleteOnTermination: - type: boolean - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - instances: - format: int32 - minimum: 1 - type: integer - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - failureRetries: - format: int32 - type: integer - hadoopConf: - additionalProperties: - type: string - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - type: string - imagePullSecrets: - items: - type: string - type: array - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - type: string - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - metricsPropertiesFile: - type: string - prometheus: - properties: - configFile: - type: string - configuration: - type: string - jmxExporterJar: - type: string - port: - format: int32 - maximum: 49151 - minimum: 1024 - type: integer - required: - - jmxExporterJar - type: object - required: - - exposeDriverMetrics - - exposeExecutorMetrics - type: object - nodeSelector: - additionalProperties: - type: string - type: object - pythonVersion: - enum: - - "2" - - "3" - type: string - restartPolicy: - properties: - onFailureRetries: - format: int32 - minimum: 0 - type: integer - onFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - onSubmissionFailureRetries: - format: int32 - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - type: - enum: - - Never - - Always - - OnFailure - type: string - type: object - retryInterval: - format: int64 - type: integer - serviceAccount: - type: string - sparkConf: - additionalProperties: - type: string - type: object - sparkConfigMap: - type: string - sparkVersion: - type: string - timeToLiveSeconds: - format: int64 - type: integer - type: - enum: - - Java - - Python - - Scala - - R - type: string - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - type: string - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - required: - - sources - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - driver - - executor - - sparkVersion - - type - type: object - status: - properties: - applicationState: - properties: - errorMessage: - type: string - state: - type: string - required: - - state - type: object - driverInfo: - properties: - podName: - type: string - webUIAddress: - type: string - webUIIngressAddress: - type: string - webUIIngressName: - type: string - webUIPort: - format: int32 - type: integer - webUIServiceName: - type: string - type: object - executionAttempts: - format: int32 - type: integer - executorState: - additionalProperties: - type: string - type: object - sparkApplicationId: - type: string - submissionAttempts: - format: int32 - type: integer - submissionID: - type: string - submissionTime: - format: date-time - nullable: true - type: string - terminationTime: - format: date-time - nullable: true - type: string - required: - - driverInfo - type: object - required: - - metadata - - spec - type: object - version: v1beta2 - versions: - - name: v1beta2 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: pytorchjobs.kubeflow.org -spec: - additionalPrinterColumns: - - JSONPath: .status.conditions[-1:].type - name: State - type: string - - JSONPath: .metadata.creationTimestamp - name: Age - type: date - group: kubeflow.org - names: - kind: PyTorchJob - plural: pytorchjobs - singular: pytorchjob - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - properties: - pytorchReplicaSpecs: - properties: - Master: - properties: - replicas: - maximum: 1 - minimum: 1 - type: integer - Worker: - properties: - replicas: - minimum: 1 - type: integer - versions: - - name: v1 - served: true - storage: true ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: datacatalog - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flyte-pod-webhook - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::111222333456:role/flyte-operator - name: flyteadmin - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - annotations: - eks.amazonaws.com/role-arn: arn:aws:iam::111222333456:role/flyte-operator - name: flytepropeller - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: sparkoperator - namespace: sparkoperator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flyte-pod-webhook - namespace: flyte -rules: -- apiGroups: - - '*' - resources: - - mutatingwebhookconfigurations - - secrets - - pods - - replicasets/finalizers - verbs: - - get - - create - - update - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flyteadmin - namespace: flyte -rules: -- apiGroups: - - "" - - flyte.lyft.com - - rbac.authorization.k8s.io - resources: - - configmaps - - flyteworkflows - - namespaces - - pods - - resourcequotas - - roles - - rolebindings - - secrets - - services - - serviceaccounts - - spark-role - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flytepropeller -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - delete - - patch -- apiGroups: - - '*' - resources: - - '*' - verbs: - - get - - list - - watch - - create - - update - - delete - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - create - - delete - - update -- apiGroups: - - flyte.lyft.com - resources: - - flyteworkflows - - flyteworkflows/finalizers - verbs: - - get - - list - - watch - - create - - update - - delete - - patch - - post - - deletecollection ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pytorchjobs-admin: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kustomize.component: pytorch-operator - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true" - name: kubeflow-pytorchjobs-admin -rules: [] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kustomize.component: pytorch-operator - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pytorchjobs-admin: "true" - name: kubeflow-pytorchjobs-edit -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - - pytorchjobs/status - verbs: - - get - - list - - watch - - create - - delete - - deletecollection - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kustomize.component: pytorch-operator - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" - name: kubeflow-pytorchjobs-view -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - - pytorchjobs/status - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sparkoperator -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - '*' -- apiGroups: - - "" - resources: - - services - - configmaps - - secrets - verbs: - - create - - get - - delete -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - create - - get - - delete -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - get - - update - - delete - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - update - - delete -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - verbs: - - create - - get - - update - - delete -- apiGroups: - - sparkoperator.k8s.io - resources: - - sparkapplications - - scheduledsparkapplications - - sparkapplications/status - - scheduledsparkapplications/status - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - - pytorchjobs/status - verbs: - - '*' -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - '*' -- apiGroups: - - "" - resources: - - pods - - services - - endpoints - - events - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flyte-pod-webhook - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flyte-pod-webhook -subjects: -- kind: ServiceAccount - name: flyte-pod-webhook - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flyteadmin-binding - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flyteadmin -subjects: -- kind: ServiceAccount - name: flyteadmin - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flytepropeller - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flytepropeller -subjects: -- kind: ServiceAccount - name: flytepropeller - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: sparkoperator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: sparkoperator -subjects: -- kind: ServiceAccount - name: sparkoperator - namespace: sparkoperator ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: pytorch-operator -subjects: -- kind: ServiceAccount - name: pytorch-operator - namespace: kubeflow ---- -apiVersion: v1 -data: - aa_namespace.yaml: | - apiVersion: v1 - kind: Namespace - metadata: - name: {{ namespace }} - spec: - finalizers: - - kubernetes - ab_project-resource-quota.yaml: "apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }} \nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }} \n limits.memory: {{ projectQuotaMemory }}\n\n" - ad_spark-role.yaml: | - apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: spark-role - namespace: {{ namespace }} - rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - '*' - - apiGroups: - - "" - resources: - - services - verbs: - - '*' - - apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' - ae_spark-service-account.yaml: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: spark - namespace: {{ namespace }} - af_spark-role-binding.yaml: "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: spark-role-binding\n namespace: {{ namespace }} \nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: spark-role\nsubjects:\n- kind: ServiceAccount\n name: spark\n namespace: {{ namespace }}\n\n" -kind: ConfigMap -metadata: - name: clusterresource-template-4fbh4bk26k - namespace: flyte ---- -apiVersion: v1 -data: - db.yaml: | - database: - port: 5432 - username: flyte - # TODO Change this to match aurora or rds postgres endpoint - host: flyteadmin-cluster.cluster-456123e6ivib.us-west-2.rds.amazonaws.com - dbname: flytedatacatalog - passwordPath: /etc/db/pass.txt - logger.yaml: | - # TODO this is used to control the log level - logger: - show-source: true - level: 2 - server.yaml: | - datacatalog: - storage-prefix: metadata/datacatalog - metrics-scope: "datacatalog" - profiler-port: 10254 - application: - grpcPort: 8089 - storage.yaml: | - storage: - type: stow - stow: - kind: s3 - config: - auth_type: iam - region: us-east-2 - # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store - container: "flyte-demo" - # NOTE this cache configuration is purely for propeller. But since we are having a common storage - # config, we are configuring this value. In production create a separate storage config for - # propeller and increase the cache size - cache: - max_size_mbs: 512 - target_gc_percent: 70 - limits: - maxDownloadMBs: 10 -kind: ConfigMap -metadata: - name: datacatalog-config-fktmhdgfk8 - namespace: flyte ---- -apiVersion: v1 -data: - cluster_resources.yaml: | - cluster_resources: - templatePath: "/etc/flyte/clusterresource/templates" - customData: - - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" - refreshInterval: 1m - db.yaml: | - database: - port: 5432 - username: flyte - # TODO Change this to match aurora or rds postgres endpoint - host: flyteadmin-cluster.cluster-456123e6ivib.us-west-2.rds.amazonaws.com - dbname: flyte - passwordPath: /etc/db/pass.txt - domain.yaml: | - domains: - - id: development - name: development - - id: staging - name: staging - - id: production - name: production - logger.yaml: | - # TODO this is used to control the log level - logger: - show-source: true - level: 2 - server.yaml: | - server: - httpPort: 8088 - grpcPort: 8089 - security: - # Controls whether to serve requests over SSL/TLS. - secure: false - # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. - useAuth: false - allowCors: true - allowedOrigins: - # Accepting all domains for Sandbox installation - - "*" - allowedHeaders: - - "Content-Type" - - "flyte-authorization" - auth: - authorizedUris: - # This should point at your public http Uri. - - https://localhost:30081 - # This will be used by internal services in the same namespace as flyteadmin - - http://flyteadmin:80 - # This will be used by internal services in the same cluster but different namespaces - - http://flyteadmin.flyte.svc.cluster.local:80 - - # Controls app authentication config - appAuth: - thirdPartyConfig: - flyteClient: - clientId: flytectl - redirectUri: https://localhost:53593/callback - scopes: - - offline - - all - # Controls user authentication - userAuth: - openId: - baseUrl: https://accounts.google.com - scopes: - - profile - - openid - clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com - flyteadmin: - roleNameKey: "iam.amazonaws.com/role" - storage.yaml: | - storage: - type: stow - stow: - kind: s3 - config: - auth_type: iam - region: us-east-2 - # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store - container: "flyte-demo" - # NOTE this cache configuration is purely for propeller. But since we are having a common storage - # config, we are configuring this value. In production create a separate storage config for - # propeller and increase the cache size - cache: - max_size_mbs: 512 - target_gc_percent: 70 - limits: - maxDownloadMBs: 10 - task_resource_defaults.yaml: | - task_resources: - defaults: - cpu: 1000m - memory: 1000Mi - storage: 1000Mi - limits: - cpu: 2 - memory: 8Gi - storage: 2000Mi - gpu: 1 -kind: ConfigMap -metadata: - name: flyte-admin-config-29g4dtt8tc - namespace: flyte ---- -apiVersion: v1 -data: - BASE_URL: /console - CONFIG_DIR: /etc/flyte/config -kind: ConfigMap -metadata: - name: flyte-console-config - namespace: flyte ---- -apiVersion: v1 -data: - admin.yaml: | - event: - type: admin - rate: 500 - capacity: 1000 - admin: - endpoint: flyteadmin:81 - insecure: true - catalog.yaml: | - catalog-cache: - endpoint: datacatalog:89 - type: datacatalog - insecure: true - catalog_cache.yaml: "plugins:\n catalogCache: \n reader:\n maxItems: 10000\n writer:\n maxItems: 10000\n" - copilot.yaml: | - plugins: - k8s: - co-pilot: - name: "flyte-copilot-" - image: "cr.flyte.org/flyteorg/flytecopilot:v0.0.15" - start-timeout: "30s" - core.yaml: | - propeller: - rawoutput-prefix: s3://my-s3-bucket/ - metadata-prefix: metadata/propeller - workers: 40 - gc-interval: 12h - max-workflow-retries: 50 - workflow-reeval-duration: 30s - downstream-eval-duration: 30s - limit-namespace: "all" - prof-port: 10254 - metrics-prefix: flyte - enable-admin-launcher: true - leader-election: - lock-config-map: - name: propeller-leader - namespace: flyte - enabled: true - lease-duration: 15s - renew-deadline: 10s - retry-period: 2s - kube-client-config: - qps: 100 - burst: 25 - timeout: 30s - queue: - type: batch - batching-interval: 2s - batch-size: -1 - queue: - type: maxof - rate: 100 - capacity: 1000 - base-delay: 5s - max-delay: 120s - sub-queue: - type: bucket - rate: 100 - capacity: 1000 - workflowStore: - policy: "ResourceVersionCache" - enabled_plugins.yaml: | - tasks: - max-plugin-phase-versions: 1000000 - task-plugins: - enabled-plugins: - - container - - sidecar - - spark - - k8s-array - - pytorch - default-for-task-types: - container: container - sidecar: sidecar - spark: spark - container_array: k8s-array - pytorch: pytorch - k8s.yaml: | - plugins: - k8s: - default-env-vars: - - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - - FLYTE_AWS_ACCESS_KEY_ID: minio - - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage - default-cpus: 100m - default-memory: 100Mi - logger.yaml: | - # TODO this is used to control the log level - logger: - show-source: true - level: 2 - qubole.yaml: | - plugins: - qubole: - quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" - resource_manager.yaml: | - propeller: - resourcemanager: - type: redis - resourceMaxQuota: 10000 - redis: - hostPath: redis-resource-manager:6379 - hostKey: mypassword - spark.yaml: | - plugins: - spark: - spark-config-default: - # We override the default credentials chain provider for Hadoop so that - # it can use the serviceAccount based IAM role or ec2 metadata based. - # This is more in line with how AWS works - - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" - - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.multipart.threshold: "536870912" - - spark.blacklist.enabled: "true" - - spark.blacklist.timeout: "5m" - - spark.task.maxfailures: "8" - storage.yaml: | - storage: - type: stow - stow: - kind: s3 - config: - auth_type: iam - region: us-east-2 - # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store - container: "flyte-demo" - # NOTE this cache configuration is purely for propeller. But since we are having a common storage - # config, we are configuring this value. In production create a separate storage config for - # propeller and increase the cache size - cache: - max_size_mbs: 512 - target_gc_percent: 70 - limits: - maxDownloadMBs: 10 - task_logs.yaml: | - plugins: - logs: - # Log links can link to multiple options - # #1 Kubernetes dashboard is disabled in GCP - kubernetes-enabled: false - # #2 AWS Cloudwatch - cloudwatch-enabled: true - # TODO Add region and log group - cloudwatch-region: - cloudwatch-log-group: -kind: ConfigMap -metadata: - name: flyte-propeller-config-dtkccdkch8 - namespace: flyte ---- -apiVersion: v1 -data: - admin.yaml: | - admin: - clientId: flytepropeller - clientSecretLocation: /etc/secrets/client_secret - endpoint: flyteadmin:81 - insecure: true - event: - capacity: 1000 - rate: 500 - type: admin - db.yaml: | - database: - dbname: postgres - host: postgres - port: 5432 - username: postgres - logger.yaml: | - logger: - level: 4 - show-source: true -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: flyte - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: flytescheduler - helm.sh/chart: flyte-v0.1.10 - name: flyte-scheduler-config - namespace: flyte ---- -apiVersion: v1 -data: - pass.txt: eW91cnBhc3N3b3Jk -kind: Secret -metadata: - name: db-pass-bthd2588cc - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-admin-auth - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-pod-webhook - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-secret-auth - namespace: flyte -stringData: - client_secret: foobar -type: Opaque ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - contour.heptio.com/upstream-protocol.h2c: grpc - name: datacatalog - namespace: flyte -spec: - ports: - - name: http - port: 88 - protocol: TCP - targetPort: 8088 - - name: grpc - port: 89 - protocol: TCP - targetPort: 8089 - selector: - app: datacatalog ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - cloud.google.com/load-balancer-type: Internal - name: datacatalog-metrics - namespace: flyte -spec: - ports: - - name: http-metrics - port: 10254 - protocol: TCP - selector: - app: datacatalog - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - projectcontour.io/upstream-protocol.h2c: grpc - name: flyte-pod-webhook - namespace: flyte -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: 9443 - selector: - app: flyte-pod-webhook ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - external-dns.alpha.kubernetes.io/hostname: flyteadmin.subdomain.mydomain.com - projectcontour.io/upstream-protocol.h2c: grpc - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" - service.beta.kubernetes.io/aws-load-balancer-extra-security-groups: sg-...,sg-...,sg-... - name: flyteadmin - namespace: flyte -spec: - loadBalancerSourceRanges: - - 0.0.0.0 - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 8088 - - name: grpc - port: 81 - protocol: TCP - targetPort: 8089 - selector: - app: flyteadmin - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - external-dns.alpha.kubernetes.io/hostname: flyte.subdomain.mydomain.com - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" - service.beta.kubernetes.io/aws-load-balancer-extra-security-groups: sg-....,sg-...,sg-... - name: flyteconsole - namespace: flyte -spec: - loadBalancerSourceRanges: - - 0.0.0.0 - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 8080 - selector: - app: flyteconsole - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - cloud.google.com/load-balancer-type: Internal - name: flytepropeller - namespace: flyte -spec: - ports: - - name: http-metrics - port: 10254 - protocol: TCP - selector: - app: flytepropeller - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: redis-resource-manager - name: redis-resource-manager - namespace: flyte -spec: - ports: - - name: redis - port: 6379 - protocol: TCP - targetPort: redis - selector: - app: redis-resource-manager - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "8443" - prometheus.io/scrape: "true" - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator - namespace: kubeflow -spec: - ports: - - name: monitoring-port - port: 8443 - targetPort: 8443 - selector: - kustomize.component: pytorch-operator - name: pytorch-operator - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - name: spark-webhook - namespace: sparkoperator -spec: - ports: - - name: webhook - port: 443 - targetPort: 8080 - selector: - app.kubernetes.io/name: sparkoperator ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: datacatalog - name: datacatalog - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: datacatalog - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: datacatalog - app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.3.0 - spec: - containers: - - command: - - datacatalog - - --config - - /etc/datacatalog/config/*.yaml - - serve - image: cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: datacatalog - ports: - - containerPort: 8088 - - containerPort: 8089 - resources: - limits: - cpu: "2" - ephemeral-storage: 1000Mi - memory: 1Gi - volumeMounts: - - mountPath: /etc/datacatalog/config - name: config-volume - - mountPath: /etc/db - name: db-pass - initContainers: - - command: - - datacatalog - - --config - - /etc/datacatalog/config/*.yaml - - migrate - - run - image: cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: run-migrations - volumeMounts: - - mountPath: /etc/datacatalog/config - name: config-volume - - mountPath: /etc/db - name: db-pass - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: datacatalog - volumes: - - emptyDir: {} - name: shared-data - - configMap: - name: datacatalog-config-fktmhdgfk8 - name: config-volume - - name: db-pass - secret: - secretName: db-pass-bthd2588cc ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyte-pod-webhook - name: flyte-pod-webhook - namespace: flyte -spec: - selector: - matchLabels: - app: flyte-pod-webhook - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flyte-pod-webhook - app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: 0.5.13 - spec: - containers: - - args: - - webhook - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: webhook - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - readOnly: true - - mountPath: /etc/webhook/certs - name: webhook-certs - readOnly: true - initContainers: - - args: - - webhook - - init-certs - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: generate-secrets - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flyte-pod-webhook - volumes: - - configMap: - name: flyte-propeller-config-dtkccdkch8 - name: config-volume - - name: webhook-certs - secret: - secretName: flyte-pod-webhook ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyteadmin - name: flyteadmin - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flyteadmin - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flyteadmin - app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.4.13 - spec: - containers: - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - serve - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: flyteadmin - ports: - - containerPort: 8088 - - containerPort: 8089 - resources: - limits: - cpu: "2" - ephemeral-storage: 1Gi - memory: 1Gi - volumeMounts: - - mountPath: /srv/flyte - name: shared-data - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - mountPath: /etc/secrets/ - name: auth - - command: - - sh - - -c - - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh - env: - - name: PAGE_TITLE - value: Flyte Admin OpenAPI - - name: SPEC_URL - value: /api/v1/openapi - - name: PORT - value: "8087" - image: docker.io/redocly/redoc - imagePullPolicy: IfNotPresent - name: redoc - ports: - - containerPort: 8087 - resources: - limits: - cpu: "0.1" - memory: 200Mi - initContainers: - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - migrate - - run - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: run-migrations - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - migrate - - seed-projects - - flytesnacks - - flytetester - - flyteexamples - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: seed-projects - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - clusterresource - - sync - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: sync-cluster-resources - volumeMounts: - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - args: - - flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/scratch/secrets && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/scratch/secrets - command: - - /bin/sh - - -c - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: generate-secrets - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/scratch - name: scratch - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flyteadmin - volumes: - - emptyDir: {} - name: shared-data - - emptyDir: {} - name: scratch - - configMap: - name: flyte-admin-config-29g4dtt8tc - name: config-volume - - configMap: - name: clusterresource-template-4fbh4bk26k - name: resource-templates - - name: db-pass - secret: - secretName: db-pass-bthd2588cc - - name: auth - secret: - secretName: flyte-admin-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyteconsole - name: flyteconsole - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flyteconsole - template: - metadata: - labels: - app: flyteconsole - app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.19.0 - spec: - containers: - - envFrom: - - configMapRef: - name: flyte-console-config - image: cr.flyte.org/flyteorg/flyteconsole:v1.11.0 - name: flyteconsole - ports: - - containerPort: 8080 - volumeMounts: - - mountPath: /srv/flyte - name: shared-data - securityContext: - fsGroupChangePolicy: Always - runAsUser: 1000 - volumes: - - emptyDir: {} - name: shared-data ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flytepropeller - name: flytepropeller - namespace: flyte -spec: - selector: - matchLabels: - app: flytepropeller - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flytepropeller - app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.7.1 - spec: - containers: - - args: - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: flytepropeller - ports: - - containerPort: 10254 - resources: - limits: - cpu: "2" - ephemeral-storage: 1Gi - memory: 4Gi - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/secrets/ - name: auth - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flytepropeller - volumes: - - configMap: - name: flyte-propeller-config-dtkccdkch8 - name: config-volume - - name: auth - secret: - secretName: flyte-secret-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flytescheduler - name: flytescheduler - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flytescheduler - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flytescheduler - app.kubernetes.io/name: flytescheduler - app.kubernetes.io/version: 0.3.4 - spec: - containers: - - command: - - flytescheduler - - run - - --config - - /etc/flyte/config/*.yaml - image: cr.flyte.org/flyteorg/flytescheduler:v0.6.49 - imagePullPolicy: IfNotPresent - name: flytescheduler - resources: - limits: - cpu: 250m - ephemeral-storage: 100Mi - memory: 500Mi - requests: - cpu: 10m - ephemeral-storage: 50Mi - memory: 50Mi - volumeMounts: - - mountPath: /etc/secrets/ - name: auth - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - initContainers: - - command: - - flytescheduler - - precheck - - --config - - /etc/flyte/config/*.yaml - image: cr.flyte.org/flyteorg/flytescheduler:v0.6.49 - imagePullPolicy: IfNotPresent - name: flytescheduler-check - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - serviceAccountName: flyteadmin - volumes: - - configMap: - name: flyte-scheduler-config - name: config-volume - - name: db-pass - secret: - secretName: db-pass-bthd2588cc - - name: auth - secret: - secretName: flyte-secret-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - kustomize.component: pytorch-operator - name: pytorch-operator - namespace: kubeflow -spec: - replicas: 1 - selector: - matchLabels: - kustomize.component: pytorch-operator - name: pytorch-operator - template: - metadata: - labels: - kustomize.component: pytorch-operator - name: pytorch-operator - spec: - containers: - - command: - - /pytorch-operator.v1 - - --alsologtostderr - - -v=1 - - --monitoring-port=8443 - env: - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: gcr.io/kubeflow-images-public/pytorch-operator:v1.0.0-g047cf0f - name: pytorch-operator - serviceAccountName: pytorch-operator ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - name: sparkoperator - namespace: sparkoperator -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: sparkoperator - strategy: - type: Recreate - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - spec: - containers: - - args: - - -logtostderr - - -v=2 - - -controller-threads=20 - - -enable-metrics=true - - '-metrics-prefix=service:' - - -metrics-labels=task_name - - -metrics-labels=workflow_name - - -enable-webhook=true - - -webhook-svc-namespace=sparkoperator - command: - - /usr/bin/spark-operator - image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5 - imagePullPolicy: Always - name: sparkoperator-unknown - ports: - - containerPort: 10254 - - containerPort: 8080 - volumeMounts: - - mountPath: /etc/webhook-certs - name: webhook-certs - serviceAccountName: sparkoperator - volumes: - - name: webhook-certs - secret: - secretName: spark-webhook-certs ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: redis-resource-manager - serviceName: redis-resource-manager - template: - metadata: - labels: - app: redis-resource-manager - spec: - containers: - - env: - - name: REDIS_PASSWORD - value: mypassword - image: ecr.flyte.org/bitnami/redis:6.2.5-debian-10-r59 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - name: redis-resource-manager - ports: - - containerPort: 6379 - name: redis - protocol: TCP - readinessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 200m - memory: 128Mi - volumeMounts: - - mountPath: /bitnami - name: redis-data - dnsPolicy: ClusterFirst - restartPolicy: Always - volumes: - - emptyDir: {} - name: redis-data ---- -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: syncresources - namespace: flyte -spec: - jobTemplate: - spec: - template: - spec: - containers: - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - clusterresource - - sync - image: cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1 - imagePullPolicy: IfNotPresent - name: sync-cluster-resources - volumeMounts: - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - restartPolicy: OnFailure - serviceAccountName: flyteadmin - volumes: - - configMap: - name: clusterresource-template-4fbh4bk26k - name: resource-templates - - configMap: - name: flyte-admin-config-29g4dtt8tc - name: config-volume - - name: db-pass - secret: - secretName: db-pass-bthd2588cc - schedule: '*/1 * * * *' ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - name: sparkoperator-init - namespace: sparkoperator -spec: - backoffLimit: 3 - template: - metadata: - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - spec: - containers: - - command: - - /usr/bin/gencerts.sh - - --namespace - - sparkoperator - - -p - image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5 - imagePullPolicy: IfNotPresent - name: main - restartPolicy: Never - serviceAccountName: sparkoperator ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 - alb.ingress.kubernetes.io/group.name: flytesystem - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/tags: service_instance=production - kubernetes.io/ingress.class: alb - labels: - app: flyteadmin - name: flytesystem - namespace: flyte -spec: - rules: - - http: - paths: - - backend: - service: - name: ssl-redirect - port: - name: use-annotation - path: /* - pathType: ImplementationSpecific - - backend: - service: - name: flyteconsole - port: - number: 80 - path: /console - pathType: ImplementationSpecific - - backend: - service: - name: flyteconsole - port: - number: 80 - path: /console/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /api/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /healthcheck - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /v1/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 87 - path: /openapi/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /.well-known/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /login - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /login/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /logout - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /logout/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /callback - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /callback/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /me - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /config - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /config/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /oauth2 - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /oauth2/* - pathType: ImplementationSpecific ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/backend-protocol-version: GRPC - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 - alb.ingress.kubernetes.io/group.name: flytesystem - alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]' - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/tags: service_instance=production - kubernetes.io/ingress.class: alb - nginx.ingress.kubernetes.io/ssl-redirect: "false" - labels: - app: flyteadmin - name: flytesystem-grpc - namespace: flyte -spec: - rules: - - http: - paths: - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.SignalService/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.AdminService/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.AuthMetadataService/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.IdentityService/* - pathType: ImplementationSpecific diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml index b60694f9471..9e813c42753 100644 --- a/deployment/eks/flyte_helm_controlplane_generated.yaml +++ b/deployment/eks/flyte_helm_controlplane_generated.yaml @@ -574,7 +574,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -595,7 +595,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -613,7 +613,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -630,7 +630,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -657,7 +657,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -762,7 +762,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -818,7 +818,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -892,7 +892,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -910,7 +910,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -993,7 +993,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1013,7 +1013,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml index 90d327cb9eb..865b72da243 100644 --- a/deployment/eks/flyte_helm_dataplane_generated.yaml +++ b/deployment/eks/flyte_helm_dataplane_generated.yaml @@ -94,7 +94,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -154,6 +154,7 @@ data: - container - sidecar - k8s-array + - agent-service k8s.yaml: | plugins: k8s: @@ -427,7 +428,7 @@ spec: template: metadata: annotations: - configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" + configChecksum: "64a0a51a13929685cf9e74444b216f25bfc9dbdb542d6d491db345519111197" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -453,7 +454,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -507,9 +508,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b1 + app.kubernetes.io/version: v1.11.0 annotations: - configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" + configChecksum: "64a0a51a13929685cf9e74444b216f25bfc9dbdb542d6d491db345519111197" spec: securityContext: fsGroup: 65534 @@ -521,7 +522,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -548,7 +549,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index 8f34f4f2361..bb0b5e966c3 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -460,7 +460,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -520,6 +520,7 @@ data: - container - sidecar - k8s-array + - agent-service k8s.yaml: | plugins: k8s: @@ -899,7 +900,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -920,7 +921,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -938,7 +939,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -955,7 +956,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -982,7 +983,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1087,7 +1088,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1143,7 +1144,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1217,7 +1218,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1235,7 +1236,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1318,7 +1319,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1338,7 +1339,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -1397,7 +1398,7 @@ spec: template: metadata: annotations: - configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" + configChecksum: "64a0a51a13929685cf9e74444b216f25bfc9dbdb542d6d491db345519111197" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1423,7 +1424,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1477,9 +1478,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b1 + app.kubernetes.io/version: v1.11.0 annotations: - configChecksum: "6f925c4627a6ae0040ea2073d2f5faca3fbfa82c90d90eae474819f23d52212" + configChecksum: "64a0a51a13929685cf9e74444b216f25bfc9dbdb542d6d491db345519111197" spec: securityContext: fsGroup: 65534 @@ -1491,7 +1492,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1518,7 +1519,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml deleted file mode 100644 index 01d708d860b..00000000000 --- a/deployment/gcp/flyte_generated.yaml +++ /dev/null @@ -1,9553 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: flyte ---- -apiVersion: v1 -kind: Namespace -metadata: - name: kubeflow ---- -apiVersion: v1 -kind: Namespace -metadata: - name: sparkoperator ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: flyteworkflows.flyte.lyft.com -spec: - group: flyte.lyft.com - names: - kind: FlyteWorkflow - plural: flyteworkflows - shortNames: - - fly - singular: flyteworkflow - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - type: object - x-kubernetes-preserve-unknown-fields: true - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (unknown) - creationTimestamp: null - name: scheduledsparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: ScheduledSparkApplication - listKind: ScheduledSparkApplicationList - plural: scheduledsparkapplications - shortNames: - - scheduledsparkapp - singular: scheduledsparkapplication - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - concurrencyPolicy: - type: string - failedRunHistoryLimit: - format: int32 - type: integer - schedule: - type: string - successfulRunHistoryLimit: - format: int32 - type: integer - suspend: - type: boolean - template: - properties: - arguments: - items: - type: string - type: array - batchScheduler: - type: string - batchSchedulerOptions: - properties: - priorityClassName: - type: string - queue: - type: string - type: object - deps: - properties: - files: - items: - type: string - type: array - jars: - items: - type: string - type: array - pyFiles: - items: - type: string - type: array - type: object - driver: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - type: string - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - serviceAccount: - type: string - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - executor: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - deleteOnTermination: - type: boolean - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - instances: - format: int32 - minimum: 1 - type: integer - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - failureRetries: - format: int32 - type: integer - hadoopConf: - additionalProperties: - type: string - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - type: string - imagePullSecrets: - items: - type: string - type: array - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - type: string - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - metricsPropertiesFile: - type: string - prometheus: - properties: - configFile: - type: string - configuration: - type: string - jmxExporterJar: - type: string - port: - format: int32 - maximum: 49151 - minimum: 1024 - type: integer - required: - - jmxExporterJar - type: object - required: - - exposeDriverMetrics - - exposeExecutorMetrics - type: object - nodeSelector: - additionalProperties: - type: string - type: object - pythonVersion: - enum: - - "2" - - "3" - type: string - restartPolicy: - properties: - onFailureRetries: - format: int32 - minimum: 0 - type: integer - onFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - onSubmissionFailureRetries: - format: int32 - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - type: - enum: - - Never - - Always - - OnFailure - type: string - type: object - retryInterval: - format: int64 - type: integer - serviceAccount: - type: string - sparkConf: - additionalProperties: - type: string - type: object - sparkConfigMap: - type: string - sparkVersion: - type: string - timeToLiveSeconds: - format: int64 - type: integer - type: - enum: - - Java - - Python - - Scala - - R - type: string - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - type: string - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - required: - - sources - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - driver - - executor - - sparkVersion - - type - type: object - required: - - schedule - - template - type: object - status: - properties: - lastRun: - format: date-time - nullable: true - type: string - lastRunName: - type: string - nextRun: - format: date-time - nullable: true - type: string - pastFailedRunNames: - items: - type: string - type: array - pastSuccessfulRunNames: - items: - type: string - type: array - reason: - type: string - scheduleState: - type: string - type: object - required: - - metadata - - spec - type: object - version: v1beta2 - versions: - - name: v1beta2 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (unknown) - creationTimestamp: null - name: sparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: SparkApplication - listKind: SparkApplicationList - plural: sparkapplications - shortNames: - - sparkapp - singular: sparkapplication - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - arguments: - items: - type: string - type: array - batchScheduler: - type: string - batchSchedulerOptions: - properties: - priorityClassName: - type: string - queue: - type: string - type: object - deps: - properties: - files: - items: - type: string - type: array - jars: - items: - type: string - type: array - pyFiles: - items: - type: string - type: array - type: object - driver: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - type: string - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - serviceAccount: - type: string - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - executor: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - deleteOnTermination: - type: boolean - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - instances: - format: int32 - minimum: 1 - type: integer - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - failureRetries: - format: int32 - type: integer - hadoopConf: - additionalProperties: - type: string - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - type: string - imagePullSecrets: - items: - type: string - type: array - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - type: string - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - metricsPropertiesFile: - type: string - prometheus: - properties: - configFile: - type: string - configuration: - type: string - jmxExporterJar: - type: string - port: - format: int32 - maximum: 49151 - minimum: 1024 - type: integer - required: - - jmxExporterJar - type: object - required: - - exposeDriverMetrics - - exposeExecutorMetrics - type: object - nodeSelector: - additionalProperties: - type: string - type: object - pythonVersion: - enum: - - "2" - - "3" - type: string - restartPolicy: - properties: - onFailureRetries: - format: int32 - minimum: 0 - type: integer - onFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - onSubmissionFailureRetries: - format: int32 - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - type: - enum: - - Never - - Always - - OnFailure - type: string - type: object - retryInterval: - format: int64 - type: integer - serviceAccount: - type: string - sparkConf: - additionalProperties: - type: string - type: object - sparkConfigMap: - type: string - sparkVersion: - type: string - timeToLiveSeconds: - format: int64 - type: integer - type: - enum: - - Java - - Python - - Scala - - R - type: string - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - type: string - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - required: - - sources - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - driver - - executor - - sparkVersion - - type - type: object - status: - properties: - applicationState: - properties: - errorMessage: - type: string - state: - type: string - required: - - state - type: object - driverInfo: - properties: - podName: - type: string - webUIAddress: - type: string - webUIIngressAddress: - type: string - webUIIngressName: - type: string - webUIPort: - format: int32 - type: integer - webUIServiceName: - type: string - type: object - executionAttempts: - format: int32 - type: integer - executorState: - additionalProperties: - type: string - type: object - sparkApplicationId: - type: string - submissionAttempts: - format: int32 - type: integer - submissionID: - type: string - submissionTime: - format: date-time - nullable: true - type: string - terminationTime: - format: date-time - nullable: true - type: string - required: - - driverInfo - type: object - required: - - metadata - - spec - type: object - version: v1beta2 - versions: - - name: v1beta2 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: pytorchjobs.kubeflow.org -spec: - additionalPrinterColumns: - - JSONPath: .status.conditions[-1:].type - name: State - type: string - - JSONPath: .metadata.creationTimestamp - name: Age - type: date - group: kubeflow.org - names: - kind: PyTorchJob - plural: pytorchjobs - singular: pytorchjob - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - spec: - properties: - pytorchReplicaSpecs: - properties: - Master: - properties: - replicas: - maximum: 1 - minimum: 1 - type: integer - Worker: - properties: - replicas: - minimum: 1 - type: integer - versions: - - name: v1 - served: true - storage: true ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: datacatalog - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flyte-pod-webhook - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flyteadmin - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flytepropeller - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator - namespace: kubeflow ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: sparkoperator - namespace: sparkoperator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flyte-pod-webhook - namespace: flyte -rules: -- apiGroups: - - '*' - resources: - - mutatingwebhookconfigurations - - secrets - - pods - - replicasets/finalizers - verbs: - - get - - create - - update - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flyteadmin - namespace: flyte -rules: -- apiGroups: - - "" - - flyte.lyft.com - - rbac.authorization.k8s.io - resources: - - configmaps - - flyteworkflows - - namespaces - - pods - - resourcequotas - - roles - - rolebindings - - secrets - - services - - serviceaccounts - - spark-role - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flytepropeller -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - delete - - patch -- apiGroups: - - '*' - resources: - - '*' - verbs: - - get - - list - - watch - - create - - update - - delete - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - create - - delete - - update -- apiGroups: - - flyte.lyft.com - resources: - - flyteworkflows - - flyteworkflows/finalizers - verbs: - - get - - list - - watch - - create - - update - - delete - - patch - - post - - deletecollection ---- -aggregationRule: - clusterRoleSelectors: - - matchLabels: - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pytorchjobs-admin: "true" -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kustomize.component: pytorch-operator - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-admin: "true" - name: kubeflow-pytorchjobs-admin -rules: [] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kustomize.component: pytorch-operator - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-edit: "true" - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-pytorchjobs-admin: "true" - name: kubeflow-pytorchjobs-edit -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - - pytorchjobs/status - verbs: - - get - - list - - watch - - create - - delete - - deletecollection - - patch - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - kustomize.component: pytorch-operator - rbac.authorization.kubeflow.org/aggregate-to-kubeflow-view: "true" - name: kubeflow-pytorchjobs-view -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - - pytorchjobs/status - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sparkoperator -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - '*' -- apiGroups: - - "" - resources: - - services - - configmaps - - secrets - verbs: - - create - - get - - delete -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - create - - get - - delete -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - batch - resources: - - jobs - verbs: - - create - - get - - update - - delete - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - create - - get - - update - - delete -- apiGroups: - - admissionregistration.k8s.io - resources: - - mutatingwebhookconfigurations - verbs: - - create - - get - - update - - delete -- apiGroups: - - sparkoperator.k8s.io - resources: - - sparkapplications - - scheduledsparkapplications - - sparkapplications/status - - scheduledsparkapplications/status - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator -rules: -- apiGroups: - - kubeflow.org - resources: - - pytorchjobs - - pytorchjobs/status - verbs: - - '*' -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - '*' -- apiGroups: - - "" - resources: - - pods - - services - - endpoints - - events - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flyte-pod-webhook - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flyte-pod-webhook -subjects: -- kind: ServiceAccount - name: flyte-pod-webhook - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flyteadmin-binding - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flyteadmin -subjects: -- kind: ServiceAccount - name: flyteadmin - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flytepropeller - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flytepropeller -subjects: -- kind: ServiceAccount - name: flytepropeller - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: sparkoperator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: sparkoperator -subjects: -- kind: ServiceAccount - name: sparkoperator - namespace: sparkoperator ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: pytorch-operator -subjects: -- kind: ServiceAccount - name: pytorch-operator - namespace: kubeflow ---- -apiVersion: v1 -data: - aa_namespace.yaml: | - apiVersion: v1 - kind: Namespace - metadata: - name: {{ namespace }} - spec: - finalizers: - - kubernetes - ab_project-resource-quota.yaml: "apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }} \nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }} \n limits.memory: {{ projectQuotaMemory }}\n\n" - ad_spark-role.yaml: | - apiVersion: rbac.authorization.k8s.io/v1 - kind: Role - metadata: - name: spark-role - namespace: {{ namespace }} - rules: - - apiGroups: - - "" - resources: - - pods - verbs: - - '*' - - apiGroups: - - "" - resources: - - services - verbs: - - '*' - - apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' - ae_spark-service-account.yaml: | - apiVersion: v1 - kind: ServiceAccount - metadata: - name: spark - namespace: {{ namespace }} - af_spark-role-binding.yaml: "apiVersion: rbac.authorization.k8s.io/v1\nkind: RoleBinding\nmetadata:\n name: spark-role-binding\n namespace: {{ namespace }} \nroleRef:\n apiGroup: rbac.authorization.k8s.io\n kind: Role\n name: spark-role\nsubjects:\n- kind: ServiceAccount\n name: spark\n namespace: {{ namespace }}\n\n" -kind: ConfigMap -metadata: - name: clusterresource-template-4fbh4bk26k - namespace: flyte ---- -apiVersion: v1 -data: - db.yaml: | - database: - port: 5432 - username: flyte - host: cloudsqlproxy - dbname: datacatalog - options: "sslmode=disable" - passwordPath: /etc/db/pass.txt - logger.yaml: | - # TODO this is used to control the log level - logger: - show-source: true - level: 2 - server.yaml: | - datacatalog: - storage-prefix: metadata/datacatalog - metrics-scope: "datacatalog" - profiler-port: 10254 - application: - grpcPort: 8089 - storage.yaml: | - storage: - type: stow - stow: - kind: google - config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/cloud-platform - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" - # NOTE this cache configuration is purely for propeller. But since we are having a common storage - # config, we are configuring this value. In production create a separate storage config for - # propeller and increase the cache size - cache: - max_size_mbs: 512 - target_gc_percent: 70 - limits: - maxDownloadMBs: 10 -kind: ConfigMap -metadata: - name: datacatalog-config-mk4gcdf6db - namespace: flyte ---- -apiVersion: v1 -data: - cluster_resources.yaml: | - cluster_resources: - templatePath: "/etc/flyte/clusterresource/templates" - customData: - - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" - refreshInterval: 1m - db.yaml: | - database: - port: 5432 - username: flyte - host: cloudsqlproxy - dbname: flyte - passwordPath: /etc/db/pass.txt - domain.yaml: | - domains: - - id: development - name: development - - id: staging - name: staging - - id: production - name: production - logger.yaml: | - # TODO this is used to control the log level - logger: - show-source: true - level: 2 - server.yaml: | - server: - httpPort: 8088 - grpcPort: 8089 - security: - # Controls whether to serve requests over SSL/TLS. - secure: false - # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. - useAuth: false - allowCors: true - allowedOrigins: - # Accepting all domains for Sandbox installation - - "*" - allowedHeaders: - - "Content-Type" - - "flyte-authorization" - auth: - authorizedUris: - # This should point at your public http Uri. - - https://localhost:30081 - # This will be used by internal services in the same namespace as flyteadmin - - http://flyteadmin:80 - # This will be used by internal services in the same cluster but different namespaces - - http://flyteadmin.flyte.svc.cluster.local:80 - - # Controls app authentication config - appAuth: - thirdPartyConfig: - flyteClient: - clientId: flytectl - redirectUri: https://localhost:53593/callback - scopes: - - offline - - all - # Controls user authentication - userAuth: - openId: - baseUrl: https://accounts.google.com - scopes: - - profile - - openid - clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com - flyteadmin: - roleNameKey: "iam.amazonaws.com/role" - storage.yaml: | - storage: - type: stow - stow: - kind: google - config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/cloud-platform - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" - # NOTE this cache configuration is purely for propeller. But since we are having a common storage - # config, we are configuring this value. In production create a separate storage config for - # propeller and increase the cache size - cache: - max_size_mbs: 512 - target_gc_percent: 70 - limits: - maxDownloadMBs: 10 - task_resource_defaults.yaml: | - task_resources: - defaults: - cpu: 1000m - memory: 1000Mi - storage: 1000Mi - limits: - cpu: 2 - memory: 8Gi - storage: 2000Mi - gpu: 1 -kind: ConfigMap -metadata: - name: flyte-admin-config-gf99k75c82 - namespace: flyte ---- -apiVersion: v1 -data: - BASE_URL: /console - CONFIG_DIR: /etc/flyte/config -kind: ConfigMap -metadata: - name: flyte-console-config - namespace: flyte ---- -apiVersion: v1 -data: - admin.yaml: | - event: - type: admin - rate: 500 - capacity: 1000 - admin: - endpoint: flyteadmin:81 - insecure: true - catalog.yaml: | - catalog-cache: - endpoint: datacatalog:89 - type: datacatalog - insecure: true - catalog_cache.yaml: "plugins:\n catalogCache: \n reader:\n maxItems: 10000\n writer:\n maxItems: 10000\n" - copilot.yaml: | - plugins: - k8s: - co-pilot: - name: "flyte-copilot-" - image: "cr.flyte.org/flyteorg/flytecopilot:v0.0.15" - start-timeout: "30s" - core.yaml: | - propeller: - rawoutput-prefix: gs://my-s3-bucket/ - metadata-prefix: metadata/propeller - workers: 40 - gc-interval: 12h - max-workflow-retries: 50 - workflow-reeval-duration: 30s - downstream-eval-duration: 30s - limit-namespace: "all" - prof-port: 10254 - metrics-prefix: flyte - enable-admin-launcher: true - leader-election: - lock-config-map: - name: propeller-leader - namespace: flyte - enabled: true - lease-duration: 15s - renew-deadline: 10s - retry-period: 2s - kube-client-config: - qps: 100 - burst: 25 - timeout: 30s - queue: - type: batch - batching-interval: 2s - batch-size: -1 - queue: - type: maxof - rate: 100 - capacity: 1000 - base-delay: 5s - max-delay: 120s - sub-queue: - type: bucket - rate: 100 - capacity: 1000 - workflowStore: - policy: "ResourceVersionCache" - enabled_plugins.yaml: | - tasks: - max-plugin-phase-versions: 1000000 - task-plugins: - enabled-plugins: - - container - - sidecar - - spark - - k8s-array - - pytorch - default-for-task-types: - container: container - sidecar: sidecar - spark: spark - container_array: k8s-array - pytorch: pytorch - k8s.yaml: | - plugins: - k8s: - default-env-vars: - - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - - FLYTE_AWS_ACCESS_KEY_ID: minio - - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage - default-cpus: 100m - default-memory: 100Mi - logger.yaml: | - # TODO this is used to control the log level - logger: - show-source: true - level: 2 - qubole.yaml: | - plugins: - qubole: - quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" - resource_manager.yaml: | - propeller: - resourcemanager: - type: redis - resourceMaxQuota: 10000 - redis: - hostPath: redis-resource-manager:6379 - hostKey: mypassword - spark.yaml: | - plugins: - spark: - spark-config-default: - - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" - - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.multipart.threshold: "536870912" - - spark.blacklist.enabled: "true" - - spark.blacklist.timeout: "5m" - - spark.task.maxfailures: "8" - storage.yaml: | - storage: - type: stow - stow: - kind: google - config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/cloud-platform - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" - # NOTE this cache configuration is purely for propeller. But since we are having a common storage - # config, we are configuring this value. In production create a separate storage config for - # propeller and increase the cache size - cache: - max_size_mbs: 512 - target_gc_percent: 70 - limits: - maxDownloadMBs: 10 - task_logs.yaml: | - plugins: - logs: - # Log links can link to multiple options - # #1 Kubernetes dashboard is disabled in GCP - kubernetes-enabled: false - # #2 GCP stackdriver - stackdriver-enabled: true - # TODO: replace with the GCP project ID - gcp-project: - stackdriver-logresourcename: k8s_container -kind: ConfigMap -metadata: - name: flyte-propeller-config-kgbdtkgf56 - namespace: flyte ---- -apiVersion: v1 -data: - admin.yaml: | - admin: - clientId: flytepropeller - clientSecretLocation: /etc/secrets/client_secret - endpoint: flyteadmin:81 - insecure: true - event: - capacity: 1000 - rate: 500 - type: admin - db.yaml: | - database: - dbname: postgres - host: postgres - port: 5432 - username: postgres - logger.yaml: | - logger: - level: 4 - show-source: true -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: flyte - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: flytescheduler - helm.sh/chart: flyte-v0.1.10 - name: flyte-scheduler-config - namespace: flyte ---- -apiVersion: v1 -data: - pass.txt: eW91cnBhc3N3b3Jk -kind: Secret -metadata: - name: db-pass-bthd2588cc - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-admin-auth - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-pod-webhook - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-secret-auth - namespace: flyte -stringData: - client_secret: foobar -type: Opaque ---- -apiVersion: v1 -kind: Service -metadata: - name: cloudsqlproxy - namespace: flyte -spec: - ports: - - name: http - port: 5432 - protocol: TCP - selector: - app: cloudsqlproxy ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - contour.heptio.com/upstream-protocol.h2c: grpc - name: datacatalog - namespace: flyte -spec: - ports: - - name: http - port: 88 - protocol: TCP - targetPort: 8088 - - name: grpc - port: 89 - protocol: TCP - targetPort: 8089 - selector: - app: datacatalog ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - cloud.google.com/load-balancer-type: Internal - name: datacatalog-metrics - namespace: flyte -spec: - ports: - - name: http-metrics - port: 10254 - protocol: TCP - selector: - app: datacatalog - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - projectcontour.io/upstream-protocol.h2c: grpc - name: flyte-pod-webhook - namespace: flyte -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: 9443 - selector: - app: flyte-pod-webhook ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - cloud.google.com/load-balancer-type: Internal - projectcontour.io/upstream-protocol.h2c: grpc - name: flyteadmin - namespace: flyte -spec: - ports: - - name: redoc - port: 87 - protocol: TCP - targetPort: 8087 - - name: http-metrics - port: 10254 - protocol: TCP - - name: http - port: 80 - protocol: TCP - targetPort: 8088 - - name: grpc - port: 81 - protocol: TCP - targetPort: 8089 - selector: - app: flyteadmin - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - cloud.google.com/load-balancer-type: Internal - name: flyteconsole - namespace: flyte -spec: - ports: - - name: redoc - port: 87 - protocol: TCP - targetPort: 8087 - - name: http-metrics - port: 10254 - protocol: TCP - - port: 80 - protocol: TCP - targetPort: 8080 - selector: - app: flyteconsole - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - cloud.google.com/load-balancer-type: Internal - name: flytepropeller - namespace: flyte -spec: - ports: - - name: http-metrics - port: 10254 - protocol: TCP - selector: - app: flytepropeller - type: LoadBalancer ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: redis-resource-manager - name: redis-resource-manager - namespace: flyte -spec: - ports: - - name: redis - port: 6379 - protocol: TCP - targetPort: redis - selector: - app: redis-resource-manager - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "8443" - prometheus.io/scrape: "true" - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator - namespace: kubeflow -spec: - ports: - - name: monitoring-port - port: 8443 - targetPort: 8443 - selector: - kustomize.component: pytorch-operator - name: pytorch-operator - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - name: spark-webhook - namespace: sparkoperator -spec: - ports: - - name: webhook - port: 443 - targetPort: 8080 - selector: - app.kubernetes.io/name: sparkoperator ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: cloudsqlproxy - name: cloudsqlproxy - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: cloudsqlproxy - template: - metadata: - labels: - app: cloudsqlproxy - spec: - containers: - - command: - - /cloud_sql_proxy - - -instances=::flyte=tcp:0.0.0.0:5432 - image: gcr.io/cloudsql-docker/gce-proxy:1.16 - imagePullPolicy: IfNotPresent - name: cloudsql-proxy - ports: - - containerPort: 5432 ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: datacatalog - name: datacatalog - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: datacatalog - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: datacatalog - app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.3.0 - spec: - containers: - - command: - - datacatalog - - --config - - /etc/datacatalog/config/*.yaml - - serve - image: cr.flyte.org/flyteorg/datacatalog:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: datacatalog - ports: - - containerPort: 8088 - - containerPort: 8089 - resources: - limits: - cpu: "2" - ephemeral-storage: 1000Mi - memory: 1Gi - volumeMounts: - - mountPath: /etc/datacatalog/config - name: config-volume - - mountPath: /etc/db - name: db-pass - initContainers: - - command: - - datacatalog - - --config - - /etc/datacatalog/config/*.yaml - - migrate - - run - image: cr.flyte.org/flyteorg/datacatalog:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: run-migrations - volumeMounts: - - mountPath: /etc/datacatalog/config - name: config-volume - - mountPath: /etc/db - name: db-pass - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: datacatalog - volumes: - - emptyDir: {} - name: shared-data - - configMap: - name: datacatalog-config-mk4gcdf6db - name: config-volume - - name: db-pass - secret: - secretName: db-pass-bthd2588cc ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyte-pod-webhook - name: flyte-pod-webhook - namespace: flyte -spec: - selector: - matchLabels: - app: flyte-pod-webhook - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flyte-pod-webhook - app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: 0.5.13 - spec: - containers: - - args: - - webhook - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: webhook - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - readOnly: true - - mountPath: /etc/webhook/certs - name: webhook-certs - readOnly: true - initContainers: - - args: - - webhook - - init-certs - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: generate-secrets - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flyte-pod-webhook - volumes: - - configMap: - name: flyte-propeller-config-kgbdtkgf56 - name: config-volume - - name: webhook-certs - secret: - secretName: flyte-pod-webhook ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyteadmin - name: flyteadmin - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flyteadmin - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flyteadmin - app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.4.13 - spec: - containers: - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - serve - image: cr.flyte.org/flyteorg/flyteadmin:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: flyteadmin - ports: - - containerPort: 8088 - - containerPort: 8089 - resources: - limits: - cpu: "2" - ephemeral-storage: 1Gi - memory: 1Gi - volumeMounts: - - mountPath: /srv/flyte - name: shared-data - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - mountPath: /etc/secrets/ - name: auth - - command: - - sh - - -c - - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh - env: - - name: PAGE_TITLE - value: Flyte Admin OpenAPI - - name: SPEC_URL - value: /api/v1/openapi - - name: PORT - value: "8087" - image: docker.io/redocly/redoc - imagePullPolicy: IfNotPresent - name: redoc - ports: - - containerPort: 8087 - resources: - limits: - cpu: "0.1" - memory: 200Mi - initContainers: - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - migrate - - run - image: cr.flyte.org/flyteorg/flyteadmin:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: run-migrations - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - migrate - - seed-projects - - flytesnacks - - flytetester - - flyteexamples - image: cr.flyte.org/flyteorg/flyteadmin:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: seed-projects - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - clusterresource - - sync - image: cr.flyte.org/flyteorg/flyteadmin:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: sync-cluster-resources - volumeMounts: - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - args: - - flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/scratch/secrets && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/scratch/secrets - command: - - /bin/sh - - -c - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flyteadmin:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: generate-secrets - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/scratch - name: scratch - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flyteadmin - volumes: - - emptyDir: {} - name: shared-data - - emptyDir: {} - name: scratch - - configMap: - name: flyte-admin-config-gf99k75c82 - name: config-volume - - configMap: - name: clusterresource-template-4fbh4bk26k - name: resource-templates - - name: db-pass - secret: - secretName: db-pass-bthd2588cc - - name: auth - secret: - secretName: flyte-admin-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyteconsole - name: flyteconsole - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flyteconsole - template: - metadata: - labels: - app: flyteconsole - app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.19.0 - spec: - containers: - - envFrom: - - configMapRef: - name: flyte-console-config - image: cr.flyte.org/flyteorg/flyteconsole:v1.10.2 - name: flyteconsole - ports: - - containerPort: 8080 - volumeMounts: - - mountPath: /srv/flyte - name: shared-data - securityContext: - fsGroupChangePolicy: Always - runAsUser: 1000 - volumes: - - emptyDir: {} - name: shared-data ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flytepropeller - name: flytepropeller - namespace: flyte -spec: - selector: - matchLabels: - app: flytepropeller - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flytepropeller - app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.7.1 - spec: - containers: - - args: - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: flytepropeller - ports: - - containerPort: 10254 - resources: - limits: - cpu: "2" - ephemeral-storage: 1Gi - memory: 4Gi - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/secrets/ - name: auth - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flytepropeller - volumes: - - configMap: - name: flyte-propeller-config-kgbdtkgf56 - name: config-volume - - name: auth - secret: - secretName: flyte-secret-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flytescheduler - name: flytescheduler - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flytescheduler - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flytescheduler - app.kubernetes.io/name: flytescheduler - app.kubernetes.io/version: 0.3.4 - spec: - containers: - - command: - - flytescheduler - - run - - --config - - /etc/flyte/config/*.yaml - image: cr.flyte.org/flyteorg/flytescheduler:v0.6.49 - imagePullPolicy: IfNotPresent - name: flytescheduler - resources: - limits: - cpu: 250m - ephemeral-storage: 100Mi - memory: 500Mi - requests: - cpu: 10m - ephemeral-storage: 50Mi - memory: 50Mi - volumeMounts: - - mountPath: /etc/secrets/ - name: auth - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - initContainers: - - command: - - flytescheduler - - precheck - - --config - - /etc/flyte/config/*.yaml - image: cr.flyte.org/flyteorg/flytescheduler:v0.6.49 - imagePullPolicy: IfNotPresent - name: flytescheduler-check - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - serviceAccountName: flyteadmin - volumes: - - configMap: - name: flyte-scheduler-config - name: config-volume - - name: db-pass - secret: - secretName: db-pass-bthd2588cc - - name: auth - secret: - secretName: flyte-secret-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - kustomize.component: pytorch-operator - name: pytorch-operator - namespace: kubeflow -spec: - replicas: 1 - selector: - matchLabels: - kustomize.component: pytorch-operator - name: pytorch-operator - template: - metadata: - labels: - kustomize.component: pytorch-operator - name: pytorch-operator - spec: - containers: - - command: - - /pytorch-operator.v1 - - --alsologtostderr - - -v=1 - - --monitoring-port=8443 - env: - - name: MY_POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: MY_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - image: gcr.io/kubeflow-images-public/pytorch-operator:v1.0.0-g047cf0f - name: pytorch-operator - serviceAccountName: pytorch-operator ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - name: sparkoperator - namespace: sparkoperator -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: sparkoperator - strategy: - type: Recreate - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - spec: - containers: - - args: - - -logtostderr - - -v=2 - - -controller-threads=20 - - -enable-metrics=true - - '-metrics-prefix=service:' - - -metrics-labels=task_name - - -metrics-labels=workflow_name - - -enable-webhook=true - - -webhook-svc-namespace=sparkoperator - command: - - /usr/bin/spark-operator - image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5 - imagePullPolicy: Always - name: sparkoperator-unknown - ports: - - containerPort: 10254 - - containerPort: 8080 - volumeMounts: - - mountPath: /etc/webhook-certs - name: webhook-certs - serviceAccountName: sparkoperator - volumes: - - name: webhook-certs - secret: - secretName: spark-webhook-certs ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: redis-resource-manager - serviceName: redis-resource-manager - template: - metadata: - labels: - app: redis-resource-manager - spec: - containers: - - env: - - name: REDIS_PASSWORD - value: mypassword - image: ecr.flyte.org/bitnami/redis:6.2.5-debian-10-r59 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - name: redis-resource-manager - ports: - - containerPort: 6379 - name: redis - protocol: TCP - readinessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 200m - memory: 128Mi - volumeMounts: - - mountPath: /bitnami - name: redis-data - dnsPolicy: ClusterFirst - restartPolicy: Always - volumes: - - emptyDir: {} - name: redis-data ---- -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: syncresources - namespace: flyte -spec: - jobTemplate: - spec: - template: - spec: - containers: - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - clusterresource - - sync - image: cr.flyte.org/flyteorg/flyteadmin:v1.10.7-b2 - imagePullPolicy: IfNotPresent - name: sync-cluster-resources - volumeMounts: - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - restartPolicy: OnFailure - serviceAccountName: flyteadmin - volumes: - - configMap: - name: clusterresource-template-4fbh4bk26k - name: resource-templates - - configMap: - name: flyte-admin-config-gf99k75c82 - name: config-volume - - name: db-pass - secret: - secretName: db-pass-bthd2588cc - schedule: '*/1 * * * *' ---- -apiVersion: batch/v1 -kind: Job -metadata: - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - name: sparkoperator-init - namespace: sparkoperator -spec: - backoffLimit: 3 - template: - metadata: - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - spec: - containers: - - command: - - /usr/bin/gencerts.sh - - --namespace - - sparkoperator - - -p - image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5 - imagePullPolicy: IfNotPresent - name: main - restartPolicy: Never - serviceAccountName: sparkoperator ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/app-root: /console - nginx.ingress.kubernetes.io/ssl-redirect: "false" - name: flytesystem - namespace: flyte -spec: - rules: - - http: - paths: - - backend: - service: - name: flyteconsole - port: - number: 80 - path: /__webpack_hmr - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.AdminService - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.AuthMetadataService - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.IdentityService - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.SignalService - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 87 - path: /openapi - pathType: ImplementationSpecific - - backend: - service: - name: flyteconsole - port: - number: 80 - path: /console - pathType: ImplementationSpecific - - backend: - service: - name: flyteconsole - port: - number: 80 - path: /console/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /api - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /api/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /healthcheck - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /v1/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 87 - path: /openapi/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /.well-known/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /login - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /login/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /logout - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /logout/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /callback - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /callback/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /me - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /config - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /config/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /oauth2 - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /oauth2/* - pathType: ImplementationSpecific diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml index 7e0b58b832b..281fcf87a1e 100644 --- a/deployment/gcp/flyte_helm_controlplane_generated.yaml +++ b/deployment/gcp/flyte_helm_controlplane_generated.yaml @@ -589,7 +589,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -610,7 +610,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -628,7 +628,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -645,7 +645,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -672,7 +672,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -777,7 +777,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -833,7 +833,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -907,7 +907,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -925,7 +925,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1008,7 +1008,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1028,7 +1028,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml index d19a4d48c45..a0349f78c41 100644 --- a/deployment/gcp/flyte_helm_dataplane_generated.yaml +++ b/deployment/gcp/flyte_helm_dataplane_generated.yaml @@ -94,7 +94,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -154,6 +154,7 @@ data: - container - sidecar - k8s-array + - agent-service k8s.yaml: | plugins: k8s: @@ -435,7 +436,7 @@ spec: template: metadata: annotations: - configChecksum: "9c6856d58b3168a752486076eccbc6783bbb94d62b47139c6cab6e1fe12174c" + configChecksum: "cc8b6585fb0703724b6418fea14f52893fdd6913d5aee513867e84d66e23d72" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -460,7 +461,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -514,9 +515,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b1 + app.kubernetes.io/version: v1.11.0 annotations: - configChecksum: "9c6856d58b3168a752486076eccbc6783bbb94d62b47139c6cab6e1fe12174c" + configChecksum: "cc8b6585fb0703724b6418fea14f52893fdd6913d5aee513867e84d66e23d72" spec: securityContext: fsGroup: 65534 @@ -528,7 +529,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -555,7 +556,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index 5b8757459c1..165712c85d3 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -473,7 +473,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -533,6 +533,7 @@ data: - container - sidecar - k8s-array + - agent-service k8s.yaml: | plugins: k8s: @@ -922,7 +923,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -943,7 +944,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -961,7 +962,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -978,7 +979,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -1005,7 +1006,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -1110,7 +1111,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -1166,7 +1167,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -1240,7 +1241,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -1258,7 +1259,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -1341,7 +1342,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -1361,7 +1362,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -1420,7 +1421,7 @@ spec: template: metadata: annotations: - configChecksum: "9c6856d58b3168a752486076eccbc6783bbb94d62b47139c6cab6e1fe12174c" + configChecksum: "cc8b6585fb0703724b6418fea14f52893fdd6913d5aee513867e84d66e23d72" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1445,7 +1446,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -1499,9 +1500,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b1 + app.kubernetes.io/version: v1.11.0 annotations: - configChecksum: "9c6856d58b3168a752486076eccbc6783bbb94d62b47139c6cab6e1fe12174c" + configChecksum: "cc8b6585fb0703724b6418fea14f52893fdd6913d5aee513867e84d66e23d72" spec: securityContext: fsGroup: 65534 @@ -1513,7 +1514,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -1540,7 +1541,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml index 1aeec5ee6de..b3b130b3a04 100644 --- a/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml +++ b/deployment/sandbox-binary/flyte_sandbox_binary_helm_generated.yaml @@ -116,7 +116,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0" k8s-array: logs: config: @@ -358,7 +358,7 @@ spec: app.kubernetes.io/instance: flyte app.kubernetes.io/component: flyte-binary annotations: - checksum/configuration: 9c864736e7d08baebc6de026def33e542b35097dd158017c89f85d7672401b92 + checksum/configuration: 8d7f2229ae4d345e158f32f67615656d9a8d7590cef80a4c1b6f8eb333fcd896 checksum/configuration-secret: d5d93f4e67780b21593dc3799f0f6682aab0765e708e4020939975d14d44f929 checksum/cluster-resource-templates: 7dfa59f3d447e9c099b8f8ffad3af466fecbc9cf9f8c97295d9634254a55d4ae spec: diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml deleted file mode 100644 index cac33303fd3..00000000000 --- a/deployment/sandbox/flyte_generated.yaml +++ /dev/null @@ -1,3965 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: flyte ---- -apiVersion: v1 -kind: Namespace -metadata: - name: kubernetes-dashboard ---- -apiVersion: v1 -kind: Namespace -metadata: - name: projectcontour ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null - name: extensionservices.projectcontour.io -spec: - group: projectcontour.io - names: - kind: ExtensionService - listKind: ExtensionServiceList - plural: extensionservices - shortNames: - - extensionservice - - extensionservices - singular: extensionservice - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: ExtensionService is the schema for the Contour extension services API. An ExtensionService resource binds a network service to the Contour API so that Contour API features can be implemented by collaborating components. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. - properties: - loadBalancerPolicy: - description: The policy for load balancing GRPC service requests. Note that the `Cookie` and `RequestHash` load balancing strategies cannot be used here. - properties: - requestHashPolicies: - description: RequestHashPolicies contains a list of hash policies to apply when the `RequestHash` load balancing strategy is chosen. If an element of the supplied list of hash policies is invalid, it will be ignored. If the list of hash policies is empty after validation, the load balancing strategy will fall back the the default `RoundRobin`. - items: - description: RequestHashPolicy contains configuration for an individual hash policy on a request attribute. - properties: - headerHashOptions: - description: HeaderHashOptions should be set when request header hash based load balancing is desired. It must be the only hash option field set, otherwise this request hash policy object will be ignored. - properties: - headerName: - description: HeaderName is the name of the HTTP request header that will be used to calculate the hash key. If the header specified is not present on a request, no hash will be produced. - minLength: 1 - type: string - type: object - terminal: - description: Terminal is a flag that allows for short-circuiting computing of a hash for a given request. If set to true, and the request attribute specified in the attribute hash options is present, no further hash policies will be used to calculate a hash for the request. - type: boolean - type: object - type: array - strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, and `RequestHash`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. - type: string - type: object - protocol: - description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. - enum: - - h2 - - h2c - type: string - protocolVersion: - description: This field sets the version of the GRPC protocol that Envoy uses to send requests to the extension service. Since Contour always uses the v3 Envoy API, this is currently fixed at "v3". However, other protocol options will be available in future. - enum: - - v3 - type: string - services: - description: Services specifies the set of Kubernetes Service resources that receive GRPC extension API requests. If no weights are specified for any of the entries in this array, traffic will be spread evenly across all the services. Otherwise, traffic is balanced proportionally to the Weight field in each entry. - items: - description: ExtensionServiceTarget defines an Kubernetes Service to target with extension service traffic. - properties: - name: - description: Name is the name of Kubernetes service that will accept service traffic. - type: string - port: - description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. - exclusiveMaximum: true - maximum: 65536 - minimum: 1 - type: integer - weight: - description: Weight defines proportion of traffic to balance to the Kubernetes Service. - format: int32 - type: integer - required: - - name - - port - type: object - minItems: 1 - type: array - timeoutPolicy: - description: The timeout policy for requests to the services. - properties: - idle: - description: Timeout after which, if there are no active requests for this route, the connection between Envoy and the backend or Envoy and the external client will be closed. If not specified, there is no per-route idle timeout, though a connection manager-wide stream_idle_timeout default of 5m still applies. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - response: - description: Timeout for receiving a response from the server after processing a request from client. If not supplied, Envoy's default value of 15s applies. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - type: object - validation: - description: UpstreamValidation defines how to verify the backend service's certificate - properties: - caSecret: - description: Name of the Kubernetes secret be used to validate the certificate presented by the backend - type: string - subjectName: - description: Key which is expected to be present in the 'subjectAltName' of the presented certificate - type: string - required: - - caSecret - - subjectName - type: object - required: - - services - type: object - status: - description: ExtensionServiceStatus defines the observed state of an ExtensionService resource. - properties: - conditions: - description: "Conditions contains the current status of the ExtensionService resource. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. \n Contour will not modify any other Conditions set in this block, in case some other controller wants to add a Condition." - items: - description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." - properties: - errors: - description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - warnings: - description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - name: flyteworkflows.flyte.lyft.com -spec: - group: flyte.lyft.com - names: - kind: FlyteWorkflow - plural: flyteworkflows - shortNames: - - fly - singular: flyteworkflow - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - type: object - x-kubernetes-preserve-unknown-fields: true - served: true - storage: true ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null - name: httpproxies.projectcontour.io -spec: - group: projectcontour.io - names: - kind: HTTPProxy - listKind: HTTPProxyList - plural: httpproxies - shortNames: - - proxy - - proxies - singular: httpproxy - preserveUnknownFields: false - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Fully qualified domain name - jsonPath: .spec.virtualhost.fqdn - name: FQDN - type: string - - description: Secret with TLS credentials - jsonPath: .spec.virtualhost.tls.secretName - name: TLS Secret - type: string - - description: The current status of the HTTPProxy - jsonPath: .status.currentStatus - name: Status - type: string - - description: Description of the current status - jsonPath: .status.description - name: Status Description - type: string - name: v1 - schema: - openAPIV3Schema: - description: HTTPProxy is an Ingress CRD specification. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HTTPProxySpec defines the spec of the CRD. - properties: - includes: - description: Includes allow for specific routing configuration to be included from another HTTPProxy, possibly in another namespace. - items: - description: Include describes a set of policies that can be applied to an HTTPProxy in a namespace. - properties: - conditions: - description: 'Conditions are a set of rules that are applied to included HTTPProxies. In effect, they are added onto the Conditions of included HTTPProxy Route structs. When applied, they are merged using AND, with one exception: There can be only one Prefix MatchCondition per Conditions slice. More than one Prefix, or contradictory Conditions, will make the include invalid.' - items: - description: MatchCondition are a general holder for matching rules for HTTPProxies. One of Prefix or Header must be provided. - properties: - header: - description: Header specifies the header condition to match. - properties: - contains: - description: Contains specifies a substring that must be present in the header value. - type: string - exact: - description: Exact specifies a string that the header value must be equal to. - type: string - name: - description: Name is the name of the header to match against. Name is required. Header names are case insensitive. - type: string - notcontains: - description: NotContains specifies a substring that must not be present in the header value. - type: string - notexact: - description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value. - type: string - present: - description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent. - type: boolean - required: - - name - type: object - prefix: - description: Prefix defines a prefix match for a request. - type: string - type: object - type: array - name: - description: Name of the HTTPProxy - type: string - namespace: - description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. - type: string - required: - - name - type: object - type: array - routes: - description: Routes are the ingress routes. If TCPProxy is present, Routes is ignored. - items: - description: Route contains the set of routes for a virtual host. - properties: - authPolicy: - description: AuthPolicy updates the authorization policy that was set on the root HTTPProxy object for client requests that match this route. - properties: - context: - additionalProperties: - type: string - description: Context is a set of key/value pairs that are sent to the authentication server in the check request. If a context is provided at an enclosing scope, the entries are merged such that the inner scope overrides matching keys from the outer scope. - type: object - disabled: - description: When true, this field disables client request authentication for the scope of the policy. - type: boolean - type: object - conditions: - description: 'Conditions are a set of rules that are applied to a Route. When applied, they are merged using AND, with one exception: There can be only one Prefix MatchCondition per Conditions slice. More than one Prefix, or contradictory Conditions, will make the route invalid.' - items: - description: MatchCondition are a general holder for matching rules for HTTPProxies. One of Prefix or Header must be provided. - properties: - header: - description: Header specifies the header condition to match. - properties: - contains: - description: Contains specifies a substring that must be present in the header value. - type: string - exact: - description: Exact specifies a string that the header value must be equal to. - type: string - name: - description: Name is the name of the header to match against. Name is required. Header names are case insensitive. - type: string - notcontains: - description: NotContains specifies a substring that must not be present in the header value. - type: string - notexact: - description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value. - type: string - present: - description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent. - type: boolean - required: - - name - type: object - prefix: - description: Prefix defines a prefix match for a request. - type: string - type: object - type: array - enableWebsockets: - description: Enables websocket support for the route. - type: boolean - healthCheckPolicy: - description: The health check policy for this route. - properties: - healthyThresholdCount: - description: The number of healthy health checks required before a host is marked healthy - format: int64 - minimum: 0 - type: integer - host: - description: The value of the host header in the HTTP health check request. If left empty (default value), the name "contour-envoy-healthcheck" will be used. - type: string - intervalSeconds: - description: The interval (seconds) between health checks - format: int64 - type: integer - path: - description: HTTP endpoint used to perform health checks on upstream service - type: string - timeoutSeconds: - description: The time to wait (seconds) for a health check response - format: int64 - type: integer - unhealthyThresholdCount: - description: The number of unhealthy health checks required before a host is marked unhealthy - format: int64 - minimum: 0 - type: integer - required: - - path - type: object - loadBalancerPolicy: - description: The load balancing policy for this route. - properties: - requestHashPolicies: - description: RequestHashPolicies contains a list of hash policies to apply when the `RequestHash` load balancing strategy is chosen. If an element of the supplied list of hash policies is invalid, it will be ignored. If the list of hash policies is empty after validation, the load balancing strategy will fall back the the default `RoundRobin`. - items: - description: RequestHashPolicy contains configuration for an individual hash policy on a request attribute. - properties: - headerHashOptions: - description: HeaderHashOptions should be set when request header hash based load balancing is desired. It must be the only hash option field set, otherwise this request hash policy object will be ignored. - properties: - headerName: - description: HeaderName is the name of the HTTP request header that will be used to calculate the hash key. If the header specified is not present on a request, no hash will be produced. - minLength: 1 - type: string - type: object - terminal: - description: Terminal is a flag that allows for short-circuiting computing of a hash for a given request. If set to true, and the request attribute specified in the attribute hash options is present, no further hash policies will be used to calculate a hash for the request. - type: boolean - type: object - type: array - strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, and `RequestHash`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. - type: string - type: object - pathRewritePolicy: - description: The policy for rewriting the path of the request URL after the request has been routed to a Service. - properties: - replacePrefix: - description: ReplacePrefix describes how the path prefix should be replaced. - items: - description: ReplacePrefix describes a path prefix replacement. - properties: - prefix: - description: "Prefix specifies the URL path prefix to be replaced. \n If Prefix is specified, it must exactly match the MatchCondition prefix that is rendered by the chain of including HTTPProxies and only that path prefix will be replaced by Replacement. This allows HTTPProxies that are included through multiple roots to only replace specific path prefixes, leaving others unmodified. \n If Prefix is not specified, all routing prefixes rendered by the include chain will be replaced." - minLength: 1 - type: string - replacement: - description: Replacement is the string that the routing path prefix will be replaced with. This must not be empty. - minLength: 1 - type: string - required: - - replacement - type: object - type: array - type: object - permitInsecure: - description: Allow this path to respond to insecure requests over HTTP which are normally not permitted when a `virtualhost.tls` block is present. - type: boolean - rateLimitPolicy: - description: The policy for rate limiting on the route. - properties: - global: - description: Global defines global rate limiting parameters, i.e. parameters defining descriptors that are sent to an external rate limit service (RLS) for a rate limit decision on each request. - properties: - descriptors: - description: Descriptors defines the list of descriptors that will be generated and sent to the rate limit service. Each descriptor contains 1+ key-value pair entries. - items: - description: RateLimitDescriptor defines a list of key-value pair generators. - properties: - entries: - description: Entries is the list of key-value pair generators. - items: - description: RateLimitDescriptorEntry is a key-value pair generator. Exactly one field on this struct must be non-nil. - properties: - genericKey: - description: GenericKey defines a descriptor entry with a static key and value. - properties: - key: - description: Key defines the key of the descriptor entry. If not set, the key is set to "generic_key". - type: string - value: - description: Value defines the value of the descriptor entry. - minLength: 1 - type: string - type: object - remoteAddress: - description: RemoteAddress defines a descriptor entry with a key of "remote_address" and a value equal to the client's IP address (from x-forwarded-for). - type: object - requestHeader: - description: RequestHeader defines a descriptor entry that's populated only if a given header is present on the request. The descriptor key is static, and the descriptor value is equal to the value of the header. - properties: - descriptorKey: - description: DescriptorKey defines the key to use on the descriptor entry. - minLength: 1 - type: string - headerName: - description: HeaderName defines the name of the header to look for on the request. - minLength: 1 - type: string - type: object - type: object - minItems: 1 - type: array - type: object - minItems: 1 - type: array - type: object - local: - description: Local defines local rate limiting parameters, i.e. parameters for rate limiting that occurs within each Envoy pod as requests are handled. - properties: - burst: - description: Burst defines the number of requests above the requests per unit that should be allowed within a short period of time. - format: int32 - type: integer - requests: - description: Requests defines how many requests per unit of time should be allowed before rate limiting occurs. - format: int32 - minimum: 1 - type: integer - responseHeadersToAdd: - description: ResponseHeadersToAdd is an optional list of response headers to set when a request is rate-limited. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - responseStatusCode: - description: ResponseStatusCode is the HTTP status code to use for responses to rate-limited requests. Codes must be in the 400-599 range (inclusive). If not specified, the Envoy default of 429 (Too Many Requests) is used. - format: int32 - maximum: 599 - minimum: 400 - type: integer - unit: - description: Unit defines the period of time within which requests over the limit will be rate limited. Valid values are "second", "minute" and "hour". - enum: - - second - - minute - - hour - type: string - required: - - requests - - unit - type: object - type: object - requestHeadersPolicy: - description: The policy for managing request headers during proxying. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - responseHeadersPolicy: - description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - retryPolicy: - description: The retry policy for this route. - properties: - count: - description: NumRetries is maximum allowed number of retries. If not supplied, the number of retries is one. - format: int64 - minimum: 0 - type: integer - perTryTimeout: - description: PerTryTimeout specifies the timeout per retry attempt. Ignored if NumRetries is not supplied. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - retriableStatusCodes: - description: "RetriableStatusCodes specifies the HTTP status codes that should be retried. \n This field is only respected when you include `retriable-status-codes` in the `RetryOn` field." - items: - format: int32 - type: integer - type: array - retryOn: - description: "RetryOn specifies the conditions on which to retry a request. \n Supported [HTTP conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-on): \n - `5xx` - `gateway-error` - `reset` - `connect-failure` - `retriable-4xx` - `refused-stream` - `retriable-status-codes` - `retriable-headers` \n Supported [gRPC conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-grpc-on): \n - `cancelled` - `deadline-exceeded` - `internal` - `resource-exhausted` - `unavailable`" - items: - description: RetryOn is a string type alias with validation to ensure that the value is valid. - enum: - - 5xx - - gateway-error - - reset - - connect-failure - - retriable-4xx - - refused-stream - - retriable-status-codes - - retriable-headers - - cancelled - - deadline-exceeded - - internal - - resource-exhausted - - unavailable - type: string - type: array - type: object - services: - description: Services are the services to proxy traffic. - items: - description: Service defines an Kubernetes Service to proxy traffic. - properties: - mirror: - description: If Mirror is true the Service will receive a read only mirror of the traffic for this route. - type: boolean - name: - description: Name is the name of Kubernetes service to proxy traffic. Names defined here will be used to look up corresponding endpoints which contain the ips to route. - type: string - port: - description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. - exclusiveMaximum: true - maximum: 65536 - minimum: 1 - type: integer - protocol: - description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. - enum: - - h2 - - h2c - - tls - type: string - requestHeadersPolicy: - description: The policy for managing request headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - responseHeadersPolicy: - description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - validation: - description: UpstreamValidation defines how to verify the backend service's certificate - properties: - caSecret: - description: Name of the Kubernetes secret be used to validate the certificate presented by the backend - type: string - subjectName: - description: Key which is expected to be present in the 'subjectAltName' of the presented certificate - type: string - required: - - caSecret - - subjectName - type: object - weight: - description: Weight defines percentage of traffic to balance traffic - format: int64 - minimum: 0 - type: integer - required: - - name - - port - type: object - minItems: 1 - type: array - timeoutPolicy: - description: The timeout policy for this route. - properties: - idle: - description: Timeout after which, if there are no active requests for this route, the connection between Envoy and the backend or Envoy and the external client will be closed. If not specified, there is no per-route idle timeout, though a connection manager-wide stream_idle_timeout default of 5m still applies. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - response: - description: Timeout for receiving a response from the server after processing a request from client. If not supplied, Envoy's default value of 15s applies. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - type: object - required: - - services - type: object - type: array - tcpproxy: - description: TCPProxy holds TCP proxy information. - properties: - healthCheckPolicy: - description: The health check policy for this tcp proxy - properties: - healthyThresholdCount: - description: The number of healthy health checks required before a host is marked healthy - format: int32 - type: integer - intervalSeconds: - description: The interval (seconds) between health checks - format: int64 - type: integer - timeoutSeconds: - description: The time to wait (seconds) for a health check response - format: int64 - type: integer - unhealthyThresholdCount: - description: The number of unhealthy health checks required before a host is marked unhealthy - format: int32 - type: integer - type: object - include: - description: Include specifies that this tcpproxy should be delegated to another HTTPProxy. - properties: - name: - description: Name of the child HTTPProxy - type: string - namespace: - description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. - type: string - required: - - name - type: object - includes: - description: "IncludesDeprecated allow for specific routing configuration to be appended to another HTTPProxy in another namespace. \n Exists due to a mistake when developing HTTPProxy and the field was marked plural when it should have been singular. This field should stay to not break backwards compatibility to v1 users." - properties: - name: - description: Name of the child HTTPProxy - type: string - namespace: - description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. - type: string - required: - - name - type: object - loadBalancerPolicy: - description: The load balancing policy for the backend services. Note that the `Cookie` and `RequestHash` load balancing strategies cannot be used here. - properties: - requestHashPolicies: - description: RequestHashPolicies contains a list of hash policies to apply when the `RequestHash` load balancing strategy is chosen. If an element of the supplied list of hash policies is invalid, it will be ignored. If the list of hash policies is empty after validation, the load balancing strategy will fall back the the default `RoundRobin`. - items: - description: RequestHashPolicy contains configuration for an individual hash policy on a request attribute. - properties: - headerHashOptions: - description: HeaderHashOptions should be set when request header hash based load balancing is desired. It must be the only hash option field set, otherwise this request hash policy object will be ignored. - properties: - headerName: - description: HeaderName is the name of the HTTP request header that will be used to calculate the hash key. If the header specified is not present on a request, no hash will be produced. - minLength: 1 - type: string - type: object - terminal: - description: Terminal is a flag that allows for short-circuiting computing of a hash for a given request. If set to true, and the request attribute specified in the attribute hash options is present, no further hash policies will be used to calculate a hash for the request. - type: boolean - type: object - type: array - strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, and `RequestHash`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. - type: string - type: object - services: - description: Services are the services to proxy traffic - items: - description: Service defines an Kubernetes Service to proxy traffic. - properties: - mirror: - description: If Mirror is true the Service will receive a read only mirror of the traffic for this route. - type: boolean - name: - description: Name is the name of Kubernetes service to proxy traffic. Names defined here will be used to look up corresponding endpoints which contain the ips to route. - type: string - port: - description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. - exclusiveMaximum: true - maximum: 65536 - minimum: 1 - type: integer - protocol: - description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. - enum: - - h2 - - h2c - - tls - type: string - requestHeadersPolicy: - description: The policy for managing request headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - responseHeadersPolicy: - description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - validation: - description: UpstreamValidation defines how to verify the backend service's certificate - properties: - caSecret: - description: Name of the Kubernetes secret be used to validate the certificate presented by the backend - type: string - subjectName: - description: Key which is expected to be present in the 'subjectAltName' of the presented certificate - type: string - required: - - caSecret - - subjectName - type: object - weight: - description: Weight defines percentage of traffic to balance traffic - format: int64 - minimum: 0 - type: integer - required: - - name - - port - type: object - type: array - type: object - virtualhost: - description: Virtualhost appears at most once. If it is present, the object is considered to be a "root" HTTPProxy. - properties: - authorization: - description: This field configures an extension service to perform authorization for this virtual host. Authorization can only be configured on virtual hosts that have TLS enabled. If the TLS configuration requires client certificate /validation, the client certificate is always included in the authentication check request. - properties: - authPolicy: - description: AuthPolicy sets a default authorization policy for client requests. This policy will be used unless overridden by individual routes. - properties: - context: - additionalProperties: - type: string - description: Context is a set of key/value pairs that are sent to the authentication server in the check request. If a context is provided at an enclosing scope, the entries are merged such that the inner scope overrides matching keys from the outer scope. - type: object - disabled: - description: When true, this field disables client request authentication for the scope of the policy. - type: boolean - type: object - extensionRef: - description: ExtensionServiceRef specifies the extension resource that will authorize client requests. - properties: - apiVersion: - description: API version of the referent. If this field is not specified, the default "projectcontour.io/v1alpha1" will be used - minLength: 1 - type: string - name: - description: "Name of the referent. \n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" - minLength: 1 - type: string - namespace: - description: "Namespace of the referent. If this field is not specifies, the namespace of the resource that targets the referent will be used. \n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/" - minLength: 1 - type: string - type: object - failOpen: - description: If FailOpen is true, the client request is forwarded to the upstream service even if the authorization server fails to respond. This field should not be set in most cases. It is intended for use only while migrating applications from internal authorization to Contour external authorization. - type: boolean - responseTimeout: - description: ResponseTimeout configures maximum time to wait for a check response from the authorization server. Timeout durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". The string "infinity" is also a valid input and specifies no timeout. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - required: - - extensionRef - type: object - corsPolicy: - description: Specifies the cross-origin policy to apply to the VirtualHost. - properties: - allowCredentials: - description: Specifies whether the resource allows credentials. - type: boolean - allowHeaders: - description: AllowHeaders specifies the content for the *access-control-allow-headers* header. - items: - description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. - pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ - type: string - type: array - allowMethods: - description: AllowMethods specifies the content for the *access-control-allow-methods* header. - items: - description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. - pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ - type: string - type: array - allowOrigin: - description: AllowOrigin specifies the origins that will be allowed to do CORS requests. "*" means allow any origin. - items: - type: string - type: array - exposeHeaders: - description: ExposeHeaders Specifies the content for the *access-control-expose-headers* header. - items: - description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. - pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ - type: string - type: array - maxAge: - description: MaxAge indicates for how long the results of a preflight request can be cached. MaxAge durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Only positive values are allowed while 0 disables the cache requiring a preflight OPTIONS check for all cross-origin requests. - type: string - required: - - allowMethods - - allowOrigin - type: object - fqdn: - description: The fully qualified domain name of the root of the ingress tree all leaves of the DAG rooted at this object relate to the fqdn. - type: string - rateLimitPolicy: - description: The policy for rate limiting on the virtual host. - properties: - global: - description: Global defines global rate limiting parameters, i.e. parameters defining descriptors that are sent to an external rate limit service (RLS) for a rate limit decision on each request. - properties: - descriptors: - description: Descriptors defines the list of descriptors that will be generated and sent to the rate limit service. Each descriptor contains 1+ key-value pair entries. - items: - description: RateLimitDescriptor defines a list of key-value pair generators. - properties: - entries: - description: Entries is the list of key-value pair generators. - items: - description: RateLimitDescriptorEntry is a key-value pair generator. Exactly one field on this struct must be non-nil. - properties: - genericKey: - description: GenericKey defines a descriptor entry with a static key and value. - properties: - key: - description: Key defines the key of the descriptor entry. If not set, the key is set to "generic_key". - type: string - value: - description: Value defines the value of the descriptor entry. - minLength: 1 - type: string - type: object - remoteAddress: - description: RemoteAddress defines a descriptor entry with a key of "remote_address" and a value equal to the client's IP address (from x-forwarded-for). - type: object - requestHeader: - description: RequestHeader defines a descriptor entry that's populated only if a given header is present on the request. The descriptor key is static, and the descriptor value is equal to the value of the header. - properties: - descriptorKey: - description: DescriptorKey defines the key to use on the descriptor entry. - minLength: 1 - type: string - headerName: - description: HeaderName defines the name of the header to look for on the request. - minLength: 1 - type: string - type: object - type: object - minItems: 1 - type: array - type: object - minItems: 1 - type: array - type: object - local: - description: Local defines local rate limiting parameters, i.e. parameters for rate limiting that occurs within each Envoy pod as requests are handled. - properties: - burst: - description: Burst defines the number of requests above the requests per unit that should be allowed within a short period of time. - format: int32 - type: integer - requests: - description: Requests defines how many requests per unit of time should be allowed before rate limiting occurs. - format: int32 - minimum: 1 - type: integer - responseHeadersToAdd: - description: ResponseHeadersToAdd is an optional list of response headers to set when a request is rate-limited. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - responseStatusCode: - description: ResponseStatusCode is the HTTP status code to use for responses to rate-limited requests. Codes must be in the 400-599 range (inclusive). If not specified, the Envoy default of 429 (Too Many Requests) is used. - format: int32 - maximum: 599 - minimum: 400 - type: integer - unit: - description: Unit defines the period of time within which requests over the limit will be rate limited. Valid values are "second", "minute" and "hour". - enum: - - second - - minute - - hour - type: string - required: - - requests - - unit - type: object - type: object - tls: - description: If present the fields describes TLS properties of the virtual host. The SNI names that will be matched on are described in fqdn, the tls.secretName secret must contain a certificate that itself contains a name that matches the FQDN. - properties: - clientValidation: - description: "ClientValidation defines how to verify the client certificate when an external client establishes a TLS connection to Envoy. \n This setting: \n 1. Enables TLS client certificate validation. 2. Requires clients to present a TLS certificate (i.e. not optional validation). 3. Specifies how the client certificate will be validated." - properties: - caSecret: - description: Name of a Kubernetes secret that contains a CA certificate bundle. The client certificate must validate against the certificates in the bundle. - minLength: 1 - type: string - required: - - caSecret - type: object - enableFallbackCertificate: - description: EnableFallbackCertificate defines if the vhost should allow a default certificate to be applied which handles all requests which don't match the SNI defined in this vhost. - type: boolean - minimumProtocolVersion: - description: MinimumProtocolVersion is the minimum TLS version this vhost should negotiate. Valid options are `1.2` (default) and `1.3`. Any other value defaults to TLS 1.2. - type: string - passthrough: - description: Passthrough defines whether the encrypted TLS handshake will be passed through to the backing cluster. Either Passthrough or SecretName must be specified, but not both. - type: boolean - secretName: - description: SecretName is the name of a TLS secret in the current namespace. Either SecretName or Passthrough must be specified, but not both. If specified, the named secret must contain a matching certificate for the virtual host's FQDN. - type: string - type: object - required: - - fqdn - type: object - type: object - status: - description: Status is a container for computed information about the HTTPProxy. - properties: - conditions: - description: "Conditions contains information about the current status of the HTTPProxy, in an upstream-friendly container. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, and vice versa. \n Contour will leave untouched any other Conditions set in this block, in case some other controller wants to add a Condition. \n If you are another controller owner and wish to add a condition, you *should* namespace your condition with a label, like `controller.domain.com/ConditionName`." - items: - description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." - properties: - errors: - description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - warnings: - description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - currentStatus: - type: string - description: - type: string - loadBalancer: - description: LoadBalancer contains the current status of the load balancer. - properties: - ingress: - description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. - items: - description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' - properties: - hostname: - description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) - type: string - ip: - description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) - type: string - ports: - description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it - items: - properties: - error: - description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - port: - description: Port is the port number of the service port of which status is recorded here - format: int32 - type: integer - protocol: - default: TCP - description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' - type: string - required: - - port - - protocol - type: object - type: array - x-kubernetes-list-type: atomic - type: object - type: array - type: object - type: object - required: - - metadata - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.4.1 - creationTimestamp: null - name: tlscertificatedelegations.projectcontour.io -spec: - group: projectcontour.io - names: - kind: TLSCertificateDelegation - listKind: TLSCertificateDelegationList - plural: tlscertificatedelegations - shortNames: - - tlscerts - singular: tlscertificatedelegation - preserveUnknownFields: false - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: TLSCertificateDelegation is an TLS Certificate Delegation CRD specification. See design/tls-certificate-delegation.md for details. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: TLSCertificateDelegationSpec defines the spec of the CRD - properties: - delegations: - items: - description: CertificateDelegation maps the authority to reference a secret in the current namespace to a set of namespaces. - properties: - secretName: - description: required, the name of a secret in the current namespace. - type: string - targetNamespaces: - description: required, the namespaces the authority to reference the the secret will be delegated to. If TargetNamespaces is nil or empty, the CertificateDelegation' is ignored. If the TargetNamespace list contains the character, "*" the secret will be delegated to all namespaces. - items: - type: string - type: array - required: - - secretName - - targetNamespaces - type: object - type: array - required: - - delegations - type: object - status: - description: TLSCertificateDelegationStatus allows for the status of the delegation to be presented to the user. - properties: - conditions: - description: "Conditions contains information about the current status of the HTTPProxy, in an upstream-friendly container. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, and vice versa. \n Contour will leave untouched any other Conditions set in this block, in case some other controller wants to add a Condition. \n If you are another controller owner and wish to add a condition, you *should* namespace your condition with a label, like `controller.domain.com\\ConditionName`." - items: - description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." - properties: - errors: - description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - warnings: - description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - required: - - metadata - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: datacatalog - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flyte-pod-webhook - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flyteadmin - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flytepropeller - namespace: flyte ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: contour - namespace: projectcontour ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: contour-certgen - namespace: projectcontour ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: envoy - namespace: projectcontour ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard -rules: -- apiGroups: - - "" - resourceNames: - - kubernetes-dashboard-key-holder - - kubernetes-dashboard-certs - - kubernetes-dashboard-csrf - resources: - - secrets - verbs: - - get - - update - - delete -- apiGroups: - - "" - resourceNames: - - kubernetes-dashboard-settings - resources: - - configmaps - verbs: - - get - - update -- apiGroups: - - "" - resourceNames: - - heapster - - dashboard-metrics-scraper - resources: - - services - verbs: - - proxy -- apiGroups: - - "" - resourceNames: - - heapster - - 'http:heapster:' - - 'https:heapster:' - - dashboard-metrics-scraper - - http:dashboard-metrics-scraper - resources: - - services/proxy - verbs: - - get ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: contour-certgen - namespace: projectcontour -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flyte-pod-webhook - namespace: flyte -rules: -- apiGroups: - - '*' - resources: - - mutatingwebhookconfigurations - - secrets - - pods - - replicasets/finalizers - verbs: - - get - - create - - update - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flyteadmin - namespace: flyte -rules: -- apiGroups: - - "" - - flyte.lyft.com - - rbac.authorization.k8s.io - resources: - - configmaps - - flyteworkflows - - namespaces - - pods - - resourcequotas - - roles - - rolebindings - - secrets - - services - - serviceaccounts - - spark-role - verbs: - - '*' ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: contour -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - create - - get - - update -- apiGroups: - - networking.x-k8s.io - resources: - - backendpolicies - - gateways - - httproutes - - tlsroutes - verbs: - - get - - list - - watch -- apiGroups: - - projectcontour.io - resources: - - extensionservices - verbs: - - get - - list - - watch -- apiGroups: - - projectcontour.io - resources: - - extensionservices/status - verbs: - - create - - get - - update -- apiGroups: - - projectcontour.io - resources: - - httpproxies - - tlscertificatedelegations - verbs: - - get - - list - - watch -- apiGroups: - - projectcontour.io - resources: - - httpproxies/status - verbs: - - create - - get - - update ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: flytepropeller -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - delete - - patch -- apiGroups: - - '*' - resources: - - '*' - verbs: - - get - - list - - watch - - create - - update - - delete - - patch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - create - - delete - - update -- apiGroups: - - flyte.lyft.com - resources: - - flyteworkflows - - flyteworkflows/finalizers - verbs: - - get - - list - - watch - - create - - update - - delete - - patch - - post - - deletecollection ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard -rules: -- apiGroups: - - metrics.k8s.io - resources: - - pods - - nodes - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: kubernetes-dashboard -subjects: -- kind: ServiceAccount - name: kubernetes-dashboard - namespace: kubernetes-dashboard ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: contour - namespace: projectcontour -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: contour-certgen -subjects: -- kind: ServiceAccount - name: contour-certgen - namespace: projectcontour ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flyte-pod-webhook - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flyte-pod-webhook -subjects: -- kind: ServiceAccount - name: flyte-pod-webhook - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flyteadmin-binding - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flyteadmin -subjects: -- kind: ServiceAccount - name: flyteadmin - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flytepropeller - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flytepropeller -subjects: -- kind: ServiceAccount - name: flytepropeller - namespace: flyte ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: contour -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: contour -subjects: -- kind: ServiceAccount - name: contour - namespace: projectcontour ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubernetes-dashboard -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: kubernetes-dashboard -subjects: -- kind: ServiceAccount - name: kubernetes-dashboard - namespace: kubernetes-dashboard ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubernetes-dashboard-admin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- kind: ServiceAccount - name: kubernetes-dashboard - namespace: kubernetes-dashboard ---- -apiVersion: v1 -data: - aa_namespace.yaml: | - apiVersion: v1 - kind: Namespace - metadata: - name: {{ namespace }} - spec: - finalizers: - - kubernetes - ab_project-resource-quota.yaml: "apiVersion: v1\nkind: ResourceQuota\nmetadata:\n name: project-quota\n namespace: {{ namespace }} \nspec:\n hard:\n limits.cpu: {{ projectQuotaCpu }} \n limits.memory: {{ projectQuotaMemory }}\n\n" -kind: ConfigMap -metadata: - name: clusterresource-template-dk5mbchdmt - namespace: flyte ---- -apiVersion: v1 -data: - db.yaml: | - database: - port: 5432 - username: postgres - host: postgres - dbname: datacatalog - options: sslmode=disable - logger.yaml: | - logger: - show-source: true - level: 2 - server.yaml: | - datacatalog: - storage-prefix: metadata/datacatalog - metrics-scope: "datacatalog" - profiler-port: 10254 - application: - grpcPort: 8089 - storage.yaml: |+ - storage: - type: minio - connection: - access-key: minio - auth-type: accesskey - secret-key: miniostorage - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - container: "my-s3-bucket" - -kind: ConfigMap -metadata: - name: datacatalog-config-64k8dg9gck - namespace: flyte ---- -apiVersion: v1 -data: - cluster_resources.yaml: | - cluster_resources: - templatePath: "/etc/flyte/clusterresource/templates" - customData: - - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" - refreshInterval: 1m - db.yaml: | - database: - port: 5432 - username: postgres - host: postgres - dbname: postgres - options: sslmode=disable - domain.yaml: | - domains: - - id: development - name: development - - id: staging - name: staging - - id: production - name: production - logger.yaml: | - logger: - show-source: true - level: 2 - server.yaml: | - server: - httpPort: 8088 - grpcPort: 8089 - security: - # Controls whether to serve requests over SSL/TLS. - secure: false - # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. - useAuth: false - allowCors: true - allowedOrigins: - # Accepting all domains for Sandbox installation - - "*" - allowedHeaders: - - "Content-Type" - - "flyte-authorization" - auth: - authorizedUris: - # This should point at your public http Uri. - - https://localhost:30081 - # This will be used by internal services in the same namespace as flyteadmin - - http://flyteadmin:80 - # This will be used by internal services in the same cluster but different namespaces - - http://flyteadmin.flyte.svc.cluster.local:80 - - # Controls app authentication config - appAuth: - thirdPartyConfig: - flyteClient: - clientId: flytectl - redirectUri: https://localhost:53593/callback - scopes: - - offline - - all - # Controls user authentication - userAuth: - openId: - baseUrl: https://accounts.google.com - scopes: - - profile - - openid - clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com - flyteadmin: - roleNameKey: "iam.amazonaws.com/role" - storage.yaml: |+ - storage: - type: minio - connection: - access-key: minio - auth-type: accesskey - secret-key: miniostorage - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - container: "my-s3-bucket" - - task_resource_defaults.yaml: | - task_resources: - defaults: - cpu: 100m - memory: 200Mi - storage: 5Mi - limits: - cpu: 2 - memory: 1Gi - storage: 20Mi - gpu: 1 -kind: ConfigMap -metadata: - name: flyte-admin-config-dbg8dt2dgb - namespace: flyte ---- -apiVersion: v1 -data: - BASE_URL: /console - CONFIG_DIR: /etc/flyte/config -kind: ConfigMap -metadata: - name: flyte-console-config - namespace: flyte ---- -apiVersion: v1 -data: - admin.yaml: | - event: - type: admin - rate: 500 - capacity: 1000 - admin: - endpoint: flyteadmin:81 - insecure: true - catalog.yaml: | - catalog-cache: - endpoint: datacatalog:89 - type: datacatalog - insecure: true - copilot.yaml: | - plugins: - k8s: - co-pilot: - name: "flyte-copilot-" - image: "cr.flyte.org/flyteorg/flytecopilot:v0.0.15" - start-timeout: "30s" - core.yaml: | - propeller: - rawoutput-prefix: s3://my-s3-bucket/ - metadata-prefix: metadata/propeller - workers: 4 - max-workflow-retries: 30 - workflow-reeval-duration: 30s - downstream-eval-duration: 30s - limit-namespace: "all" - prof-port: 10254 - metrics-prefix: flyte - enable-admin-launcher: true - leader-election: - lock-config-map: - name: propeller-leader - namespace: flyte - enabled: true - lease-duration: 15s - renew-deadline: 10s - retry-period: 2s - queue: - type: batch - batching-interval: 2s - batch-size: -1 - queue: - type: maxof - rate: 100 - capacity: 1000 - base-delay: 5s - max-delay: 120s - sub-queue: - type: bucket - rate: 10 - capacity: 100 - webhook: - certDir: /etc/webhook/certs - serviceName: flyte-pod-webhook - enabled_plugins.yaml: | - tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array - k8s.yaml: | - plugins: - k8s: - default-env-vars: - - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - - FLYTE_AWS_ACCESS_KEY_ID: minio - - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage - default-cpus: 100m - default-memory: 200Mi - logger.yaml: | - logger: - show-source: true - level: 2 - resource_manager.yaml: | - propeller: - resourcemanager: - type: noop - storage.yaml: |+ - storage: - type: minio - connection: - access-key: minio - auth-type: accesskey - secret-key: miniostorage - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - container: "my-s3-bucket" - - task_logs.yaml: | - plugins: - logs: - kubernetes-enabled: true - kubernetes-template-uri: "http://localhost:30082/#/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}" -kind: ConfigMap -metadata: - name: flyte-propeller-config-g52db28fmc - namespace: flyte ---- -apiVersion: v1 -data: - admin.yaml: | - admin: - clientId: flytepropeller - clientSecretLocation: /etc/secrets/client_secret - endpoint: flyteadmin:81 - insecure: true - event: - capacity: 1000 - rate: 500 - type: admin - db.yaml: | - database: - dbname: postgres - host: postgres - port: 5432 - username: postgres - logger.yaml: | - logger: - level: 4 - show-source: true -kind: ConfigMap -metadata: - labels: - app.kubernetes.io/instance: flyte - app.kubernetes.io/managed-by: Helm - app.kubernetes.io/name: flytescheduler - helm.sh/chart: flyte-v0.1.10 - name: flyte-scheduler-config - namespace: flyte ---- -apiVersion: v1 -kind: ConfigMap -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard-settings - namespace: kubernetes-dashboard ---- -apiVersion: v1 -data: - contour.yaml: | - # - # server: - # determine which XDS Server implementation to utilize in Contour. - # xds-server-type: contour - # - # Specify the gateway-api Gateway Contour should watch. - # gateway: - # name: contour - # namespace: projectcontour - # - # should contour expect to be running inside a k8s cluster - # incluster: true - # - # path to kubeconfig (if not running inside a k8s cluster) - # kubeconfig: /path/to/.kube/config - # - # Disable RFC-compliant behavior to strip "Content-Length" header if - # "Tranfer-Encoding: chunked" is also set. - # disableAllowChunkedLength: false - # Disable HTTPProxy permitInsecure field - disablePermitInsecure: false - tls: - # minimum TLS version that Contour will negotiate - # minimum-protocol-version: "1.2" - # TLS ciphers to be supported by Envoy TLS listeners when negotiating - # TLS 1.2. - # cipher-suites: - # - '[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]' - # - '[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]' - # - 'ECDHE-ECDSA-AES256-GCM-SHA384' - # - 'ECDHE-RSA-AES256-GCM-SHA384' - # Defines the Kubernetes name/namespace matching a secret to use - # as the fallback certificate when requests which don't match the - # SNI defined for a vhost. - fallback-certificate: - # name: fallback-secret-name - # namespace: projectcontour - envoy-client-certificate: - # name: envoy-client-cert-secret-name - # namespace: projectcontour - # The following config shows the defaults for the leader election. - # leaderelection: - # configmap-name: leader-elect - # configmap-namespace: projectcontour - ### Logging options - # Default setting - accesslog-format: envoy - # To enable JSON logging in Envoy - # accesslog-format: json - # The default fields that will be logged are specified below. - # To customise this list, just add or remove entries. - # The canonical list is available at - # https://godoc.org/github.com/projectcontour/contour/internal/envoy#JSONFields - # json-fields: - # - "@timestamp" - # - "authority" - # - "bytes_received" - # - "bytes_sent" - # - "downstream_local_address" - # - "downstream_remote_address" - # - "duration" - # - "method" - # - "path" - # - "protocol" - # - "request_id" - # - "requested_server_name" - # - "response_code" - # - "response_flags" - # - "uber_trace_id" - # - "upstream_cluster" - # - "upstream_host" - # - "upstream_local_address" - # - "upstream_service_time" - # - "user_agent" - # - "x_forwarded_for" - # - # default-http-versions: - # - "HTTP/2" - # - "HTTP/1.1" - # - # The following shows the default proxy timeout settings. - # timeouts: - # request-timeout: infinity - # connection-idle-timeout: 60s - # stream-idle-timeout: 5m - # max-connection-duration: infinity - # delayed-close-timeout: 1s - # connection-shutdown-grace-period: 5s - # - # Envoy cluster settings. - # cluster: - # configure the cluster dns lookup family - # valid options are: auto (default), v4, v6 - # dns-lookup-family: auto - # - # Envoy network settings. - # network: - # Configure the number of additional ingress proxy hops from the - # right side of the x-forwarded-for HTTP header to trust. - # num-trusted-hops: 0 - # - # Configure an optional global rate limit service. - # rateLimitService: - # Identifies the extension service defining the rate limit service, - # formatted as /. - # extensionService: projectcontour/ratelimit - # Defines the rate limit domain to pass to the rate limit service. - # Acts as a container for a set of rate limit definitions within - # the RLS. - # domain: contour - # Defines whether to allow requests to proceed when the rate limit - # service fails to respond with a valid rate limit decision within - # the timeout defined on the extension service. - # failOpen: false -kind: ConfigMap -metadata: - name: contour - namespace: projectcontour ---- -apiVersion: v1 -data: - pass.txt: YXdlc29tZXNhdWNl -kind: Secret -metadata: - name: db-pass-9dgchhk2bm - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-admin-auth - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-pod-webhook - namespace: flyte -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - name: flyte-secret-auth - namespace: flyte -stringData: - client_secret: foobar -type: Opaque ---- -apiVersion: v1 -data: - password: bXl1c2Vy - user_secret: bXlzZWNyZXQ= - username: bXl1c2Vy -kind: Secret -metadata: - name: user-info - namespace: flyte -type: Opaque ---- -apiVersion: v1 -data: - csrf: "" -kind: Secret -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard-csrf - namespace: kubernetes-dashboard -type: Opaque ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard-key-holder - namespace: kubernetes-dashboard -type: Opaque ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - contour.heptio.com/upstream-protocol.h2c: grpc - name: datacatalog - namespace: flyte -spec: - ports: - - name: http - port: 88 - protocol: TCP - targetPort: 8088 - - name: grpc - port: 89 - protocol: TCP - targetPort: 8089 - selector: - app: datacatalog ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - projectcontour.io/upstream-protocol.h2c: grpc - name: flyte-pod-webhook - namespace: flyte -spec: - ports: - - name: https - port: 443 - protocol: TCP - targetPort: 9443 - selector: - app: flyte-pod-webhook ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - projectcontour.io/upstream-protocol.h2c: grpc - name: flyteadmin - namespace: flyte -spec: - ports: - - name: redoc - port: 87 - protocol: TCP - targetPort: 8087 - - name: http - port: 80 - protocol: TCP - targetPort: 8088 - - name: grpc - port: 81 - protocol: TCP - targetPort: 8089 - selector: - app: flyteadmin ---- -apiVersion: v1 -kind: Service -metadata: - name: flyteconsole - namespace: flyte -spec: - ports: - - port: 80 - protocol: TCP - targetPort: 8080 - selector: - app: flyteconsole ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: flyte -spec: - externalName: minio - ports: - - name: minio-api - port: 9000 - - name: minio-console - port: 9001 - selector: - app: minio ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: minio - name: minio-direct - namespace: flyte -spec: - ports: - - nodePort: 30084 - port: 9000 - protocol: TCP - selector: - app: minio - type: NodePort ---- -apiVersion: v1 -kind: Service -metadata: - name: postgres - namespace: flyte -spec: - ports: - - port: 5432 - selector: - app: postgres ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: postgres - name: postgres-direct - namespace: flyte -spec: - ports: - - nodePort: 30083 - port: 5432 - protocol: TCP - selector: - app: postgres - type: NodePort ---- -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: dashboard-metrics-scraper - name: dashboard-metrics-scraper - namespace: kubernetes-dashboard -spec: - ports: - - port: 8000 - targetPort: 8000 - selector: - k8s-app: dashboard-metrics-scraper ---- -apiVersion: v1 -kind: Service -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard -spec: - ports: - - name: http - nodePort: 30082 - port: 80 - protocol: TCP - targetPort: 9090 - selector: - k8s-app: kubernetes-dashboard - type: NodePort ---- -apiVersion: v1 -kind: Service -metadata: - name: contour - namespace: projectcontour -spec: - ports: - - name: xds - port: 8001 - protocol: TCP - targetPort: 8001 - selector: - app: contour - type: ClusterIP ---- -apiVersion: v1 -kind: Service -metadata: - annotations: - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp - name: envoy - namespace: projectcontour -spec: - externalTrafficPolicy: Local - ports: - - name: http - nodePort: 30081 - port: 80 - protocol: TCP - - name: https - port: 443 - protocol: TCP - selector: - app: envoy - type: NodePort ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: datacatalog - name: datacatalog - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: datacatalog - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: datacatalog - app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.3.0 - spec: - containers: - - command: - - datacatalog - - --config - - /etc/datacatalog/config/*.yaml - - serve - image: cr.flyte.org/flyteorg/datacatalog:v1.0.51 - imagePullPolicy: IfNotPresent - name: datacatalog - ports: - - containerPort: 8088 - - containerPort: 8089 - volumeMounts: - - mountPath: /etc/datacatalog/config - name: config-volume - - mountPath: /etc/db - name: db-pass - initContainers: - - command: - - datacatalog - - --config - - /etc/datacatalog/config/*.yaml - - migrate - - run - image: cr.flyte.org/flyteorg/datacatalog:v1.0.51 - imagePullPolicy: IfNotPresent - name: run-migrations - volumeMounts: - - mountPath: /etc/datacatalog/config - name: config-volume - - mountPath: /etc/db - name: db-pass - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: datacatalog - volumes: - - emptyDir: {} - name: shared-data - - configMap: - name: datacatalog-config-64k8dg9gck - name: config-volume - - name: db-pass - secret: - secretName: db-pass-9dgchhk2bm ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyte-pod-webhook - name: flyte-pod-webhook - namespace: flyte -spec: - selector: - matchLabels: - app: flyte-pod-webhook - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flyte-pod-webhook - app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: 0.5.13 - spec: - containers: - - args: - - webhook - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.1.116 - imagePullPolicy: IfNotPresent - name: webhook - volumeMounts: - - mountPath: /etc/secrets/user-info - name: sample-secrets - readOnly: true - - mountPath: /etc/flyte/config - name: config-volume - readOnly: true - - mountPath: /etc/webhook/certs - name: webhook-certs - readOnly: true - initContainers: - - args: - - webhook - - init-certs - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.1.116 - imagePullPolicy: IfNotPresent - name: generate-secrets - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flyte-pod-webhook - volumes: - - name: sample-secrets - secret: - secretName: user-info - - configMap: - name: flyte-propeller-config-g52db28fmc - name: config-volume - - name: webhook-certs - secret: - secretName: flyte-pod-webhook ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyteadmin - name: flyteadmin - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flyteadmin - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flyteadmin - app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.4.13 - spec: - containers: - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - serve - image: cr.flyte.org/flyteorg/flyteadmin:v1.1.118 - imagePullPolicy: IfNotPresent - name: flyteadmin - ports: - - containerPort: 8088 - - containerPort: 8089 - resources: - limits: - cpu: "0.1" - ephemeral-storage: 100Mi - memory: 200Mi - volumeMounts: - - mountPath: /srv/flyte - name: shared-data - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - mountPath: /etc/secrets/ - name: auth - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - clusterresource - - run - image: cr.flyte.org/flyteorg/flyteadmin:v1.1.118 - imagePullPolicy: IfNotPresent - name: sync-cluster-resources-goroutine - volumeMounts: - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume - - command: - - sh - - -c - - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh - env: - - name: PAGE_TITLE - value: Flyte Admin OpenAPI - - name: SPEC_URL - value: /api/v1/openapi - - name: PORT - value: "8087" - image: docker.io/redocly/redoc - imagePullPolicy: IfNotPresent - name: redoc - ports: - - containerPort: 8087 - resources: - limits: - cpu: "0.1" - memory: 200Mi - initContainers: - - command: - - sh - - -c - - until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done; - image: ecr.flyte.org/ubuntu/postgres:13-21.04_beta - name: check-db-ready - securityContext: - runAsUser: 999 - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - migrate - - run - image: cr.flyte.org/flyteorg/flyteadmin:v1.1.118 - imagePullPolicy: IfNotPresent - name: run-migrations - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - migrate - - seed-projects - - flytesnacks - - flyteexamples - image: cr.flyte.org/flyteorg/flyteadmin:v1.1.118 - imagePullPolicy: IfNotPresent - name: seed-projects - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - clusterresource - - sync - image: cr.flyte.org/flyteorg/flyteadmin:v1.1.118 - imagePullPolicy: IfNotPresent - name: sync-cluster-resources - volumeMounts: - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - - args: - - flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/scratch/secrets && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/scratch/secrets - command: - - /bin/sh - - -c - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flyteadmin:v1.1.118 - imagePullPolicy: IfNotPresent - name: generate-secrets - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/scratch - name: scratch - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flyteadmin - volumes: - - configMap: - name: clusterresource-template-dk5mbchdmt - name: resource-templates - - emptyDir: {} - name: shared-data - - emptyDir: {} - name: scratch - - configMap: - name: flyte-admin-config-dbg8dt2dgb - name: config-volume - - name: db-pass - secret: - secretName: db-pass-9dgchhk2bm - - name: auth - secret: - secretName: flyte-admin-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flyteconsole - name: flyteconsole - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flyteconsole - template: - metadata: - labels: - app: flyteconsole - app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.19.0 - spec: - containers: - - envFrom: - - configMapRef: - name: flyte-console-config - image: cr.flyte.org/flyteorg/flyteconsole:v1.9.2 - name: flyteconsole - ports: - - containerPort: 8080 - volumeMounts: - - mountPath: /srv/flyte - name: shared-data - securityContext: - fsGroupChangePolicy: Always - runAsUser: 1000 - volumes: - - emptyDir: {} - name: shared-data ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flytepropeller - name: flytepropeller - namespace: flyte -spec: - selector: - matchLabels: - app: flytepropeller - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flytepropeller - app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.7.1 - spec: - containers: - - args: - - --config - - /etc/flyte/config/*.yaml - command: - - flytepropeller - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: cr.flyte.org/flyteorg/flytepropeller:v1.1.116 - imagePullPolicy: IfNotPresent - name: flytepropeller - ports: - - containerPort: 10254 - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/secrets/ - name: auth - securityContext: - fsGroup: 65534 - fsGroupChangePolicy: Always - runAsUser: 1001 - serviceAccountName: flytepropeller - volumes: - - configMap: - name: flyte-propeller-config-g52db28fmc - name: config-volume - - name: auth - secret: - secretName: flyte-secret-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: flytescheduler - name: flytescheduler - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: flytescheduler - template: - metadata: - annotations: - prometheus.io/path: /metrics - prometheus.io/port: "10254" - prometheus.io/scrape: "true" - labels: - app: flytescheduler - app.kubernetes.io/name: flytescheduler - app.kubernetes.io/version: 0.3.4 - spec: - containers: - - command: - - flytescheduler - - run - - --config - - /etc/flyte/config/*.yaml - image: cr.flyte.org/flyteorg/flytescheduler:v0.6.49 - imagePullPolicy: IfNotPresent - name: flytescheduler - resources: - limits: - cpu: 250m - ephemeral-storage: 100Mi - memory: 500Mi - requests: - cpu: 10m - ephemeral-storage: 50Mi - memory: 50Mi - volumeMounts: - - mountPath: /etc/secrets/ - name: auth - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - initContainers: - - command: - - flytescheduler - - precheck - - --config - - /etc/flyte/config/*.yaml - image: cr.flyte.org/flyteorg/flytescheduler:v0.6.49 - imagePullPolicy: IfNotPresent - name: flytescheduler-check - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - serviceAccountName: flyteadmin - volumes: - - configMap: - name: flyte-scheduler-config - name: config-volume - - name: db-pass - secret: - secretName: db-pass-9dgchhk2bm - - name: auth - secret: - secretName: flyte-secret-auth ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: flyte -spec: - selector: - matchLabels: - app: minio - template: - metadata: - labels: - app: minio - spec: - containers: - - env: - - name: MINIO_ACCESS_KEY - value: minio - - name: MINIO_SECRET_KEY - value: miniostorage - - name: MINIO_DEFAULT_BUCKETS - value: my-s3-bucket - image: ecr.flyte.org/bitnami/minio:2021.10.13-debian-10-r0 - name: minio - ports: - - containerPort: 9000 - name: minio - - containerPort: 9001 - name: minio-console - volumeMounts: - - mountPath: /data - name: minio-storage - securityContext: - fsGroup: 1001 - fsGroupChangePolicy: OnRootMismatch - runAsUser: 1001 - volumes: - - emptyDir: {} - name: minio-storage ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres - namespace: flyte -spec: - selector: - matchLabels: - app: postgres - template: - metadata: - labels: - app: postgres - spec: - containers: - - env: - - name: POSTGRES_HOST_AUTH_METHOD - value: trust - image: ecr.flyte.org/ubuntu/postgres:13-21.04_beta - name: postgres - ports: - - containerPort: 5432 - name: postgres - volumeMounts: - - mountPath: /var/lib/postgresql/data - name: postgres-storage - volumes: - - emptyDir: {} - name: postgres-storage ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - k8s-app: dashboard-metrics-scraper - name: dashboard-metrics-scraper - namespace: kubernetes-dashboard -spec: - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - k8s-app: dashboard-metrics-scraper - template: - metadata: - annotations: - seccomp.security.alpha.kubernetes.io/pod: runtime/default - labels: - k8s-app: dashboard-metrics-scraper - spec: - containers: - - image: kubernetesui/metrics-scraper:v1.0.6 - livenessProbe: - httpGet: - path: / - port: 8000 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 30 - name: dashboard-metrics-scraper - ports: - - containerPort: 8000 - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsGroup: 2001 - runAsUser: 1001 - volumeMounts: - - mountPath: /tmp - name: tmp-volume - nodeSelector: - kubernetes.io/os: linux - serviceAccountName: kubernetes-dashboard - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - emptyDir: {} - name: tmp-volume ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - k8s-app: kubernetes-dashboard - name: kubernetes-dashboard - namespace: kubernetes-dashboard -spec: - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - k8s-app: kubernetes-dashboard - template: - metadata: - labels: - k8s-app: kubernetes-dashboard - spec: - containers: - - args: - - --namespace=kubernetes-dashboard - - --enable-insecure-login - - --enable-skip-login - - --disable-settings-authorizer - image: kubernetesui/dashboard:v2.2.0 - livenessProbe: - httpGet: - path: / - port: 9090 - initialDelaySeconds: 30 - timeoutSeconds: 30 - name: kubernetes-dashboard - ports: - - containerPort: 9090 - protocol: TCP - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - runAsGroup: 2001 - runAsUser: 1001 - volumeMounts: - - mountPath: /tmp - name: tmp-volume - nodeSelector: - kubernetes.io/os: linux - serviceAccountName: kubernetes-dashboard - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - volumes: - - emptyDir: {} - name: tmp-volume ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: contour - name: contour - namespace: projectcontour -spec: - replicas: 2 - selector: - matchLabels: - app: contour - strategy: - rollingUpdate: - maxSurge: 50% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/port: "8000" - prometheus.io/scrape: "true" - labels: - app: contour - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app: contour - topologyKey: kubernetes.io/hostname - weight: 100 - containers: - - args: - - serve - - --incluster - - --xds-address=0.0.0.0 - - --xds-port=8001 - - --envoy-service-http-port=80 - - --envoy-service-https-port=443 - - --contour-cafile=/certs/ca.crt - - --contour-cert-file=/certs/tls.crt - - --contour-key-file=/certs/tls.key - - --config-path=/config/contour.yaml - command: - - contour - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - image: docker.io/projectcontour/contour:v1.13.1 - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /healthz - port: 8000 - name: contour - ports: - - containerPort: 8001 - name: xds - protocol: TCP - - containerPort: 8000 - name: metrics - protocol: TCP - - containerPort: 6060 - name: debug - protocol: TCP - readinessProbe: - initialDelaySeconds: 15 - periodSeconds: 10 - tcpSocket: - port: 8001 - volumeMounts: - - mountPath: /certs - name: contourcert - readOnly: true - - mountPath: /config - name: contour-config - readOnly: true - dnsPolicy: ClusterFirst - securityContext: - runAsGroup: 65534 - runAsNonRoot: true - runAsUser: 65534 - serviceAccountName: contour - volumes: - - name: contourcert - secret: - secretName: contourcert - - configMap: - defaultMode: 420 - items: - - key: contour.yaml - path: contour.yaml - name: contour - name: contour-config ---- -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: syncresources - namespace: flyte -spec: - jobTemplate: - spec: - template: - spec: - containers: - - command: - - flyteadmin - - --config - - /etc/flyte/config/*.yaml - - clusterresource - - sync - image: cr.flyte.org/flyteorg/flyteadmin:v1.1.118 - imagePullPolicy: IfNotPresent - name: sync-cluster-resources - volumeMounts: - - mountPath: /etc/flyte/clusterresource/templates - name: resource-templates - - mountPath: /etc/flyte/config - name: config-volume - - mountPath: /etc/db - name: db-pass - restartPolicy: OnFailure - serviceAccountName: flyteadmin - volumes: - - configMap: - name: clusterresource-template-dk5mbchdmt - name: resource-templates - - configMap: - name: flyte-admin-config-dbg8dt2dgb - name: config-volume - - name: db-pass - secret: - secretName: db-pass-9dgchhk2bm - schedule: '*/1 * * * *' ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: envoy - name: envoy - namespace: projectcontour -spec: - selector: - matchLabels: - app: envoy - template: - metadata: - annotations: - prometheus.io/path: /stats/prometheus - prometheus.io/port: "8002" - prometheus.io/scrape: "true" - labels: - app: envoy - spec: - automountServiceAccountToken: false - containers: - - args: - - envoy - - shutdown-manager - command: - - /bin/contour - image: docker.io/projectcontour/contour:v1.13.1 - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /bin/contour - - envoy - - shutdown - livenessProbe: - httpGet: - path: /healthz - port: 8090 - initialDelaySeconds: 3 - periodSeconds: 10 - name: shutdown-manager - - args: - - -c - - /config/envoy.json - - --service-cluster $(CONTOUR_NAMESPACE) - - --service-node $(ENVOY_POD_NAME) - - --log-level info - command: - - envoy - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: ENVOY_POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - image: docker.io/envoyproxy/envoy:v1.17.1 - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - httpGet: - path: /shutdown - port: 8090 - scheme: HTTP - name: envoy - ports: - - containerPort: 80 - hostPort: 80 - name: http - protocol: TCP - - containerPort: 443 - hostPort: 443 - name: https - protocol: TCP - readinessProbe: - httpGet: - path: /ready - port: 8002 - initialDelaySeconds: 3 - periodSeconds: 4 - volumeMounts: - - mountPath: /config - name: envoy-config - readOnly: true - - mountPath: /certs - name: envoycert - readOnly: true - initContainers: - - args: - - bootstrap - - /config/envoy.json - - --xds-address=contour - - --xds-port=8001 - - --xds-resource-version=v3 - - --resources-dir=/config/resources - - --envoy-cafile=/certs/ca.crt - - --envoy-cert-file=/certs/tls.crt - - --envoy-key-file=/certs/tls.key - command: - - contour - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: docker.io/projectcontour/contour:v1.13.1 - imagePullPolicy: IfNotPresent - name: envoy-initconfig - volumeMounts: - - mountPath: /config - name: envoy-config - - mountPath: /certs - name: envoycert - readOnly: true - restartPolicy: Always - serviceAccountName: envoy - terminationGracePeriodSeconds: 300 - volumes: - - emptyDir: {} - name: envoy-config - - name: envoycert - secret: - secretName: envoycert - updateStrategy: - rollingUpdate: - maxUnavailable: 10% - type: RollingUpdate ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: contour-certgen-v1.13.1 - namespace: projectcontour -spec: - backoffLimit: 1 - completions: 1 - parallelism: 1 - template: - metadata: - labels: - app: contour-certgen - spec: - containers: - - command: - - contour - - certgen - - --kube - - --incluster - - --overwrite - - --secrets-format=compact - - --namespace=$(CONTOUR_NAMESPACE) - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - image: docker.io/projectcontour/contour:v1.13.1 - imagePullPolicy: Always - name: contour - restartPolicy: Never - securityContext: - runAsGroup: 65534 - runAsNonRoot: true - runAsUser: 65534 - serviceAccountName: contour-certgen - ttlSecondsAfterFinished: 0 ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/app-root: /console - nginx.ingress.kubernetes.io/ssl-redirect: "false" - name: flytesystem - namespace: flyte -spec: - rules: - - http: - paths: - - backend: - service: - name: flyteconsole - port: - number: 80 - path: /__webpack_hmr - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.AdminService - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.AuthMetadataService - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.IdentityService - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.SignalService - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 87 - path: /openapi - pathType: ImplementationSpecific - - backend: - service: - name: flyteconsole - port: - number: 80 - path: /console - pathType: ImplementationSpecific - - backend: - service: - name: flyteconsole - port: - number: 80 - path: /console/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /api - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /api/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /healthcheck - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /v1/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 87 - path: /openapi/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /.well-known/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /login - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /login/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /logout - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /logout/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /callback - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /callback/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /me - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /config - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /config/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /oauth2 - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 80 - path: /oauth2/* - pathType: ImplementationSpecific ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - nginx.ingress.kubernetes.io/ssl-redirect: "false" - name: minio - namespace: flyte -spec: - rules: - - http: - paths: - - backend: - service: - name: minio - port: - number: 9001 - path: /minio - pathType: ImplementationSpecific diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index 1959cc5bd81..9d032a0735c 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -585,7 +585,7 @@ data: plugins: k8s: co-pilot: - image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1 + image: cr.flyte.org/flyteorg/flytecopilot:v1.11.0 name: flyte-copilot- start-timeout: 30s core.yaml: | @@ -6705,7 +6705,7 @@ spec: - /etc/flyte/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations securityContext: @@ -6725,7 +6725,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: seed-projects securityContext: @@ -6742,7 +6742,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - sync - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources securityContext: @@ -6758,7 +6758,7 @@ spec: - mountPath: /etc/secrets/ name: admin-secrets - name: generate-secrets - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" command: ["/bin/sh", "-c"] args: @@ -6785,7 +6785,7 @@ spec: - --config - /etc/flyte/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: flyteadmin ports: @@ -6880,7 +6880,7 @@ spec: - /etc/flyte/config/*.yaml - clusterresource - run - image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flyteadmin:v1.11.0" imagePullPolicy: "IfNotPresent" name: sync-cluster-resources volumeMounts: @@ -6933,7 +6933,7 @@ spec: seLinuxOptions: type: spc_t containers: - - image: "cr.flyte.org/flyteorg/flyteconsole:v1.11.0" + - image: "cr.flyte.org/flyteorg/flyteconsole:v1.10.3" imagePullPolicy: "IfNotPresent" name: flyteconsole envFrom: @@ -7005,7 +7005,7 @@ spec: - /etc/datacatalog/config/*.yaml - migrate - run - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: run-migrations volumeMounts: @@ -7022,7 +7022,7 @@ spec: - --config - /etc/datacatalog/config/*.yaml - serve - image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/datacatalog:v1.11.0" imagePullPolicy: "IfNotPresent" name: datacatalog ports: @@ -7095,7 +7095,7 @@ spec: - precheck - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler-check securityContext: @@ -7114,7 +7114,7 @@ spec: - run - --config - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytescheduler:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytescheduler ports: @@ -7170,7 +7170,7 @@ spec: template: metadata: annotations: - configChecksum: "a82c947246fac99ad98c86bc01cdd99569b2ea11f25d3e1361cd40d145bed87" + configChecksum: "ecefcb1d6df38b32a9ec77d91de63cd40da935a3239b70a5bb0396435878c80" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -7195,7 +7195,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" name: flytepropeller ports: @@ -7242,9 +7242,9 @@ spec: labels: app: flyte-pod-webhook app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: v1.11.0-b1 + app.kubernetes.io/version: v1.11.0 annotations: - configChecksum: "a82c947246fac99ad98c86bc01cdd99569b2ea11f25d3e1361cd40d145bed87" + configChecksum: "ecefcb1d6df38b32a9ec77d91de63cd40da935a3239b70a5bb0396435878c80" spec: securityContext: fsGroup: 65534 @@ -7256,7 +7256,7 @@ spec: serviceAccountName: flyte-pod-webhook initContainers: - name: generate-secrets - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller @@ -7283,7 +7283,7 @@ spec: mountPath: /etc/flyte/config containers: - name: webhook - image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytepropeller:v1.11.0" imagePullPolicy: "IfNotPresent" command: - flytepropeller diff --git a/docker/sandbox-bundled/README.md b/docker/sandbox-bundled/README.md deleted file mode 100644 index 17ca62b0020..00000000000 --- a/docker/sandbox-bundled/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Flyte Deployment Sandbox - -First make images -``` -ytong@Yees-MBP:~/go/src/github.com/flyteorg/flyte/docker/sandbox-ultra [flyte-sandbox] (cicd-sandbox-lite) $ make images -``` - -then build the k3s image. -``` -ytong@Yees-MBP:~/go/src/github.com/flyteorg/flyte/docker/sandbox-ultra [] (cicd-sandbox-lite) $ docker buildx build --file images/dockerfiles/k3s/Dockerfile --platform linux/arm64,linux/amd64 --push --tag ghcr.io/flyteorg/flyte-sandbox-lite:ultra7 . -``` diff --git a/docker/sandbox-bundled/manifests/complete-agent.yaml b/docker/sandbox-bundled/manifests/complete-agent.yaml index 5f72d69c20c..925836b897e 100644 --- a/docker/sandbox-bundled/manifests/complete-agent.yaml +++ b/docker/sandbox-bundled/manifests/complete-agent.yaml @@ -468,7 +468,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0" k8s-array: logs: config: @@ -816,7 +816,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: NTRkQThQbW1FNDlVQ1B4Rw== + haSharedSecret: bUJTMk9LZGQyaTB6VkRSaA== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1246,7 +1246,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 286b381aca83e796ac69572dba6d2071241be79d4f00f3c2af55415a95157efd + checksum/configuration: 2aaf6d07c01e76a5e97b2c5aa1e581ee7e2e9929d3690e71eceb6b0bdfb7b646 checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1412,7 +1412,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 7ce58af0efd945f657619718c017f528d7055b8069cab01aed85e29639bd6d6a + checksum/secret: 1c9189b243db03d412ad40e2973fb6b1dbe18a4a65c99ecc3c6a09d42808d3a7 labels: app: docker-registry release: flyte-sandbox @@ -1755,12 +1755,17 @@ spec: value: minio - name: FLYTE_AWS_SECRET_ACCESS_KEY value: miniostorage - image: ghcr.io/flyteorg/flyteagent:1.10.7 + image: ghcr.io/flyteorg/flyteagent:1.10.8b4 imagePullPolicy: IfNotPresent name: flyteagent ports: - containerPort: 8000 name: agent-grpc + readinessProbe: + grpc: + port: 8000 + initialDelaySeconds: 1 + periodSeconds: 3 resources: limits: cpu: 500m diff --git a/docker/sandbox-bundled/manifests/complete.yaml b/docker/sandbox-bundled/manifests/complete.yaml index cacefc4730f..bfce156c3d4 100644 --- a/docker/sandbox-bundled/manifests/complete.yaml +++ b/docker/sandbox-bundled/manifests/complete.yaml @@ -457,7 +457,7 @@ data: stackdriver-enabled: false k8s: co-pilot: - image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0-b1" + image: "cr.flyte.org/flyteorg/flytecopilot:v1.11.0" k8s-array: logs: config: @@ -796,7 +796,7 @@ type: Opaque --- apiVersion: v1 data: - haSharedSecret: c21mSkU1U0c0S0J0Q2lRRg== + haSharedSecret: djBGOWN4R0VYa1lOOHJGYQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -1194,7 +1194,7 @@ spec: metadata: annotations: checksum/cluster-resource-templates: 6fd9b172465e3089fcc59f738b92b8dc4d8939360c19de8ee65f68b0e7422035 - checksum/configuration: 1362bc266b00c161aac0a24f7db312b9d6b9e68d7d8d8859ec8e171b9dc3e2bd + checksum/configuration: cd9caceec9bd91bdf1eedb10aee289a53786fc70df8c3f4951881abb9f937c49 checksum/configuration-secret: 09216ffaa3d29e14f88b1f30af580d02a2a5e014de4d750b7f275cc07ed4e914 labels: app.kubernetes.io/component: flyte-binary @@ -1360,7 +1360,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: 67f0b60e6520f3ae46a9f396411386d473441e7733c2544f6bface0f3b8475d9 + checksum/secret: d18d66b9d050f0206137c4e6633af2c4b9905d07e460984f7df5dbf5175675c3 labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-bundled/manifests/dev.yaml b/docker/sandbox-bundled/manifests/dev.yaml index 5e0017253ed..c0f5be7f87b 100644 --- a/docker/sandbox-bundled/manifests/dev.yaml +++ b/docker/sandbox-bundled/manifests/dev.yaml @@ -499,7 +499,7 @@ metadata: --- apiVersion: v1 data: - haSharedSecret: YjFqbVE2UE05QnpwMVVVeg== + haSharedSecret: VHZrRVVvTjFlNFNHSjF2dQ== proxyPassword: "" proxyUsername: "" kind: Secret @@ -934,7 +934,7 @@ spec: metadata: annotations: checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81 - checksum/secret: ec699a25c688a15358899da82f9fa9bb199fe085f200f00facc09b448ad80369 + checksum/secret: 60dbc3fbd016623bae860a7a22b554908f71e864127a947c9ea7f700df32ed3f labels: app: docker-registry release: flyte-sandbox diff --git a/docker/sandbox-lite/flyte-entrypoint-dind.sh b/docker/sandbox-lite/flyte-entrypoint-dind.sh deleted file mode 100755 index 0585624970c..00000000000 --- a/docker/sandbox-lite/flyte-entrypoint-dind.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/sh - -set -euo pipefail - -# Apply cgroup v2 hack -cgroup-v2-hack.sh - -trap 'pkill -P $$' EXIT -FLYTE_TIMEOUT=${FLYTE_TIMEOUT:-600} - -monitor() { - while : ; do - for pid in $@ ; do - kill -0 $pid &> /dev/null || exit 1 - done - - sleep 1 - done -} - -# Start docker daemon -echo "Starting Docker daemon..." -file="/var/run/docker.pid" -if [ -f "$file" ] ; then - rm "$file" -fi -dockerd &> /var/log/dockerd.log & -DOCKERD_PID=$! -timeout "$FLYTE_TIMEOUT" sh -c "until docker info &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for dockerd to start"; exit 1 ) -echo "Done." - -# Start k3s -echo "Starting k3s cluster..." -KUBERNETES_API_PORT=${KUBERNETES_API_PORT:-6443} -k3s server --docker --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log & -K3S_PID=$! -timeout "$FLYTE_TIMEOUT" sh -c "until k3s kubectl get node $HOSTNAME &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to start"; exit 1 ) -k3s kubectl wait node $HOSTNAME --for condition=Ready --timeout ${FLYTE_TIMEOUT}s &> /dev/null || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to be ready"; exit 1 ) -echo "Done." - -# Deploy flyte -echo "Deploying Flyte..." -charts="/flyteorg/share/flyte-deps" -version="" - -if [[ $FLYTE_TEST = "release" ]] -then - helm repo add flyteorg https://flyteorg.github.io/flyte - helm fetch flyteorg/flyte-deps --version=$FLYTE_VERSION - version="--version $FLYTE_VERSION" - charts="flyteorg/flyte-deps" -fi - -if [[ $FLYTE_TEST = "local" ]] -then - helm dep update $charts -fi -helm upgrade -n flyte --create-namespace flyte $charts --kubeconfig /etc/rancher/k3s/k3s.yaml --install --wait - -timeout "$FLYTE_TIMEOUT" sh -c "until k3s kubectl get namespace flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Flyte namespace to be created"; exit 1 ) -timeout "$FLYTE_TIMEOUT" sh -c "until k3s kubectl rollout status deployment minio -n flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the minio rollout to be created"; exit 1 ) -timeout "$FLYTE_TIMEOUT" sh -c "until k3s kubectl rollout status deployment postgres -n flyte &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the postgres rollout to be created"; exit 1 ) - -k3s kubectl wait --for=condition=available deployment/minio deployment/postgres -n flyte --timeout=5m || ( echo >&2 "Timed out while waiting for the Flyte deployment to start"; exit 1 ) -# Create directory to store certificate -mkdir -p /tmp/k8s-webhook-server/serving-certs - -# With flytectl sandbox --source flag, we mount the root volume to user source dir that will create helm & k8s cache specific directory. -# In Linux, These file belongs to root user that is different then current user -# In this case during fast serialization, Pyflyte will through error because of permission denied -rm -rf /root/.cache /root/.kube /root/.config - -if [[ $FLYTE_DEV = "True" ]] -then - # Namespaces must be manually created since cluster resource manager is disabled by default - k3s kubectl create ns flytesnacks-development - # Monitor running processes. Exit when the first process exits. - monitor ${DOCKERD_PID} ${K3S_PID} -else - flyte start --config /flyteorg/share/flyte.yaml & - FLYTE_PID=$! - # Monitor running processes. Exit when the first process exits. - monitor ${DOCKERD_PID} ${K3S_PID} ${FLYTE_PID} -fi diff --git a/docker/sandbox-lite/templates/namespace.yaml b/docker/sandbox-lite/templates/namespace.yaml deleted file mode 100644 index 90b53893ac4..00000000000 --- a/docker/sandbox-lite/templates/namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ namespace }} -spec: - finalizers: - - kubernetes \ No newline at end of file diff --git a/docker/sandbox-lite/templates/project_resource_quota.yaml b/docker/sandbox-lite/templates/project_resource_quota.yaml deleted file mode 100644 index 12e77115030..00000000000 --- a/docker/sandbox-lite/templates/project_resource_quota.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: ResourceQuota -metadata: - name: project-quota - namespace: {{ namespace }} -spec: - hard: - limits.cpu: {{ projectQuotaCpu }} - limits.memory: {{ projectQuotaMemory }} \ No newline at end of file diff --git a/docs/community/contribute.rst b/docs/community/contribute.rst index e866be5a2c9..993cb6bca52 100644 --- a/docs/community/contribute.rst +++ b/docs/community/contribute.rst @@ -151,7 +151,7 @@ To understand how the below components interact with each other, refer to :ref:` * - `Repo `__ * - **Purpose**: Deployment, Documentation, and Issues - * - **Languages**: Kustomize & RST + * - **Languages**: RST To build the Flyte docs locally you will need the following prerequisites: diff --git a/docs/conf.py b/docs/conf.py index 6d20ecf4b93..cca6cc266ae 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,7 +35,7 @@ # The short X.Y version version = "" # The full version, including alpha/beta/rc tags -release = "1.11.0-b1" +release = "1.11.0" # -- General configuration --------------------------------------------------- diff --git a/docs/deployment/configuration/customizable_resources.rst b/docs/deployment/configuration/customizable_resources.rst index 6fb1318ac65..29bb7c8ca98 100644 --- a/docs/deployment/configuration/customizable_resources.rst +++ b/docs/deployment/configuration/customizable_resources.rst @@ -170,7 +170,7 @@ apply. .. note:: The template values, for example, ``projectQuotaCpu`` or ``projectQuotaMemory`` are free-form strings. - Ensure that they match the template placeholders in your `template file `__ + Ensure that they match the template placeholders in your values file (e.g. `values-eks.yaml `__) for your changes to take effect and custom values to be substituted. You can view all custom cluster-resource-attributes by visiting ``protocol://`` diff --git a/docs/deployment/configuration/general.rst b/docs/deployment/configuration/general.rst index 0ce9d259a4c..0b97f8b8ff5 100644 --- a/docs/deployment/configuration/general.rst +++ b/docs/deployment/configuration/general.rst @@ -202,10 +202,10 @@ An example PodTemplate is shown: template: metadata: labels: - - foo + foo: from-pod-template annotations: - - foo: initial-value - - bar: initial-value + foo: initial-value + bar: initial-value spec: containers: - name: default @@ -221,10 +221,10 @@ Pod Labels, Annotations, and enables the host networking. plugins: k8s: default-labels: - - bar + bar: from-default-label default-annotations: - - foo: overridden-value - - baz: non-overridden-value + foo: overridden-value + baz: non-overridden-value enable-host-networking-pod: true To construct a Pod, FlytePropeller initializes a Pod definition using the default @@ -241,12 +241,12 @@ The resultant Pod using the above default PodTemplate and K8s Plugin configurati name: example-pod namespace: flytesnacks-development labels: - - foo # maintained initial value - - bar # value appended by k8s plugin configuration + foo: from-pod-template # maintained initial value + bar: from-default-label # value appended by k8s plugin configuration annotations: - - foo: overridden-value # value overridden by k8s plugin configuration - - bar: initial-value # maintained initial value - - baz: non-overridden-value # value added by k8s plugin configuration + foo: overridden-value # value overridden by k8s plugin configuration + bar: initial-value # maintained initial value + baz: non-overridden-value # value added by k8s plugin configuration spec: containers: - name: ax9kd5xb4p8r45bpdv7v-n0-0 @@ -280,8 +280,8 @@ of the task. For example: template: metadata: annotations: - - annotation_1: initial-value - - bar: initial-value + annotation_1: initial-value + bar: initial-value spec: containers: - name: default @@ -328,12 +328,12 @@ The resultant Pod is as follows: name: example-pod namespace: flytesnacks-development labels: - - label_1: value-1 # from Compile-time value - - label_2: value-2 # from Compile-time value + label_1: value-1 # from Compile-time value + label_2: value-2 # from Compile-time value annotations: - - annotation_1: value-1 # value overridden by Compile-time PodTemplate - - annotation_2: value-2 # from Compile-time PodTemplate - - bar: initial-value # from Runtime PodTemplate + annotation_1: value-1 # value overridden by Compile-time PodTemplate + annotation_2: value-2 # from Compile-time PodTemplate + bar: initial-value # from Runtime PodTemplate spec: containers: - name: default @@ -398,12 +398,12 @@ And a Runtime PodTemplate: template: metadata: labels: - - label_1: value-runtime - - label_2: value-runtime - - label_3: value-runtime + label_1: value-runtime + label_2: value-runtime + label_3: value-runtime annotations: - - foo: value-runtime - - bar: value-runtime + foo: value-runtime + bar: value-runtime spec: containers: - name: default @@ -418,10 +418,10 @@ And the following K8s Plugin Configuration: plugins: k8s: default-labels: - - label_1: value-plugin + label_1: value-plugin default-annotations: - - annotation_1: value-plugin - - baz: value-plugin + annotation_1: value-plugin + baz: value-plugin The resultant pod for that task is as follows: @@ -433,14 +433,14 @@ The resultant pod for that task is as follows: name: example-pod namespace: flytesnacks-development labels: - - label_1: value-plugin - - label_2: value-compile + label_1: value-plugin + label_2: value-compile annotations: - - annotation_1: value-plugin - - annotation_2: value-compile - - foo: value-runtime - - bar: value-runtime - - baz: value-plugin + annotation_1: value-plugin + annotation_2: value-compile + foo: value-runtime + bar: value-runtime + baz: value-plugin spec: containers: - name: default diff --git a/docs/deployment/deployment/index.rst b/docs/deployment/deployment/index.rst index eb06d0a6c0c..0a44f437ef4 100644 --- a/docs/deployment/deployment/index.rst +++ b/docs/deployment/deployment/index.rst @@ -93,8 +93,7 @@ There are three different paths for deploying a Flyte cluster: Helm ==== -Flyte uses `Helm `__ as the K8s release packaging solution, though you may still see some old -`Kustomize `__ artifacts in the `flyte `__ repo. The core Flyte +Flyte uses `Helm `__ as the K8s release packaging solution. The core Flyte team maintains Helm charts that correspond with the latter two deployment paths. .. note:: diff --git a/docs/user_guide/advanced_composition/index.md b/docs/user_guide/advanced_composition/index.md index 26eb8df33cf..3d0f14fc83c 100644 --- a/docs/user_guide/advanced_composition/index.md +++ b/docs/user_guide/advanced_composition/index.md @@ -16,6 +16,7 @@ chaining_flyte_entities subworkflows dynamic_workflows map_tasks +nested_parallelization eager_workflows decorating_tasks decorating_workflows diff --git a/docs/user_guide/advanced_composition/nested_parallelization.md b/docs/user_guide/advanced_composition/nested_parallelization.md new file mode 100644 index 00000000000..73fd921b2e4 --- /dev/null +++ b/docs/user_guide/advanced_composition/nested_parallelization.md @@ -0,0 +1,177 @@ +(nested_parallelization)= + +# Nested parallelization + +```{eval-rst} +.. tags:: Advanced +``` + +For exceptionally large or complicated workflows that can't be adequately implemented as dynamic workflows or map tasks, it can be beneficial to have multiple levels of workflow parallelization. + +This is useful for multiple reasons: +- Better code organization +- Better code reuse +- Better testing +- Better debugging +- Better monitoring, since each subworkflow can be run independently and monitored independently +- Better performance and scale, since each subworkflow is executed as a separate workflow and thus can be distributed among different flytepropeller workers and shards. This allows for better parallelism and scale. + +## Nested dynamic workflows + +You can use nested dynamic workflows to break down a large workflow into smaller workflows and then compose them together to form a hierarchy. In this example, a top-level workflow uses two levels of dynamic workflows to process a list through some simple addition tasks and then flatten the list again. + +### Example code + +```python +""" +A core workflow parallelized as six items with a chunk size of two will be structured as follows: + +multi_wf -> level1 -> level2 -> core_wf -> step1 -> step2 + -> core_wf -> step1 -> step2 + level2 -> core_wf -> step1 -> step2 + -> core_wf -> step1 -> step2 + level2 -> core_wf -> step1 -> step2 + -> core_wf -> step1 -> step2 +""" + +import typing +from flytekit import task, workflow, dynamic, LaunchPlan + + +@task +def step1(a: int) -> int: + return a + 1 + + +@task +def step2(a: int) -> int: + return a + 2 + + +@workflow +def core_wf(a: int) -> int: + return step2(a=step1(a=a)) + + +core_wf_lp = LaunchPlan.get_or_create(core_wf) + + +@dynamic +def level2(l: typing.List[int]) -> typing.List[int]: + return [core_wf_lp(a=a) for a in l] + + +@task +def reduce(l: typing.List[typing.List[int]]) -> typing.List[int]: + f = [] + for i in l: + f.extend(i) + return f + + +@dynamic +def level1(l: typing.List[int], chunk: int) -> typing.List[int]: + v = [] + for i in range(0, len(l), chunk): + v.append(level2(l=l[i:i + chunk])) + return reduce(l=v) + + +@workflow +def multi_wf(l: typing.List[int], chunk: int) -> typing.List[int]: + return level1(l=l, chunk=chunk) +``` + + +### Flyte console + +Here is a visual representation of the execution of nested dynamic workflows in the Flyte console: + +:::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/nested_parallel_top_level.png?raw=true +:alt: Nested Parallelization UI View +:class: with-shadow +::: + +In each level2 node at the top level, two core workflows are run in parallel: + +:::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/nested_parallel_inner_dynamic_anno.png?raw=true +:alt: Inner dynamic workflow +:class: with-shadow +::: + +Finally, in each core workflow, the two tasks are executed in series: + +:::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/nested_parallel_subworkflow.png?raw=true +:alt: Core workflow +:class: with-shadow +::: + +## Mixed parallelism + +This example is similar to nested dynamic workflows, but instead of using a dynamic workflow to parallelize a core workflow with serial tasks, we use a core workflow to call a map task, which processes both inputs in parallel. This workflow has one less layer of parallelism, so the outputs won't be the same as those of the nested parallelization example, but it does still demonstrate how you can mix these different approaches to achieve concurrency. + +### Example code + +```python +""" +A core workflow parallelized as six items with a chunk size of two will be structured as follows: + +multi_wf -> level1 -> level2 -> mappable + -> mappable + level2 -> mappable + -> mappable + level2 -> mappable + -> mappable +""" +import typing +from flytekit import task, workflow, dynamic, map_task + + +@task +def mappable(a: int) -> int: + return a + 2 + + +@workflow +def level2(l: typing.List[int]) -> typing.List[int]: + return map_task(mappable)(a=l) + + +@task +def reduce(l: typing.List[typing.List[int]]) -> typing.List[int]: + f = [] + for i in l: + f.extend(i) + return f + + +@dynamic +def level1(l: typing.List[int], chunk: int) -> typing.List[int]: + v = [] + for i in range(0, len(l), chunk): + v.append(level2(l=l[i : i + chunk])) + return reduce(l=v) + + +@workflow +def multi_wf(l: typing.List[int], chunk: int) -> typing.List[int]: + return level1(l=l, chunk=chunk) + +``` + +### Flyte console + +While the top-level dynamic workflow will be exactly the same as the nested dynamic workflows example, the inner map task nodes will be visible as links in the sidebar: + +:::{figure} https://raw.githubusercontent.com/flyteorg/static-resources/main/flytesnacks/user_guide/nested_parallel_inner_map.png?raw=true +:alt: Inner Map Task +:class: with-shadow +::: + +## Design considerations + +While you can nest even further if needed, or incorporate map tasks if your inputs are all the same type, the design of your workflow should be informed by the actual data you're processing. For example, if you have a big library of music from which you'd like to extract the lyrics, the first level could loop through all the albums, and the second level could process each song. + +If you're just processing an enormous list of the same input, it's best to keep your code simple and let the scheduler handle optimizing the execution. Additionally, unless you need dynamic workflow features like mixing and matching inputs and outputs, it's usually most efficient to use a map task, which has the added benefit of keeping the UI clean. + +You can also choose to limit the scale of parallel execution at a few levels. The `max_parallelism` attribute can be applied at the workflow level and will limit the number of parallel tasks being executed. (This is set to 25 by default.) Within map tasks, you can specify a `concurrency` argument, which will limit the number of mapped tasks that can run in parallel at any given time. \ No newline at end of file diff --git a/docs/user_guide/customizing_dependencies/imagespec.md b/docs/user_guide/customizing_dependencies/imagespec.md index 5a9ef93736e..12d9295b1e3 100644 --- a/docs/user_guide/customizing_dependencies/imagespec.md +++ b/docs/user_guide/customizing_dependencies/imagespec.md @@ -160,3 +160,17 @@ If you only want to build the image without registering the workflow, you can us ``` pyflyte build --remote image_spec.py wf ``` + ++++ + +In some cases, you may want to force an image to rebuild, even if the image spec hasn’t changed. If you want to overwrite an existing image, you can pass the `FLYTE_FORCE_PUSH_IMAGE_SPEC=True` to `pyflyte` command or add `force_push()` to the ImageSpec. + +```bash +FLYTE_FORCE_PUSH_IMAGE_SPEC=True pyflyte run --remote image_spec.py wf +``` + +or + +```python +image = ImageSpec(registry="ghcr.io/flyteorg", packages=["pandas"]).force_push() +``` diff --git a/docs/user_guide/productionizing/secrets.md b/docs/user_guide/productionizing/secrets.md index dba145c483d..d631594cc72 100644 --- a/docs/user_guide/productionizing/secrets.md +++ b/docs/user_guide/productionizing/secrets.md @@ -372,8 +372,6 @@ The following secret managers are available at the time of writing: - [AWS Secret Manager](https://docs.aws.amazon.com/secretsmanager/latest/userguide/create_secret.html): `flyte-pod-webhook` will add the AWS Secret Manager sidecar container to a task Pod which will mount the secret. - [Vault Agent Injector](https://developer.hashicorp.com/vault/tutorials/getting-started/getting-started-first-secret#write-a-secret) : `flyte-pod-webhook` will annotate the task Pod with the respective Vault annotations that trigger an existing Vault Agent Injector to retrieve the specified secret Key from a vault path defined as secret Group. -You can configure the additional secret manager by defining `secretManagerType` to be either 'K8s', 'AWS' or 'Vault' in -the [core config](https://github.com/flyteorg/flyte/blob/master/kustomize/base/single_cluster/headless/config/propeller/core.yaml#L34) of the Flytepropeller. When using the K8s secret manager plugin, which is enabled by default, the secrets need to be available in the same namespace as the task execution (for example `flytesnacks-development`). K8s secrets can be mounted as either files or injected as environment variables into the task pod, diff --git a/flyteadmin/dataproxy/service.go b/flyteadmin/dataproxy/service.go index 3d657c6958b..5bb7a16632e 100644 --- a/flyteadmin/dataproxy/service.go +++ b/flyteadmin/dataproxy/service.go @@ -49,11 +49,13 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp // If it doesn't exist, then proceed as normal. if len(req.Project) == 0 || len(req.Domain) == 0 { + logger.Infof(ctx, "project and domain are required parameters. Project [%v]. Domain [%v]", req.Project, req.Domain) return nil, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "project and domain are required parameters") } // At least one of the hash or manually given prefix must be provided. if len(req.FilenameRoot) == 0 && len(req.ContentMd5) == 0 { + logger.Infof(ctx, "content_md5 or filename_root is a required parameter. FilenameRoot [%v], ContentMD5 [%v]", req.FilenameRoot, req.ContentMd5) return nil, errors.NewFlyteAdminErrorf(codes.InvalidArgument, "content_md5 or filename_root is a required parameter") } @@ -63,10 +65,12 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp knownLocation, err := createStorageLocation(ctx, s.dataStore, s.cfg.Upload, req.Project, req.Domain, req.FilenameRoot, req.Filename) if err != nil { + logger.Errorf(ctx, "failed to create storage location. Error %v", err) return nil, errors.NewFlyteAdminErrorf(codes.Internal, "failed to create storage location, Error: %v", err) } metadata, err := s.dataStore.Head(ctx, knownLocation) if err != nil { + logger.Errorf(ctx, "failed to check if file exists at location [%s], Error: %v", knownLocation.String(), err) return nil, errors.NewFlyteAdminErrorf(codes.Internal, "failed to check if file exists at location [%s], Error: %v", knownLocation.String(), err) } if metadata.Exists() { @@ -76,12 +80,18 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp if len(req.ContentMd5) == 0 { return nil, errors.NewFlyteAdminErrorf(codes.AlreadyExists, "file already exists at location [%v], specify a matching hash if you wish to rewrite", knownLocation) } - // Re-encode the hash 3-ways to support matching, hex, base32 and base64 - hexDigest := hex.EncodeToString(req.ContentMd5) - base32Digest := base32.StdEncoding.EncodeToString(req.ContentMd5) base64Digest := base64.StdEncoding.EncodeToString(req.ContentMd5) - if hexDigest != metadata.Etag() && base32Digest != metadata.Etag() && base64Digest != metadata.Etag() { - logger.Debugf(ctx, "File already exists at location [%v] but hashes do not match", knownLocation) + if len(metadata.ContentMD5()) == 0 { + // For backward compatibility, dataproxy assumes that the Etag exists if ContentMD5 is not in the metadata. + // Data proxy won't allow people to overwrite the file if both the Etag and the ContentMD5 do not exist. + hexDigest := hex.EncodeToString(req.ContentMd5) + base32Digest := base32.StdEncoding.EncodeToString(req.ContentMd5) + if hexDigest != metadata.Etag() && base32Digest != metadata.Etag() && base64Digest != metadata.Etag() { + logger.Errorf(ctx, "File already exists at location [%v] but hashes do not match", knownLocation) + return nil, errors.NewFlyteAdminErrorf(codes.AlreadyExists, "file already exists at location [%v], specify a matching hash if you wish to rewrite", knownLocation) + } + } else if base64Digest != metadata.ContentMD5() { + logger.Errorf(ctx, "File already exists at location [%v] but hashes do not match", knownLocation) return nil, errors.NewFlyteAdminErrorf(codes.AlreadyExists, "file already exists at location [%v], specify a matching hash if you wish to rewrite", knownLocation) } logger.Debugf(ctx, "File already exists at location [%v] but allowing rewrite", knownLocation) @@ -105,7 +115,7 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp req.Filename = rand.String(s.cfg.Upload.DefaultFileNameLength) } - md5 := base64.StdEncoding.EncodeToString(req.ContentMd5) + base64digestMD5 := base64.StdEncoding.EncodeToString(req.ContentMd5) var prefix string if len(req.FilenameRoot) > 0 { @@ -117,16 +127,19 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp storagePath, err := createStorageLocation(ctx, s.dataStore, s.cfg.Upload, req.Project, req.Domain, prefix, req.Filename) if err != nil { + logger.Errorf(ctx, "failed to create shardedStorageLocation. Error %v", err) return nil, errors.NewFlyteAdminErrorf(codes.Internal, "failed to create shardedStorageLocation, Error: %v", err) } resp, err := s.dataStore.CreateSignedURL(ctx, storagePath, storage.SignedURLProperties{ - Scope: stow.ClientMethodPut, - ExpiresIn: req.ExpiresIn.AsDuration(), - ContentMD5: md5, + Scope: stow.ClientMethodPut, + ExpiresIn: req.ExpiresIn.AsDuration(), + ContentMD5: base64digestMD5, + AddContentMD5Metadata: req.AddContentMd5Metadata, }) if err != nil { + logger.Errorf(ctx, "failed to create signed url. Error:", err) return nil, errors.NewFlyteAdminErrorf(codes.Internal, "failed to create a signed url. Error: %v", err) } @@ -134,6 +147,7 @@ func (s Service) CreateUploadLocation(ctx context.Context, req *service.CreateUp SignedUrl: resp.URL.String(), NativeUrl: storagePath.String(), ExpiresAt: timestamppb.New(time.Now().Add(req.ExpiresIn.AsDuration())), + Headers: resp.RequiredRequestHeaders, }, nil } diff --git a/flyteadmin/go.mod b/flyteadmin/go.mod index badc2a3c884..58ebd90a845 100644 --- a/flyteadmin/go.mod +++ b/flyteadmin/go.mod @@ -17,7 +17,7 @@ require ( github.com/flyteorg/flyte/flyteplugins v0.0.0-00010101000000-000000000000 github.com/flyteorg/flyte/flytepropeller v0.0.0-00010101000000-000000000000 github.com/flyteorg/flyte/flytestdlib v0.0.0-00010101000000-000000000000 - github.com/flyteorg/stow v0.3.8 + github.com/flyteorg/stow v0.3.10 github.com/ghodss/yaml v1.0.0 github.com/go-gormigrate/gormigrate/v2 v2.1.1 github.com/golang-jwt/jwt/v4 v4.5.0 diff --git a/flyteadmin/go.sum b/flyteadmin/go.sum index eb2eb483299..fa9c065e82a 100644 --- a/flyteadmin/go.sum +++ b/flyteadmin/go.sum @@ -237,8 +237,8 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flyteorg/stow v0.3.8 h1:4a6BtfgDR86fUwa48DkkZTcp6WK4oQXSfewPd/kN0Z4= -github.com/flyteorg/stow v0.3.8/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= +github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8= +github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= diff --git a/flyteadmin/pkg/data/implementations/noop_remote_url_test.go b/flyteadmin/pkg/data/implementations/noop_remote_url_test.go index 0c05c2a3919..965dc9eeb20 100644 --- a/flyteadmin/pkg/data/implementations/noop_remote_url_test.go +++ b/flyteadmin/pkg/data/implementations/noop_remote_url_test.go @@ -14,6 +14,10 @@ const noopFileSize = int64(1256) type MockMetadata struct{} +func (m MockMetadata) ContentMD5() string { + return "" +} + func (m MockMetadata) Exists() bool { return true } diff --git a/flytecopilot/go.mod b/flytecopilot/go.mod index 254e636513a..de6c77795c9 100644 --- a/flytecopilot/go.mod +++ b/flytecopilot/go.mod @@ -39,7 +39,7 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/flyteorg/stow v0.3.8 // indirect + github.com/flyteorg/stow v0.3.10 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect diff --git a/flytecopilot/go.sum b/flytecopilot/go.sum index 076bd774a82..061cd63e87b 100644 --- a/flytecopilot/go.sum +++ b/flytecopilot/go.sum @@ -103,8 +103,8 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flyteorg/stow v0.3.8 h1:4a6BtfgDR86fUwa48DkkZTcp6WK4oQXSfewPd/kN0Z4= -github.com/flyteorg/stow v0.3.8/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= +github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8= +github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= diff --git a/flyteidl/gen/pb-es/flyteidl/service/dataproxy_pb.ts b/flyteidl/gen/pb-es/flyteidl/service/dataproxy_pb.ts index 2b4cf829abb..8ce9d574351 100644 --- a/flyteidl/gen/pb-es/flyteidl/service/dataproxy_pb.ts +++ b/flyteidl/gen/pb-es/flyteidl/service/dataproxy_pb.ts @@ -60,6 +60,13 @@ export class CreateUploadLocationResponse extends Message headers = 4; + */ + headers: { [key: string]: string } = {}; + constructor(data?: PartialMessage) { super(); proto3.util.initPartial(data, this); @@ -71,6 +78,7 @@ export class CreateUploadLocationResponse extends Message): CreateUploadLocationResponse { @@ -152,6 +160,15 @@ export class CreateUploadLocationRequest extends Message) { super(); proto3.util.initPartial(data, this); @@ -166,6 +183,7 @@ export class CreateUploadLocationRequest extends Message): CreateUploadLocationRequest { diff --git a/flyteidl/gen/pb-go/flyteidl/service/dataproxy.pb.go b/flyteidl/gen/pb-go/flyteidl/service/dataproxy.pb.go index 3acf2559cf5..802d4080b44 100644 --- a/flyteidl/gen/pb-go/flyteidl/service/dataproxy.pb.go +++ b/flyteidl/gen/pb-go/flyteidl/service/dataproxy.pb.go @@ -86,6 +86,8 @@ type CreateUploadLocationResponse struct { NativeUrl string `protobuf:"bytes,2,opt,name=native_url,json=nativeUrl,proto3" json:"native_url,omitempty"` // ExpiresAt defines when will the signed URL expires. ExpiresAt *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=expires_at,json=expiresAt,proto3" json:"expires_at,omitempty"` + // Data proxy generates these headers for client, and they have to add these headers to the request when uploading the file. + Headers map[string]string `protobuf:"bytes,4,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *CreateUploadLocationResponse) Reset() { @@ -141,6 +143,13 @@ func (x *CreateUploadLocationResponse) GetExpiresAt() *timestamppb.Timestamp { return nil } +func (x *CreateUploadLocationResponse) GetHeaders() map[string]string { + if x != nil { + return x.Headers + } + return nil +} + // CreateUploadLocationRequest specified request for the CreateUploadLocation API. // The implementation in data proxy service will create the s3 location with some server side configured prefixes, // and then: @@ -173,6 +182,10 @@ type CreateUploadLocationRequest struct { // in data proxy config. This option is useful when uploading multiple files. // +optional FilenameRoot string `protobuf:"bytes,6,opt,name=filename_root,json=filenameRoot,proto3" json:"filename_root,omitempty"` + // If true, the data proxy will add content_md5 to the metadata to the signed URL and + // it will force clients to add this metadata to the object. + // This make sure dataproxy is backward compatible with the old flytekit. + AddContentMd5Metadata bool `protobuf:"varint,7,opt,name=add_content_md5_metadata,json=addContentMd5Metadata,proto3" json:"add_content_md5_metadata,omitempty"` } func (x *CreateUploadLocationRequest) Reset() { @@ -249,6 +262,13 @@ func (x *CreateUploadLocationRequest) GetFilenameRoot() string { return "" } +func (x *CreateUploadLocationRequest) GetAddContentMd5Metadata() bool { + if x != nil { + return x.AddContentMd5Metadata + } + return false +} + // CreateDownloadLocationRequest specified request for the CreateDownloadLocation API. // // Deprecated: Marked as deprecated in flyteidl/service/dataproxy.proto. @@ -762,7 +782,7 @@ var file_flyteidl_service_dataproxy_proto_rawDesc = []byte{ 0x72, 0x65, 0x2f, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x63, 0x6f, 0x72, 0x65, 0x2f, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x22, 0x97, 0x01, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, 0x6c, 0x6f, + 0x6f, 0x22, 0xaa, 0x02, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x72, @@ -771,163 +791,176 @@ var file_flyteidl_service_dataproxy_proto_rawDesc = []byte{ 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x22, 0xeb, 0x01, 0x0a, 0x1b, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, - 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, - 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x1a, 0x0a, - 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x0a, 0x65, 0x78, 0x70, - 0x69, 0x72, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, - 0x73, 0x49, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x6d, - 0x64, 0x35, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, - 0x74, 0x4d, 0x64, 0x35, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, - 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x66, 0x69, 0x6c, - 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x22, 0x7c, 0x0a, 0x1d, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, - 0x74, 0x69, 0x76, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x38, 0x0a, 0x0a, 0x65, 0x78, 0x70, - 0x69, 0x72, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, - 0x73, 0x49, 0x6e, 0x3a, 0x02, 0x18, 0x01, 0x22, 0x7e, 0x0a, 0x1e, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x69, 0x67, - 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, - 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x72, 0x6c, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, - 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, - 0x73, 0x41, 0x74, 0x3a, 0x02, 0x18, 0x01, 0x22, 0xfa, 0x01, 0x0a, 0x19, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x43, 0x0a, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, - 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x66, + 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x12, 0x55, 0x0a, 0x07, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, - 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x61, 0x72, - 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x0a, 0x65, 0x78, - 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, - 0x65, 0x73, 0x49, 0x6e, 0x12, 0x54, 0x0a, 0x11, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x26, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, - 0x4e, 0x6f, 0x64, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x48, 0x00, 0x52, 0x0f, 0x6e, 0x6f, 0x64, 0x65, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x22, 0xc7, 0x01, 0x0a, 0x1a, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, - 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x21, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, - 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x09, 0x73, 0x69, 0x67, - 0x6e, 0x65, 0x64, 0x55, 0x72, 0x6c, 0x12, 0x3d, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, - 0x73, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x42, 0x02, 0x18, 0x01, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, - 0x72, 0x65, 0x73, 0x41, 0x74, 0x12, 0x47, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x5f, 0x73, 0x69, 0x67, - 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, - 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x2e, 0x50, 0x72, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x73, 0x52, - 0x0d, 0x70, 0x72, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x72, 0x6c, 0x73, 0x22, 0x69, - 0x0a, 0x0d, 0x50, 0x72, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x73, 0x12, - 0x1d, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x72, 0x6c, 0x12, 0x39, - 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, - 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x22, 0x2d, 0x0a, 0x0e, 0x47, 0x65, 0x74, - 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x66, - 0x6c, 0x79, 0x74, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x66, 0x6c, 0x79, 0x74, 0x65, 0x55, 0x72, 0x6c, 0x22, 0xd6, 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, - 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x0b, - 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x19, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, - 0x65, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x48, 0x00, 0x52, 0x0a, - 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x12, 0x49, 0x0a, 0x0f, 0x70, 0x72, - 0x65, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x50, 0x72, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, - 0x55, 0x52, 0x4c, 0x73, 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, - 0x64, 0x55, 0x72, 0x6c, 0x73, 0x12, 0x32, 0x0a, 0x07, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, - 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x48, 0x00, - 0x52, 0x07, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, - 0x61, 0x2a, 0x43, 0x0a, 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, - 0x65, 0x12, 0x1b, 0x0a, 0x17, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x5f, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x16, - 0x0a, 0x12, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x44, 0x45, 0x43, 0x4b, 0x10, 0x01, 0x32, 0x84, 0x07, 0x0a, 0x10, 0x44, 0x61, 0x74, 0x61, 0x50, - 0x72, 0x6f, 0x78, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0xf0, 0x01, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, - 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, 0x6c, - 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0x79, 0x92, 0x41, 0x4d, 0x1a, 0x4b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x73, 0x20, 0x61, 0x20, 0x77, 0x72, 0x69, 0x74, 0x65, 0x2d, 0x6f, 0x6e, 0x6c, 0x79, 0x20, 0x68, - 0x74, 0x74, 0x70, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x68, 0x61, - 0x74, 0x20, 0x69, 0x73, 0x20, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x20, - 0x66, 0x6f, 0x72, 0x20, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x20, 0x61, 0x74, 0x20, 0x72, 0x75, 0x6e, - 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x23, 0x3a, 0x01, 0x2a, 0x22, 0x1e, - 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x70, 0x72, 0x6f, 0x78, - 0x79, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x75, 0x72, 0x6e, 0x12, 0xa9, - 0x02, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, - 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2f, 0x2e, 0x66, 0x6c, 0x79, 0x74, - 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x66, 0x6c, 0x79, - 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xab, 0x01, 0x92, - 0x41, 0x7f, 0x1a, 0x7d, 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x3a, 0x20, - 0x50, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x20, 0x75, 0x73, 0x65, 0x20, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x20, 0x69, 0x6e, - 0x73, 0x74, 0x65, 0x61, 0x64, 0x2e, 0x20, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, - 0x20, 0x72, 0x65, 0x61, 0x64, 0x2d, 0x6f, 0x6e, 0x6c, 0x79, 0x20, 0x68, 0x74, 0x74, 0x70, 0x20, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x68, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x73, 0x1a, 0x3a, 0x0a, 0x0c, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0xa4, + 0x02, 0x0a, 0x1b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, + 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x6f, 0x6d, 0x61, + 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, + 0x12, 0x1a, 0x0a, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x38, 0x0a, 0x0a, + 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x70, + 0x69, 0x72, 0x65, 0x73, 0x49, 0x6e, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, + 0x74, 0x5f, 0x6d, 0x64, 0x35, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x4d, 0x64, 0x35, 0x12, 0x23, 0x0a, 0x0d, 0x66, 0x69, 0x6c, 0x65, 0x6e, + 0x61, 0x6d, 0x65, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x66, 0x69, 0x6c, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x37, 0x0a, 0x18, + 0x61, 0x64, 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x64, 0x35, 0x5f, + 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, + 0x61, 0x64, 0x64, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4d, 0x64, 0x35, 0x4d, 0x65, 0x74, + 0x61, 0x64, 0x61, 0x74, 0x61, 0x22, 0x7c, 0x0a, 0x1d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, + 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6e, 0x61, 0x74, 0x69, 0x76, 0x65, + 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6e, 0x61, 0x74, 0x69, + 0x76, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x38, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, + 0x5f, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x49, 0x6e, 0x3a, + 0x02, 0x18, 0x01, 0x22, 0x7e, 0x0a, 0x1e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, + 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x5f, + 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x65, + 0x64, 0x55, 0x72, 0x6c, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, + 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, 0x74, 0x3a, + 0x02, 0x18, 0x01, 0x22, 0xfa, 0x01, 0x0a, 0x19, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, + 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x43, 0x0a, 0x0d, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x74, 0x79, + 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1e, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, + 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x41, 0x72, 0x74, 0x69, + 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0c, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, + 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, + 0x73, 0x5f, 0x69, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x49, 0x6e, + 0x12, 0x54, 0x0a, 0x11, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x66, 0x6c, + 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4e, 0x6f, 0x64, 0x65, + 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, + 0x69, 0x65, 0x72, 0x48, 0x00, 0x52, 0x0f, 0x6e, 0x6f, 0x64, 0x65, 0x45, 0x78, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x42, 0x08, 0x0a, 0x06, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, + 0x22, 0xc7, 0x01, 0x0a, 0x1a, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, + 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x21, 0x0a, 0x0a, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x09, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, + 0x72, 0x6c, 0x12, 0x3d, 0x0a, 0x0a, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, + 0x6d, 0x70, 0x42, 0x02, 0x18, 0x01, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73, 0x41, + 0x74, 0x12, 0x47, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x5f, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x5f, + 0x75, 0x72, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x66, 0x6c, 0x79, + 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x50, 0x72, + 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x73, 0x52, 0x0d, 0x70, 0x72, 0x65, + 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x72, 0x6c, 0x73, 0x22, 0x69, 0x0a, 0x0d, 0x50, 0x72, + 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x73, + 0x69, 0x67, 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x09, 0x73, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x72, 0x6c, 0x12, 0x39, 0x0a, 0x0a, 0x65, 0x78, + 0x70, 0x69, 0x72, 0x65, 0x73, 0x5f, 0x61, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x65, 0x78, 0x70, 0x69, + 0x72, 0x65, 0x73, 0x41, 0x74, 0x22, 0x2d, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x66, 0x6c, 0x79, 0x74, 0x65, + 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x66, 0x6c, 0x79, 0x74, + 0x65, 0x55, 0x72, 0x6c, 0x22, 0xd6, 0x01, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x0b, 0x6c, 0x69, 0x74, 0x65, + 0x72, 0x61, 0x6c, 0x5f, 0x6d, 0x61, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, + 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, 0x72, 0x65, 0x2e, 0x4c, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x48, 0x00, 0x52, 0x0a, 0x6c, 0x69, 0x74, 0x65, + 0x72, 0x61, 0x6c, 0x4d, 0x61, 0x70, 0x12, 0x49, 0x0a, 0x0f, 0x70, 0x72, 0x65, 0x5f, 0x73, 0x69, + 0x67, 0x6e, 0x65, 0x64, 0x5f, 0x75, 0x72, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1f, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x2e, 0x50, 0x72, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x52, 0x4c, 0x73, + 0x48, 0x00, 0x52, 0x0d, 0x70, 0x72, 0x65, 0x53, 0x69, 0x67, 0x6e, 0x65, 0x64, 0x55, 0x72, 0x6c, + 0x73, 0x12, 0x32, 0x0a, 0x07, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x63, 0x6f, + 0x72, 0x65, 0x2e, 0x4c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x69, + 0x74, 0x65, 0x72, 0x61, 0x6c, 0x42, 0x06, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, 0x2a, 0x43, 0x0a, + 0x0c, 0x41, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, + 0x17, 0x41, 0x52, 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, + 0x4e, 0x44, 0x45, 0x46, 0x49, 0x4e, 0x45, 0x44, 0x10, 0x00, 0x12, 0x16, 0x0a, 0x12, 0x41, 0x52, + 0x54, 0x49, 0x46, 0x41, 0x43, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x45, 0x43, 0x4b, + 0x10, 0x01, 0x32, 0x84, 0x07, 0x0a, 0x10, 0x44, 0x61, 0x74, 0x61, 0x50, 0x72, 0x6f, 0x78, 0x79, + 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0xf0, 0x01, 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x2d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, + 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x2e, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x4c, + 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x79, 0x92, 0x41, 0x4d, 0x1a, 0x4b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x20, + 0x77, 0x72, 0x69, 0x74, 0x65, 0x2d, 0x6f, 0x6e, 0x6c, 0x79, 0x20, 0x68, 0x74, 0x74, 0x70, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x69, 0x73, 0x20, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x20, 0x61, 0x74, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, - 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x20, 0x12, 0x1e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, - 0x2f, 0x64, 0x61, 0x74, 0x61, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, - 0x61, 0x63, 0x74, 0x5f, 0x75, 0x72, 0x6e, 0x88, 0x02, 0x01, 0x12, 0xea, 0x01, 0x0a, 0x12, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, - 0x6b, 0x12, 0x2b, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, - 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, - 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, - 0x4c, 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x79, 0x92, 0x41, - 0x4c, 0x1a, 0x4a, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x20, 0x72, 0x65, 0x61, + 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x23, 0x3a, 0x01, 0x2a, 0x22, 0x1e, 0x2f, 0x61, 0x70, 0x69, + 0x2f, 0x76, 0x31, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x61, 0x72, + 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x75, 0x72, 0x6e, 0x12, 0xa9, 0x02, 0x0a, 0x16, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2f, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, + 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, + 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x30, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, + 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xab, 0x01, 0x92, 0x41, 0x7f, 0x1a, 0x7d, + 0x44, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x3a, 0x20, 0x50, 0x6c, 0x65, 0x61, + 0x73, 0x65, 0x20, 0x75, 0x73, 0x65, 0x20, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, + 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x20, 0x69, 0x6e, 0x73, 0x74, 0x65, 0x61, + 0x64, 0x2e, 0x20, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x20, 0x72, 0x65, 0x61, 0x64, 0x2d, 0x6f, 0x6e, 0x6c, 0x79, 0x20, 0x68, 0x74, 0x74, 0x70, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x69, 0x73, 0x20, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, 0x62, 0x6c, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x20, 0x61, 0x74, 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x82, 0xd3, 0xe4, - 0x93, 0x02, 0x24, 0x3a, 0x01, 0x2a, 0x22, 0x1f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, - 0x64, 0x61, 0x74, 0x61, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, - 0x63, 0x74, 0x5f, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x64, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x44, 0x61, - 0x74, 0x61, 0x12, 0x20, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, - 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x14, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0e, 0x12, - 0x0c, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x42, 0xc6, 0x01, - 0x0a, 0x14, 0x63, 0x6f, 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x42, 0x0e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x72, 0x6f, 0x78, - 0x79, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, - 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, - 0x79, 0x74, 0x65, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, - 0x2f, 0x70, 0x62, 0x2d, 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, - 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x53, 0x58, 0xaa, 0x02, 0x10, - 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0xca, 0x02, 0x10, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x53, 0x65, 0x72, 0x76, - 0x69, 0x63, 0x65, 0xe2, 0x02, 0x1c, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0xea, 0x02, 0x11, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x93, 0x02, 0x20, 0x12, 0x1e, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x64, 0x61, 0x74, + 0x61, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, + 0x75, 0x72, 0x6e, 0x88, 0x02, 0x01, 0x12, 0xea, 0x01, 0x0a, 0x12, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, 0x12, 0x2b, 0x2e, + 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, + 0x69, 0x6e, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2c, 0x2e, 0x66, 0x6c, 0x79, + 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6e, 0x6b, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x79, 0x92, 0x41, 0x4c, 0x1a, 0x4a, 0x43, + 0x72, 0x65, 0x61, 0x74, 0x65, 0x73, 0x20, 0x61, 0x20, 0x72, 0x65, 0x61, 0x64, 0x2d, 0x6f, 0x6e, + 0x6c, 0x79, 0x20, 0x68, 0x74, 0x74, 0x70, 0x20, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x20, 0x74, 0x68, 0x61, 0x74, 0x20, 0x69, 0x73, 0x20, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x69, + 0x62, 0x6c, 0x65, 0x20, 0x66, 0x6f, 0x72, 0x20, 0x74, 0x61, 0x73, 0x6b, 0x73, 0x20, 0x61, 0x74, + 0x20, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x2e, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x24, 0x3a, + 0x01, 0x2a, 0x22, 0x1f, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x64, 0x61, 0x74, 0x61, + 0x70, 0x72, 0x6f, 0x78, 0x79, 0x2f, 0x61, 0x72, 0x74, 0x69, 0x66, 0x61, 0x63, 0x74, 0x5f, 0x6c, + 0x69, 0x6e, 0x6b, 0x12, 0x64, 0x0a, 0x07, 0x47, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12, 0x20, + 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, + 0x65, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x21, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0x2e, 0x47, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x14, 0x82, 0xd3, 0xe4, 0x93, 0x02, 0x0e, 0x12, 0x0c, 0x2f, 0x61, 0x70, + 0x69, 0x2f, 0x76, 0x31, 0x2f, 0x64, 0x61, 0x74, 0x61, 0x42, 0xc6, 0x01, 0x0a, 0x14, 0x63, 0x6f, + 0x6d, 0x2e, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x42, 0x0e, 0x44, 0x61, 0x74, 0x61, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x3d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x6f, 0x72, 0x67, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x2f, + 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x70, 0x62, 0x2d, + 0x67, 0x6f, 0x2f, 0x66, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x2f, 0x73, 0x65, 0x72, 0x76, + 0x69, 0x63, 0x65, 0xa2, 0x02, 0x03, 0x46, 0x53, 0x58, 0xaa, 0x02, 0x10, 0x46, 0x6c, 0x79, 0x74, + 0x65, 0x69, 0x64, 0x6c, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0xca, 0x02, 0x10, 0x46, + 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0xe2, + 0x02, 0x1c, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x5c, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, + 0x11, 0x46, 0x6c, 0x79, 0x74, 0x65, 0x69, 0x64, 0x6c, 0x3a, 0x3a, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -943,7 +976,7 @@ func file_flyteidl_service_dataproxy_proto_rawDescGZIP() []byte { } var file_flyteidl_service_dataproxy_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_flyteidl_service_dataproxy_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_flyteidl_service_dataproxy_proto_msgTypes = make([]protoimpl.MessageInfo, 10) var file_flyteidl_service_dataproxy_proto_goTypes = []interface{}{ (ArtifactType)(0), // 0: flyteidl.service.ArtifactType (*CreateUploadLocationResponse)(nil), // 1: flyteidl.service.CreateUploadLocationResponse @@ -955,39 +988,41 @@ var file_flyteidl_service_dataproxy_proto_goTypes = []interface{}{ (*PreSignedURLs)(nil), // 7: flyteidl.service.PreSignedURLs (*GetDataRequest)(nil), // 8: flyteidl.service.GetDataRequest (*GetDataResponse)(nil), // 9: flyteidl.service.GetDataResponse - (*timestamppb.Timestamp)(nil), // 10: google.protobuf.Timestamp - (*durationpb.Duration)(nil), // 11: google.protobuf.Duration - (*core.NodeExecutionIdentifier)(nil), // 12: flyteidl.core.NodeExecutionIdentifier - (*core.LiteralMap)(nil), // 13: flyteidl.core.LiteralMap - (*core.Literal)(nil), // 14: flyteidl.core.Literal + nil, // 10: flyteidl.service.CreateUploadLocationResponse.HeadersEntry + (*timestamppb.Timestamp)(nil), // 11: google.protobuf.Timestamp + (*durationpb.Duration)(nil), // 12: google.protobuf.Duration + (*core.NodeExecutionIdentifier)(nil), // 13: flyteidl.core.NodeExecutionIdentifier + (*core.LiteralMap)(nil), // 14: flyteidl.core.LiteralMap + (*core.Literal)(nil), // 15: flyteidl.core.Literal } var file_flyteidl_service_dataproxy_proto_depIdxs = []int32{ - 10, // 0: flyteidl.service.CreateUploadLocationResponse.expires_at:type_name -> google.protobuf.Timestamp - 11, // 1: flyteidl.service.CreateUploadLocationRequest.expires_in:type_name -> google.protobuf.Duration - 11, // 2: flyteidl.service.CreateDownloadLocationRequest.expires_in:type_name -> google.protobuf.Duration - 10, // 3: flyteidl.service.CreateDownloadLocationResponse.expires_at:type_name -> google.protobuf.Timestamp - 0, // 4: flyteidl.service.CreateDownloadLinkRequest.artifact_type:type_name -> flyteidl.service.ArtifactType - 11, // 5: flyteidl.service.CreateDownloadLinkRequest.expires_in:type_name -> google.protobuf.Duration - 12, // 6: flyteidl.service.CreateDownloadLinkRequest.node_execution_id:type_name -> flyteidl.core.NodeExecutionIdentifier - 10, // 7: flyteidl.service.CreateDownloadLinkResponse.expires_at:type_name -> google.protobuf.Timestamp - 7, // 8: flyteidl.service.CreateDownloadLinkResponse.pre_signed_urls:type_name -> flyteidl.service.PreSignedURLs - 10, // 9: flyteidl.service.PreSignedURLs.expires_at:type_name -> google.protobuf.Timestamp - 13, // 10: flyteidl.service.GetDataResponse.literal_map:type_name -> flyteidl.core.LiteralMap - 7, // 11: flyteidl.service.GetDataResponse.pre_signed_urls:type_name -> flyteidl.service.PreSignedURLs - 14, // 12: flyteidl.service.GetDataResponse.literal:type_name -> flyteidl.core.Literal - 2, // 13: flyteidl.service.DataProxyService.CreateUploadLocation:input_type -> flyteidl.service.CreateUploadLocationRequest - 3, // 14: flyteidl.service.DataProxyService.CreateDownloadLocation:input_type -> flyteidl.service.CreateDownloadLocationRequest - 5, // 15: flyteidl.service.DataProxyService.CreateDownloadLink:input_type -> flyteidl.service.CreateDownloadLinkRequest - 8, // 16: flyteidl.service.DataProxyService.GetData:input_type -> flyteidl.service.GetDataRequest - 1, // 17: flyteidl.service.DataProxyService.CreateUploadLocation:output_type -> flyteidl.service.CreateUploadLocationResponse - 4, // 18: flyteidl.service.DataProxyService.CreateDownloadLocation:output_type -> flyteidl.service.CreateDownloadLocationResponse - 6, // 19: flyteidl.service.DataProxyService.CreateDownloadLink:output_type -> flyteidl.service.CreateDownloadLinkResponse - 9, // 20: flyteidl.service.DataProxyService.GetData:output_type -> flyteidl.service.GetDataResponse - 17, // [17:21] is the sub-list for method output_type - 13, // [13:17] is the sub-list for method input_type - 13, // [13:13] is the sub-list for extension type_name - 13, // [13:13] is the sub-list for extension extendee - 0, // [0:13] is the sub-list for field type_name + 11, // 0: flyteidl.service.CreateUploadLocationResponse.expires_at:type_name -> google.protobuf.Timestamp + 10, // 1: flyteidl.service.CreateUploadLocationResponse.headers:type_name -> flyteidl.service.CreateUploadLocationResponse.HeadersEntry + 12, // 2: flyteidl.service.CreateUploadLocationRequest.expires_in:type_name -> google.protobuf.Duration + 12, // 3: flyteidl.service.CreateDownloadLocationRequest.expires_in:type_name -> google.protobuf.Duration + 11, // 4: flyteidl.service.CreateDownloadLocationResponse.expires_at:type_name -> google.protobuf.Timestamp + 0, // 5: flyteidl.service.CreateDownloadLinkRequest.artifact_type:type_name -> flyteidl.service.ArtifactType + 12, // 6: flyteidl.service.CreateDownloadLinkRequest.expires_in:type_name -> google.protobuf.Duration + 13, // 7: flyteidl.service.CreateDownloadLinkRequest.node_execution_id:type_name -> flyteidl.core.NodeExecutionIdentifier + 11, // 8: flyteidl.service.CreateDownloadLinkResponse.expires_at:type_name -> google.protobuf.Timestamp + 7, // 9: flyteidl.service.CreateDownloadLinkResponse.pre_signed_urls:type_name -> flyteidl.service.PreSignedURLs + 11, // 10: flyteidl.service.PreSignedURLs.expires_at:type_name -> google.protobuf.Timestamp + 14, // 11: flyteidl.service.GetDataResponse.literal_map:type_name -> flyteidl.core.LiteralMap + 7, // 12: flyteidl.service.GetDataResponse.pre_signed_urls:type_name -> flyteidl.service.PreSignedURLs + 15, // 13: flyteidl.service.GetDataResponse.literal:type_name -> flyteidl.core.Literal + 2, // 14: flyteidl.service.DataProxyService.CreateUploadLocation:input_type -> flyteidl.service.CreateUploadLocationRequest + 3, // 15: flyteidl.service.DataProxyService.CreateDownloadLocation:input_type -> flyteidl.service.CreateDownloadLocationRequest + 5, // 16: flyteidl.service.DataProxyService.CreateDownloadLink:input_type -> flyteidl.service.CreateDownloadLinkRequest + 8, // 17: flyteidl.service.DataProxyService.GetData:input_type -> flyteidl.service.GetDataRequest + 1, // 18: flyteidl.service.DataProxyService.CreateUploadLocation:output_type -> flyteidl.service.CreateUploadLocationResponse + 4, // 19: flyteidl.service.DataProxyService.CreateDownloadLocation:output_type -> flyteidl.service.CreateDownloadLocationResponse + 6, // 20: flyteidl.service.DataProxyService.CreateDownloadLink:output_type -> flyteidl.service.CreateDownloadLinkResponse + 9, // 21: flyteidl.service.DataProxyService.GetData:output_type -> flyteidl.service.GetDataResponse + 18, // [18:22] is the sub-list for method output_type + 14, // [14:18] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name } func init() { file_flyteidl_service_dataproxy_proto_init() } @@ -1119,7 +1154,7 @@ func file_flyteidl_service_dataproxy_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_flyteidl_service_dataproxy_proto_rawDesc, NumEnums: 1, - NumMessages: 9, + NumMessages: 10, NumExtensions: 0, NumServices: 1, }, diff --git a/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json b/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json index e23e22df706..b26138d8b72 100644 --- a/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json +++ b/flyteidl/gen/pb-go/gateway/flyteidl/service/dataproxy.swagger.json @@ -748,6 +748,10 @@ "filename_root": { "type": "string", "title": "If present, data proxy will use this string in lieu of the md5 hash in the path. When the filename is also included\nthis makes the upload location deterministic. The native url will still be prefixed by the upload location prefix\nin data proxy config. This option is useful when uploading multiple files.\n+optional" + }, + "add_content_md5_metadata": { + "type": "boolean", + "description": "If true, the data proxy will add content_md5 to the metadata to the signed URL and\nit will force clients to add this metadata to the object.\nThis make sure dataproxy is backward compatible with the old flytekit." } }, "description": "CreateUploadLocationRequest specified request for the CreateUploadLocation API.\nThe implementation in data proxy service will create the s3 location with some server side configured prefixes,\nand then:\n - project/domain/(a deterministic str representation of the content_md5)/filename (if present); OR\n - project/domain/filename_root (if present)/filename (if present)." @@ -767,6 +771,13 @@ "type": "string", "format": "date-time", "description": "ExpiresAt defines when will the signed URL expires." + }, + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "description": "Data proxy generates these headers for client, and they have to add these headers to the request when uploading the file." } } }, diff --git a/flyteidl/gen/pb-js/flyteidl.d.ts b/flyteidl/gen/pb-js/flyteidl.d.ts index 5cc6b36eeb1..8c01d5f2847 100644 --- a/flyteidl/gen/pb-js/flyteidl.d.ts +++ b/flyteidl/gen/pb-js/flyteidl.d.ts @@ -22747,6 +22747,9 @@ export namespace flyteidl { /** CreateUploadLocationResponse expiresAt */ expiresAt?: (google.protobuf.ITimestamp|null); + + /** CreateUploadLocationResponse headers */ + headers?: ({ [k: string]: string }|null); } /** Represents a CreateUploadLocationResponse. */ @@ -22767,6 +22770,9 @@ export namespace flyteidl { /** CreateUploadLocationResponse expiresAt. */ public expiresAt?: (google.protobuf.ITimestamp|null); + /** CreateUploadLocationResponse headers. */ + public headers: { [k: string]: string }; + /** * Creates a new CreateUploadLocationResponse instance using the specified properties. * @param [properties] Properties to set @@ -22820,6 +22826,9 @@ export namespace flyteidl { /** CreateUploadLocationRequest filenameRoot */ filenameRoot?: (string|null); + + /** CreateUploadLocationRequest addContentMd5Metadata */ + addContentMd5Metadata?: (boolean|null); } /** Represents a CreateUploadLocationRequest. */ @@ -22849,6 +22858,9 @@ export namespace flyteidl { /** CreateUploadLocationRequest filenameRoot. */ public filenameRoot: string; + /** CreateUploadLocationRequest addContentMd5Metadata. */ + public addContentMd5Metadata: boolean; + /** * Creates a new CreateUploadLocationRequest instance using the specified properties. * @param [properties] Properties to set diff --git a/flyteidl/gen/pb-js/flyteidl.js b/flyteidl/gen/pb-js/flyteidl.js index 3400b971b3a..fffe3309026 100644 --- a/flyteidl/gen/pb-js/flyteidl.js +++ b/flyteidl/gen/pb-js/flyteidl.js @@ -53330,6 +53330,7 @@ * @property {string|null} [signedUrl] CreateUploadLocationResponse signedUrl * @property {string|null} [nativeUrl] CreateUploadLocationResponse nativeUrl * @property {google.protobuf.ITimestamp|null} [expiresAt] CreateUploadLocationResponse expiresAt + * @property {Object.|null} [headers] CreateUploadLocationResponse headers */ /** @@ -53341,6 +53342,7 @@ * @param {flyteidl.service.ICreateUploadLocationResponse=} [properties] Properties to set */ function CreateUploadLocationResponse(properties) { + this.headers = {}; if (properties) for (var keys = Object.keys(properties), i = 0; i < keys.length; ++i) if (properties[keys[i]] != null) @@ -53371,6 +53373,14 @@ */ CreateUploadLocationResponse.prototype.expiresAt = null; + /** + * CreateUploadLocationResponse headers. + * @member {Object.} headers + * @memberof flyteidl.service.CreateUploadLocationResponse + * @instance + */ + CreateUploadLocationResponse.prototype.headers = $util.emptyObject; + /** * Creates a new CreateUploadLocationResponse instance using the specified properties. * @function create @@ -53401,6 +53411,9 @@ writer.uint32(/* id 2, wireType 2 =*/18).string(message.nativeUrl); if (message.expiresAt != null && message.hasOwnProperty("expiresAt")) $root.google.protobuf.Timestamp.encode(message.expiresAt, writer.uint32(/* id 3, wireType 2 =*/26).fork()).ldelim(); + if (message.headers != null && message.hasOwnProperty("headers")) + for (var keys = Object.keys(message.headers), i = 0; i < keys.length; ++i) + writer.uint32(/* id 4, wireType 2 =*/34).fork().uint32(/* id 1, wireType 2 =*/10).string(keys[i]).uint32(/* id 2, wireType 2 =*/18).string(message.headers[keys[i]]).ldelim(); return writer; }; @@ -53418,7 +53431,7 @@ CreateUploadLocationResponse.decode = function decode(reader, length) { if (!(reader instanceof $Reader)) reader = $Reader.create(reader); - var end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.service.CreateUploadLocationResponse(); + var end = length === undefined ? reader.len : reader.pos + length, message = new $root.flyteidl.service.CreateUploadLocationResponse(), key; while (reader.pos < end) { var tag = reader.uint32(); switch (tag >>> 3) { @@ -53431,6 +53444,14 @@ case 3: message.expiresAt = $root.google.protobuf.Timestamp.decode(reader, reader.uint32()); break; + case 4: + reader.skip().pos++; + if (message.headers === $util.emptyObject) + message.headers = {}; + key = reader.string(); + reader.pos++; + message.headers[key] = reader.string(); + break; default: reader.skipType(tag & 7); break; @@ -53461,6 +53482,14 @@ if (error) return "expiresAt." + error; } + if (message.headers != null && message.hasOwnProperty("headers")) { + if (!$util.isObject(message.headers)) + return "headers: object expected"; + var key = Object.keys(message.headers); + for (var i = 0; i < key.length; ++i) + if (!$util.isString(message.headers[key[i]])) + return "headers: string{k:string} expected"; + } return null; }; @@ -53479,6 +53508,7 @@ * @property {google.protobuf.IDuration|null} [expiresIn] CreateUploadLocationRequest expiresIn * @property {Uint8Array|null} [contentMd5] CreateUploadLocationRequest contentMd5 * @property {string|null} [filenameRoot] CreateUploadLocationRequest filenameRoot + * @property {boolean|null} [addContentMd5Metadata] CreateUploadLocationRequest addContentMd5Metadata */ /** @@ -53544,6 +53574,14 @@ */ CreateUploadLocationRequest.prototype.filenameRoot = ""; + /** + * CreateUploadLocationRequest addContentMd5Metadata. + * @member {boolean} addContentMd5Metadata + * @memberof flyteidl.service.CreateUploadLocationRequest + * @instance + */ + CreateUploadLocationRequest.prototype.addContentMd5Metadata = false; + /** * Creates a new CreateUploadLocationRequest instance using the specified properties. * @function create @@ -53580,6 +53618,8 @@ writer.uint32(/* id 5, wireType 2 =*/42).bytes(message.contentMd5); if (message.filenameRoot != null && message.hasOwnProperty("filenameRoot")) writer.uint32(/* id 6, wireType 2 =*/50).string(message.filenameRoot); + if (message.addContentMd5Metadata != null && message.hasOwnProperty("addContentMd5Metadata")) + writer.uint32(/* id 7, wireType 0 =*/56).bool(message.addContentMd5Metadata); return writer; }; @@ -53619,6 +53659,9 @@ case 6: message.filenameRoot = reader.string(); break; + case 7: + message.addContentMd5Metadata = reader.bool(); + break; default: reader.skipType(tag & 7); break; @@ -53658,6 +53701,9 @@ if (message.filenameRoot != null && message.hasOwnProperty("filenameRoot")) if (!$util.isString(message.filenameRoot)) return "filenameRoot: string expected"; + if (message.addContentMd5Metadata != null && message.hasOwnProperty("addContentMd5Metadata")) + if (typeof message.addContentMd5Metadata !== "boolean") + return "addContentMd5Metadata: boolean expected"; return null; }; diff --git a/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.py b/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.py index feced8c21ec..f4c25e43ae4 100644 --- a/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.py +++ b/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.py @@ -19,7 +19,7 @@ from flyteidl.core import literals_pb2 as flyteidl_dot_core_dot_literals__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n flyteidl/service/dataproxy.proto\x12\x10\x66lyteidl.service\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1c\x66lyteidl/core/literals.proto\"\x97\x01\n\x1c\x43reateUploadLocationResponse\x12\x1d\n\nsigned_url\x18\x01 \x01(\tR\tsignedUrl\x12\x1d\n\nnative_url\x18\x02 \x01(\tR\tnativeUrl\x12\x39\n\nexpires_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\"\xeb\x01\n\x1b\x43reateUploadLocationRequest\x12\x18\n\x07project\x18\x01 \x01(\tR\x07project\x12\x16\n\x06\x64omain\x18\x02 \x01(\tR\x06\x64omain\x12\x1a\n\x08\x66ilename\x18\x03 \x01(\tR\x08\x66ilename\x12\x38\n\nexpires_in\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn\x12\x1f\n\x0b\x63ontent_md5\x18\x05 \x01(\x0cR\ncontentMd5\x12#\n\rfilename_root\x18\x06 \x01(\tR\x0c\x66ilenameRoot\"|\n\x1d\x43reateDownloadLocationRequest\x12\x1d\n\nnative_url\x18\x01 \x01(\tR\tnativeUrl\x12\x38\n\nexpires_in\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn:\x02\x18\x01\"~\n\x1e\x43reateDownloadLocationResponse\x12\x1d\n\nsigned_url\x18\x01 \x01(\tR\tsignedUrl\x12\x39\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt:\x02\x18\x01\"\xfa\x01\n\x19\x43reateDownloadLinkRequest\x12\x43\n\rartifact_type\x18\x01 \x01(\x0e\x32\x1e.flyteidl.service.ArtifactTypeR\x0c\x61rtifactType\x12\x38\n\nexpires_in\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn\x12T\n\x11node_execution_id\x18\x03 \x01(\x0b\x32&.flyteidl.core.NodeExecutionIdentifierH\x00R\x0fnodeExecutionIdB\x08\n\x06source\"\xc7\x01\n\x1a\x43reateDownloadLinkResponse\x12!\n\nsigned_url\x18\x01 \x03(\tB\x02\x18\x01R\tsignedUrl\x12=\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x02\x18\x01R\texpiresAt\x12G\n\x0fpre_signed_urls\x18\x03 \x01(\x0b\x32\x1f.flyteidl.service.PreSignedURLsR\rpreSignedUrls\"i\n\rPreSignedURLs\x12\x1d\n\nsigned_url\x18\x01 \x03(\tR\tsignedUrl\x12\x39\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\"-\n\x0eGetDataRequest\x12\x1b\n\tflyte_url\x18\x01 \x01(\tR\x08\x66lyteUrl\"\xd6\x01\n\x0fGetDataResponse\x12<\n\x0bliteral_map\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.LiteralMapH\x00R\nliteralMap\x12I\n\x0fpre_signed_urls\x18\x02 \x01(\x0b\x32\x1f.flyteidl.service.PreSignedURLsH\x00R\rpreSignedUrls\x12\x32\n\x07literal\x18\x03 \x01(\x0b\x32\x16.flyteidl.core.LiteralH\x00R\x07literalB\x06\n\x04\x64\x61ta*C\n\x0c\x41rtifactType\x12\x1b\n\x17\x41RTIFACT_TYPE_UNDEFINED\x10\x00\x12\x16\n\x12\x41RTIFACT_TYPE_DECK\x10\x01\x32\x84\x07\n\x10\x44\x61taProxyService\x12\xf0\x01\n\x14\x43reateUploadLocation\x12-.flyteidl.service.CreateUploadLocationRequest\x1a..flyteidl.service.CreateUploadLocationResponse\"y\x92\x41M\x1aKCreates a write-only http location that is accessible for tasks at runtime.\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/api/v1/dataproxy/artifact_urn\x12\xa9\x02\n\x16\x43reateDownloadLocation\x12/.flyteidl.service.CreateDownloadLocationRequest\x1a\x30.flyteidl.service.CreateDownloadLocationResponse\"\xab\x01\x88\x02\x01\x92\x41\x7f\x1a}Deprecated: Please use CreateDownloadLink instead. Creates a read-only http location that is accessible for tasks at runtime.\x82\xd3\xe4\x93\x02 \x12\x1e/api/v1/dataproxy/artifact_urn\x12\xea\x01\n\x12\x43reateDownloadLink\x12+.flyteidl.service.CreateDownloadLinkRequest\x1a,.flyteidl.service.CreateDownloadLinkResponse\"y\x92\x41L\x1aJCreates a read-only http location that is accessible for tasks at runtime.\x82\xd3\xe4\x93\x02$:\x01*\"\x1f/api/v1/dataproxy/artifact_link\x12\x64\n\x07GetData\x12 .flyteidl.service.GetDataRequest\x1a!.flyteidl.service.GetDataResponse\"\x14\x82\xd3\xe4\x93\x02\x0e\x12\x0c/api/v1/dataB\xc6\x01\n\x14\x63om.flyteidl.serviceB\x0e\x44\x61taproxyProtoP\x01Z=github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service\xa2\x02\x03\x46SX\xaa\x02\x10\x46lyteidl.Service\xca\x02\x10\x46lyteidl\\Service\xe2\x02\x1c\x46lyteidl\\Service\\GPBMetadata\xea\x02\x11\x46lyteidl::Serviceb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n flyteidl/service/dataproxy.proto\x12\x10\x66lyteidl.service\x1a\x1cgoogle/api/annotations.proto\x1a.protoc-gen-openapiv2/options/annotations.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1e\x66lyteidl/core/identifier.proto\x1a\x1c\x66lyteidl/core/literals.proto\"\xaa\x02\n\x1c\x43reateUploadLocationResponse\x12\x1d\n\nsigned_url\x18\x01 \x01(\tR\tsignedUrl\x12\x1d\n\nnative_url\x18\x02 \x01(\tR\tnativeUrl\x12\x39\n\nexpires_at\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\x12U\n\x07headers\x18\x04 \x03(\x0b\x32;.flyteidl.service.CreateUploadLocationResponse.HeadersEntryR\x07headers\x1a:\n\x0cHeadersEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\"\xa4\x02\n\x1b\x43reateUploadLocationRequest\x12\x18\n\x07project\x18\x01 \x01(\tR\x07project\x12\x16\n\x06\x64omain\x18\x02 \x01(\tR\x06\x64omain\x12\x1a\n\x08\x66ilename\x18\x03 \x01(\tR\x08\x66ilename\x12\x38\n\nexpires_in\x18\x04 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn\x12\x1f\n\x0b\x63ontent_md5\x18\x05 \x01(\x0cR\ncontentMd5\x12#\n\rfilename_root\x18\x06 \x01(\tR\x0c\x66ilenameRoot\x12\x37\n\x18\x61\x64\x64_content_md5_metadata\x18\x07 \x01(\x08R\x15\x61\x64\x64\x43ontentMd5Metadata\"|\n\x1d\x43reateDownloadLocationRequest\x12\x1d\n\nnative_url\x18\x01 \x01(\tR\tnativeUrl\x12\x38\n\nexpires_in\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn:\x02\x18\x01\"~\n\x1e\x43reateDownloadLocationResponse\x12\x1d\n\nsigned_url\x18\x01 \x01(\tR\tsignedUrl\x12\x39\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt:\x02\x18\x01\"\xfa\x01\n\x19\x43reateDownloadLinkRequest\x12\x43\n\rartifact_type\x18\x01 \x01(\x0e\x32\x1e.flyteidl.service.ArtifactTypeR\x0c\x61rtifactType\x12\x38\n\nexpires_in\x18\x02 \x01(\x0b\x32\x19.google.protobuf.DurationR\texpiresIn\x12T\n\x11node_execution_id\x18\x03 \x01(\x0b\x32&.flyteidl.core.NodeExecutionIdentifierH\x00R\x0fnodeExecutionIdB\x08\n\x06source\"\xc7\x01\n\x1a\x43reateDownloadLinkResponse\x12!\n\nsigned_url\x18\x01 \x03(\tB\x02\x18\x01R\tsignedUrl\x12=\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x02\x18\x01R\texpiresAt\x12G\n\x0fpre_signed_urls\x18\x03 \x01(\x0b\x32\x1f.flyteidl.service.PreSignedURLsR\rpreSignedUrls\"i\n\rPreSignedURLs\x12\x1d\n\nsigned_url\x18\x01 \x03(\tR\tsignedUrl\x12\x39\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampR\texpiresAt\"-\n\x0eGetDataRequest\x12\x1b\n\tflyte_url\x18\x01 \x01(\tR\x08\x66lyteUrl\"\xd6\x01\n\x0fGetDataResponse\x12<\n\x0bliteral_map\x18\x01 \x01(\x0b\x32\x19.flyteidl.core.LiteralMapH\x00R\nliteralMap\x12I\n\x0fpre_signed_urls\x18\x02 \x01(\x0b\x32\x1f.flyteidl.service.PreSignedURLsH\x00R\rpreSignedUrls\x12\x32\n\x07literal\x18\x03 \x01(\x0b\x32\x16.flyteidl.core.LiteralH\x00R\x07literalB\x06\n\x04\x64\x61ta*C\n\x0c\x41rtifactType\x12\x1b\n\x17\x41RTIFACT_TYPE_UNDEFINED\x10\x00\x12\x16\n\x12\x41RTIFACT_TYPE_DECK\x10\x01\x32\x84\x07\n\x10\x44\x61taProxyService\x12\xf0\x01\n\x14\x43reateUploadLocation\x12-.flyteidl.service.CreateUploadLocationRequest\x1a..flyteidl.service.CreateUploadLocationResponse\"y\x92\x41M\x1aKCreates a write-only http location that is accessible for tasks at runtime.\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/api/v1/dataproxy/artifact_urn\x12\xa9\x02\n\x16\x43reateDownloadLocation\x12/.flyteidl.service.CreateDownloadLocationRequest\x1a\x30.flyteidl.service.CreateDownloadLocationResponse\"\xab\x01\x88\x02\x01\x92\x41\x7f\x1a}Deprecated: Please use CreateDownloadLink instead. Creates a read-only http location that is accessible for tasks at runtime.\x82\xd3\xe4\x93\x02 \x12\x1e/api/v1/dataproxy/artifact_urn\x12\xea\x01\n\x12\x43reateDownloadLink\x12+.flyteidl.service.CreateDownloadLinkRequest\x1a,.flyteidl.service.CreateDownloadLinkResponse\"y\x92\x41L\x1aJCreates a read-only http location that is accessible for tasks at runtime.\x82\xd3\xe4\x93\x02$:\x01*\"\x1f/api/v1/dataproxy/artifact_link\x12\x64\n\x07GetData\x12 .flyteidl.service.GetDataRequest\x1a!.flyteidl.service.GetDataResponse\"\x14\x82\xd3\xe4\x93\x02\x0e\x12\x0c/api/v1/dataB\xc6\x01\n\x14\x63om.flyteidl.serviceB\x0e\x44\x61taproxyProtoP\x01Z=github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service\xa2\x02\x03\x46SX\xaa\x02\x10\x46lyteidl.Service\xca\x02\x10\x46lyteidl\\Service\xe2\x02\x1c\x46lyteidl\\Service\\GPBMetadata\xea\x02\x11\x46lyteidl::Serviceb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -28,6 +28,8 @@ DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\n\024com.flyteidl.serviceB\016DataproxyProtoP\001Z=github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service\242\002\003FSX\252\002\020Flyteidl.Service\312\002\020Flyteidl\\Service\342\002\034Flyteidl\\Service\\GPBMetadata\352\002\021Flyteidl::Service' + _CREATEUPLOADLOCATIONRESPONSE_HEADERSENTRY._options = None + _CREATEUPLOADLOCATIONRESPONSE_HEADERSENTRY._serialized_options = b'8\001' _CREATEDOWNLOADLOCATIONREQUEST._options = None _CREATEDOWNLOADLOCATIONREQUEST._serialized_options = b'\030\001' _CREATEDOWNLOADLOCATIONRESPONSE._options = None @@ -44,26 +46,28 @@ _DATAPROXYSERVICE.methods_by_name['CreateDownloadLink']._serialized_options = b'\222AL\032JCreates a read-only http location that is accessible for tasks at runtime.\202\323\344\223\002$:\001*\"\037/api/v1/dataproxy/artifact_link' _DATAPROXYSERVICE.methods_by_name['GetData']._options = None _DATAPROXYSERVICE.methods_by_name['GetData']._serialized_options = b'\202\323\344\223\002\016\022\014/api/v1/data' - _globals['_ARTIFACTTYPE']._serialized_start=1731 - _globals['_ARTIFACTTYPE']._serialized_end=1798 + _globals['_ARTIFACTTYPE']._serialized_start=1935 + _globals['_ARTIFACTTYPE']._serialized_end=2002 _globals['_CREATEUPLOADLOCATIONRESPONSE']._serialized_start=260 - _globals['_CREATEUPLOADLOCATIONRESPONSE']._serialized_end=411 - _globals['_CREATEUPLOADLOCATIONREQUEST']._serialized_start=414 - _globals['_CREATEUPLOADLOCATIONREQUEST']._serialized_end=649 - _globals['_CREATEDOWNLOADLOCATIONREQUEST']._serialized_start=651 - _globals['_CREATEDOWNLOADLOCATIONREQUEST']._serialized_end=775 - _globals['_CREATEDOWNLOADLOCATIONRESPONSE']._serialized_start=777 - _globals['_CREATEDOWNLOADLOCATIONRESPONSE']._serialized_end=903 - _globals['_CREATEDOWNLOADLINKREQUEST']._serialized_start=906 - _globals['_CREATEDOWNLOADLINKREQUEST']._serialized_end=1156 - _globals['_CREATEDOWNLOADLINKRESPONSE']._serialized_start=1159 - _globals['_CREATEDOWNLOADLINKRESPONSE']._serialized_end=1358 - _globals['_PRESIGNEDURLS']._serialized_start=1360 - _globals['_PRESIGNEDURLS']._serialized_end=1465 - _globals['_GETDATAREQUEST']._serialized_start=1467 - _globals['_GETDATAREQUEST']._serialized_end=1512 - _globals['_GETDATARESPONSE']._serialized_start=1515 - _globals['_GETDATARESPONSE']._serialized_end=1729 - _globals['_DATAPROXYSERVICE']._serialized_start=1801 - _globals['_DATAPROXYSERVICE']._serialized_end=2701 + _globals['_CREATEUPLOADLOCATIONRESPONSE']._serialized_end=558 + _globals['_CREATEUPLOADLOCATIONRESPONSE_HEADERSENTRY']._serialized_start=500 + _globals['_CREATEUPLOADLOCATIONRESPONSE_HEADERSENTRY']._serialized_end=558 + _globals['_CREATEUPLOADLOCATIONREQUEST']._serialized_start=561 + _globals['_CREATEUPLOADLOCATIONREQUEST']._serialized_end=853 + _globals['_CREATEDOWNLOADLOCATIONREQUEST']._serialized_start=855 + _globals['_CREATEDOWNLOADLOCATIONREQUEST']._serialized_end=979 + _globals['_CREATEDOWNLOADLOCATIONRESPONSE']._serialized_start=981 + _globals['_CREATEDOWNLOADLOCATIONRESPONSE']._serialized_end=1107 + _globals['_CREATEDOWNLOADLINKREQUEST']._serialized_start=1110 + _globals['_CREATEDOWNLOADLINKREQUEST']._serialized_end=1360 + _globals['_CREATEDOWNLOADLINKRESPONSE']._serialized_start=1363 + _globals['_CREATEDOWNLOADLINKRESPONSE']._serialized_end=1562 + _globals['_PRESIGNEDURLS']._serialized_start=1564 + _globals['_PRESIGNEDURLS']._serialized_end=1669 + _globals['_GETDATAREQUEST']._serialized_start=1671 + _globals['_GETDATAREQUEST']._serialized_end=1716 + _globals['_GETDATARESPONSE']._serialized_start=1719 + _globals['_GETDATARESPONSE']._serialized_end=1933 + _globals['_DATAPROXYSERVICE']._serialized_start=2005 + _globals['_DATAPROXYSERVICE']._serialized_end=2905 # @@protoc_insertion_point(module_scope) diff --git a/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.pyi b/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.pyi index 40245087ab4..50c2bf3c5ed 100644 --- a/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.pyi +++ b/flyteidl/gen/pb_python/flyteidl/service/dataproxy_pb2.pyi @@ -20,30 +20,41 @@ ARTIFACT_TYPE_UNDEFINED: ArtifactType ARTIFACT_TYPE_DECK: ArtifactType class CreateUploadLocationResponse(_message.Message): - __slots__ = ["signed_url", "native_url", "expires_at"] + __slots__ = ["signed_url", "native_url", "expires_at", "headers"] + class HeadersEntry(_message.Message): + __slots__ = ["key", "value"] + KEY_FIELD_NUMBER: _ClassVar[int] + VALUE_FIELD_NUMBER: _ClassVar[int] + key: str + value: str + def __init__(self, key: _Optional[str] = ..., value: _Optional[str] = ...) -> None: ... SIGNED_URL_FIELD_NUMBER: _ClassVar[int] NATIVE_URL_FIELD_NUMBER: _ClassVar[int] EXPIRES_AT_FIELD_NUMBER: _ClassVar[int] + HEADERS_FIELD_NUMBER: _ClassVar[int] signed_url: str native_url: str expires_at: _timestamp_pb2.Timestamp - def __init__(self, signed_url: _Optional[str] = ..., native_url: _Optional[str] = ..., expires_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ...) -> None: ... + headers: _containers.ScalarMap[str, str] + def __init__(self, signed_url: _Optional[str] = ..., native_url: _Optional[str] = ..., expires_at: _Optional[_Union[_timestamp_pb2.Timestamp, _Mapping]] = ..., headers: _Optional[_Mapping[str, str]] = ...) -> None: ... class CreateUploadLocationRequest(_message.Message): - __slots__ = ["project", "domain", "filename", "expires_in", "content_md5", "filename_root"] + __slots__ = ["project", "domain", "filename", "expires_in", "content_md5", "filename_root", "add_content_md5_metadata"] PROJECT_FIELD_NUMBER: _ClassVar[int] DOMAIN_FIELD_NUMBER: _ClassVar[int] FILENAME_FIELD_NUMBER: _ClassVar[int] EXPIRES_IN_FIELD_NUMBER: _ClassVar[int] CONTENT_MD5_FIELD_NUMBER: _ClassVar[int] FILENAME_ROOT_FIELD_NUMBER: _ClassVar[int] + ADD_CONTENT_MD5_METADATA_FIELD_NUMBER: _ClassVar[int] project: str domain: str filename: str expires_in: _duration_pb2.Duration content_md5: bytes filename_root: str - def __init__(self, project: _Optional[str] = ..., domain: _Optional[str] = ..., filename: _Optional[str] = ..., expires_in: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., content_md5: _Optional[bytes] = ..., filename_root: _Optional[str] = ...) -> None: ... + add_content_md5_metadata: bool + def __init__(self, project: _Optional[str] = ..., domain: _Optional[str] = ..., filename: _Optional[str] = ..., expires_in: _Optional[_Union[_duration_pb2.Duration, _Mapping]] = ..., content_md5: _Optional[bytes] = ..., filename_root: _Optional[str] = ..., add_content_md5_metadata: bool = ...) -> None: ... class CreateDownloadLocationRequest(_message.Message): __slots__ = ["native_url", "expires_in"] diff --git a/flyteidl/gen/pb_rust/flyteidl.service.rs b/flyteidl/gen/pb_rust/flyteidl.service.rs index c4271703330..e53abba678d 100644 --- a/flyteidl/gen/pb_rust/flyteidl.service.rs +++ b/flyteidl/gen/pb_rust/flyteidl.service.rs @@ -85,6 +85,9 @@ pub struct CreateUploadLocationResponse { /// ExpiresAt defines when will the signed URL expires. #[prost(message, optional, tag="3")] pub expires_at: ::core::option::Option<::prost_types::Timestamp>, + /// Data proxy generates these headers for client, and they have to add these headers to the request when uploading the file. + #[prost(map="string, string", tag="4")] + pub headers: ::std::collections::HashMap<::prost::alloc::string::String, ::prost::alloc::string::String>, } /// CreateUploadLocationRequest specified request for the CreateUploadLocation API. /// The implementation in data proxy service will create the s3 location with some server side configured prefixes, @@ -122,6 +125,11 @@ pub struct CreateUploadLocationRequest { /// +optional #[prost(string, tag="6")] pub filename_root: ::prost::alloc::string::String, + /// If true, the data proxy will add content_md5 to the metadata to the signed URL and + /// it will force clients to add this metadata to the object. + /// This make sure dataproxy is backward compatible with the old flytekit. + #[prost(bool, tag="7")] + pub add_content_md5_metadata: bool, } /// CreateDownloadLocationRequest specified request for the CreateDownloadLocation API. #[allow(clippy::derive_partial_eq_without_eq)] diff --git a/flyteidl/go.mod b/flyteidl/go.mod index dcc6b2ccaf0..2187b2aeab8 100644 --- a/flyteidl/go.mod +++ b/flyteidl/go.mod @@ -45,7 +45,7 @@ require ( github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/flyteorg/stow v0.3.8 // indirect + github.com/flyteorg/stow v0.3.10 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/flyteidl/go.sum b/flyteidl/go.sum index 0fa41c00049..5b4cf356d3b 100644 --- a/flyteidl/go.sum +++ b/flyteidl/go.sum @@ -105,8 +105,8 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flyteorg/stow v0.3.8 h1:4a6BtfgDR86fUwa48DkkZTcp6WK4oQXSfewPd/kN0Z4= -github.com/flyteorg/stow v0.3.8/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= +github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8= +github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= diff --git a/flyteidl/protos/flyteidl/service/dataproxy.proto b/flyteidl/protos/flyteidl/service/dataproxy.proto index 622910f6450..b24c794cc88 100644 --- a/flyteidl/protos/flyteidl/service/dataproxy.proto +++ b/flyteidl/protos/flyteidl/service/dataproxy.proto @@ -20,6 +20,9 @@ message CreateUploadLocationResponse { // ExpiresAt defines when will the signed URL expires. google.protobuf.Timestamp expires_at = 3; + + // Data proxy generates these headers for client, and they have to add these headers to the request when uploading the file. + map headers = 4; } // CreateUploadLocationRequest specified request for the CreateUploadLocation API. @@ -55,6 +58,11 @@ message CreateUploadLocationRequest { // in data proxy config. This option is useful when uploading multiple files. // +optional string filename_root = 6; + + // If true, the data proxy will add content_md5 to the metadata to the signed URL and + // it will force clients to add this metadata to the object. + // This make sure dataproxy is backward compatible with the old flytekit. + bool add_content_md5_metadata = 7; } // CreateDownloadLocationRequest specified request for the CreateDownloadLocation API. @@ -67,7 +75,6 @@ message CreateDownloadLocationRequest { // exceeds the platform allowed max. // +optional. The default value comes from a global config. google.protobuf.Duration expires_in = 2; - } message CreateDownloadLocationResponse { diff --git a/flyteplugins/go.mod b/flyteplugins/go.mod index 4b9e7d5f474..be1331697b7 100644 --- a/flyteplugins/go.mod +++ b/flyteplugins/go.mod @@ -64,7 +64,7 @@ require ( github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect github.com/fatih/color v1.13.0 // indirect - github.com/flyteorg/stow v0.3.8 // indirect + github.com/flyteorg/stow v0.3.10 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-logr/logr v1.2.4 // indirect diff --git a/flyteplugins/go.sum b/flyteplugins/go.sum index 89e2a0ee49c..662b36b87e8 100644 --- a/flyteplugins/go.sum +++ b/flyteplugins/go.sum @@ -138,8 +138,8 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flyteorg/stow v0.3.8 h1:4a6BtfgDR86fUwa48DkkZTcp6WK4oQXSfewPd/kN0Z4= -github.com/flyteorg/stow v0.3.8/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= +github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8= +github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= diff --git a/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go b/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go index 99a3ccdf7a9..9482b3df953 100644 --- a/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go +++ b/flyteplugins/go/tasks/pluginmachinery/internal/webapi/launcher.go @@ -2,9 +2,9 @@ package webapi import ( "context" - pluginErrors "github.com/flyteorg/flyte/flyteplugins/go/tasks/errors" "time" + pluginErrors "github.com/flyteorg/flyte/flyteplugins/go/tasks/errors" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/webapi" "github.com/flyteorg/flyte/flytestdlib/cache" diff --git a/flyteplugins/go/tasks/pluginmachinery/ioutils/remote_file_output_reader_test.go b/flyteplugins/go/tasks/pluginmachinery/ioutils/remote_file_output_reader_test.go index 0e7ae179d92..251a3adc55e 100644 --- a/flyteplugins/go/tasks/pluginmachinery/ioutils/remote_file_output_reader_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/ioutils/remote_file_output_reader_test.go @@ -14,9 +14,14 @@ import ( ) type MemoryMetadata struct { - exists bool - size int64 - etag string + exists bool + size int64 + etag string + contentMD5 string +} + +func (m MemoryMetadata) ContentMD5() string { + return m.contentMD5 } func (m MemoryMetadata) Size() int64 { diff --git a/flyteplugins/go/tasks/plugins/array/k8s/management_test.go b/flyteplugins/go/tasks/plugins/array/k8s/management_test.go index 0eaed654672..c5d52efda71 100644 --- a/flyteplugins/go/tasks/plugins/array/k8s/management_test.go +++ b/flyteplugins/go/tasks/plugins/array/k8s/management_test.go @@ -27,9 +27,10 @@ import ( ) type metadata struct { - exists bool - size int64 - etag string + exists bool + size int64 + etag string + contentMD5 string } func (m metadata) Exists() bool { @@ -44,6 +45,10 @@ func (m metadata) Etag() string { return m.etag } +func (m metadata) ContentMD5() string { + return m.contentMD5 +} + func createSampleContainerTask() *core2.Container { return &core2.Container{ Command: []string{"cmd"}, @@ -127,7 +132,7 @@ func getMockTaskExecutionContext(ctx context.Context, parallelism int) *mocks.Ta matchedBy := mock.MatchedBy(func(s storage.DataReference) bool { return true }) - composedProtobufStore.On("Head", mock.Anything, matchedBy).Return(metadata{true, 0, ""}, nil) + composedProtobufStore.On("Head", mock.Anything, matchedBy).Return(metadata{true, 0, "", ""}, nil) dataStore := &storage.DataStore{ ComposedProtobufStore: composedProtobufStore, ReferenceConstructor: &storage.URLPathConstructor{}, diff --git a/flyteplugins/go/tasks/plugins/webapi/agent/client.go b/flyteplugins/go/tasks/plugins/webapi/agent/client.go index b525acc5c3d..9f4409c8e6b 100644 --- a/flyteplugins/go/tasks/plugins/webapi/agent/client.go +++ b/flyteplugins/go/tasks/plugins/webapi/agent/client.go @@ -99,6 +99,7 @@ func getFinalContext(ctx context.Context, operation string, agent *Deployment) ( } func initializeAgentRegistry(cs *ClientSet) (Registry, error) { + logger.Infof(context.Background(), "Initializing agent registry") agentRegistry := make(Registry) cfg := GetConfig() var agentDeployments []*Deployment @@ -156,6 +157,8 @@ func initializeAgentRegistry(cs *ClientSet) (Registry, error) { } func initializeClients(ctx context.Context) (*ClientSet, error) { + logger.Infof(ctx, "Initializing agent clients") + asyncAgentClients := make(map[string]service.AsyncAgentServiceClient) syncAgentClients := make(map[string]service.SyncAgentServiceClient) agentMetadataClients := make(map[string]service.AgentMetadataServiceClient) diff --git a/flyteplugins/go/tasks/plugins/webapi/agent/plugin.go b/flyteplugins/go/tasks/plugins/webapi/agent/plugin.go index 11ef7871b3d..c4decd04ef6 100644 --- a/flyteplugins/go/tasks/plugins/webapi/agent/plugin.go +++ b/flyteplugins/go/tasks/plugins/webapi/agent/plugin.go @@ -4,13 +4,13 @@ import ( "context" "encoding/gob" "fmt" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service" "time" "golang.org/x/exp/maps" "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/admin" flyteIdl "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/core" + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/service" pluginErrors "github.com/flyteorg/flyte/flyteplugins/go/tasks/errors" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery" "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core" diff --git a/flytepropeller/events/local_eventsink.go b/flytepropeller/events/local_eventsink.go index 39e3ff5cac6..fdcd5408a49 100644 --- a/flytepropeller/events/local_eventsink.go +++ b/flytepropeller/events/local_eventsink.go @@ -7,8 +7,9 @@ import ( "os" "sync" - "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event" "github.com/golang/protobuf/proto" + + "github.com/flyteorg/flyte/flyteidl/gen/pb-go/flyteidl/event" ) type localSink struct { diff --git a/flytepropeller/go.mod b/flytepropeller/go.mod index a3be11827c5..f541398b58d 100644 --- a/flytepropeller/go.mod +++ b/flytepropeller/go.mod @@ -72,7 +72,7 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/flyteorg/stow v0.3.8 // indirect + github.com/flyteorg/stow v0.3.10 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/flytepropeller/go.sum b/flytepropeller/go.sum index 04f1dc08f37..b929dc11b5d 100644 --- a/flytepropeller/go.sum +++ b/flytepropeller/go.sum @@ -144,8 +144,8 @@ github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJ github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/flyteorg/stow v0.3.8 h1:4a6BtfgDR86fUwa48DkkZTcp6WK4oQXSfewPd/kN0Z4= -github.com/flyteorg/stow v0.3.8/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= +github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8= +github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= diff --git a/flytepropeller/pkg/compiler/test/compiler_test.go b/flytepropeller/pkg/compiler/test/compiler_test.go index 115d4bbf3ef..ae0322b66b5 100644 --- a/flytepropeller/pkg/compiler/test/compiler_test.go +++ b/flytepropeller/pkg/compiler/test/compiler_test.go @@ -4,7 +4,6 @@ import ( "bytes" "encoding/json" "flag" - "github.com/flyteorg/flyte/flytepropeller/pkg/visualize" "io/ioutil" "os" "path/filepath" @@ -27,6 +26,7 @@ import ( "github.com/flyteorg/flyte/flytepropeller/pkg/compiler/common" "github.com/flyteorg/flyte/flytepropeller/pkg/compiler/errors" "github.com/flyteorg/flyte/flytepropeller/pkg/compiler/transformers/k8s" + "github.com/flyteorg/flyte/flytepropeller/pkg/visualize" ) var update = flag.Bool("update", false, "Update .golden files") diff --git a/flytepropeller/pkg/compiler/validators/utils_test.go b/flytepropeller/pkg/compiler/validators/utils_test.go index 29daed99ae9..4a37f100dc9 100644 --- a/flytepropeller/pkg/compiler/validators/utils_test.go +++ b/flytepropeller/pkg/compiler/validators/utils_test.go @@ -1,9 +1,9 @@ package validators import ( - "github.com/golang/protobuf/proto" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "github.com/flyteorg/flyte/flyteidl/clients/go/coreutils" diff --git a/flytepropeller/pkg/compiler/workflow_compiler_test.go b/flytepropeller/pkg/compiler/workflow_compiler_test.go index 1e1e99fb022..eaffec3e218 100644 --- a/flytepropeller/pkg/compiler/workflow_compiler_test.go +++ b/flytepropeller/pkg/compiler/workflow_compiler_test.go @@ -2,10 +2,10 @@ package compiler import ( "fmt" - "github.com/golang/protobuf/proto" "strings" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/util/sets" diff --git a/flytepropeller/pkg/controller/nodes/dynamic/dynamic_workflow_test.go b/flytepropeller/pkg/controller/nodes/dynamic/dynamic_workflow_test.go index 3c016fe6bf9..4011ffb9564 100644 --- a/flytepropeller/pkg/controller/nodes/dynamic/dynamic_workflow_test.go +++ b/flytepropeller/pkg/controller/nodes/dynamic/dynamic_workflow_test.go @@ -584,6 +584,10 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t type existsMetadata struct{} +func (e existsMetadata) ContentMD5() string { + return "" +} + func (e existsMetadata) Exists() bool { return false } diff --git a/flytepropeller/pkg/controller/nodes/executor.go b/flytepropeller/pkg/controller/nodes/executor.go index 0c98aeeb5fc..cf8c62cfad5 100644 --- a/flytepropeller/pkg/controller/nodes/executor.go +++ b/flytepropeller/pkg/controller/nodes/executor.go @@ -791,7 +791,7 @@ func isTimeoutExpired(queuedAt *metav1.Time, timeout time.Duration) bool { return false } -func (c *nodeExecutor) isEligibleForRetry(nCtx interfaces.NodeExecutionContext, nodeStatus v1alpha1.ExecutableNodeStatus, err *core.ExecutionError) (currentAttempt, maxAttempts uint32, isEligible bool) { +func (c *nodeExecutor) isEligibleForRetry(nCtx interfaces.NodeExecutionContext, nodeStatus v1alpha1.ExecutableNodeStatus, err *core.ExecutionError) (currentAttempt uint32, maxAttempts uint32, isEligible bool) { if config.GetConfig().NodeConfig.IgnoreRetryCause { currentAttempt = nodeStatus.GetAttempts() + 1 } else { diff --git a/flytepropeller/pkg/controller/nodes/executor_test.go b/flytepropeller/pkg/controller/nodes/executor_test.go index 2bc552bab03..237928a937c 100644 --- a/flytepropeller/pkg/controller/nodes/executor_test.go +++ b/flytepropeller/pkg/controller/nodes/executor_test.go @@ -2541,6 +2541,10 @@ func init() { type existsMetadata struct{} +func (e existsMetadata) ContentMD5() string { + return "" +} + func (e existsMetadata) Exists() bool { return false } diff --git a/flytestdlib/go.mod b/flytestdlib/go.mod index 947fbe4b5c7..d275261dc89 100644 --- a/flytestdlib/go.mod +++ b/flytestdlib/go.mod @@ -9,7 +9,7 @@ require ( github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607 github.com/fatih/color v1.13.0 github.com/fatih/structtag v1.2.0 - github.com/flyteorg/stow v0.3.8 + github.com/flyteorg/stow v0.3.10 github.com/fsnotify/fsnotify v1.6.0 github.com/ghodss/yaml v1.0.0 github.com/go-gormigrate/gormigrate/v2 v2.1.1 diff --git a/flytestdlib/go.sum b/flytestdlib/go.sum index baacaca1e51..ee195ca9fc1 100644 --- a/flytestdlib/go.sum +++ b/flytestdlib/go.sum @@ -115,8 +115,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4= github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= -github.com/flyteorg/stow v0.3.8 h1:4a6BtfgDR86fUwa48DkkZTcp6WK4oQXSfewPd/kN0Z4= -github.com/flyteorg/stow v0.3.8/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= +github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8= +github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= diff --git a/flytestdlib/storage/mem_store.go b/flytestdlib/storage/mem_store.go index abe4de12cd4..a95a0a49ca2 100644 --- a/flytestdlib/storage/mem_store.go +++ b/flytestdlib/storage/mem_store.go @@ -19,9 +19,10 @@ type InMemoryStore struct { } type MemoryMetadata struct { - exists bool - size int64 - etag string + exists bool + size int64 + etag string + contentMD5 string } func (m MemoryMetadata) Size() int64 { @@ -36,6 +37,10 @@ func (m MemoryMetadata) Etag() string { return m.etag } +func (m MemoryMetadata) ContentMD5() string { + return m.contentMD5 +} + func (s *InMemoryStore) Head(ctx context.Context, reference DataReference) (Metadata, error) { data, found := s.cache[reference] var hash [md5.Size]byte diff --git a/flytestdlib/storage/storage.go b/flytestdlib/storage/storage.go index a022ce3e624..3e84cb7acb3 100644 --- a/flytestdlib/storage/storage.go +++ b/flytestdlib/storage/storage.go @@ -34,6 +34,10 @@ type Metadata interface { Exists() bool Size() int64 Etag() string + // ContentMD5 retrieves the value of a special metadata tag added by the system that + // contains the MD5 of the uploaded file. If there is no metadata attached + // or that `FlyteContentMD5` key isn't set, ContentMD5 will return empty. + ContentMD5() string } // DataStore is a simplified interface for accessing and storing data in one of the Cloud stores. @@ -52,10 +56,13 @@ type SignedURLProperties struct { ExpiresIn time.Duration // ContentMD5 defines the expected hash of the generated file. It's strongly recommended setting it. ContentMD5 string + // AddContentMD5Metadata Add ContentMD5 to the metadata of signed URL if true. + AddContentMD5Metadata bool } type SignedURLResponse struct { - URL url.URL + URL url.URL + RequiredRequestHeaders map[string]string } //go:generate mockery -name RawStore -case=underscore diff --git a/flytestdlib/storage/stow_store.go b/flytestdlib/storage/stow_store.go index 701604d4370..092ac1bbed7 100644 --- a/flytestdlib/storage/stow_store.go +++ b/flytestdlib/storage/stow_store.go @@ -6,6 +6,7 @@ import ( "io" "net/url" "strconv" + "strings" "sync" "time" @@ -27,9 +28,8 @@ import ( "github.com/flyteorg/stow/swift" ) -const ( - FailureTypeLabel contextutils.Key = "failure_type" -) +const FailureTypeLabel contextutils.Key = "failure_type" +const FlyteContentMD5 = "flyteContentMD5" var fQNFn = map[string]func(string) DataReference{ s3.Kind: func(bucket string) DataReference { @@ -104,9 +104,10 @@ type stowMetrics struct { // StowMetadata that will be returned type StowMetadata struct { - exists bool - size int64 - etag string + exists bool + size int64 + etag string + contentMD5 string } func (s StowMetadata) Size() int64 { @@ -121,6 +122,10 @@ func (s StowMetadata) Etag() string { return s.etag } +func (s StowMetadata) ContentMD5() string { + return s.contentMD5 +} + // Implements DataStore to talk to stow location store. type StowStore struct { copyImpl @@ -221,12 +226,19 @@ func (s *StowStore) Head(ctx context.Context, reference DataReference) (Metadata // Err will be caught below } else if etag, err := item.ETag(); err != nil { // Err will be caught below + } else if metadata, err := item.Metadata(); err != nil { + // Err will be caught below } else { t.Stop() + contentMD5, ok := metadata[strings.ToLower(FlyteContentMD5)].(string) + if !ok { + logger.Warningf(ctx, "Failed to cast contentMD5 [%v] to string", contentMD5) + } return StowMetadata{ - exists: true, - size: size, - etag: etag, + exists: true, + size: size, + etag: etag, + contentMD5: contentMD5, }, nil } } @@ -345,22 +357,24 @@ func (s *StowStore) CreateSignedURL(ctx context.Context, reference DataReference return SignedURLResponse{}, err } - urlStr, err := c.PreSignRequest(ctx, properties.Scope, key, stow.PresignRequestParams{ - ExpiresIn: properties.ExpiresIn, - ContentMD5: properties.ContentMD5, + res, err := c.PreSignRequest(ctx, properties.Scope, key, stow.PresignRequestParams{ + ExpiresIn: properties.ExpiresIn, + ContentMD5: properties.ContentMD5, + AddContentMD5Metadata: properties.AddContentMD5Metadata, }) if err != nil { return SignedURLResponse{}, err } - urlVal, err := url.Parse(urlStr) + urlVal, err := url.Parse(res.Url) if err != nil { return SignedURLResponse{}, err } return SignedURLResponse{ - URL: *urlVal, + URL: *urlVal, + RequiredRequestHeaders: res.RequiredRequestHeaders, }, nil } diff --git a/flytestdlib/storage/stow_store_test.go b/flytestdlib/storage/stow_store_test.go index d888977c577..99678eb8adc 100644 --- a/flytestdlib/storage/stow_store_test.go +++ b/flytestdlib/storage/stow_store_test.go @@ -53,8 +53,8 @@ type mockStowContainer struct { // CreateSignedURL creates a signed url with the provided properties. func (m mockStowContainer) PreSignRequest(_ context.Context, _ stow.ClientMethod, s string, - _ stow.PresignRequestParams) (url string, err error) { - return s, nil + _ stow.PresignRequestParams) (response stow.PresignResponse, err error) { + return stow.PresignResponse{Url: s}, nil } func (m mockStowContainer) ID() string { diff --git a/go.mod b/go.mod index 0280ec305c7..37e95136eaf 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( github.com/felixge/httpsnoop v1.0.3 // indirect github.com/flyteorg/flyte/flyteidl v0.0.0-00010101000000-000000000000 // indirect github.com/flyteorg/flyte/flyteplugins v0.0.0-00010101000000-000000000000 // indirect - github.com/flyteorg/stow v0.3.8 // indirect + github.com/flyteorg/stow v0.3.10 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-gormigrate/gormigrate/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index d3fdd20bd81..8048229f181 100644 --- a/go.sum +++ b/go.sum @@ -261,8 +261,8 @@ github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/flyteorg/stow v0.3.8 h1:4a6BtfgDR86fUwa48DkkZTcp6WK4oQXSfewPd/kN0Z4= -github.com/flyteorg/stow v0.3.8/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= +github.com/flyteorg/stow v0.3.10 h1:uEe+tI+CGKn21H93uXp9z05hqynEki2BO9KkW/GweY8= +github.com/flyteorg/stow v0.3.10/go.mod h1:fArjMpsYJNWkp/hyDKKdbcv07gxbuLmKFcb7YT1aSOM= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= diff --git a/kustomize/README.md b/kustomize/README.md deleted file mode 100644 index 69a9c10d0ce..00000000000 --- a/kustomize/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Install Flyte using Kustomize -Flyte can be deployed to a kubernetes cluster using a generated deployment yaml file. This file is generated using [Kustomize](https://kubectl.docs.kubernetes.io/guides/introduction/kustomize/). -Please refer to Kustomize documentation to see how it works. - -In brief, Kustomize allows composing a deployment yaml using multiple components. In Flyte all the core components are listed under [Base Components](./base). The Base components also consist of -a composed set of components that can be deployed to a [Single Kubernetes cluster](./base/single_cluster). This deployment configures various components using [Flyte Configuration -system](todo). - -The *Single Cluster* configuration on its own is not deployable. But individual [overlays](./overlays) are deployable. - -Refer to -1. [Base Components](./base): If you want to build your own overlay start here -1. [overlays](./overlays): If you want to build on top of an existing overlay start here diff --git a/kustomize/base/README.md b/kustomize/base/README.md deleted file mode 100644 index b0bb224d24a..00000000000 --- a/kustomize/base/README.md +++ /dev/null @@ -1,19 +0,0 @@ -[Back to main menu](../) -# Base Components for Flyte -These deployments provide individual deployment units of the Flyte Backend. - -As a user it might be preferable to use the `single_cluster` deployment base to create an overlay on top of, or directly edit on top of one of the existing overlays. - -## To create a new flyte overlay for one K8s cluster - Start here -- [Single Cluster Flyte Deployment configuration](./single_cluster) - -## To create a completely custom overlay refer to components -1. FlyteAdmin [Deployment](./admindeployment) | [ServiceAccount](./adminserviceaccount) -1. [Core Flyte namespace creation](./namespace) -1. [FlytePropeller](./propeller) & its [CRD](./wf_crd) -1. [DataCatalog](./datacatalog) -1. [FlyteConsole](./console) -1. [Overall Ingress for Flyte (optional)](./ingress) -1. [Additional plugin components for Flyte using K8s operators](./operators) - diff --git a/kustomize/base/addons/cloudsqlproxy/deployment.yaml b/kustomize/base/addons/cloudsqlproxy/deployment.yaml deleted file mode 100644 index dda31d1004a..00000000000 --- a/kustomize/base/addons/cloudsqlproxy/deployment.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: cloudsqlproxy - namespace: flyte - labels: - app: cloudsqlproxy -spec: - replicas: 1 - selector: - matchLabels: - app: cloudsqlproxy - template: - metadata: - labels: - app: cloudsqlproxy - spec: - containers: - - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.16 - imagePullPolicy: IfNotPresent - # TODO: replace with the GCP project ID and with the region where - # Cloud SQL runs - command: ["/cloud_sql_proxy", "-instances=::flyte=tcp:0.0.0.0:5432"] - ports: - - containerPort: 5432 diff --git a/kustomize/base/addons/cloudsqlproxy/kustomization.yaml b/kustomize/base/addons/cloudsqlproxy/kustomization.yaml deleted file mode 100644 index 6d1374a18e3..00000000000 --- a/kustomize/base/addons/cloudsqlproxy/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - deployment.yaml - - service.yaml diff --git a/kustomize/base/addons/cloudsqlproxy/service.yaml b/kustomize/base/addons/cloudsqlproxy/service.yaml deleted file mode 100644 index 68ba5d2213d..00000000000 --- a/kustomize/base/addons/cloudsqlproxy/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: cloudsqlproxy - namespace: flyte -spec: - selector: - app: cloudsqlproxy - ports: - - name: http - protocol: TCP - port: 5432 diff --git a/kustomize/base/addons/contour_ingress_controller/kustomization.yaml b/kustomize/base/addons/contour_ingress_controller/kustomization.yaml deleted file mode 100644 index e75e1378049..00000000000 --- a/kustomize/base/addons/contour_ingress_controller/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: -- https://raw.githubusercontent.com/projectcontour/contour/release-1.13/examples/render/contour.yaml -patchesStrategicMerge: -- service.yaml diff --git a/kustomize/base/addons/contour_ingress_controller/service.yaml b/kustomize/base/addons/contour_ingress_controller/service.yaml deleted file mode 100644 index a848798e11d..00000000000 --- a/kustomize/base/addons/contour_ingress_controller/service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Service -apiVersion: v1 -kind: Service -metadata: - name: envoy - namespace: projectcontour -spec: - # use NodePort to make sure the service is accessible - type: NodePort - ports: - - port: 80 - name: http - protocol: TCP - nodePort: 30081 diff --git a/kustomize/base/addons/database/database.yaml b/kustomize/base/addons/database/database.yaml deleted file mode 100644 index de17063c4cf..00000000000 --- a/kustomize/base/addons/database/database.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres - namespace: flyte -spec: - selector: - matchLabels: - app: postgres - template: - metadata: - labels: - app: postgres - spec: - volumes: - - name: postgres-storage - emptyDir: {} - containers: - - image: postgres - name: postgres - env: - - name: POSTGRES_HOST_AUTH_METHOD - value: trust - ports: - - containerPort: 5432 - name: postgres - volumeMounts: - - name: postgres-storage - mountPath: /var/lib/postgresql/data ---- -apiVersion: v1 -kind: Service -metadata: - name: postgres - namespace: flyte -spec: - ports: - - port: 5432 - selector: - app: postgres diff --git a/kustomize/base/addons/database/kustomization.yaml b/kustomize/base/addons/database/kustomization.yaml deleted file mode 100644 index ca2ede8d46f..00000000000 --- a/kustomize/base/addons/database/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- database.yaml diff --git a/kustomize/base/addons/kubernetes_dashboard/clusterrolebinding.yaml b/kustomize/base/addons/kubernetes_dashboard/clusterrolebinding.yaml deleted file mode 100644 index 76fd3fc503c..00000000000 --- a/kustomize/base/addons/kubernetes_dashboard/clusterrolebinding.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: kubernetes-dashboard-admin -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-admin -subjects: -- kind: ServiceAccount - name: kubernetes-dashboard - namespace: kubernetes-dashboard diff --git a/kustomize/base/addons/kubernetes_dashboard/deployment.yaml b/kustomize/base/addons/kubernetes_dashboard/deployment.yaml deleted file mode 100644 index 49299fec087..00000000000 --- a/kustomize/base/addons/kubernetes_dashboard/deployment.yaml +++ /dev/null @@ -1,15 +0,0 @@ -kind: Deployment -apiVersion: apps/v1 -metadata: - name: kubernetes-dashboard - namespace: kubernetes-dashboard -spec: - template: - spec: - containers: - - name: kubernetes-dashboard - args: - - --namespace=kubernetes-dashboard - - --enable-insecure-login - - --enable-skip-login - - --disable-settings-authorizer diff --git a/kustomize/base/addons/kubernetes_dashboard/kustomization.yaml b/kustomize/base/addons/kubernetes_dashboard/kustomization.yaml deleted file mode 100644 index 22384dfe0f9..00000000000 --- a/kustomize/base/addons/kubernetes_dashboard/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -resources: - # TODO (jeev): Figure out how to rev this automatically -- https://raw.githubusercontent.com/kubernetes/dashboard/v2.2.0/aio/deploy/alternative.yaml -- clusterrolebinding.yaml -patchesStrategicMerge: -- deployment.yaml -- service.yaml diff --git a/kustomize/base/addons/kubernetes_dashboard/service.yaml b/kustomize/base/addons/kubernetes_dashboard/service.yaml deleted file mode 100644 index c08a9308c85..00000000000 --- a/kustomize/base/addons/kubernetes_dashboard/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kubernetes-dashboard - namespace: kubernetes-dashboard -spec: - type: NodePort - ports: - - port: 80 - name: http - protocol: TCP - nodePort: 30082 diff --git a/kustomize/base/addons/redis/deployment.yaml b/kustomize/base/addons/redis/deployment.yaml deleted file mode 100644 index 1f1381dae8e..00000000000 --- a/kustomize/base/addons/redis/deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: redis-resource-manager - serviceName: redis-resource-manager - template: - metadata: - labels: - app: redis-resource-manager - spec: - containers: - - env: - - name: REDIS_PASSWORD - value: mypassword - image: redis - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - name: redis-resource-manager - ports: - - containerPort: 6379 - name: redis - protocol: TCP - readinessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 200m - memory: 128Mi - volumeMounts: - - mountPath: /bitnami - name: redis-data - dnsPolicy: ClusterFirst - restartPolicy: Always - volumes: - - name: redis-data - emptyDir: {} diff --git a/kustomize/base/addons/redis/kustomization.yaml b/kustomize/base/addons/redis/kustomization.yaml deleted file mode 100644 index a944d005ca7..00000000000 --- a/kustomize/base/addons/redis/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: -- deployment.yaml -- service.yaml diff --git a/kustomize/base/addons/redis/service.yaml b/kustomize/base/addons/redis/service.yaml deleted file mode 100644 index 8c86264ae37..00000000000 --- a/kustomize/base/addons/redis/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: redis-resource-manager - name: redis-resource-manager - namespace: flyte -spec: - ports: - - name: redis - port: 6379 - protocol: TCP - targetPort: redis - selector: - app: redis-resource-manager - type: ClusterIP diff --git a/kustomize/base/addons/storage/kustomization.yaml b/kustomize/base/addons/storage/kustomization.yaml deleted file mode 100644 index 9a80610c4e1..00000000000 --- a/kustomize/base/addons/storage/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- storage.yaml diff --git a/kustomize/base/addons/storage/storage.yaml b/kustomize/base/addons/storage/storage.yaml deleted file mode 100644 index 6c895bc8ff3..00000000000 --- a/kustomize/base/addons/storage/storage.yaml +++ /dev/null @@ -1,74 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: flyte -spec: - selector: - matchLabels: - app: minio - template: - metadata: - labels: - app: minio - spec: - securityContext: - fsGroup: 1001 - runAsUser: 1001 - fsGroupChangePolicy: "OnRootMismatch" - volumes: - - name: minio-storage - emptyDir: {} - containers: - - image: minio - name: minio - env: - - name: MINIO_ACCESS_KEY - value: minio - - name: MINIO_SECRET_KEY - value: miniostorage - - name: MINIO_DEFAULT_BUCKETS - value: my-s3-bucket - ports: - - containerPort: 9000 - name: minio - - containerPort: 9001 - name: minio-console - volumeMounts: - - name: minio-storage - mountPath: /data ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: flyte -spec: - externalName: minio - ports: - - name: minio-api - port: 9000 - - name: minio-console - port: 9001 - selector: - app: minio ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: minio - namespace: flyte - annotations: - nginx.ingress.kubernetes.io/ssl-redirect: "false" -spec: - rules: - - http: - paths: - - path: /minio - pathType: ImplementationSpecific - backend: - service: - name: minio - port: - number: 9001 diff --git a/kustomize/base/admindeployment/auth_secret.yaml b/kustomize/base/admindeployment/auth_secret.yaml deleted file mode 100644 index 9a1281d7767..00000000000 --- a/kustomize/base/admindeployment/auth_secret.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: flyte-admin-auth - namespace: flyte -type: Opaque diff --git a/kustomize/base/admindeployment/clustersync/cron.yaml b/kustomize/base/admindeployment/clustersync/cron.yaml deleted file mode 100644 index e3fbfdfa692..00000000000 --- a/kustomize/base/admindeployment/clustersync/cron.yaml +++ /dev/null @@ -1,35 +0,0 @@ -apiVersion: batch/v1beta1 -kind: CronJob -metadata: - name: syncresources - namespace: flyte -spec: - schedule: "*/1 * * * *" - jobTemplate: - spec: - template: - spec: - serviceAccountName: flyteadmin - containers: - - name: sync-cluster-resources - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] - volumeMounts: - - name: resource-templates - mountPath: /etc/flyte/clusterresource/templates - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - volumes: - - name: resource-templates - configMap: - name: clusterresource-template - - name: config-volume - configMap: - name: flyte-admin-config - - name: db-pass - secret: - secretName: db-pass - restartPolicy: OnFailure diff --git a/kustomize/base/admindeployment/clustersync/kustomization.yaml b/kustomize/base/admindeployment/clustersync/kustomization.yaml deleted file mode 100644 index 1ade31b55b8..00000000000 --- a/kustomize/base/admindeployment/clustersync/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- cron.yaml diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml deleted file mode 100644 index 561e4490347..00000000000 --- a/kustomize/base/admindeployment/deployment.yaml +++ /dev/null @@ -1,157 +0,0 @@ -# Create the actual deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flyteadmin - namespace: flyte - labels: - app: flyteadmin -spec: - replicas: 1 - selector: - matchLabels: - app: flyteadmin - template: - metadata: - labels: - app: flyteadmin - app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.4.13 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "10254" - prometheus.io/path: "/metrics" - spec: - securityContext: - fsGroup: 65534 - runAsUser: 1001 - fsGroupChangePolicy: "Always" - serviceAccountName: flyteadmin - volumes: - - name: shared-data - emptyDir: {} - - emptyDir: {} - name: scratch - - name: config-volume - configMap: - name: flyte-admin-config - - name: resource-templates - configMap: - name: clusterresource-template - - name: db-pass - secret: - secretName: db-pass - - name: auth - secret: - secretName: flyte-admin-auth - initContainers: - - name: run-migrations - image: flyteadmin:v0.6.49 - imagePullPolicy: IfNotPresent - command: - [ - "flyteadmin", - "--config", - "/etc/flyte/config/*.yaml", - "migrate", - "run", - ] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - # Optional, These just seed the project - TODO move them to only - - name: seed-projects - image: flyteadmin:v0.6.49 - imagePullPolicy: IfNotPresent - command: - [ - "flyteadmin", - "--config", - "/etc/flyte/config/*.yaml", - "migrate", - "seed-projects", - "flytesnacks", - "flytetester", - "flyteexamples", - ] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - - name: sync-cluster-resources - image: flyteadmin:v0.6.49 - imagePullPolicy: IfNotPresent - command: - [ - "flyteadmin", - "--config", - "/etc/flyte/config/*.yaml", - "clusterresource", - "sync", - ] - volumeMounts: - - name: resource-templates - mountPath: /etc/flyte/clusterresource/templates - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - - name: generate-secrets - image: flyteadmin:v0.6.49 - imagePullPolicy: IfNotPresent - command: ["/bin/sh", "-c"] - args: - [ - "flyteadmin --config=/etc/flyte/config/*.yaml secrets init --localPath /etc/scratch/secrets && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/scratch/secrets", - ] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: scratch - mountPath: /etc/scratch - env: - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - containers: - - name: flyteadmin - image: flyteadmin:v0.6.49 - imagePullPolicy: IfNotPresent - command: - ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "serve"] - ports: - - containerPort: 8088 - - containerPort: 8089 - volumeMounts: - - name: shared-data - mountPath: /srv/flyte - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - - name: auth - mountPath: /etc/secrets/ - - name: redoc - image: docker.io/redocly/redoc - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8087 - resources: - limits: - memory: "200Mi" - cpu: "0.1" - command: - - sh - - -c - - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh - env: - - name: PAGE_TITLE - value: "Flyte Admin OpenAPI" - - name: SPEC_URL - value: "/api/v1/openapi" - - name: PORT - value: "8087" diff --git a/kustomize/base/admindeployment/kustomization.yaml b/kustomize/base/admindeployment/kustomization.yaml deleted file mode 100644 index c47adf129de..00000000000 --- a/kustomize/base/admindeployment/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: - - auth_secret.yaml - - deployment.yaml - - service.yaml diff --git a/kustomize/base/admindeployment/service.yaml b/kustomize/base/admindeployment/service.yaml deleted file mode 100644 index 275f1d2185f..00000000000 --- a/kustomize/base/admindeployment/service.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Service -apiVersion: v1 -kind: Service -metadata: - name: flyteadmin - namespace: flyte - annotations: - # This tells contour to use a H2 connection for the port associated - # with the name 'grpc' under spec/ports. - # For more information, refer to - # https://github.com/heptio/contour/blob/master/docs/annotations.md#contour-specific-service-annotations - # # Following this issue - the annotation was updated https://github.com/projectcontour/contour/issues/2092 - projectcontour.io/upstream-protocol.h2c: "grpc" -spec: - selector: - app: flyteadmin - ports: - - name: http - protocol: TCP - port: 80 - targetPort: 8088 - - name: grpc - protocol: TCP - port: 81 - targetPort: 8089 diff --git a/kustomize/base/adminserviceaccount/adminserviceaccount.yaml b/kustomize/base/adminserviceaccount/adminserviceaccount.yaml deleted file mode 100644 index bdaae60df42..00000000000 --- a/kustomize/base/adminserviceaccount/adminserviceaccount.yaml +++ /dev/null @@ -1,50 +0,0 @@ -# ClusterRole for flyteadmin -# https://kubernetes.io/docs/admin/authorization/rbac/ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flyteadmin - namespace: flyte -rules: -#Allow Access to all resources under flyte.lyft.com and the core API group: "" - - apiGroups: - - "" - - flyte.lyft.com - - rbac.authorization.k8s.io - resources: - - configmaps - - flyteworkflows - - namespaces - - pods - - resourcequotas - - roles - - rolebindings - - secrets - - services - - serviceaccounts - - spark-role - verbs: - - "*" - ---- -# Create a Service Account for FltyeAdmin -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flyteadmin - namespace: flyte - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: flyteadmin-binding - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flyteadmin -subjects: - - kind: ServiceAccount - name: flyteadmin - namespace: flyte diff --git a/kustomize/base/adminserviceaccount/kustomization.yaml b/kustomize/base/adminserviceaccount/kustomization.yaml deleted file mode 100644 index 3d4938e1a1e..00000000000 --- a/kustomize/base/adminserviceaccount/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- adminserviceaccount.yaml diff --git a/kustomize/base/console/configmap.yaml b/kustomize/base/console/configmap.yaml deleted file mode 100644 index 0fb42a0c1fa..00000000000 --- a/kustomize/base/console/configmap.yaml +++ /dev/null @@ -1,10 +0,0 @@ -kind: ConfigMap -apiVersion: v1 -metadata: - name: flyte-console-config - namespace: flyte -data: - # This determines the base url used for all console links. It must match the route specified in ingress.yaml - BASE_URL: /console - CONFIG_DIR: /etc/flyte/config - diff --git a/kustomize/base/console/deployment.yaml b/kustomize/base/console/deployment.yaml deleted file mode 100644 index 0e9e4405349..00000000000 --- a/kustomize/base/console/deployment.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Create the actual deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flyteconsole - namespace: flyte - labels: - app: flyteconsole -spec: - replicas: 1 - selector: - matchLabels: - app: flyteconsole - template: - metadata: - labels: - app: flyteconsole - app.kubernetes.io/name: flyteconsole - app.kubernetes.io/version: 0.19.0 - spec: - securityContext: - runAsUser: 1000 - fsGroupChangePolicy: "Always" - volumes: - - name: shared-data - emptyDir: {} - containers: - - name: flyteconsole - image: flyteconsole:v0.19.0 - # args: [] - ports: - - containerPort: 8080 - volumeMounts: - - name: shared-data - mountPath: /srv/flyte - envFrom: - - configMapRef: - name: flyte-console-config diff --git a/kustomize/base/console/kustomization.yaml b/kustomize/base/console/kustomization.yaml deleted file mode 100644 index 81d586a3cd1..00000000000 --- a/kustomize/base/console/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: -- deployment.yaml -- service.yaml -- configmap.yaml diff --git a/kustomize/base/console/service.yaml b/kustomize/base/console/service.yaml deleted file mode 100644 index e6d75ffc2fd..00000000000 --- a/kustomize/base/console/service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Service -apiVersion: v1 -kind: Service -metadata: - name: flyteconsole - namespace: flyte -spec: - selector: - app: flyteconsole - ports: - - protocol: TCP - port: 80 - targetPort: 8080 - diff --git a/kustomize/base/datacatalog/deployment.yaml b/kustomize/base/datacatalog/deployment.yaml deleted file mode 100644 index 22b2e372350..00000000000 --- a/kustomize/base/datacatalog/deployment.yaml +++ /dev/null @@ -1,60 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datacatalog - namespace: flyte - labels: - app: datacatalog -spec: - replicas: 1 - selector: - matchLabels: - app: datacatalog - template: - metadata: - labels: - app: datacatalog - app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.3.0 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "10254" - prometheus.io/path: "/metrics" - spec: - securityContext: - fsGroup: 65534 - runAsUser: 1001 - fsGroupChangePolicy: "Always" - serviceAccountName: datacatalog - volumes: - - name: shared-data - emptyDir: {} - - name: config-volume - configMap: - name: datacatalog-config - - name: db-pass - secret: - secretName: db-pass - initContainers: - - name: run-migrations - image: datacatalog:v0.3.0 - imagePullPolicy: IfNotPresent - command: ["datacatalog", "--config", "/etc/datacatalog/config/*.yaml", "migrate", "run"] - volumeMounts: - - name: config-volume - mountPath: /etc/datacatalog/config - - name: db-pass - mountPath: /etc/db - containers: - - name: datacatalog - image: datacatalog:v0.3.0 - imagePullPolicy: IfNotPresent - command: ["datacatalog", "--config", "/etc/datacatalog/config/*.yaml", "serve"] - ports: - - containerPort: 8088 - - containerPort: 8089 - volumeMounts: - - name: config-volume - mountPath: /etc/datacatalog/config - - name: db-pass - mountPath: /etc/db diff --git a/kustomize/base/datacatalog/kustomization.yaml b/kustomize/base/datacatalog/kustomization.yaml deleted file mode 100644 index c38e72a98de..00000000000 --- a/kustomize/base/datacatalog/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: -- rbac.yaml -- deployment.yaml -- service.yaml diff --git a/kustomize/base/datacatalog/rbac.yaml b/kustomize/base/datacatalog/rbac.yaml deleted file mode 100644 index 208fb8d1ef8..00000000000 --- a/kustomize/base/datacatalog/rbac.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: datacatalog - namespace: flyte diff --git a/kustomize/base/datacatalog/service.yaml b/kustomize/base/datacatalog/service.yaml deleted file mode 100644 index 11c9b58ea25..00000000000 --- a/kustomize/base/datacatalog/service.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Service -apiVersion: v1 -kind: Service -metadata: - name: datacatalog - namespace: flyte - annotations: - # This tells contour to use a H2 connection for the port associated - # with the name 'grpc' under spec/ports. - # For more information, refer to - # https://github.com/heptio/contour/blob/master/docs/annotations.md#contour-specific-service-annotations - contour.heptio.com/upstream-protocol.h2c: "grpc" -spec: - selector: - app: datacatalog - ports: - - name: http - protocol: TCP - port: 88 - targetPort: 8088 - - name: grpc - protocol: TCP - port: 89 - targetPort: 8089 diff --git a/kustomize/base/ingress/ingress.yaml b/kustomize/base/ingress/ingress.yaml deleted file mode 100644 index 99aec4b3230..00000000000 --- a/kustomize/base/ingress/ingress.yaml +++ /dev/null @@ -1,193 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: flytesystem - namespace: flyte - annotations: - nginx.ingress.kubernetes.io/ssl-redirect: "false" - nginx.ingress.kubernetes.io/app-root: /console -spec: - rules: - - http: - paths: - # This is useful only for sandbox mode and should be templatized/removed in non-sandbox environments - - path: /__webpack_hmr - pathType: ImplementationSpecific - backend: - service: - name: flyteconsole - port: - number: 80 - # NOTE: Port 81 in flyteadmin is the GRPC server port for - # FlyteAdmin. - - path: /flyteidl.service.AdminService - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 81 - - path: /flyteidl.service.AuthMetadataService - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 81 - - path: /flyteidl.service.IdentityService - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 81 - - path: /flyteidl.service.SignalService - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 81 - # Port 87 in FlyteAdmin maps to the redoc container. - - path: /openapi - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 87 - # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml - - path: /console - pathType: ImplementationSpecific - backend: - service: - name: flyteconsole - port: - number: 80 - - path: /console/* - pathType: ImplementationSpecific - backend: - service: - name: flyteconsole - port: - number: 80 - - path: /api - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /api/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /healthcheck - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /v1/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - # Port 87 in FlyteAdmin maps to the redoc container. - - path: /openapi/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 87 - - path: /.well-known/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /login - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /login/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /logout - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /logout/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /callback - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /callback/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /me - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /config - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /config/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /oauth2 - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /oauth2/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 diff --git a/kustomize/base/ingress/kustomization.yaml b/kustomize/base/ingress/kustomization.yaml deleted file mode 100644 index 14d8f3a54fd..00000000000 --- a/kustomize/base/ingress/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ingress.yaml diff --git a/kustomize/base/namespace/kustomization.yaml b/kustomize/base/namespace/kustomization.yaml deleted file mode 100644 index bf20f4df68b..00000000000 --- a/kustomize/base/namespace/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- namespace.yaml diff --git a/kustomize/base/namespace/namespace.yaml b/kustomize/base/namespace/namespace.yaml deleted file mode 100644 index ca27d7f8850..00000000000 --- a/kustomize/base/namespace/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: flyte diff --git a/kustomize/base/operators/README.md b/kustomize/base/operators/README.md deleted file mode 100644 index 4f6516eb9d4..00000000000 --- a/kustomize/base/operators/README.md +++ /dev/null @@ -1 +0,0 @@ -# Install plugins diff --git a/kustomize/base/operators/kfoperators/namespace/kustomization.yaml b/kustomize/base/operators/kfoperators/namespace/kustomization.yaml deleted file mode 100644 index bf20f4df68b..00000000000 --- a/kustomize/base/operators/kfoperators/namespace/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- namespace.yaml diff --git a/kustomize/base/operators/kfoperators/namespace/namespace.yaml b/kustomize/base/operators/kfoperators/namespace/namespace.yaml deleted file mode 100644 index 7a940e4673d..00000000000 --- a/kustomize/base/operators/kfoperators/namespace/namespace.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kubeflow diff --git a/kustomize/base/operators/kfoperators/pytorch/kustomization.yaml b/kustomize/base/operators/kfoperators/pytorch/kustomization.yaml deleted file mode 100644 index 7b00dcfb6e8..00000000000 --- a/kustomize/base/operators/kfoperators/pytorch/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -bases: - - ../namespace - - github.com/kubeflow/manifests/pytorch-job/pytorch-job-crds/base?ref=v1.0-branch - - github.com/kubeflow/manifests/pytorch-job/pytorch-operator/base?ref=v1.0-branch diff --git a/kustomize/base/operators/kfoperators/tensorflow/kustomization.yaml b/kustomize/base/operators/kfoperators/tensorflow/kustomization.yaml deleted file mode 100644 index 9d7d7b7def0..00000000000 --- a/kustomize/base/operators/kfoperators/tensorflow/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -bases: - - ../namespace - - github.com/kubeflow/manifests/tf-training/tf-job-crds/base?ref=v1.0.0 - - github.com/kubeflow/manifests/tf-training/tf-job-operator/base?ref=v1.0.0 diff --git a/kustomize/base/operators/sagemaker/kustomization.yaml b/kustomize/base/operators/sagemaker/kustomization.yaml deleted file mode 100644 index c9ea0aaaba9..00000000000 --- a/kustomize/base/operators/sagemaker/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -resources: - - https://raw.githubusercontent.com/aws/amazon-sagemaker-operator-for-k8s/v1.2.1/release/rolebased/installer.yaml diff --git a/kustomize/base/operators/spark/deployment.yaml b/kustomize/base/operators/spark/deployment.yaml deleted file mode 100644 index e8195ef1a21..00000000000 --- a/kustomize/base/operators/spark/deployment.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: sparkoperator - namespace: sparkoperator - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: sparkoperator - strategy: - type: Recreate - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "10254" - prometheus.io/path: "/metrics" - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - spec: - serviceAccountName: sparkoperator - volumes: - - name: webhook-certs - secret: - secretName: spark-webhook-certs - containers: - - name: sparkoperator-unknown - image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5 - volumeMounts: - - name: webhook-certs - mountPath: /etc/webhook-certs - imagePullPolicy: Always - command: ["/usr/bin/spark-operator"] - ports: - - containerPort: 10254 - - containerPort: 8080 - args: - - -logtostderr - - -v=2 - - -controller-threads=20 - - -enable-metrics=true - - "-metrics-prefix=service:" - - -metrics-labels=task_name - - -metrics-labels=workflow_name - - -enable-webhook=true - - -webhook-svc-namespace=sparkoperator diff --git a/kustomize/base/operators/spark/kustomization.yaml b/kustomize/base/operators/spark/kustomization.yaml deleted file mode 100644 index cfcdd57ab5b..00000000000 --- a/kustomize/base/operators/spark/kustomization.yaml +++ /dev/null @@ -1,15 +0,0 @@ -resources: -- spark-operator.yaml -- sparkapplications-crd.yaml -- scheduledsparkapplications-crd.yaml -- deployment.yaml -- webhook.yaml - -vars: - - name: NAMESPACE - objref: - kind: Deployment - name: sparkoperator - apiVersion: apps/v1 - fieldref: - fieldpath: metadata.namespace diff --git a/kustomize/base/operators/spark/scheduledsparkapplications-crd.yaml b/kustomize/base/operators/spark/scheduledsparkapplications-crd.yaml deleted file mode 100644 index 3bda6787875..00000000000 --- a/kustomize/base/operators/spark/scheduledsparkapplications-crd.yaml +++ /dev/null @@ -1,3762 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (unknown) - creationTimestamp: null - name: scheduledsparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: ScheduledSparkApplication - listKind: ScheduledSparkApplicationList - plural: scheduledsparkapplications - shortNames: - - scheduledsparkapp - singular: scheduledsparkapplication - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - concurrencyPolicy: - type: string - failedRunHistoryLimit: - format: int32 - type: integer - schedule: - type: string - successfulRunHistoryLimit: - format: int32 - type: integer - suspend: - type: boolean - template: - properties: - arguments: - items: - type: string - type: array - batchScheduler: - type: string - batchSchedulerOptions: - properties: - priorityClassName: - type: string - queue: - type: string - type: object - deps: - properties: - files: - items: - type: string - type: array - jars: - items: - type: string - type: array - pyFiles: - items: - type: string - type: array - type: object - driver: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - type: string - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - serviceAccount: - type: string - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - executor: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - deleteOnTermination: - type: boolean - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - instances: - format: int32 - minimum: 1 - type: integer - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - failureRetries: - format: int32 - type: integer - hadoopConf: - additionalProperties: - type: string - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - type: string - imagePullSecrets: - items: - type: string - type: array - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - type: string - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - metricsPropertiesFile: - type: string - prometheus: - properties: - configFile: - type: string - configuration: - type: string - jmxExporterJar: - type: string - port: - format: int32 - maximum: 49151 - minimum: 1024 - type: integer - required: - - jmxExporterJar - type: object - required: - - exposeDriverMetrics - - exposeExecutorMetrics - type: object - nodeSelector: - additionalProperties: - type: string - type: object - pythonVersion: - enum: - - "2" - - "3" - type: string - restartPolicy: - properties: - onFailureRetries: - format: int32 - minimum: 0 - type: integer - onFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - onSubmissionFailureRetries: - format: int32 - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - type: - enum: - - Never - - Always - - OnFailure - type: string - type: object - retryInterval: - format: int64 - type: integer - serviceAccount: - type: string - sparkConf: - additionalProperties: - type: string - type: object - sparkConfigMap: - type: string - sparkVersion: - type: string - timeToLiveSeconds: - format: int64 - type: integer - type: - enum: - - Java - - Python - - Scala - - R - type: string - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - type: string - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - required: - - sources - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - driver - - executor - - sparkVersion - - type - type: object - required: - - schedule - - template - type: object - status: - properties: - lastRun: - format: date-time - nullable: true - type: string - lastRunName: - type: string - nextRun: - format: date-time - nullable: true - type: string - pastFailedRunNames: - items: - type: string - type: array - pastSuccessfulRunNames: - items: - type: string - type: array - reason: - type: string - scheduleState: - type: string - type: object - required: - - metadata - - spec - type: object - version: v1beta2 - versions: - - name: v1beta2 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/kustomize/base/operators/spark/spark-operator.yaml b/kustomize/base/operators/spark/spark-operator.yaml deleted file mode 100644 index 43b481768f5..00000000000 --- a/kustomize/base/operators/spark/spark-operator.yaml +++ /dev/null @@ -1,56 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: sparkoperator ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: sparkoperator - namespace: sparkoperator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: sparkoperator -rules: -- apiGroups: [""] - resources: ["pods"] - verbs: ["*"] -- apiGroups: [""] - resources: ["services", "configmaps", "secrets"] - verbs: ["create", "get", "delete"] -- apiGroups: ["extensions"] - resources: ["ingresses"] - verbs: ["create", "get", "delete"] -- apiGroups: [""] - resources: ["nodes"] - verbs: ["get"] -- apiGroups: ["batch"] - resources: ["jobs"] - verbs: ["create", "get", "update", "delete", "list", "watch"] -- apiGroups: [""] - resources: ["events"] - verbs: ["create", "update", "patch"] -- apiGroups: ["apiextensions.k8s.io"] - resources: ["customresourcedefinitions"] - verbs: ["create", "get", "update", "delete"] -- apiGroups: ["admissionregistration.k8s.io"] - resources: ["mutatingwebhookconfigurations"] - verbs: ["create", "get", "update", "delete"] -- apiGroups: ["sparkoperator.k8s.io"] - resources: ["sparkapplications", "scheduledsparkapplications", "sparkapplications/status", "scheduledsparkapplications/status"] - verbs: ["*"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: sparkoperator -subjects: - - kind: ServiceAccount - name: sparkoperator - namespace: sparkoperator -roleRef: - kind: ClusterRole - name: sparkoperator - apiGroup: rbac.authorization.k8s.io diff --git a/kustomize/base/operators/spark/sparkapplications-crd.yaml b/kustomize/base/operators/spark/sparkapplications-crd.yaml deleted file mode 100644 index 895d4d00375..00000000000 --- a/kustomize/base/operators/spark/sparkapplications-crd.yaml +++ /dev/null @@ -1,3771 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: (unknown) - creationTimestamp: null - name: sparkapplications.sparkoperator.k8s.io -spec: - group: sparkoperator.k8s.io - names: - kind: SparkApplication - listKind: SparkApplicationList - plural: sparkapplications - shortNames: - - sparkapp - singular: sparkapplication - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - type: string - kind: - type: string - metadata: - type: object - spec: - properties: - arguments: - items: - type: string - type: array - batchScheduler: - type: string - batchSchedulerOptions: - properties: - priorityClassName: - type: string - queue: - type: string - type: object - deps: - properties: - files: - items: - type: string - type: array - jars: - items: - type: string - type: array - pyFiles: - items: - type: string - type: array - type: object - driver: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - podName: - pattern: '[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*' - type: string - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - serviceAccount: - type: string - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - executor: - properties: - affinity: - properties: - nodeAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - preference: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - weight: - format: int32 - type: integer - required: - - preference - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - properties: - nodeSelectorTerms: - items: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchFields: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - type: object - type: array - required: - - nodeSelectorTerms - type: object - type: object - podAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: - properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: - format: int32 - type: integer - required: - - podAffinityTerm - - weight - type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - configMaps: - items: - properties: - name: - type: string - path: - type: string - required: - - name - - path - type: object - type: array - coreLimit: - type: string - coreRequest: - type: string - cores: - format: int32 - minimum: 1 - type: integer - deleteOnTermination: - type: boolean - dnsConfig: - properties: - nameservers: - items: - type: string - type: array - options: - items: - properties: - name: - type: string - value: - type: string - type: object - type: array - searches: - items: - type: string - type: array - type: object - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - envSecretKeyRefs: - additionalProperties: - properties: - key: - type: string - name: - type: string - required: - - key - - name - type: object - type: object - envVars: - additionalProperties: - type: string - type: object - gpu: - properties: - name: - type: string - quantity: - format: int64 - type: integer - required: - - name - - quantity - type: object - hostNetwork: - type: boolean - image: - type: string - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - instances: - format: int32 - minimum: 1 - type: integer - javaOptions: - type: string - labels: - additionalProperties: - type: string - type: object - memory: - type: string - memoryOverhead: - type: string - nodeSelector: - additionalProperties: - type: string - type: object - schedulerName: - type: string - secrets: - items: - properties: - name: - type: string - path: - type: string - secretType: - type: string - required: - - name - - path - - secretType - type: object - type: array - securityContext: - properties: - fsGroup: - format: int64 - type: integer - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - supplementalGroups: - items: - format: int64 - type: integer - type: array - sysctls: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - type: object - sidecars: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: - properties: - name: - type: string - value: - type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - secretKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - type: object - required: - - name - type: object - type: array - envFrom: - items: - properties: - configMapRef: - properties: - name: - type: string - optional: - type: boolean - type: object - prefix: - type: string - secretRef: - properties: - name: - type: string - optional: - type: boolean - type: object - type: object - type: array - image: - type: string - imagePullPolicy: - type: string - lifecycle: - properties: - postStart: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - preStop: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - type: object - type: object - livenessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - name: - type: string - ports: - items: - properties: - containerPort: - format: int32 - type: integer - hostIP: - type: string - hostPort: - format: int32 - type: integer - name: - type: string - protocol: - type: string - required: - - containerPort - - protocol - type: object - type: array - x-kubernetes-list-map-keys: - - containerPort - - protocol - x-kubernetes-list-type: map - readinessProbe: - properties: - exec: - properties: - command: - items: - type: string - type: array - type: object - failureThreshold: - format: int32 - type: integer - httpGet: - properties: - host: - type: string - httpHeaders: - items: - properties: - name: - type: string - value: - type: string - required: - - name - - value - type: object - type: array - path: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - scheme: - type: string - required: - - port - type: object - initialDelaySeconds: - format: int32 - type: integer - periodSeconds: - format: int32 - type: integer - successThreshold: - format: int32 - type: integer - tcpSocket: - properties: - host: - type: string - port: - anyOf: - - type: integer - - type: string - x-kubernetes-int-or-string: true - required: - - port - type: object - timeoutSeconds: - format: int32 - type: integer - type: object - resources: - properties: - limits: - additionalProperties: - type: string - type: object - requests: - additionalProperties: - type: string - type: object - type: object - securityContext: - properties: - allowPrivilegeEscalation: - type: boolean - capabilities: - properties: - add: - items: - type: string - type: array - drop: - items: - type: string - type: array - type: object - privileged: - type: boolean - procMount: - type: string - readOnlyRootFilesystem: - type: boolean - runAsGroup: - format: int64 - type: integer - runAsNonRoot: - type: boolean - runAsUser: - format: int64 - type: integer - seLinuxOptions: - properties: - level: - type: string - role: - type: string - type: - type: string - user: - type: string - type: object - type: object - stdin: - type: boolean - stdinOnce: - type: boolean - terminationMessagePath: - type: string - terminationMessagePolicy: - type: string - tty: - type: boolean - volumeDevices: - items: - properties: - devicePath: - type: string - name: - type: string - required: - - devicePath - - name - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - workingDir: - type: string - required: - - name - type: object - type: array - terminationGracePeriodSeconds: - format: int64 - type: integer - tolerations: - items: - properties: - effect: - type: string - key: - type: string - operator: - type: string - tolerationSeconds: - format: int64 - type: integer - value: - type: string - type: object - type: array - volumeMounts: - items: - properties: - mountPath: - type: string - mountPropagation: - type: string - name: - type: string - readOnly: - type: boolean - subPath: - type: string - required: - - mountPath - - name - type: object - type: array - type: object - failureRetries: - format: int32 - type: integer - hadoopConf: - additionalProperties: - type: string - type: object - hadoopConfigMap: - type: string - image: - type: string - imagePullPolicy: - type: string - imagePullSecrets: - items: - type: string - type: array - mainApplicationFile: - type: string - mainClass: - type: string - memoryOverheadFactor: - type: string - mode: - enum: - - cluster - - client - type: string - monitoring: - properties: - exposeDriverMetrics: - type: boolean - exposeExecutorMetrics: - type: boolean - metricsProperties: - type: string - metricsPropertiesFile: - type: string - prometheus: - properties: - configFile: - type: string - configuration: - type: string - jmxExporterJar: - type: string - port: - format: int32 - maximum: 49151 - minimum: 1024 - type: integer - required: - - jmxExporterJar - type: object - required: - - exposeDriverMetrics - - exposeExecutorMetrics - type: object - nodeSelector: - additionalProperties: - type: string - type: object - pythonVersion: - enum: - - "2" - - "3" - type: string - restartPolicy: - properties: - onFailureRetries: - format: int32 - minimum: 0 - type: integer - onFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - onSubmissionFailureRetries: - format: int32 - minimum: 0 - type: integer - onSubmissionFailureRetryInterval: - format: int64 - minimum: 1 - type: integer - type: - enum: - - Never - - Always - - OnFailure - type: string - type: object - retryInterval: - format: int64 - type: integer - serviceAccount: - type: string - sparkConf: - additionalProperties: - type: string - type: object - sparkConfigMap: - type: string - sparkVersion: - type: string - timeToLiveSeconds: - format: int64 - type: integer - type: - enum: - - Java - - Python - - Scala - - R - type: string - volumes: - items: - properties: - awsElasticBlockStore: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - azureDisk: - properties: - cachingMode: - type: string - diskName: - type: string - diskURI: - type: string - fsType: - type: string - kind: - type: string - readOnly: - type: boolean - required: - - diskName - - diskURI - type: object - azureFile: - properties: - readOnly: - type: boolean - secretName: - type: string - shareName: - type: string - required: - - secretName - - shareName - type: object - cephfs: - properties: - monitors: - items: - type: string - type: array - path: - type: string - readOnly: - type: boolean - secretFile: - type: string - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - monitors - type: object - cinder: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeID: - type: string - required: - - volumeID - type: object - configMap: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - emptyDir: - properties: - medium: - type: string - sizeLimit: - type: string - type: object - fc: - properties: - fsType: - type: string - lun: - format: int32 - type: integer - readOnly: - type: boolean - targetWWNs: - items: - type: string - type: array - wwids: - items: - type: string - type: array - type: object - flexVolume: - properties: - driver: - type: string - fsType: - type: string - options: - additionalProperties: - type: string - type: object - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - required: - - driver - type: object - flocker: - properties: - datasetName: - type: string - datasetUUID: - type: string - type: object - gcePersistentDisk: - properties: - fsType: - type: string - partition: - format: int32 - type: integer - pdName: - type: string - readOnly: - type: boolean - required: - - pdName - type: object - gitRepo: - properties: - directory: - type: string - repository: - type: string - revision: - type: string - required: - - repository - type: object - glusterfs: - properties: - endpoints: - type: string - path: - type: string - readOnly: - type: boolean - required: - - endpoints - - path - type: object - hostPath: - properties: - path: - type: string - type: - type: string - required: - - path - type: object - iscsi: - properties: - chapAuthDiscovery: - type: boolean - chapAuthSession: - type: boolean - fsType: - type: string - initiatorName: - type: string - iqn: - type: string - iscsiInterface: - type: string - lun: - format: int32 - type: integer - portals: - items: - type: string - type: array - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - targetPortal: - type: string - required: - - iqn - - lun - - targetPortal - type: object - name: - type: string - nfs: - properties: - path: - type: string - readOnly: - type: boolean - server: - type: string - required: - - path - - server - type: object - persistentVolumeClaim: - properties: - claimName: - type: string - readOnly: - type: boolean - required: - - claimName - type: object - photonPersistentDisk: - properties: - fsType: - type: string - pdID: - type: string - required: - - pdID - type: object - portworxVolume: - properties: - fsType: - type: string - readOnly: - type: boolean - volumeID: - type: string - required: - - volumeID - type: object - projected: - properties: - defaultMode: - format: int32 - type: integer - sources: - items: - properties: - configMap: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - downwardAPI: - properties: - items: - items: - properties: - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - mode: - format: int32 - type: integer - path: - type: string - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string - resource: - type: string - required: - - resource - type: object - required: - - path - type: object - type: array - type: object - secret: - properties: - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - name: - type: string - optional: - type: boolean - type: object - serviceAccountToken: - properties: - audience: - type: string - expirationSeconds: - format: int64 - type: integer - path: - type: string - required: - - path - type: object - type: object - type: array - required: - - sources - type: object - quobyte: - properties: - group: - type: string - readOnly: - type: boolean - registry: - type: string - user: - type: string - volume: - type: string - required: - - registry - - volume - type: object - rbd: - properties: - fsType: - type: string - image: - type: string - keyring: - type: string - monitors: - items: - type: string - type: array - pool: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - user: - type: string - required: - - image - - monitors - type: object - scaleIO: - properties: - fsType: - type: string - gateway: - type: string - protectionDomain: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - sslEnabled: - type: boolean - storageMode: - type: string - storagePool: - type: string - system: - type: string - volumeName: - type: string - required: - - gateway - - secretRef - - system - type: object - secret: - properties: - defaultMode: - format: int32 - type: integer - items: - items: - properties: - key: - type: string - mode: - format: int32 - type: integer - path: - type: string - required: - - key - - path - type: object - type: array - optional: - type: boolean - secretName: - type: string - type: object - storageos: - properties: - fsType: - type: string - readOnly: - type: boolean - secretRef: - properties: - name: - type: string - type: object - volumeName: - type: string - volumeNamespace: - type: string - type: object - vsphereVolume: - properties: - fsType: - type: string - storagePolicyID: - type: string - storagePolicyName: - type: string - volumePath: - type: string - required: - - volumePath - type: object - required: - - name - type: object - type: array - required: - - driver - - executor - - sparkVersion - - type - type: object - status: - properties: - applicationState: - properties: - errorMessage: - type: string - state: - type: string - required: - - state - type: object - driverInfo: - properties: - podName: - type: string - webUIAddress: - type: string - webUIIngressAddress: - type: string - webUIIngressName: - type: string - webUIPort: - format: int32 - type: integer - webUIServiceName: - type: string - type: object - executionAttempts: - format: int32 - type: integer - executorState: - additionalProperties: - type: string - type: object - sparkApplicationId: - type: string - submissionAttempts: - format: int32 - type: integer - submissionID: - type: string - submissionTime: - format: date-time - nullable: true - type: string - terminationTime: - format: date-time - nullable: true - type: string - required: - - driverInfo - type: object - required: - - metadata - - spec - type: object - version: v1beta2 - versions: - - name: v1beta2 - served: true - storage: true -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/kustomize/base/operators/spark/webhook.yaml b/kustomize/base/operators/spark/webhook.yaml deleted file mode 100644 index d09106c6e7c..00000000000 --- a/kustomize/base/operators/spark/webhook.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: sparkoperator-init - namespace: sparkoperator - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 -spec: - backoffLimit: 3 - template: - metadata: - labels: - app.kubernetes.io/name: sparkoperator - app.kubernetes.io/version: v2.4.5-v1beta2 - spec: - serviceAccountName: sparkoperator - restartPolicy: Never - containers: - - name: main - image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5 - imagePullPolicy: IfNotPresent - command: ["/usr/bin/gencerts.sh","--namespace", "$(NAMESPACE)", "-p"] ---- -kind: Service -apiVersion: v1 -metadata: - name: spark-webhook - namespace: sparkoperator -spec: - ports: - - port: 443 - targetPort: 8080 - name: webhook - selector: - app.kubernetes.io/name: sparkoperator diff --git a/kustomize/base/pod_webhook/deployment.yaml b/kustomize/base/pod_webhook/deployment.yaml deleted file mode 100644 index 4209031b9bd..00000000000 --- a/kustomize/base/pod_webhook/deployment.yaml +++ /dev/null @@ -1,83 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flyte-pod-webhook - namespace: flyte - labels: - app: flyte-pod-webhook -spec: - selector: - matchLabels: - app: flyte-pod-webhook - template: - metadata: - labels: - app: flyte-pod-webhook - app.kubernetes.io/name: flyte-pod-webhook - app.kubernetes.io/version: 0.5.13 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "10254" - prometheus.io/path: "/metrics" - spec: - securityContext: - fsGroup: 65534 - runAsUser: 1001 - fsGroupChangePolicy: "Always" - serviceAccountName: flyte-pod-webhook - initContainers: - - name: generate-secrets - image: flytepropeller:v0.5.13 - imagePullPolicy: IfNotPresent - command: - - flytepropeller - args: - - webhook - - init-certs - - --config - - /etc/flyte/config/*.yaml - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - containers: - - name: webhook - image: flytepropeller:v0.5.13 - imagePullPolicy: IfNotPresent - command: - - flytepropeller - args: - - webhook - - --config - - /etc/flyte/config/*.yaml - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - readOnly: true - - name: webhook-certs - mountPath: /etc/webhook/certs - readOnly: true - volumes: - - name: config-volume - configMap: - name: flyte-propeller-config - - name: webhook-certs - secret: - secretName: flyte-pod-webhook diff --git a/kustomize/base/pod_webhook/kustomization.yaml b/kustomize/base/pod_webhook/kustomization.yaml deleted file mode 100644 index c1b978c89d3..00000000000 --- a/kustomize/base/pod_webhook/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: - - deployment.yaml - - rbac.yaml - - secret.yaml - - service.yaml diff --git a/kustomize/base/pod_webhook/rbac.yaml b/kustomize/base/pod_webhook/rbac.yaml deleted file mode 100644 index 94b3ded0fa2..00000000000 --- a/kustomize/base/pod_webhook/rbac.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Create a ClusterRole for the webhook -# https://kubernetes.io/docs/admin/authorization/rbac/ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flyte-pod-webhook - namespace: flyte -rules: - - apiGroups: - - "*" - resources: - - mutatingwebhookconfigurations - - secrets - - pods - - replicasets/finalizers - verbs: - - get - - create - - update - - patch ---- -# Create a Service Account for webhook -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flyte-pod-webhook - namespace: flyte ---- -# Create a binding from Role -> ServiceAccount -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flyte-pod-webhook - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flyte-pod-webhook -subjects: - - kind: ServiceAccount - name: flyte-pod-webhook - namespace: flyte diff --git a/kustomize/base/pod_webhook/secret.yaml b/kustomize/base/pod_webhook/secret.yaml deleted file mode 100644 index b5f37860ceb..00000000000 --- a/kustomize/base/pod_webhook/secret.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: flyte-pod-webhook - namespace: flyte -type: Opaque diff --git a/kustomize/base/pod_webhook/service.yaml b/kustomize/base/pod_webhook/service.yaml deleted file mode 100644 index 41d86826d72..00000000000 --- a/kustomize/base/pod_webhook/service.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: flyte-pod-webhook - namespace: flyte - annotations: - # This tells contour to use a H2 connection for the port associated - # with the name 'grpc' under spec/ports. - # For more information, refer to - # https://github.com/heptio/contour/blob/master/docs/annotations.md#contour-specific-service-annotations - # # Following this issue - the annotation was updated https://github.com/projectcontour/contour/issues/2092 - projectcontour.io/upstream-protocol.h2c: "grpc" -spec: - selector: - app: flyte-pod-webhook - ports: - - name: https - protocol: TCP - port: 443 - targetPort: 9443 diff --git a/kustomize/base/propeller/auth_secret.yaml b/kustomize/base/propeller/auth_secret.yaml deleted file mode 100644 index a6b978f605b..00000000000 --- a/kustomize/base/propeller/auth_secret.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: flyte-secret-auth - namespace: flyte -type: Opaque -stringData: - client_secret: foobar diff --git a/kustomize/base/propeller/deployment.yaml b/kustomize/base/propeller/deployment.yaml deleted file mode 100644 index 8df9e9ca075..00000000000 --- a/kustomize/base/propeller/deployment.yaml +++ /dev/null @@ -1,60 +0,0 @@ -# Create the actual deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flytepropeller - namespace: flyte - labels: - app: flytepropeller -spec: - selector: - matchLabels: - app: flytepropeller - template: - metadata: - labels: - app: flytepropeller - app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.7.1 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "10254" - prometheus.io/path: "/metrics" - spec: - securityContext: - fsGroup: 65534 - runAsUser: 1001 - fsGroupChangePolicy: "Always" - serviceAccountName: flytepropeller - volumes: - - name: config-volume - configMap: - name: flyte-propeller-config - - name: auth - secret: - secretName: flyte-secret-auth - containers: - - name: flytepropeller - image: flytepropeller:v0.7.1 - command: - - flytepropeller - args: - - --config - - /etc/flyte/config/*.yaml - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - imagePullPolicy: IfNotPresent - ports: - - containerPort: 10254 - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: auth - mountPath: /etc/secrets/ diff --git a/kustomize/base/propeller/kustomization.yaml b/kustomize/base/propeller/kustomization.yaml deleted file mode 100644 index a558adcc2cb..00000000000 --- a/kustomize/base/propeller/kustomization.yaml +++ /dev/null @@ -1,4 +0,0 @@ -resources: - - auth_secret.yaml - - deployment.yaml - - rbac.yaml diff --git a/kustomize/base/propeller/rbac.yaml b/kustomize/base/propeller/rbac.yaml deleted file mode 100644 index 3e0811ac794..00000000000 --- a/kustomize/base/propeller/rbac.yaml +++ /dev/null @@ -1,89 +0,0 @@ -# Create a ClusterRole for flytepropeller -# https://kubernetes.io/docs/admin/authorization/rbac/ -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flytepropeller -rules: -#Allow RO access to PODS - - apiGroups: - - "" - resources: - - pods - verbs: - - get - - list - - watch -#Allow Event recording access - - apiGroups: - - "" - resources: - - events - verbs: - - create - - update - - delete - - patch -#Allow Access All plugin objects - - apiGroups: - - "*" - resources: - - "*" - verbs: - - get - - list - - watch - - create - - update - - delete - - patch -#Allow Access to CRD - - apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - get - - list - - watch - - create - - delete - - update -#Allow Access to all resources under flyte.lyft.com - - apiGroups: - - flyte.lyft.com - resources: - - flyteworkflows - - flyteworkflows/finalizers - verbs: - - get - - list - - watch - - create - - update - - delete - - patch - - post - - deletecollection ---- -# Create a Service Account for Flytepropeller -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flytepropeller - namespace: flyte ---- -# Create a binding from Role -> ServiceAccount -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: flytepropeller - namespace: flyte -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: flytepropeller -subjects: -- kind: ServiceAccount - name: flytepropeller - namespace: flyte diff --git a/kustomize/base/schedulerdeployment/configmap.yaml b/kustomize/base/schedulerdeployment/configmap.yaml deleted file mode 100644 index 6a5b445af51..00000000000 --- a/kustomize/base/schedulerdeployment/configmap.yaml +++ /dev/null @@ -1,31 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: flyte-scheduler-config - namespace: flyte - labels: - app.kubernetes.io/name: flytescheduler - app.kubernetes.io/instance: flyte - helm.sh/chart: flyte-v0.1.10 - app.kubernetes.io/managed-by: Helm -data: - admin.yaml: | - admin: - clientId: flytepropeller - clientSecretLocation: /etc/secrets/client_secret - endpoint: flyteadmin:81 - insecure: true - event: - capacity: 1000 - rate: 500 - type: admin - db.yaml: | - database: - dbname: postgres - host: postgres - port: 5432 - username: postgres - logger.yaml: | - logger: - level: 4 - show-source: true diff --git a/kustomize/base/schedulerdeployment/deployment.yaml b/kustomize/base/schedulerdeployment/deployment.yaml deleted file mode 100644 index e919a768935..00000000000 --- a/kustomize/base/schedulerdeployment/deployment.yaml +++ /dev/null @@ -1,72 +0,0 @@ -# Create the actual deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flytescheduler - namespace: flyte - labels: - app: flytescheduler -spec: - replicas: 1 - selector: - matchLabels: - app: flytescheduler - template: - metadata: - labels: - app: flytescheduler - app.kubernetes.io/name: flytescheduler - app.kubernetes.io/version: 0.3.4 - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "10254" - prometheus.io/path: "/metrics" - spec: - serviceAccountName: flyteadmin - volumes: - - name: config-volume - configMap: - name: flyte-scheduler-config - - name: db-pass - secret: - secretName: db-pass - - name: auth - secret: - secretName: flyte-secret-auth - initContainers: - - command: - - flytescheduler - - precheck - - --config - - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v0.6.49" - imagePullPolicy: "IfNotPresent" - name: flytescheduler-check - volumeMounts: - - mountPath: /etc/flyte/config - name: config-volume - containers: - - command: - - flytescheduler - - run - - --config - - /etc/flyte/config/*.yaml - image: "cr.flyte.org/flyteorg/flytescheduler:v0.6.49" - imagePullPolicy: "IfNotPresent" - name: flytescheduler - resources: - limits: - cpu: 250m - ephemeral-storage: 100Mi - memory: 500Mi - requests: - cpu: 10m - ephemeral-storage: 50Mi - memory: 50Mi - volumeMounts: - - name: auth - mountPath: /etc/secrets/ - - mountPath: /etc/flyte/config - name: config-volume - - name: db-pass - mountPath: /etc/db diff --git a/kustomize/base/schedulerdeployment/kustomization.yaml b/kustomize/base/schedulerdeployment/kustomization.yaml deleted file mode 100644 index 898fc1587d9..00000000000 --- a/kustomize/base/schedulerdeployment/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: - - deployment.yaml - - configmap.yaml diff --git a/kustomize/base/single_cluster/README.md b/kustomize/base/single_cluster/README.md deleted file mode 100644 index 29346429457..00000000000 --- a/kustomize/base/single_cluster/README.md +++ /dev/null @@ -1,15 +0,0 @@ -[Back to Base components menu](../) -# Flyte Single cluster deployment mode -This allows flyte to be deployed in one Kubernetes cluster. -All components are deployed and can be configured to use dependencies based on the environment. - -To understand configuration of dependencies refer to sandbox or any cloud deployments - -Overlays using the single cluster mode -1. [Sandbox Overlay](../../overlays/sandbox) -1. [GCP SingleCluster Overlay](../../overlays/gcp) -1. [EKS SingleCluster Overlay](../../overlays/eks) - -These overlays are based on [Flyte Single Cluster deployment Configuration base](./complete) - -The complete per component configuration for Flyte can be found [here](./headless/config) diff --git a/kustomize/base/single_cluster/complete/README.md b/kustomize/base/single_cluster/complete/README.md deleted file mode 100644 index 052104b66e7..00000000000 --- a/kustomize/base/single_cluster/complete/README.md +++ /dev/null @@ -1,7 +0,0 @@ -[Back to Base components menu](../) -# Flyte Single cluster deployment mode -This provides a complete deployment of Flyte onto a single K8s cluster, it is based on [Headless Cluster](../headless). -The configuration for this deployment is also completely managed in the Headless Cluster configuration. - -This overlay just adds to resource - Sync controller and Flyte Console - diff --git a/kustomize/base/single_cluster/complete/kustomization.yaml b/kustomize/base/single_cluster/complete/kustomization.yaml deleted file mode 100644 index 54a1f14d87d..00000000000 --- a/kustomize/base/single_cluster/complete/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -bases: -- ../headless - -# All the resources that make up the deployment -resources: -# global resources -- ../../admindeployment/clustersync -- ../../console - -# configMapGenerator: -# TODO Flyte Console Configuration -#- name: flyte-console-config -# files: -# - ./config/console.yaml diff --git a/kustomize/base/single_cluster/headless/README.md b/kustomize/base/single_cluster/headless/README.md deleted file mode 100644 index ddc5573197f..00000000000 --- a/kustomize/base/single_cluster/headless/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# Flyte Headless cluster - -This is a complete single cluster Flyte deployment with 2 missing features -1. FlyteConsole is not installed - No UI -2. Clustersync manager is not installed - No automatic update of projects and tenant specific information - - -## USE Cases -This configuration is useful in running integration / end to end tests diff --git a/kustomize/base/single_cluster/headless/config/admin/cluster_resources.yaml b/kustomize/base/single_cluster/headless/config/admin/cluster_resources.yaml deleted file mode 100644 index bd2f1dc9d6e..00000000000 --- a/kustomize/base/single_cluster/headless/config/admin/cluster_resources.yaml +++ /dev/null @@ -1,19 +0,0 @@ -cluster_resources: - templatePath: "/etc/flyte/clusterresource/templates" - customData: - - production: - - projectQuotaCpu: - value: "5" - - projectQuotaMemory: - value: "4000Mi" - - staging: - - projectQuotaCpu: - value: "2" - - projectQuotaMemory: - value: "3000Mi" - - development: - - projectQuotaCpu: - value: "4" - - projectQuotaMemory: - value: "3000Mi" - refreshInterval: 1m diff --git a/kustomize/base/single_cluster/headless/config/admin/db.yaml b/kustomize/base/single_cluster/headless/config/admin/db.yaml deleted file mode 100644 index 5aefbfb4359..00000000000 --- a/kustomize/base/single_cluster/headless/config/admin/db.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# This should be replaced with the db endpoint -# This can be a postgres data base in the cloud like AWS Aurora / AWS RDS, Google Cloud SQL, etc -# Or can be a self hosted Postgres in the cluster -database: - port: 5432 - username: postgres - host: postgres - dbname: postgres - options: sslmode=disable - passwordPath: /etc/db/pass.txt diff --git a/kustomize/base/single_cluster/headless/config/admin/domain.yaml b/kustomize/base/single_cluster/headless/config/admin/domain.yaml deleted file mode 100644 index 9df6ef80a9b..00000000000 --- a/kustomize/base/single_cluster/headless/config/admin/domain.yaml +++ /dev/null @@ -1,7 +0,0 @@ -domains: - - id: development - name: development - - id: staging - name: staging - - id: production - name: production diff --git a/kustomize/base/single_cluster/headless/config/admin/server.yaml b/kustomize/base/single_cluster/headless/config/admin/server.yaml deleted file mode 100644 index e13f3ad156a..00000000000 --- a/kustomize/base/single_cluster/headless/config/admin/server.yaml +++ /dev/null @@ -1,43 +0,0 @@ -server: - httpPort: 8088 - grpcPort: 8089 - security: - # Controls whether to serve requests over SSL/TLS. - secure: false - # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. - useAuth: false - allowCors: true - allowedOrigins: - # Accepting all domains for Sandbox installation - - "*" - allowedHeaders: - - "Content-Type" - - "flyte-authorization" -auth: - authorizedUris: - # This should point at your public http Uri. - - https://localhost:30081 - # This will be used by internal services in the same namespace as flyteadmin - - http://flyteadmin:80 - # This will be used by internal services in the same cluster but different namespaces - - http://flyteadmin.flyte.svc.cluster.local:80 - - # Controls app authentication config - appAuth: - thirdPartyConfig: - flyteClient: - clientId: flytectl - redirectUri: https://localhost:53593/callback - scopes: - - offline - - all - # Controls user authentication - userAuth: - openId: - baseUrl: https://accounts.google.com - scopes: - - profile - - openid - clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com -flyteadmin: - roleNameKey: "iam.amazonaws.com/role" diff --git a/kustomize/base/single_cluster/headless/config/admin/task_resource_defaults.yaml b/kustomize/base/single_cluster/headless/config/admin/task_resource_defaults.yaml deleted file mode 100644 index f057182fb5a..00000000000 --- a/kustomize/base/single_cluster/headless/config/admin/task_resource_defaults.yaml +++ /dev/null @@ -1,10 +0,0 @@ -task_resources: - defaults: - cpu: 100m - memory: 100Mi - storage: 5Mi - limits: - cpu: 2 - memory: 1Gi - storage: 20Mi - gpu: 1 diff --git a/kustomize/base/single_cluster/headless/config/clusterresource-templates/aa_namespace.yaml b/kustomize/base/single_cluster/headless/config/clusterresource-templates/aa_namespace.yaml deleted file mode 100644 index 3075aa9f9ef..00000000000 --- a/kustomize/base/single_cluster/headless/config/clusterresource-templates/aa_namespace.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: {{ namespace }} -spec: - finalizers: - - kubernetes diff --git a/kustomize/base/single_cluster/headless/config/clusterresource-templates/ab_project-resource-quota.yaml b/kustomize/base/single_cluster/headless/config/clusterresource-templates/ab_project-resource-quota.yaml deleted file mode 100644 index ddfade3c292..00000000000 --- a/kustomize/base/single_cluster/headless/config/clusterresource-templates/ab_project-resource-quota.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: ResourceQuota -metadata: - name: project-quota - namespace: {{ namespace }} -spec: - hard: - limits.cpu: {{ projectQuotaCpu }} - limits.memory: {{ projectQuotaMemory }} - diff --git a/kustomize/base/single_cluster/headless/config/common/logger.yaml b/kustomize/base/single_cluster/headless/config/common/logger.yaml deleted file mode 100644 index 7fc20650ec7..00000000000 --- a/kustomize/base/single_cluster/headless/config/common/logger.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# TODO this is used to control the log level -logger: - show-source: true - level: 2 diff --git a/kustomize/base/single_cluster/headless/config/common/storage.yaml b/kustomize/base/single_cluster/headless/config/common/storage.yaml deleted file mode 100644 index 09e6ec6dec6..00000000000 --- a/kustomize/base/single_cluster/headless/config/common/storage.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# TODO This should be changed for the right storage option - e.g. -# hosted blob stores like S3, GCS, AFS etc -# k8s blob store like minio -# For all supported options look at https://github.com/lyft/flytestdlib/blob/master/storage/config.go -storage: - type: mem diff --git a/kustomize/base/single_cluster/headless/config/console/console.yaml b/kustomize/base/single_cluster/headless/config/console/console.yaml deleted file mode 100644 index d1c0cc51dcf..00000000000 --- a/kustomize/base/single_cluster/headless/config/console/console.yaml +++ /dev/null @@ -1,3 +0,0 @@ -# This determines the base url used for all console links. It must match the route specified in ingress.yaml -BASE_URL: /console -CONFIG_DIR: /etc/flyte/config diff --git a/kustomize/base/single_cluster/headless/config/datacatalog/db.yaml b/kustomize/base/single_cluster/headless/config/datacatalog/db.yaml deleted file mode 100644 index 479475aa94c..00000000000 --- a/kustomize/base/single_cluster/headless/config/datacatalog/db.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# This should be replaced with the db endpoint -# This can be a postgres data base in the cloud like AWS Aurora / AWS RDS, Google Cloud SQL, etc -# Or can be a self hosted Postgres in the cluster -database: - port: 5432 - username: postgres - host: postgres - dbname: datacatalog - options: sslmode=disable - passwordPath: /etc/db/pass.txt diff --git a/kustomize/base/single_cluster/headless/config/datacatalog/server.yaml b/kustomize/base/single_cluster/headless/config/datacatalog/server.yaml deleted file mode 100644 index 1cdd80c27f1..00000000000 --- a/kustomize/base/single_cluster/headless/config/datacatalog/server.yaml +++ /dev/null @@ -1,6 +0,0 @@ -datacatalog: - storage-prefix: metadata/datacatalog - metrics-scope: "datacatalog" - profiler-port: 10254 -application: - grpcPort: 8089 diff --git a/kustomize/base/single_cluster/headless/config/propeller/admin.yaml b/kustomize/base/single_cluster/headless/config/propeller/admin.yaml deleted file mode 100644 index 55cf01b83b0..00000000000 --- a/kustomize/base/single_cluster/headless/config/propeller/admin.yaml +++ /dev/null @@ -1,7 +0,0 @@ -event: - type: admin - rate: 500 - capacity: 1000 -admin: - endpoint: flyteadmin:81 - insecure: true diff --git a/kustomize/base/single_cluster/headless/config/propeller/catalog.yaml b/kustomize/base/single_cluster/headless/config/propeller/catalog.yaml deleted file mode 100644 index e70e7bd7701..00000000000 --- a/kustomize/base/single_cluster/headless/config/propeller/catalog.yaml +++ /dev/null @@ -1,4 +0,0 @@ -catalog-cache: - endpoint: datacatalog:89 - type: datacatalog - insecure: true diff --git a/kustomize/base/single_cluster/headless/config/propeller/core.yaml b/kustomize/base/single_cluster/headless/config/propeller/core.yaml deleted file mode 100644 index 055e7a8840f..00000000000 --- a/kustomize/base/single_cluster/headless/config/propeller/core.yaml +++ /dev/null @@ -1,36 +0,0 @@ -propeller: - rawoutput-prefix: s3://my-s3-bucket/ - metadata-prefix: metadata/propeller - workers: 4 - max-workflow-retries: 30 - workflow-reeval-duration: 30s - downstream-eval-duration: 30s - limit-namespace: "all" - prof-port: 10254 - metrics-prefix: flyte - enable-admin-launcher: true - leader-election: - lock-config-map: - name: propeller-leader - namespace: flyte - enabled: true - lease-duration: 15s - renew-deadline: 10s - retry-period: 2s - queue: - type: batch - batching-interval: 2s - batch-size: -1 - queue: - type: maxof - rate: 100 - capacity: 1000 - base-delay: 5s - max-delay: 120s - sub-queue: - type: bucket - rate: 10 - capacity: 100 -webhook: - certDir: /etc/webhook/certs - serviceName: flyte-pod-webhook diff --git a/kustomize/base/single_cluster/headless/config/propeller/enabled_plugins.yaml b/kustomize/base/single_cluster/headless/config/propeller/enabled_plugins.yaml deleted file mode 100644 index ebf4892451b..00000000000 --- a/kustomize/base/single_cluster/headless/config/propeller/enabled_plugins.yaml +++ /dev/null @@ -1,8 +0,0 @@ -## -# TODO It is important to enable the plugins that you want to deploy here. -tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array diff --git a/kustomize/base/single_cluster/headless/config/propeller/plugins/copilot.yaml b/kustomize/base/single_cluster/headless/config/propeller/plugins/copilot.yaml deleted file mode 100644 index 31f483ab6dc..00000000000 --- a/kustomize/base/single_cluster/headless/config/propeller/plugins/copilot.yaml +++ /dev/null @@ -1,6 +0,0 @@ -plugins: - k8s: - co-pilot: - name: "flyte-copilot-" - image: "cr.flyte.org/flyteorg/flytecopilot:v0.0.15" - start-timeout: "30s" diff --git a/kustomize/base/single_cluster/headless/config/propeller/plugins/k8s.yaml b/kustomize/base/single_cluster/headless/config/propeller/plugins/k8s.yaml deleted file mode 100644 index 5097741639a..00000000000 --- a/kustomize/base/single_cluster/headless/config/propeller/plugins/k8s.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# TODO set any custom env vars -plugins: - k8s: - default-env-vars: - - SOME_ENV_VARS: "These get passed to all containers" - default-cpus: 100m - default-memory: 100Mi diff --git a/kustomize/base/single_cluster/headless/config/propeller/resource_manager.yaml b/kustomize/base/single_cluster/headless/config/propeller/resource_manager.yaml deleted file mode 100644 index a4601e8f2db..00000000000 --- a/kustomize/base/single_cluster/headless/config/propeller/resource_manager.yaml +++ /dev/null @@ -1,7 +0,0 @@ -propeller: - resourcemanager: - type: redis - resourceMaxQuota: 10000 - redis: - hostPath: redis-resource-manager:6379 - hostKey: mypassword diff --git a/kustomize/base/single_cluster/headless/kustomization.yaml b/kustomize/base/single_cluster/headless/kustomization.yaml deleted file mode 100644 index 8e6067e9979..00000000000 --- a/kustomize/base/single_cluster/headless/kustomization.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -# All the resources that make up the deployment -resources: - # global resources - - ../../namespace - - ../../admindeployment - - ../../schedulerdeployment - - ../../datacatalog - - ../../wf_crd - - ../../pod_webhook - - ../../propeller - - ../../adminserviceaccount - -configMapGenerator: - # the main admin configmap - - name: flyte-admin-config - files: - - ./config/admin/server.yaml - - ./config/admin/domain.yaml - - ./config/admin/db.yaml - - ./config/admin/cluster_resources.yaml - - ./config/admin/task_resource_defaults.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - - # cluster resource templates - - name: clusterresource-template - files: - # Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". - - ./config/clusterresource-templates/aa_namespace.yaml - - ./config/clusterresource-templates/ab_project-resource-quota.yaml - - # Flyte Propeller Configuration - - name: flyte-propeller-config - files: - - ./config/propeller/core.yaml - - ./config/propeller/admin.yaml - - ./config/propeller/catalog.yaml - - ./config/propeller/resource_manager.yaml - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/plugins/copilot.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - - # TODO Flyte Console Configuration - #- name: flyte-console-config - # files: - # - ./config/console.yaml - - - name: datacatalog-config - files: - - ./config/datacatalog/server.yaml - - ./config/datacatalog/db.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - -secretGenerator: - - name: db-pass - literals: - - pass.txt="awesomesauce" diff --git a/kustomize/base/wf_crd/kustomization.yaml b/kustomize/base/wf_crd/kustomization.yaml deleted file mode 100644 index 6f680eaac23..00000000000 --- a/kustomize/base/wf_crd/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- wf_crd.yaml diff --git a/kustomize/base/wf_crd/wf_crd.yaml b/kustomize/base/wf_crd/wf_crd.yaml deleted file mode 100644 index 3cd3c3eb172..00000000000 --- a/kustomize/base/wf_crd/wf_crd.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - # name must match the spec fields below, and be in the form: . - name: flyteworkflows.flyte.lyft.com -spec: - # group name to use for REST API: /apis// - group: flyte.lyft.com - # either Namespaced or Cluster - names: - # plural name to be used in the URL: /apis/// - plural: flyteworkflows - # singular name to be used as an alias on the CLI and for display - singular: flyteworkflow - # kind is normally the CamelCased singular type. Your resource manifests use this. - kind: FlyteWorkflow - # shortNames allow shorter string to match your resource on the CLI - shortNames: - - fly - scope: Namespaced - versions: - - name: v1alpha1 - served: true - storage: true - schema: - openAPIV3Schema: - type: object - x-kubernetes-preserve-unknown-fields: true diff --git a/kustomize/overlays/README.md b/kustomize/overlays/README.md deleted file mode 100644 index 63ed238e9ea..00000000000 --- a/kustomize/overlays/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Deploy Flyte using one of the given overlays - -To start a deployment use one of the following overlays as a starting point. If you are just playing around with Flyte, use Sandbox overlay - and preferably the pre-generated yaml - available [here](../../deployment/sandbox/flyte_generated.yaml) - -1. [Sandbox](./sandbox) : A pre-configured overlay to deploy a complete standalone cluster onto a local k8s cluster -1. [GCP](./gcp): An almost configured overlay to deploy a production ready cluster to GCP. Some modifications needed for your environment -1. [EKS](./eks): An almost configured overlay to deploy a production ready cluster to AWS. Some modifications needed for your environment diff --git a/kustomize/overlays/eks/README.md b/kustomize/overlays/eks/README.md deleted file mode 100644 index 8873542337f..00000000000 --- a/kustomize/overlays/eks/README.md +++ /dev/null @@ -1,73 +0,0 @@ -[All Overlays](./) -# :construction: Amazon EKS deployment - -This overlay serves as an example to bootstrap Flyte setup on AWS. It is not -designed to work out of the box due to the need of AWS resources. Please follow the instruction -below to further configure. - -_Hint_: searching `TODO:` through this directory would help to understand what needs to be done. - -## Amazon RDS / Amazon Aurora - -A few things are required for this overlay to function: - -* Two databases named as `flyte` and `datacatalog` -* A database user named as `flyte` -* Password of the database user can be added to either to [kustomization.yaml](kustomization.yaml) or you can create a new file and change the secretGenerator tag to use files. (Refer to kustomize documentation) -* Service account(s) associated with `flyteadmin` and `datacatalog` pods (either as GKE cluster - service account or through workload identity) should have `Cloud SQL Editor` role - -## Create S3 bucket -1. Create a S3 bucket named as `flyte` (if other name replace it next) -1. Replace in [config/common/storage.yaml](flyte/config/common/storage.yaml) if using a bucket other than Flyte then replace the bucket name too - -## flyteadmin - -flyteadmin configuration is derived from the [single cluster](../../base/single_cluster) overlay, with only modification to [database configuration db.yaml](flyte/config/admin/db.yaml) - -**Advanced / OPTIONAL** -1. The default CORS setting in flyteAdmin allows cross origin requests. A more secure way would be to allow requests only from the expected domain. To do this, you will have to create a new *server.yaml* -similar to [base/single_cluster/headless/config](../../base/single_cluster/headless/config) under config/admin and then set -`server -> security -> allowedOrigins`. - -## flyteconsole - -[flyteconsole configmap](console/config.yaml) needs to be updated with flyteadmin internal load -balancer IP address or the DNS name associated with it if any. - -flyteconsole is exposed as a service using [internal load balancer](https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing). - -## flytepropeller - -flytepropeller configuration is derived from the [single cluster](../../base/single_cluster) overlay, with only modification to the config for performance tuning and logs -For logs configuration Replace `` in [config/propeller/plugins/task_logs.yaml](flyte/config/propeller/plugins/task_logs.yaml) to use CloudWatch - -Some important points - -* Storage configuration is shared with Admin and Catalog. Ideally in production Propeller should have its own configuration with real high cache size. - -* By default, three plugins are enabled: -1. container -2. k8s-array -3. sidecar - -## datacatalog - -datacatalog configuration is derived from the [single cluster](../../base/single_cluster) overlay, with only modification to [database configuration db.yaml](flyte/config/datacatalog/db.yaml) - - -## How to build your overlay -To build your overlay there are 2 options -1. Build it in your own repo Example coming soon :construction: -1. hack it in your clone of Flyte repo in place of EKS overlay. In this case just navigate to the root of the repo and run -```bash -$ make kustomize -``` -If all goes well a new overlay composite should be generated in [/deployment/eks/flyte_generated.yaml](../../../deployment/eks/flyte_generated.yaml) - -## Now ship it - -``` shell -make -kubectl apply -f deployment/gcp/flyte_generated.yaml -``` diff --git a/kustomize/overlays/eks/flyte/admin/deployment.yaml b/kustomize/overlays/eks/flyte/admin/deployment.yaml deleted file mode 100644 index b581a3dba1f..00000000000 --- a/kustomize/overlays/eks/flyte/admin/deployment.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flyteadmin - namespace: flyte -spec: - template: - spec: - containers: - - name: flyteadmin - resources: - limits: - memory: "1Gi" - cpu: "2" - ephemeral-storage: "1Gi" diff --git a/kustomize/overlays/eks/flyte/admin/service.yaml b/kustomize/overlays/eks/flyte/admin/service.yaml deleted file mode 100644 index de61a74fde8..00000000000 --- a/kustomize/overlays/eks/flyte/admin/service.yaml +++ /dev/null @@ -1,28 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" - # TODO add security groups - service.beta.kubernetes.io/aws-load-balancer-extra-security-groups: "sg-...,sg-...,sg-..." - # TODO alter domain - external-dns.alpha.kubernetes.io/hostname: "flyteadmin.subdomain.mydomain.com" - name: flyteadmin - namespace: flyte -spec: - loadBalancerSourceRanges: - # TODO change source ip range if desired - - 0.0.0.0 - ports: - # TODO do multiple ports work for ELB - - name: http - port: 80 - protocol: TCP - targetPort: 8088 - - name: grpc - port: 80 - protocol: TCP - targetPort: 8089 - selector: - app: flyteadmin - type: LoadBalancer diff --git a/kustomize/overlays/eks/flyte/admin/serviceaccount.yaml b/kustomize/overlays/eks/flyte/admin/serviceaccount.yaml deleted file mode 100644 index cf30eaa74ad..00000000000 --- a/kustomize/overlays/eks/flyte/admin/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flyteadmin - namespace: flyte - annotations: - eks.amazonaws.com/role-arn: "arn:aws:iam::111222333456:role/flyte-operator" - diff --git a/kustomize/overlays/eks/flyte/config/admin/db.yaml b/kustomize/overlays/eks/flyte/config/admin/db.yaml deleted file mode 100644 index d44ef2329d8..00000000000 --- a/kustomize/overlays/eks/flyte/config/admin/db.yaml +++ /dev/null @@ -1,7 +0,0 @@ -database: - port: 5432 - username: flyte - # TODO Change this to match aurora or rds postgres endpoint - host: flyteadmin-cluster.cluster-456123e6ivib.us-west-2.rds.amazonaws.com - dbname: flyte - passwordPath: /etc/db/pass.txt diff --git a/kustomize/overlays/eks/flyte/config/admin/task_resource_defaults.yaml b/kustomize/overlays/eks/flyte/config/admin/task_resource_defaults.yaml deleted file mode 100644 index 06870c232bf..00000000000 --- a/kustomize/overlays/eks/flyte/config/admin/task_resource_defaults.yaml +++ /dev/null @@ -1,10 +0,0 @@ -task_resources: - defaults: - cpu: 1000m - memory: 1000Mi - storage: 1000Mi - limits: - cpu: 2 - memory: 8Gi - storage: 2000Mi - gpu: 1 diff --git a/kustomize/overlays/eks/flyte/config/clusterresource-templates/ad_spark-role.yaml b/kustomize/overlays/eks/flyte/config/clusterresource-templates/ad_spark-role.yaml deleted file mode 100644 index 51998e30510..00000000000 --- a/kustomize/overlays/eks/flyte/config/clusterresource-templates/ad_spark-role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: spark-role - namespace: {{ namespace }} -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - '*' -- apiGroups: - - "" - resources: - - services - verbs: - - '*' -- apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' diff --git a/kustomize/overlays/eks/flyte/config/clusterresource-templates/ae_spark-service-account.yaml b/kustomize/overlays/eks/flyte/config/clusterresource-templates/ae_spark-service-account.yaml deleted file mode 100644 index dddd8d27428..00000000000 --- a/kustomize/overlays/eks/flyte/config/clusterresource-templates/ae_spark-service-account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: spark - namespace: {{ namespace }} diff --git a/kustomize/overlays/eks/flyte/config/clusterresource-templates/af_spark-role-binding.yaml b/kustomize/overlays/eks/flyte/config/clusterresource-templates/af_spark-role-binding.yaml deleted file mode 100644 index 2e9c8ae7655..00000000000 --- a/kustomize/overlays/eks/flyte/config/clusterresource-templates/af_spark-role-binding.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: spark-role-binding - namespace: {{ namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role -subjects: -- kind: ServiceAccount - name: spark - namespace: {{ namespace }} - diff --git a/kustomize/overlays/eks/flyte/config/common/storage.yaml b/kustomize/overlays/eks/flyte/config/common/storage.yaml deleted file mode 100644 index 39dd3931122..00000000000 --- a/kustomize/overlays/eks/flyte/config/common/storage.yaml +++ /dev/null @@ -1,17 +0,0 @@ -storage: - type: stow - stow: - kind: s3 - config: - auth_type: iam - region: us-east-2 - # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store - container: "flyte-demo" - # NOTE this cache configuration is purely for propeller. But since we are having a common storage - # config, we are configuring this value. In production create a separate storage config for - # propeller and increase the cache size - cache: - max_size_mbs: 512 - target_gc_percent: 70 - limits: - maxDownloadMBs: 10 diff --git a/kustomize/overlays/eks/flyte/config/datacatalog/db.yaml b/kustomize/overlays/eks/flyte/config/datacatalog/db.yaml deleted file mode 100644 index 6be110650dc..00000000000 --- a/kustomize/overlays/eks/flyte/config/datacatalog/db.yaml +++ /dev/null @@ -1,7 +0,0 @@ -database: - port: 5432 - username: flyte - # TODO Change this to match aurora or rds postgres endpoint - host: flyteadmin-cluster.cluster-456123e6ivib.us-west-2.rds.amazonaws.com - dbname: flytedatacatalog - passwordPath: /etc/db/pass.txt diff --git a/kustomize/overlays/eks/flyte/config/propeller/core.yaml b/kustomize/overlays/eks/flyte/config/propeller/core.yaml deleted file mode 100644 index baa8c6b2612..00000000000 --- a/kustomize/overlays/eks/flyte/config/propeller/core.yaml +++ /dev/null @@ -1,40 +0,0 @@ -propeller: - rawoutput-prefix: s3://my-s3-bucket/ - metadata-prefix: metadata/propeller - workers: 40 - gc-interval: 12h - max-workflow-retries: 50 - workflow-reeval-duration: 30s - downstream-eval-duration: 30s - limit-namespace: "all" - prof-port: 10254 - metrics-prefix: flyte - enable-admin-launcher: true - leader-election: - lock-config-map: - name: propeller-leader - namespace: flyte - enabled: true - lease-duration: 15s - renew-deadline: 10s - retry-period: 2s - kube-client-config: - qps: 100 - burst: 25 - timeout: 30s - queue: - type: batch - batching-interval: 2s - batch-size: -1 - queue: - type: maxof - rate: 100 - capacity: 1000 - base-delay: 5s - max-delay: 120s - sub-queue: - type: bucket - rate: 100 - capacity: 1000 - workflowStore: - policy: "ResourceVersionCache" diff --git a/kustomize/overlays/eks/flyte/config/propeller/enabled_plugins.yaml b/kustomize/overlays/eks/flyte/config/propeller/enabled_plugins.yaml deleted file mode 100644 index 9e4644ac18a..00000000000 --- a/kustomize/overlays/eks/flyte/config/propeller/enabled_plugins.yaml +++ /dev/null @@ -1,15 +0,0 @@ -tasks: - max-plugin-phase-versions: 1000000 - task-plugins: - enabled-plugins: - - container - - sidecar - - spark - - k8s-array - - pytorch - default-for-task-types: - container: container - sidecar: sidecar - spark: spark - container_array: k8s-array - pytorch: pytorch diff --git a/kustomize/overlays/eks/flyte/config/propeller/plugins/catalog_cache.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/catalog_cache.yaml deleted file mode 100644 index 3678fbbc895..00000000000 --- a/kustomize/overlays/eks/flyte/config/propeller/plugins/catalog_cache.yaml +++ /dev/null @@ -1,6 +0,0 @@ -plugins: - catalogCache: - reader: - maxItems: 10000 - writer: - maxItems: 10000 diff --git a/kustomize/overlays/eks/flyte/config/propeller/plugins/k8s.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/k8s.yaml deleted file mode 100644 index 8fe068397fe..00000000000 --- a/kustomize/overlays/eks/flyte/config/propeller/plugins/k8s.yaml +++ /dev/null @@ -1,8 +0,0 @@ -plugins: - k8s: - default-env-vars: - - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - - FLYTE_AWS_ACCESS_KEY_ID: minio - - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage - default-cpus: 100m - default-memory: 100Mi diff --git a/kustomize/overlays/eks/flyte/config/propeller/plugins/qubole.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/qubole.yaml deleted file mode 100644 index ea34d2752b9..00000000000 --- a/kustomize/overlays/eks/flyte/config/propeller/plugins/qubole.yaml +++ /dev/null @@ -1,3 +0,0 @@ -plugins: - qubole: - quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" diff --git a/kustomize/overlays/eks/flyte/config/propeller/plugins/spark.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/spark.yaml deleted file mode 100644 index 6b514a8c363..00000000000 --- a/kustomize/overlays/eks/flyte/config/propeller/plugins/spark.yaml +++ /dev/null @@ -1,20 +0,0 @@ -plugins: - spark: - spark-config-default: - # We override the default credentials chain provider for Hadoop so that - # it can use the serviceAccount based IAM role or ec2 metadata based. - # This is more in line with how AWS works - - spark.hadoop.fs.s3a.aws.credentials.provider: "com.amazonaws.auth.DefaultAWSCredentialsProviderChain" - - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" - - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.multipart.threshold: "536870912" - - spark.blacklist.enabled: "true" - - spark.blacklist.timeout: "5m" - - spark.task.maxfailures: "8" diff --git a/kustomize/overlays/eks/flyte/config/propeller/plugins/task_logs.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/task_logs.yaml deleted file mode 100644 index 6e46cde9c2d..00000000000 --- a/kustomize/overlays/eks/flyte/config/propeller/plugins/task_logs.yaml +++ /dev/null @@ -1,10 +0,0 @@ -plugins: - logs: - # Log links can link to multiple options - # #1 Kubernetes dashboard is disabled in GCP - kubernetes-enabled: false - # #2 AWS Cloudwatch - cloudwatch-enabled: true - # TODO Add region and log group - cloudwatch-region: - cloudwatch-log-group: diff --git a/kustomize/overlays/eks/flyte/console/service.yaml b/kustomize/overlays/eks/flyte/console/service.yaml deleted file mode 100644 index 7dcaed5d8bc..00000000000 --- a/kustomize/overlays/eks/flyte/console/service.yaml +++ /dev/null @@ -1,23 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: "600" - # TODO add your security groups here - service.beta.kubernetes.io/aws-load-balancer-extra-security-groups: "sg-....,sg-...,sg-..." - # TODO add your external dns here - external-dns.alpha.kubernetes.io/hostname: "flyte.subdomain.mydomain.com" - name: flyteconsole - namespace: flyte -spec: - loadBalancerSourceRanges: - # TODO limit source ranges if you want - - 0.0.0.0 - ports: - - name: http - port: 80 - protocol: TCP - targetPort: 8080 - selector: - app: flyteconsole - type: LoadBalancer diff --git a/kustomize/overlays/eks/flyte/datacatalog/deployment.yaml b/kustomize/overlays/eks/flyte/datacatalog/deployment.yaml deleted file mode 100644 index 1035dac5a0d..00000000000 --- a/kustomize/overlays/eks/flyte/datacatalog/deployment.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datacatalog - namespace: flyte -spec: - template: - spec: - containers: - - name: datacatalog - resources: - limits: - memory: "1Gi" - cpu: "2" - ephemeral-storage: "1000Mi" diff --git a/kustomize/overlays/eks/flyte/datacatalog/service.yaml b/kustomize/overlays/eks/flyte/datacatalog/service.yaml deleted file mode 100644 index 70e95295ab2..00000000000 --- a/kustomize/overlays/eks/flyte/datacatalog/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: datacatalog-metrics - namespace: flyte - annotations: - cloud.google.com/load-balancer-type: Internal -spec: - selector: - app: datacatalog - type: LoadBalancer - ports: - - name: http-metrics - protocol: TCP - port: 10254 diff --git a/kustomize/overlays/eks/flyte/ingress/README.md b/kustomize/overlays/eks/flyte/ingress/README.md deleted file mode 100644 index 28fa058659a..00000000000 --- a/kustomize/overlays/eks/flyte/ingress/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# :construction: Instructions to deploy ALB Ingress controller - -Follow instructions here to install ALB Ingress Controller: https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html - -Replace `alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6` in ingress.yaml and ingress_grpc.yaml with your own SSL cert (that you will create by following ALB Instructions above) diff --git a/kustomize/overlays/eks/flyte/ingress/ingress.yaml b/kustomize/overlays/eks/flyte/ingress/ingress.yaml deleted file mode 100644 index 05b6ba465e0..00000000000 --- a/kustomize/overlays/eks/flyte/ingress/ingress.yaml +++ /dev/null @@ -1,157 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: flytesystem - namespace: flyte - annotations: - kubernetes.io/ingress.class: alb - alb.ingress.kubernetes.io/tags: service_instance=production - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - - # Instruct ALB Controller to not create multiple load balancers (and hence maintain a single endpoint for both GRPC and Http) - alb.ingress.kubernetes.io/group.name: flytesystem - - # Replace certificate Arn with one deployed to your EKS cluster. Follow instructions in README.md - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 - labels: - app: flyteadmin -spec: - rules: - - http: - paths: - - path: /* - pathType: ImplementationSpecific - backend: - service: - name: ssl-redirect - port: - name: use-annotation - - path: /console - pathType: ImplementationSpecific - backend: - service: - name: flyteconsole - port: - number: 80 - - path: /console/* - pathType: ImplementationSpecific - backend: - service: - name: flyteconsole - port: - number: 80 - - path: /api/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /healthcheck - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /v1/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - # Port 87 in FlyteAdmin maps to the redoc container. - - path: /openapi/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 87 - - path: /.well-known/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /login - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /login/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /logout - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /logout/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /callback - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /callback/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /me - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /config - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /config/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /oauth2 - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 - - path: /oauth2/* - pathType: ImplementationSpecific - backend: - service: - name: flyteadmin - port: - number: 80 diff --git a/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml b/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml deleted file mode 100644 index bfc75f6d356..00000000000 --- a/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml +++ /dev/null @@ -1,52 +0,0 @@ ---- -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - annotations: - alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' - alb.ingress.kubernetes.io/backend-protocol-version: GRPC - alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]' - alb.ingress.kubernetes.io/scheme: internet-facing - alb.ingress.kubernetes.io/tags: service_instance=production - kubernetes.io/ingress.class: alb - nginx.ingress.kubernetes.io/ssl-redirect: "false" - alb.ingress.kubernetes.io/group.name: flytesystem - - # Replace certificate Arn with one deployed to your EKS cluster. Follow instructions in README.md - alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 - labels: - app: flyteadmin - name: flytesystem-grpc - namespace: flyte -spec: - rules: - - http: - paths: - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.SignalService/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.AdminService/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.AuthMetadataService/* - pathType: ImplementationSpecific - - backend: - service: - name: flyteadmin - port: - number: 81 - path: /flyteidl.service.IdentityService/* - pathType: ImplementationSpecific diff --git a/kustomize/overlays/eks/flyte/ingress/kustomization.yaml b/kustomize/overlays/eks/flyte/ingress/kustomization.yaml deleted file mode 100644 index 2cb8f8d97aa..00000000000 --- a/kustomize/overlays/eks/flyte/ingress/kustomization.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -resources: - - ingress.yaml - - ingress_grpc.yaml diff --git a/kustomize/overlays/eks/flyte/kustomization.yaml b/kustomize/overlays/eks/flyte/kustomization.yaml deleted file mode 100644 index cdc206dca1d..00000000000 --- a/kustomize/overlays/eks/flyte/kustomization.yaml +++ /dev/null @@ -1,62 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -# Override the namespace -namespace: flyte - -bases: - - ../../../base/single_cluster/complete - - ingress - -resources: - - datacatalog/service.yaml - - propeller/service.yaml - -patchesStrategicMerge: - - admin/deployment.yaml - - admin/service.yaml - - admin/serviceaccount.yaml - - datacatalog/deployment.yaml - - propeller/deployment.yaml - - propeller/serviceaccount.yaml - - console/service.yaml - -# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". - -configMapGenerator: - - behavior: merge - files: - - ./config/admin/db.yaml - - ./config/admin/task_resource_defaults.yaml - - ./config/common/storage.yaml - name: flyte-admin-config - - behavior: merge - files: - - ./config/clusterresource-templates/ad_spark-role.yaml - - ./config/clusterresource-templates/ae_spark-service-account.yaml - - ./config/clusterresource-templates/af_spark-role-binding.yaml - name: clusterresource-template - - behavior: merge - files: - - ./config/propeller/core.yaml - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/plugins/catalog_cache.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/propeller/plugins/qubole.yaml - - ./config/propeller/plugins/spark.yaml - - ./config/propeller/plugins/task_logs.yaml - - ./config/common/storage.yaml - name: flyte-propeller-config - - behavior: merge - files: - - ./config/common/storage.yaml - - ./config/datacatalog/db.yaml - name: datacatalog-config - -########### -# Use this to record the password for the Google CloudSQL Database -secretGenerator: - - name: db-pass - behavior: merge - literals: - - pass.txt="yourpassword" diff --git a/kustomize/overlays/eks/flyte/propeller/deployment.yaml b/kustomize/overlays/eks/flyte/propeller/deployment.yaml deleted file mode 100644 index 9a46601b53c..00000000000 --- a/kustomize/overlays/eks/flyte/propeller/deployment.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flytepropeller - namespace: flyte -spec: - template: - spec: - containers: - - name: flytepropeller - resources: - limits: - memory: "4Gi" - cpu: "2" - ephemeral-storage: "1Gi" diff --git a/kustomize/overlays/eks/flyte/propeller/service.yaml b/kustomize/overlays/eks/flyte/propeller/service.yaml deleted file mode 100644 index e80a8746e6d..00000000000 --- a/kustomize/overlays/eks/flyte/propeller/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: flytepropeller - namespace: flyte - annotations: - cloud.google.com/load-balancer-type: Internal -spec: - type: LoadBalancer - selector: - app: flytepropeller - ports: - - name: http-metrics - protocol: TCP - port: 10254 diff --git a/kustomize/overlays/eks/flyte/propeller/serviceaccount.yaml b/kustomize/overlays/eks/flyte/propeller/serviceaccount.yaml deleted file mode 100644 index 9272749c4b6..00000000000 --- a/kustomize/overlays/eks/flyte/propeller/serviceaccount.yaml +++ /dev/null @@ -1,8 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: flytepropeller - namespace: flyte - annotations: - # TODO the role to use - eks.amazonaws.com/role-arn: "arn:aws:iam::111222333456:role/flyte-operator" diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml deleted file mode 100644 index b29d932a808..00000000000 --- a/kustomize/overlays/eks/kustomization.yaml +++ /dev/null @@ -1,51 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -bases: - # All the resources that make up the deployment - - flyte - ######## - # Operators to be enabled - ############# - # Dependencies to be configured - # In local we use minio, but in cloud environment use S3 / GCS / AFS / Oracle Blob store etc - # This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) - # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's - # Add node ports for ease of use locally - - ../../base/operators/spark - - ../../base/operators/kfoperators/pytorch - # Optional dependency - - ../../base/addons/redis - -# Images that should be used -images: - # FlyteAdmin - - name: flyteadmin # match images with this name - newTag: v1.11.0-b1 # FLYTEADMIN_TAG override the tag - newName: cr.flyte.org/flyteorg/flyteadmin # override the name - # FlyteConsole - - name: flyteconsole # match images with this name - newTag: v1.11.0 # FLYTECONSOLE_TAG the tag - newName: cr.flyte.org/flyteorg/flyteconsole # override the namep - # Flyte DataCatalog - - name: datacatalog # match images with this name - newTag: v1.11.0-b1 # DATACATALOG_TAG override the tag - newName: cr.flyte.org/flyteorg/datacatalog # override the name - # FlytePropeller - - name: flytepropeller # match images with this name - newTag: v1.11.0-b1 # FLYTEPROPELLER_TAG override the tag - newName: cr.flyte.org/flyteorg/flytepropeller # override the name - # Webhook - - name: webhook # match images with this name - newTag: v1.11.0-b1 # FLYTEPROPELLER_TAG override the tag - newName: cr.flyte.org/flyteorg/flytepropeller # override the name - # Override postgres image to use alpine based (rather smaller) docker image - - name: postgres - newTag: 13-21.04_beta - newName: ecr.flyte.org/ubuntu/postgres - - name: minio - newTag: 2021.10.13-debian-10-r0 - newName: ecr.flyte.org/bitnami/minio - - name: redis - newTag: 6.2.5-debian-10-r59 - newName: ecr.flyte.org/bitnami/redis diff --git a/kustomize/overlays/gcp/README.md b/kustomize/overlays/gcp/README.md deleted file mode 100644 index 450f73dbfd1..00000000000 --- a/kustomize/overlays/gcp/README.md +++ /dev/null @@ -1,95 +0,0 @@ -[All Overlays](./) -# :beta: Google Cloud Platform Overlay - -This overlay serves as an example to bootstrap Flyte setup on Google Cloud Platform (GCP). It is not -designed to work out of the box due to the need of GCP resources. Please follow the instruction -below to further configure. - -_Hint_: searching `TODO:` through this directory would help to understand what needs to be done. - -## Cloud SQL - -[Cloud SQL](https://cloud.google.com/sql) is used as persistence layer. To set it up, please -follow standard GCP documentation. - -A few things are required for this overlay to function: - -* Two databases named as `flyte` and `datacatalog` -* A database user named as `flyte` -* Password of the database user can be added to either to [kustomization.yaml](kustomization.yaml) or you can create a new file and change the secretGenerator tag to use files. (Refer to kustomize documentation) -* Service account(s) associated with `flyteadmin` and `datacatalog` pods (either as GKE cluster - service account or through workload identity) should have `Cloud SQL Editor` role - -To securely access Cloud SQL instance, [Cloud SQL -Proxy](https://cloud.google.com/sql/docs/postgres/connect-admin-proxy) is launched as a pod sitting -in between Flyte and Cloud SQL instance. - -The kustomization files can be found under [cloudsqlproxy](dependencies/cloudsqlproxy/). Please note that one -needs to replace `` and `` accordingly in -[dependencies/cloudsqlproxy/deployment.yaml](cloudsqlproxy/deployment.yaml). - -## Create GCS Storage -1. Create a GCS bucket named as `flyte` in a GCP project. -1. Replace `` in [config/common/storage.yaml](flyte/config/common/storage.yaml) with the GCP project ID and if using a bucket other than Flyte then replace the bucket name too - -## flyteadmin - -flyteadmin configuration is derived from the [single cluster](../../base/single_cluster) overlay, with only modification to [database configuration db.yaml](flyte/config/admin/db.yaml) - -If one has followed [Cloud SQL](#cloud-sql) section, there is nothing to be done for database. - -**Advanced / OPTIONAL** -1. The default CORS setting in flyteAdmin allows cross origin requests. A more secure way would be to allow requests only from the expected domain. To do this, you will have to create a new *server.yaml* -similar to [base/single_cluster/headless/config](../../base/single_cluster/headless/config) under config/admin and then set -`server -> security -> allowedOrigins`. - -1. flyteadmin (including metrics endpoint) is exposed as a service using [internal load balancer](https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing). - -## flyteconsole - -[flyteconsole configmap](console/config.yaml) needs to be updated with flyteadmin internal load -balancer IP address or the DNS name associated with it if any. - -flyteconsole is exposed as a service using [internal load balancer](https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing). - -## flytepropeller - -flytepropeller configuration is derived from the [single cluster](../../base/single_cluster) overlay, with only modification to the config for performance tuning and logs -For logs configuration Replace `` in [config/propeller/plugins/task_logs.yaml](flyte/config/propeller/plugins/task_logs.yaml) with the GCP project ID - -Some important points - -* Storage configuration is shared with Admin and Catalog. Ideally in production Propeller should have its own configuration with real high cache size. - -* By default, three plugins are enabled: -1. container -2. k8s-array -3. sidecar - -* flytepropeller metrics endpoint is exposed as a service using [internal load balancer](https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing). - -## datacatalog - -datacatalog configuration is derived from the [single cluster](../../base/single_cluster) overlay, with only modification to [database configuration db.yaml](flyte/config/datacatalog/db.yaml) - -If one has followed [Cloud SQL](#cloud-sql) section, there is nothing to be done for database. - -datacatalog metrics endpoint is exposed as a service using [internal load -balancer](https://cloud.google.com/kubernetes-engine/docs/how-to/internal-load-balancing). - - -## How to build your overlay -To build your overlay there are 2 options -1. Build it in your own repo Example coming soon :construction: -1. hack it in your clone of Flyte repo in place of GCP overlay. In this case just navigate to the root of the repo and run -```bash -$ make kustomize -``` -If all goes well a new overlay composite should be generated in [/deployment/gcp/flyte_generated.yaml](../../../deployment/gcp/flyte_generated.yaml) - -## Now ship it - -``` shell -make -kubectl apply -f deployment/gcp/flyte_generated.yaml -``` diff --git a/kustomize/overlays/gcp/flyte/admin/deployment.yaml b/kustomize/overlays/gcp/flyte/admin/deployment.yaml deleted file mode 100644 index b581a3dba1f..00000000000 --- a/kustomize/overlays/gcp/flyte/admin/deployment.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flyteadmin - namespace: flyte -spec: - template: - spec: - containers: - - name: flyteadmin - resources: - limits: - memory: "1Gi" - cpu: "2" - ephemeral-storage: "1Gi" diff --git a/kustomize/overlays/gcp/flyte/admin/service.yaml b/kustomize/overlays/gcp/flyte/admin/service.yaml deleted file mode 100644 index 1af1d10ce82..00000000000 --- a/kustomize/overlays/gcp/flyte/admin/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: flyteadmin - namespace: flyte - annotations: - cloud.google.com/load-balancer-type: Internal -spec: - type: LoadBalancer - ports: - - name: redoc - protocol: TCP - port: 87 - targetPort: 8087 - - name: http-metrics - protocol: TCP - port: 10254 diff --git a/kustomize/overlays/gcp/flyte/config/admin/db.yaml b/kustomize/overlays/gcp/flyte/config/admin/db.yaml deleted file mode 100644 index 4e2741221d9..00000000000 --- a/kustomize/overlays/gcp/flyte/config/admin/db.yaml +++ /dev/null @@ -1,6 +0,0 @@ -database: - port: 5432 - username: flyte - host: cloudsqlproxy - dbname: flyte - passwordPath: /etc/db/pass.txt diff --git a/kustomize/overlays/gcp/flyte/config/admin/task_resource_defaults.yaml b/kustomize/overlays/gcp/flyte/config/admin/task_resource_defaults.yaml deleted file mode 100644 index 06870c232bf..00000000000 --- a/kustomize/overlays/gcp/flyte/config/admin/task_resource_defaults.yaml +++ /dev/null @@ -1,10 +0,0 @@ -task_resources: - defaults: - cpu: 1000m - memory: 1000Mi - storage: 1000Mi - limits: - cpu: 2 - memory: 8Gi - storage: 2000Mi - gpu: 1 diff --git a/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ad_spark-role.yaml b/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ad_spark-role.yaml deleted file mode 100644 index 51998e30510..00000000000 --- a/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ad_spark-role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: spark-role - namespace: {{ namespace }} -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - '*' -- apiGroups: - - "" - resources: - - services - verbs: - - '*' -- apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' diff --git a/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ae_spark-service-account.yaml b/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ae_spark-service-account.yaml deleted file mode 100644 index dddd8d27428..00000000000 --- a/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ae_spark-service-account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: spark - namespace: {{ namespace }} diff --git a/kustomize/overlays/gcp/flyte/config/clusterresource-templates/af_spark-role-binding.yaml b/kustomize/overlays/gcp/flyte/config/clusterresource-templates/af_spark-role-binding.yaml deleted file mode 100644 index 2e9c8ae7655..00000000000 --- a/kustomize/overlays/gcp/flyte/config/clusterresource-templates/af_spark-role-binding.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: spark-role-binding - namespace: {{ namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role -subjects: -- kind: ServiceAccount - name: spark - namespace: {{ namespace }} - diff --git a/kustomize/overlays/gcp/flyte/config/common/storage.yaml b/kustomize/overlays/gcp/flyte/config/common/storage.yaml deleted file mode 100644 index 6f20d08dcdc..00000000000 --- a/kustomize/overlays/gcp/flyte/config/common/storage.yaml +++ /dev/null @@ -1,19 +0,0 @@ -storage: - type: stow - stow: - kind: google - config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/cloud-platform - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" - # NOTE this cache configuration is purely for propeller. But since we are having a common storage - # config, we are configuring this value. In production create a separate storage config for - # propeller and increase the cache size - cache: - max_size_mbs: 512 - target_gc_percent: 70 - limits: - maxDownloadMBs: 10 diff --git a/kustomize/overlays/gcp/flyte/config/datacatalog/db.yaml b/kustomize/overlays/gcp/flyte/config/datacatalog/db.yaml deleted file mode 100644 index abfb3304592..00000000000 --- a/kustomize/overlays/gcp/flyte/config/datacatalog/db.yaml +++ /dev/null @@ -1,7 +0,0 @@ -database: - port: 5432 - username: flyte - host: cloudsqlproxy - dbname: datacatalog - options: "sslmode=disable" - passwordPath: /etc/db/pass.txt diff --git a/kustomize/overlays/gcp/flyte/config/propeller/core.yaml b/kustomize/overlays/gcp/flyte/config/propeller/core.yaml deleted file mode 100644 index b49a5f41887..00000000000 --- a/kustomize/overlays/gcp/flyte/config/propeller/core.yaml +++ /dev/null @@ -1,40 +0,0 @@ -propeller: - rawoutput-prefix: gs://my-s3-bucket/ - metadata-prefix: metadata/propeller - workers: 40 - gc-interval: 12h - max-workflow-retries: 50 - workflow-reeval-duration: 30s - downstream-eval-duration: 30s - limit-namespace: "all" - prof-port: 10254 - metrics-prefix: flyte - enable-admin-launcher: true - leader-election: - lock-config-map: - name: propeller-leader - namespace: flyte - enabled: true - lease-duration: 15s - renew-deadline: 10s - retry-period: 2s - kube-client-config: - qps: 100 - burst: 25 - timeout: 30s - queue: - type: batch - batching-interval: 2s - batch-size: -1 - queue: - type: maxof - rate: 100 - capacity: 1000 - base-delay: 5s - max-delay: 120s - sub-queue: - type: bucket - rate: 100 - capacity: 1000 - workflowStore: - policy: "ResourceVersionCache" diff --git a/kustomize/overlays/gcp/flyte/config/propeller/enabled_plugins.yaml b/kustomize/overlays/gcp/flyte/config/propeller/enabled_plugins.yaml deleted file mode 100644 index 9e4644ac18a..00000000000 --- a/kustomize/overlays/gcp/flyte/config/propeller/enabled_plugins.yaml +++ /dev/null @@ -1,15 +0,0 @@ -tasks: - max-plugin-phase-versions: 1000000 - task-plugins: - enabled-plugins: - - container - - sidecar - - spark - - k8s-array - - pytorch - default-for-task-types: - container: container - sidecar: sidecar - spark: spark - container_array: k8s-array - pytorch: pytorch diff --git a/kustomize/overlays/gcp/flyte/config/propeller/plugins/catalog_cache.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/catalog_cache.yaml deleted file mode 100644 index 3678fbbc895..00000000000 --- a/kustomize/overlays/gcp/flyte/config/propeller/plugins/catalog_cache.yaml +++ /dev/null @@ -1,6 +0,0 @@ -plugins: - catalogCache: - reader: - maxItems: 10000 - writer: - maxItems: 10000 diff --git a/kustomize/overlays/gcp/flyte/config/propeller/plugins/k8s.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/k8s.yaml deleted file mode 100644 index 8fe068397fe..00000000000 --- a/kustomize/overlays/gcp/flyte/config/propeller/plugins/k8s.yaml +++ /dev/null @@ -1,8 +0,0 @@ -plugins: - k8s: - default-env-vars: - - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - - FLYTE_AWS_ACCESS_KEY_ID: minio - - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage - default-cpus: 100m - default-memory: 100Mi diff --git a/kustomize/overlays/gcp/flyte/config/propeller/plugins/qubole.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/qubole.yaml deleted file mode 100644 index ea34d2752b9..00000000000 --- a/kustomize/overlays/gcp/flyte/config/propeller/plugins/qubole.yaml +++ /dev/null @@ -1,3 +0,0 @@ -plugins: - qubole: - quboleTokenKey: "FLYTE_QUBOLE_CLIENT_TOKEN" diff --git a/kustomize/overlays/gcp/flyte/config/propeller/plugins/spark.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/spark.yaml deleted file mode 100644 index c1ffb9209a7..00000000000 --- a/kustomize/overlays/gcp/flyte/config/propeller/plugins/spark.yaml +++ /dev/null @@ -1,16 +0,0 @@ -plugins: - spark: - spark-config-default: - - spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version: "2" - - spark.kubernetes.allocation.batch.size: "50" - - spark.hadoop.fs.s3a.acl.default: "BucketOwnerFullControl" - - spark.hadoop.fs.s3n.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3n.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem" - - spark.hadoop.fs.AbstractFileSystem.s3a.impl: "org.apache.hadoop.fs.s3a.S3A" - - spark.hadoop.fs.s3a.multipart.threshold: "536870912" - - spark.blacklist.enabled: "true" - - spark.blacklist.timeout: "5m" - - spark.task.maxfailures: "8" diff --git a/kustomize/overlays/gcp/flyte/config/propeller/plugins/task_logs.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/task_logs.yaml deleted file mode 100644 index 5a9d0714c5e..00000000000 --- a/kustomize/overlays/gcp/flyte/config/propeller/plugins/task_logs.yaml +++ /dev/null @@ -1,10 +0,0 @@ -plugins: - logs: - # Log links can link to multiple options - # #1 Kubernetes dashboard is disabled in GCP - kubernetes-enabled: false - # #2 GCP stackdriver - stackdriver-enabled: true - # TODO: replace with the GCP project ID - gcp-project: - stackdriver-logresourcename: k8s_container diff --git a/kustomize/overlays/gcp/flyte/console/service.yaml b/kustomize/overlays/gcp/flyte/console/service.yaml deleted file mode 100644 index 2d3ad158aa2..00000000000 --- a/kustomize/overlays/gcp/flyte/console/service.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - annotations: - cloud.google.com/load-balancer-type: Internal - name: flyteconsole - namespace: flyte -spec: - selector: - app: flyteconsole - type: LoadBalancer - ports: - - name: redoc - protocol: TCP - port: 87 - targetPort: 8087 - - name: http-metrics - protocol: TCP - port: 10254 diff --git a/kustomize/overlays/gcp/flyte/datacatalog/deployment.yaml b/kustomize/overlays/gcp/flyte/datacatalog/deployment.yaml deleted file mode 100644 index 1035dac5a0d..00000000000 --- a/kustomize/overlays/gcp/flyte/datacatalog/deployment.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: datacatalog - namespace: flyte -spec: - template: - spec: - containers: - - name: datacatalog - resources: - limits: - memory: "1Gi" - cpu: "2" - ephemeral-storage: "1000Mi" diff --git a/kustomize/overlays/gcp/flyte/datacatalog/service.yaml b/kustomize/overlays/gcp/flyte/datacatalog/service.yaml deleted file mode 100644 index 70e95295ab2..00000000000 --- a/kustomize/overlays/gcp/flyte/datacatalog/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: datacatalog-metrics - namespace: flyte - annotations: - cloud.google.com/load-balancer-type: Internal -spec: - selector: - app: datacatalog - type: LoadBalancer - ports: - - name: http-metrics - protocol: TCP - port: 10254 diff --git a/kustomize/overlays/gcp/flyte/kustomization.yaml b/kustomize/overlays/gcp/flyte/kustomization.yaml deleted file mode 100644 index 2e6e7e7cd8f..00000000000 --- a/kustomize/overlays/gcp/flyte/kustomization.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -# Override the namespace -namespace: flyte - -bases: -- ../../../base/single_cluster/complete - -resources: -- datacatalog/service.yaml -- propeller/service.yaml - -patchesStrategicMerge: -- admin/deployment.yaml -- admin/service.yaml -- datacatalog/deployment.yaml -- propeller/deployment.yaml -- console/service.yaml - -# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". - -configMapGenerator: -- behavior: merge - files: - - ./config/admin/db.yaml - - ./config/admin/task_resource_defaults.yaml - - ./config/common/storage.yaml - name: flyte-admin-config -- behavior: merge - files: - - ./config/clusterresource-templates/ad_spark-role.yaml - - ./config/clusterresource-templates/ae_spark-service-account.yaml - - ./config/clusterresource-templates/af_spark-role-binding.yaml - name: clusterresource-template -- behavior: merge - files: - - ./config/propeller/core.yaml - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/plugins/catalog_cache.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/propeller/plugins/qubole.yaml - - ./config/propeller/plugins/spark.yaml - - ./config/propeller/plugins/task_logs.yaml - - ./config/common/storage.yaml - name: flyte-propeller-config -- behavior: merge - files: - - ./config/common/storage.yaml - - ./config/datacatalog/db.yaml - name: datacatalog-config - -########### -# Use this to record the password for the Google CloudSQL Database -secretGenerator: -- name: db-pass - behavior: merge - literals: - - pass.txt="yourpassword" diff --git a/kustomize/overlays/gcp/flyte/propeller/deployment.yaml b/kustomize/overlays/gcp/flyte/propeller/deployment.yaml deleted file mode 100644 index 9a46601b53c..00000000000 --- a/kustomize/overlays/gcp/flyte/propeller/deployment.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flytepropeller - namespace: flyte -spec: - template: - spec: - containers: - - name: flytepropeller - resources: - limits: - memory: "4Gi" - cpu: "2" - ephemeral-storage: "1Gi" diff --git a/kustomize/overlays/gcp/flyte/propeller/service.yaml b/kustomize/overlays/gcp/flyte/propeller/service.yaml deleted file mode 100644 index e80a8746e6d..00000000000 --- a/kustomize/overlays/gcp/flyte/propeller/service.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: flytepropeller - namespace: flyte - annotations: - cloud.google.com/load-balancer-type: Internal -spec: - type: LoadBalancer - selector: - app: flytepropeller - ports: - - name: http-metrics - protocol: TCP - port: 10254 diff --git a/kustomize/overlays/gcp/kustomization.yaml b/kustomize/overlays/gcp/kustomization.yaml deleted file mode 100644 index e8dc107ffa8..00000000000 --- a/kustomize/overlays/gcp/kustomization.yaml +++ /dev/null @@ -1,53 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -bases: - # All the resources that make up the deployment - - flyte - ######## - # Operators to be enabled - ############# - # Dependencies to be configured - # In local we use minio, but in cloud environment use S3 / GCS / AFS / Oracle Blob store etc - # This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) - # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's - # Add node ports for ease of use locally - - ../../base/ingress - - ../../base/operators/spark - - ../../base/operators/kfoperators/pytorch - - ../../base/addons/cloudsqlproxy - # Optional dependency - - ../../base/addons/redis - -# Images that should be used -images: - # FlyteAdmin - - name: flyteadmin # match images with this name - newTag: v1.10.7-b2 # FLYTEADMIN_TAG override the tag - newName: cr.flyte.org/flyteorg/flyteadmin # override the name - # FlyteConsole - - name: flyteconsole # match images with this name - newTag: v1.10.2 # FLYTECONSOLE_TAG the tag - newName: cr.flyte.org/flyteorg/flyteconsole # override the namep - # Flyte DataCatalog - - name: datacatalog # match images with this name - newTag: v1.10.7-b2 # DATACATALOG_TAG override the tag - newName: cr.flyte.org/flyteorg/datacatalog # override the name - # FlytePropeller - - name: flytepropeller # match images with this name - newTag: v1.10.7-b2 # FLYTEPROPELLER_TAG override the tag - newName: cr.flyte.org/flyteorg/flytepropeller # override the name - # Webhook - - name: webhook # match images with this name - newTag: v1.10.7-b2 # FLYTEPROPELLER_TAG override the tag - newName: cr.flyte.org/flyteorg/flytepropeller # override the name - # Override postgres image to use alpine based (rather smaller) docker image - - name: postgres - newTag: 13-21.04_beta - newName: ecr.flyte.org/ubuntu/postgres - - name: minio - newTag: 2021.10.13-debian-10-r0 - newName: ecr.flyte.org/bitnami/minio - - name: redis - newTag: 6.2.5-debian-10-r59 - newName: ecr.flyte.org/bitnami/redis diff --git a/kustomize/overlays/sandbox/README.md b/kustomize/overlays/sandbox/README.md deleted file mode 100644 index 82dbef46f14..00000000000 --- a/kustomize/overlays/sandbox/README.md +++ /dev/null @@ -1,4 +0,0 @@ -[All overlays](../) -# Flyte Sandbox deployment - Kustomize overlay -This module creates a standalone Flyte cluster with all its dependencies. This is a great starting point for trying out Flyte and experimenting with Flyte. The overlay can be Kustomized and made to -scale higher, but as is is not intended for production use diff --git a/kustomize/overlays/sandbox/flyte/admin/deployment.yaml b/kustomize/overlays/sandbox/flyte/admin/deployment.yaml deleted file mode 100644 index 857439a6dca..00000000000 --- a/kustomize/overlays/sandbox/flyte/admin/deployment.yaml +++ /dev/null @@ -1,105 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flyteadmin - namespace: flyte -spec: - template: - spec: - volumes: - - name: resource-templates - configMap: - name: clusterresource-template - initContainers: - - name: check-db-ready - image: postgres:13-21.04_beta - command: [ - "sh", - "-c", - "until pg_isready -h postgres -p 5432; - do echo waiting for database; sleep 2; done;", - ] - securityContext: - runAsUser: 999 - - name: run-migrations - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: - [ - "flyteadmin", - "--config", - "/etc/flyte/config/*.yaml", - "migrate", - "run", - ] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: seed-projects - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: - [ - "flyteadmin", - "--config", - "/etc/flyte/config/*.yaml", - "migrate", - "seed-projects", - "flytesnacks", - "flyteexamples", - ] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: sync-cluster-resources - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: - [ - "flyteadmin", - "--config", - "/etc/flyte/config/*.yaml", - "clusterresource", - "sync", - ] - volumeMounts: - - name: resource-templates - mountPath: /etc/flyte/clusterresource/templates - - name: config-volume - mountPath: /etc/flyte/config - containers: - - name: flyteadmin - resources: - limits: - memory: "200Mi" - cpu: "0.1" - ephemeral-storage: "100Mi" - - name: sync-cluster-resources-goroutine - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: - [ - "flyteadmin", - "--config", - "/etc/flyte/config/*.yaml", - "clusterresource", - "run", - ] - volumeMounts: - - name: resource-templates - mountPath: /etc/flyte/clusterresource/templates - - name: config-volume - mountPath: /etc/flyte/config - ---- -apiVersion: v1 -kind: Service -metadata: - name: flyteadmin - namespace: flyte -spec: - ports: - - name: redoc - protocol: TCP - port: 87 - targetPort: 8087 diff --git a/kustomize/overlays/sandbox/flyte/config/admin/db.yaml b/kustomize/overlays/sandbox/flyte/config/admin/db.yaml deleted file mode 100644 index c71ee6e4bb4..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/admin/db.yaml +++ /dev/null @@ -1,6 +0,0 @@ -database: - port: 5432 - username: postgres - host: postgres - dbname: postgres - options: sslmode=disable diff --git a/kustomize/overlays/sandbox/flyte/config/admin/task_resource_defaults.yaml b/kustomize/overlays/sandbox/flyte/config/admin/task_resource_defaults.yaml deleted file mode 100644 index f3fd770e44c..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/admin/task_resource_defaults.yaml +++ /dev/null @@ -1,10 +0,0 @@ -task_resources: - defaults: - cpu: 100m - memory: 200Mi - storage: 5Mi - limits: - cpu: 2 - memory: 1Gi - storage: 20Mi - gpu: 1 diff --git a/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ad_spark-role.yaml b/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ad_spark-role.yaml deleted file mode 100644 index 51998e30510..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ad_spark-role.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: spark-role - namespace: {{ namespace }} -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - '*' -- apiGroups: - - "" - resources: - - services - verbs: - - '*' -- apiGroups: - - "" - resources: - - configmaps - verbs: - - '*' diff --git a/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ae_spark-service-account.yaml b/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ae_spark-service-account.yaml deleted file mode 100644 index dddd8d27428..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ae_spark-service-account.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: spark - namespace: {{ namespace }} diff --git a/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/af_spark-role-binding.yaml b/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/af_spark-role-binding.yaml deleted file mode 100644 index 6fa4f5cb8a1..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/af_spark-role-binding.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: spark-role-binding - namespace: {{ namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: spark-role -subjects: -- kind: ServiceAccount - name: spark - namespace: {{ namespace }} diff --git a/kustomize/overlays/sandbox/flyte/config/common/logger.yaml b/kustomize/overlays/sandbox/flyte/config/common/logger.yaml deleted file mode 100644 index a6854ed6cec..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/common/logger.yaml +++ /dev/null @@ -1,3 +0,0 @@ -logger: - show-source: true - level: 2 diff --git a/kustomize/overlays/sandbox/flyte/config/common/storage.yaml b/kustomize/overlays/sandbox/flyte/config/common/storage.yaml deleted file mode 100644 index 16e6f527553..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/common/storage.yaml +++ /dev/null @@ -1,11 +0,0 @@ -storage: - type: minio - connection: - access-key: minio - auth-type: accesskey - secret-key: miniostorage - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - container: "my-s3-bucket" - diff --git a/kustomize/overlays/sandbox/flyte/config/datacatalog/db.yaml b/kustomize/overlays/sandbox/flyte/config/datacatalog/db.yaml deleted file mode 100644 index 76ba812d63a..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/datacatalog/db.yaml +++ /dev/null @@ -1,6 +0,0 @@ -database: - port: 5432 - username: postgres - host: postgres - dbname: datacatalog - options: sslmode=disable diff --git a/kustomize/overlays/sandbox/flyte/config/propeller/enabled_plugins.yaml b/kustomize/overlays/sandbox/flyte/config/propeller/enabled_plugins.yaml deleted file mode 100644 index 0978255017d..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/propeller/enabled_plugins.yaml +++ /dev/null @@ -1,10 +0,0 @@ -tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array diff --git a/kustomize/overlays/sandbox/flyte/config/propeller/plugins/k8s.yaml b/kustomize/overlays/sandbox/flyte/config/propeller/plugins/k8s.yaml deleted file mode 100644 index 9f06f53fdbc..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/propeller/plugins/k8s.yaml +++ /dev/null @@ -1,8 +0,0 @@ -plugins: - k8s: - default-env-vars: - - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - - FLYTE_AWS_ACCESS_KEY_ID: minio - - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage - default-cpus: 100m - default-memory: 200Mi diff --git a/kustomize/overlays/sandbox/flyte/config/propeller/plugins/task_logs.yaml b/kustomize/overlays/sandbox/flyte/config/propeller/plugins/task_logs.yaml deleted file mode 100644 index 2768f8c95df..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/propeller/plugins/task_logs.yaml +++ /dev/null @@ -1,4 +0,0 @@ -plugins: - logs: - kubernetes-enabled: true - kubernetes-template-uri: "http://localhost:30082/#/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}" diff --git a/kustomize/overlays/sandbox/flyte/config/propeller/resource_manager.yaml b/kustomize/overlays/sandbox/flyte/config/propeller/resource_manager.yaml deleted file mode 100644 index e08575c90bb..00000000000 --- a/kustomize/overlays/sandbox/flyte/config/propeller/resource_manager.yaml +++ /dev/null @@ -1,3 +0,0 @@ -propeller: - resourcemanager: - type: noop diff --git a/kustomize/overlays/sandbox/flyte/kustomization.yaml b/kustomize/overlays/sandbox/flyte/kustomization.yaml deleted file mode 100644 index 94eb948fa2f..00000000000 --- a/kustomize/overlays/sandbox/flyte/kustomization.yaml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -# Override the namespace -namespace: flyte - -bases: - - ../../../base/single_cluster/complete - -resources: - - propeller/webhook-secret.yaml - -patchesStrategicMerge: - - admin/deployment.yaml - - propeller/webhook.yaml - -configMapGenerator: - - behavior: merge - files: - - ./config/admin/db.yaml - - ./config/admin/task_resource_defaults.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - name: flyte-admin-config - - - behavior: merge - files: - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/resource_manager.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/propeller/plugins/task_logs.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - name: flyte-propeller-config - - - behavior: merge - files: - - ./config/common/storage.yaml - - ./config/common/logger.yaml - - ./config/datacatalog/db.yaml - name: datacatalog-config diff --git a/kustomize/overlays/sandbox/flyte/propeller/webhook-secret.yaml b/kustomize/overlays/sandbox/flyte/propeller/webhook-secret.yaml deleted file mode 100644 index 4029be3716f..00000000000 --- a/kustomize/overlays/sandbox/flyte/propeller/webhook-secret.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -data: - password: bXl1c2Vy - user_secret: bXlzZWNyZXQ= - username: bXl1c2Vy -kind: Secret -metadata: - name: user-info - namespace: flyte -type: Opaque diff --git a/kustomize/overlays/sandbox/flyte/propeller/webhook.yaml b/kustomize/overlays/sandbox/flyte/propeller/webhook.yaml deleted file mode 100644 index a659e88141d..00000000000 --- a/kustomize/overlays/sandbox/flyte/propeller/webhook.yaml +++ /dev/null @@ -1,18 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flyte-pod-webhook - namespace: flyte -spec: - template: - spec: - containers: - - name: webhook - volumeMounts: - - name: sample-secrets - mountPath: /etc/secrets/user-info - readOnly: true - volumes: - - name: sample-secrets - secret: - secretName: user-info diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml deleted file mode 100644 index e03f19ed1b5..00000000000 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ /dev/null @@ -1,55 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -bases: - # All the resources that make up the deployment - - flyte - ######## - # Operators to be enabled - ############# - # Dependencies to be configured - # In local we use minio, but in cloud environment use S3 / GCS / AFS / Oracle Blob store etc - # This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) - # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's - # Add node ports for ease of use locally - - ../../base/ingress - - ../../base/addons/database - - ../../base/addons/kubernetes_dashboard - - ../../base/addons/storage - - ../../base/addons/contour_ingress_controller - -resources: - - nodeport-services.yaml - -# Images that should be used -images: - # FlyteAdmin - - name: flyteadmin # match images with this name - newTag: v1.1.118 # FLYTEADMIN_TAG override the tag - newName: cr.flyte.org/flyteorg/flyteadmin # override the name - # FlyteConsole - - name: flyteconsole # match images with this name - newTag: v1.9.2 # FLYTECONSOLE_TAG the tag - newName: cr.flyte.org/flyteorg/flyteconsole # override the namep - # Flyte DataCatalog - - name: datacatalog # match images with this name - newTag: v1.0.51 # DATACATALOG_TAG override the tag - newName: cr.flyte.org/flyteorg/datacatalog # override the name - # FlytePropeller - - name: flytepropeller # match images with this name - newTag: v1.1.116 # FLYTEPROPELLER_TAG override the tag - newName: cr.flyte.org/flyteorg/flytepropeller # override the name - # Webhook - - name: webhook # match images with this name - newTag: v1.1.116 # FLYTEPROPELLER_TAG override the tag - newName: cr.flyte.org/flyteorg/flytepropeller # override the name - # Override postgres image to use alpine based (rather smaller) docker image - - name: postgres - newTag: 13-21.04_beta - newName: ecr.flyte.org/ubuntu/postgres - - name: minio - newTag: 2021.10.13-debian-10-r0 - newName: ecr.flyte.org/bitnami/minio - - name: redis - newTag: 6.2.5-debian-10-r59 - newName: ecr.flyte.org/bitnami/redis diff --git a/kustomize/overlays/sandbox/nodeport-services.yaml b/kustomize/overlays/sandbox/nodeport-services.yaml deleted file mode 100644 index 6439a773fce..00000000000 --- a/kustomize/overlays/sandbox/nodeport-services.yaml +++ /dev/null @@ -1,51 +0,0 @@ -# For docker-desktop at least, the range of valid ports is 30000-32767, which is why we're constrained to these -# odd port numbers. Since the base ingress runs on 30081, the K8s dashboard runs on 30082, these just continue from there. -# These nodeports are exposed to save contributors the trouble of port forwarding when running locally. - -apiVersion: v1 -kind: Service -metadata: - labels: - app: postgres - name: postgres-direct - namespace: flyte -spec: - ports: - - nodePort: 30083 - port: 5432 - protocol: TCP - selector: - app: postgres - type: NodePort ---- -apiVersion: v1 -kind: Service -metadata: - labels: - app: minio - name: minio-direct - namespace: flyte -spec: - ports: - - nodePort: 30084 - port: 9000 - protocol: TCP - selector: - app: minio - type: NodePort ---- -#apiVersion: v1 -#kind: Service -#metadata: -#labels: -#k8s-app: kubernetes-dashboard -#name: dashboard-direct -#namespace: flyte -#spec: -#ports: -#- nodePort: 30082 -#port: 8443 -#protocol: TCP -#selector: -#k8s-app: kubernetes-dashboard -#type: NodePort diff --git a/kustomize/overlays/test/flyte/admin/deployment.yaml b/kustomize/overlays/test/flyte/admin/deployment.yaml deleted file mode 100644 index d0a5abfacfe..00000000000 --- a/kustomize/overlays/test/flyte/admin/deployment.yaml +++ /dev/null @@ -1,64 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: flyteadmin - namespace: flyte -spec: - template: - spec: - volumes: - - name: resource-templates - configMap: - name: clusterresource-template - initContainers: - - name: check-db-ready - image: postgres:10.1 - command: ['sh', '-c', - 'until pg_isready -h postgres -p 5432; - do echo waiting for database; sleep 2; done;'] - securityContext: - runAsUser: 999 - - name: run-migrations - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", - "migrate", "run"] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: seed-projects - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", - "migrate", "seed-projects", "flytetester", "flytesnacks"] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: sync-cluster-resources - image: flyteadmin:v0.4.13 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] - volumeMounts: - - name: resource-templates - mountPath: /etc/flyte/clusterresource/templates - - name: config-volume - mountPath: /etc/flyte/config - containers: - - name: flyteadmin - resources: - limits: - memory: "200Mi" - cpu: "0.1" - ephemeral-storage: "100Mi" ---- -apiVersion: v1 -kind: Service -metadata: - name: flyteadmin - namespace: flyte -spec: - ports: - - name: redoc - protocol: TCP - port: 87 - targetPort: 8087 diff --git a/kustomize/overlays/test/flyte/config/admin/db.yaml b/kustomize/overlays/test/flyte/config/admin/db.yaml deleted file mode 100644 index c71ee6e4bb4..00000000000 --- a/kustomize/overlays/test/flyte/config/admin/db.yaml +++ /dev/null @@ -1,6 +0,0 @@ -database: - port: 5432 - username: postgres - host: postgres - dbname: postgres - options: sslmode=disable diff --git a/kustomize/overlays/test/flyte/config/common/logger.yaml b/kustomize/overlays/test/flyte/config/common/logger.yaml deleted file mode 100644 index a6854ed6cec..00000000000 --- a/kustomize/overlays/test/flyte/config/common/logger.yaml +++ /dev/null @@ -1,3 +0,0 @@ -logger: - show-source: true - level: 2 diff --git a/kustomize/overlays/test/flyte/config/common/storage.yaml b/kustomize/overlays/test/flyte/config/common/storage.yaml deleted file mode 100644 index 16e6f527553..00000000000 --- a/kustomize/overlays/test/flyte/config/common/storage.yaml +++ /dev/null @@ -1,11 +0,0 @@ -storage: - type: minio - connection: - access-key: minio - auth-type: accesskey - secret-key: miniostorage - disable-ssl: true - endpoint: http://minio.flyte.svc.cluster.local:9000 - region: us-east-1 - container: "my-s3-bucket" - diff --git a/kustomize/overlays/test/flyte/config/datacatalog/db.yaml b/kustomize/overlays/test/flyte/config/datacatalog/db.yaml deleted file mode 100644 index 76ba812d63a..00000000000 --- a/kustomize/overlays/test/flyte/config/datacatalog/db.yaml +++ /dev/null @@ -1,6 +0,0 @@ -database: - port: 5432 - username: postgres - host: postgres - dbname: datacatalog - options: sslmode=disable diff --git a/kustomize/overlays/test/flyte/config/propeller/enabled_plugins.yaml b/kustomize/overlays/test/flyte/config/propeller/enabled_plugins.yaml deleted file mode 100644 index 0978255017d..00000000000 --- a/kustomize/overlays/test/flyte/config/propeller/enabled_plugins.yaml +++ /dev/null @@ -1,10 +0,0 @@ -tasks: - task-plugins: - enabled-plugins: - - container - - sidecar - - k8s-array - default-for-task-types: - container: container - sidecar: sidecar - container_array: k8s-array diff --git a/kustomize/overlays/test/flyte/config/propeller/plugins/k8s.yaml b/kustomize/overlays/test/flyte/config/propeller/plugins/k8s.yaml deleted file mode 100644 index 8fe068397fe..00000000000 --- a/kustomize/overlays/test/flyte/config/propeller/plugins/k8s.yaml +++ /dev/null @@ -1,8 +0,0 @@ -plugins: - k8s: - default-env-vars: - - FLYTE_AWS_ENDPOINT: "http://minio.flyte:9000" - - FLYTE_AWS_ACCESS_KEY_ID: minio - - FLYTE_AWS_SECRET_ACCESS_KEY: miniostorage - default-cpus: 100m - default-memory: 100Mi diff --git a/kustomize/overlays/test/flyte/config/propeller/resource_manager.yaml b/kustomize/overlays/test/flyte/config/propeller/resource_manager.yaml deleted file mode 100644 index e08575c90bb..00000000000 --- a/kustomize/overlays/test/flyte/config/propeller/resource_manager.yaml +++ /dev/null @@ -1,3 +0,0 @@ -propeller: - resourcemanager: - type: noop diff --git a/kustomize/overlays/test/flyte/kustomization.yaml b/kustomize/overlays/test/flyte/kustomization.yaml deleted file mode 100644 index 3bdbb899cfe..00000000000 --- a/kustomize/overlays/test/flyte/kustomization.yaml +++ /dev/null @@ -1,42 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -# Override the namespace -namespace: flyte - -bases: -- ../../../base/single_cluster/headless - -patchesStrategicMerge: -- admin/deployment.yaml - -configMapGenerator: -# the main admin configmap -- name: flyte-admin-config - behavior: merge - files: - - ./config/admin/db.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - -# Flyte Propeller Configuration -- name: flyte-propeller-config - behavior: merge - files: - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/resource_manager.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - -# TODO Flyte Console Configuration -#- name: flyte-console-config -# files: -# - ./config/console.yaml - -- name: datacatalog-config - behavior: merge - files: - - ./config/datacatalog/db.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml deleted file mode 100644 index b7511f89091..00000000000 --- a/kustomize/overlays/test/kustomization.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization - -bases: - # All the resources that make up the deployment - - flyte - ######## - # Operators to be enabled - ############# - # Dependencies to be configured - # In local we use minio, but in cloud environment use S3 / GCS / AFS / Oracle Blob store etc - # This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) - # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's - # Add node ports for ease of use locally - - ../../base/addons/database - - ../../base/addons/storage - -# Images that should be used -images: - # FlyteAdmin - - name: flyteadmin # match images with this name - newTag: v1.1.118 # FLYTEADMIN_TAG override the tag - newName: cr.flyte.org/flyteorg/flyteadmin # override the name - # FlyteConsole - - name: flyteconsole # match images with this name - newTag: v1.9.2 # FLYTECONSOLE_TAG the tag - newName: cr.flyte.org/flyteorg/flyteconsole # override the namep - # Flyte DataCatalog - - name: datacatalog # match images with this name - newTag: v1.0.51 # DATACATALOG_TAG override the tag - newName: cr.flyte.org/flyteorg/datacatalog # override the name - # FlytePropeller - - name: flytepropeller # match images with this name - newTag: v1.1.116 # FLYTEPROPELLER_TAG override the tag - newName: cr.flyte.org/flyteorg/flytepropeller # override the name - # Webhook - - name: webhook # match images with this name - newTag: v1.1.116 # FLYTEPROPELLER_TAG override the tag - newName: cr.flyte.org/flyteorg/flytepropeller # override the name - # Override postgres image to use alpine based (rather smaller) docker image - - name: postgres - newTag: 13-21.04_beta - newName: ecr.flyte.org/ubuntu/postgres - - name: minio - newTag: 2021.10.13-debian-10-r0 - newName: ecr.flyte.org/bitnami/minio - - name: redis - newTag: 6.2.5-debian-10-r59 - newName: ecr.flyte.org/bitnami/redis diff --git a/script/generate_kustomize.sh b/script/generate_kustomize.sh deleted file mode 100755 index 81a1142e009..00000000000 --- a/script/generate_kustomize.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env bash - -set -ex - -echo "Installing Kustomize" -KUSTOMIZE=_bin/kustomize -KUSTOMIZE_VERSION=${KUSTOMIZE_VERSION:-3.8.1} - -if [ -f ${KUSTOMIZE} ]; then - rm ${KUSTOMIZE} -fi -mkdir -p _bin; cd _bin -curl -s "https://raw.githubusercontent.com/\ -kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | bash -s ${KUSTOMIZE_VERSION} -cd - - -# All the overlays to be built -DEPLOYMENT=${1:-sandbox test eks gcp} - -KUSTOMIZE_OVERLAYS_ROOT=kustomize/overlays - -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" - -for deployment in ${DEPLOYMENT}; do - ${KUSTOMIZE} build ${KUSTOMIZE_OVERLAYS_ROOT}/${deployment} > ${DIR}/../deployment/${deployment}/flyte_generated.yaml -done - -# This section is used by GitHub workflow to ensure that the generation step was run -if [ -n "$DELTA_CHECK" ]; then - DIRTY=$(git status --porcelain) - if [ -n "$DIRTY" ]; then - echo "FAILED: kustomize code updated without committing generated code." - echo "Ensure make kustomize has run and all changes are committed." - DIFF=$(git diff) - echo "diff detected: $DIFF" - DIFF=$(git diff --name-only) - echo "files different: $DIFF" - exit 1 - else - echo "SUCCESS: Generated code is up to date." - fi -fi diff --git a/script/release.sh b/script/release.sh index 1f3d763feee..c6773e30407 100755 --- a/script/release.sh +++ b/script/release.sh @@ -3,13 +3,10 @@ set -ex FLYTEKIT_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytekit/releases/latest" | jq -r .tag_name | sed 's/^v//') -FLYTECONSOLE_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteconsole/releases/latest" | jq -r .tag_name) - -# bump latest release of flyte component in kustomize -grep -rlZ "newTag:[^P]*# FLYTEADMIN_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTEADMIN_TAG/newTag: ${VERSION} # FLYTEADMIN_TAG/g" {} -grep -rlZ "newTag:[^P]*# DATACATALOG_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# DATACATALOG_TAG/newTag: ${VERSION} # DATACATALOG_TAG/g" {} -grep -rlZ "newTag:[^P]*# FLYTECONSOLE_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTECONSOLE_TAG/newTag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG/g" {} -grep -rlZ "newTag:[^P]*# FLYTEPROPELLER_TAG" ./kustomize/overlays | xargs -I {} sed -i "s/newTag:[^P]*# FLYTEPROPELLER_TAG/newTag: ${VERSION} # FLYTEPROPELLER_TAG/g" {} +# The flyteconsole revamp is not released yet (we need "schedules" to be present before we can release it). In the meantime +# we are using the latest release (v1.10.3) as the tag for flyteconsole. +# FLYTECONSOLE_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteconsole/releases/latest" | jq -r .tag_name) +FLYTECONSOLE_TAG=v1.10.3 # bump latest release of flyte component in helm sed -i "s,tag:[^P]*# FLYTEADMIN_TAG,tag: ${VERSION} # FLYTEADMIN_TAG," ./charts/flyte/values.yaml