From a7ece008ae070cb1b65d5191bb10b3c0dad10a6b Mon Sep 17 00:00:00 2001 From: Ketan Umare Date: Tue, 23 Jun 2020 10:05:00 -0700 Subject: [PATCH] Flyte CoPilot: Raw container support for all K8s containers (#84) --- flyteplugins/.github/workflows/master.yml | 31 + .../.github/workflows/pull_request.yml | 19 + flyteplugins/.gitignore | 1 + flyteplugins/Dockerfile | 32 + flyteplugins/Makefile | 15 + .../boilerplate/lyft/docker_build/Makefile | 12 + .../boilerplate/lyft/docker_build/Readme.rst | 23 + .../lyft/docker_build/docker_build.sh | 67 ++ .../lyft/github_workflows/Readme.rst | 20 + .../lyft/github_workflows/master.yml | 31 + .../lyft/github_workflows/pull_request.yml | 19 + .../lyft/github_workflows/update.sh | 16 + .../golang_dockerfile/Dockerfile.GoTemplate | 32 + .../lyft/golang_dockerfile/Readme.rst | 16 + .../lyft/golang_dockerfile/update.sh | 13 + flyteplugins/boilerplate/update.cfg | 3 + flyteplugins/copilot/README.md | 52 ++ .../cmd/containerwatcher/file_watcher.go | 114 +++ .../copilot/cmd/containerwatcher/iface.go | 35 + .../cmd/containerwatcher/kubeapi_watcher.go | 85 ++ .../cmd/containerwatcher/noop_watcher.go | 20 + .../cmd/containerwatcher/spns_watcher.go | 133 ++++ flyteplugins/copilot/cmd/download.go | 121 +++ flyteplugins/copilot/cmd/download_test.go | 189 +++++ flyteplugins/copilot/cmd/root.go | 156 ++++ flyteplugins/copilot/cmd/sidecar.go | 186 +++++ flyteplugins/copilot/cmd/sidecar_test.go | 99 +++ flyteplugins/copilot/data/common.go | 19 + flyteplugins/copilot/data/download.go | 365 +++++++++ flyteplugins/copilot/data/upload.go | 200 +++++ flyteplugins/copilot/data/upload_test.go | 63 ++ flyteplugins/copilot/data/utils.go | 90 +++ flyteplugins/copilot/data/utils_test.go | 119 +++ flyteplugins/copilot/go.mod | 33 + flyteplugins/copilot/go.sum | 735 ++++++++++++++++++ flyteplugins/copilot/main.go | 16 + flyteplugins/go.mod | 13 +- flyteplugins/go.sum | 17 + .../pluginmachinery/catalog/async_client.go | 2 +- .../catalog/async_client_impl_test.go | 4 +- .../catalog/mocks/async_client.go | 6 +- .../pluginmachinery/flytek8s/config/config.go | 58 +- .../flytek8s/config/config_test.go | 12 + .../flytek8s/config/k8spluginconfig_flags.go | 5 + .../config/k8spluginconfig_flags_test.go | 110 +++ .../flytek8s/container_helper.go | 5 +- .../tasks/pluginmachinery/flytek8s/copilot.go | 269 +++++++ .../pluginmachinery/flytek8s/copilot_test.go | 584 ++++++++++++++ .../pluginmachinery/flytek8s/pod_helper.go | 31 +- .../tasks/pluginmachinery/utils/literals.go | 439 +++++++++++ .../pluginmachinery/utils/literals_test.go | 356 +++++++++ .../tasks/plugins/k8s/container/container.go | 10 +- .../plugins/k8s/container/container_test.go | 6 +- .../tasks/plugins/k8s/sidecar/sidecar_test.go | 2 + 54 files changed, 5073 insertions(+), 36 deletions(-) create mode 100644 flyteplugins/.github/workflows/master.yml create mode 100644 flyteplugins/.github/workflows/pull_request.yml create mode 100644 flyteplugins/Dockerfile create mode 100644 flyteplugins/boilerplate/lyft/docker_build/Makefile create mode 100644 flyteplugins/boilerplate/lyft/docker_build/Readme.rst create mode 100755 flyteplugins/boilerplate/lyft/docker_build/docker_build.sh create mode 100644 flyteplugins/boilerplate/lyft/github_workflows/Readme.rst create mode 100644 flyteplugins/boilerplate/lyft/github_workflows/master.yml create mode 100644 flyteplugins/boilerplate/lyft/github_workflows/pull_request.yml create mode 100755 flyteplugins/boilerplate/lyft/github_workflows/update.sh create mode 100644 flyteplugins/boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate create mode 100644 flyteplugins/boilerplate/lyft/golang_dockerfile/Readme.rst create mode 100755 flyteplugins/boilerplate/lyft/golang_dockerfile/update.sh create mode 100644 flyteplugins/copilot/README.md create mode 100644 flyteplugins/copilot/cmd/containerwatcher/file_watcher.go create mode 100644 flyteplugins/copilot/cmd/containerwatcher/iface.go create mode 100644 flyteplugins/copilot/cmd/containerwatcher/kubeapi_watcher.go create mode 100644 flyteplugins/copilot/cmd/containerwatcher/noop_watcher.go create mode 100644 flyteplugins/copilot/cmd/containerwatcher/spns_watcher.go create mode 100644 flyteplugins/copilot/cmd/download.go create mode 100644 flyteplugins/copilot/cmd/download_test.go create mode 100644 flyteplugins/copilot/cmd/root.go create mode 100644 flyteplugins/copilot/cmd/sidecar.go create mode 100644 flyteplugins/copilot/cmd/sidecar_test.go create mode 100644 flyteplugins/copilot/data/common.go create mode 100644 flyteplugins/copilot/data/download.go create mode 100644 flyteplugins/copilot/data/upload.go create mode 100644 flyteplugins/copilot/data/upload_test.go create mode 100644 flyteplugins/copilot/data/utils.go create mode 100644 flyteplugins/copilot/data/utils_test.go create mode 100644 flyteplugins/copilot/go.mod create mode 100644 flyteplugins/copilot/go.sum create mode 100644 flyteplugins/copilot/main.go create mode 100644 flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config_test.go create mode 100644 flyteplugins/go/tasks/pluginmachinery/flytek8s/copilot.go create mode 100644 flyteplugins/go/tasks/pluginmachinery/flytek8s/copilot_test.go create mode 100644 flyteplugins/go/tasks/pluginmachinery/utils/literals.go create mode 100644 flyteplugins/go/tasks/pluginmachinery/utils/literals_test.go diff --git a/flyteplugins/.github/workflows/master.yml b/flyteplugins/.github/workflows/master.yml new file mode 100644 index 0000000000..aa665e3563 --- /dev/null +++ b/flyteplugins/.github/workflows/master.yml @@ -0,0 +1,31 @@ +name: Master + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: '0' + - name: Bump version and push tag + id: bump-version + uses: anothrNick/github-tag-action@1.17.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: true + DEFAULT_BUMP: patch + - name: Push Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 + with: + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" + image_name: ${{ secrets.flytegithub_repo }}/flytecopilot + image_tag: latest,${{ github.sha }},${{ steps.bump-version.outputs.tag }} + push_git_tag: true + registry: docker.pkg.github.com + build_extra_args: "--compress=true" diff --git a/flyteplugins/.github/workflows/pull_request.yml b/flyteplugins/.github/workflows/pull_request.yml new file mode 100644 index 0000000000..0cad2c90c9 --- /dev/null +++ b/flyteplugins/.github/workflows/pull_request.yml @@ -0,0 +1,19 @@ +name: Pull Request + +on: + pull_request + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Push Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 + with: + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" + image_name: ${{ secrets.flytegithub_repo }}/flytecopilot + image_tag: ${{ github.sha }} + push_git_tag: true + registry: docker.pkg.github.com diff --git a/flyteplugins/.gitignore b/flyteplugins/.gitignore index 5871e8b15c..a436c16b9b 100644 --- a/flyteplugins/.gitignore +++ b/flyteplugins/.gitignore @@ -2,3 +2,4 @@ .idea/ vendor/ *.swp +artifacts/ diff --git a/flyteplugins/Dockerfile b/flyteplugins/Dockerfile new file mode 100644 index 0000000000..295ddd13b4 --- /dev/null +++ b/flyteplugins/Dockerfile @@ -0,0 +1,32 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +FROM golang:1.13.3-alpine3.10 as builder +RUN apk add git openssh-client make curl + +# COPY only the go mod files for efficient caching +COPY go.mod go.sum /go/src/github.com/lyft/flyteplugins/ +WORKDIR /go/src/github.com/lyft/flyteplugins + +# Pull dependencies +RUN go mod download + +# COPY the rest of the source code +COPY . /go/src/github.com/lyft/flyteplugins/ + +# This 'linux_compile' target should compile binaries to the /artifacts directory +# The main entrypoint should be compiled to /artifacts/flyteplugins +RUN make linux_compile + +# update the PATH to include the /artifacts directory +ENV PATH="/artifacts:${PATH}" + +# This will eventually move to centurylink/ca-certs:latest for minimum possible image size +FROM alpine:3.10 +COPY --from=builder /artifacts /bin + +RUN apk --update add ca-certificates + +CMD ["flyte-copilot"] diff --git a/flyteplugins/Makefile b/flyteplugins/Makefile index bf14f0fb0b..4cd7eb7264 100755 --- a/flyteplugins/Makefile +++ b/flyteplugins/Makefile @@ -1,4 +1,5 @@ export REPOSITORY=flyteplugins +include boilerplate/lyft/docker_build/Makefile include boilerplate/lyft/golang_test_targets/Makefile .PHONY: update_boilerplate @@ -10,3 +11,17 @@ generate: download_tooling clean: rm -rf bin + +.PHONY: linux_compile +linux_compile: + cd copilot; GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts/flyte-copilot .; cd - + +.PHONY: compile +compile: + mkdir -p ./artifacts + cd copilot; go build -o ../artifacts/flyte-copilot .; cd - + +cross_compile: + @glide install + @mkdir -p ./artifacts/cross + cd copilot; GOOS=linux GOARCH=amd64 go build -o ../artifacts/flyte-copilot .; cd - diff --git a/flyteplugins/boilerplate/lyft/docker_build/Makefile b/flyteplugins/boilerplate/lyft/docker_build/Makefile new file mode 100644 index 0000000000..4019dab839 --- /dev/null +++ b/flyteplugins/boilerplate/lyft/docker_build/Makefile @@ -0,0 +1,12 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +.PHONY: docker_build +docker_build: + IMAGE_NAME=$$REPOSITORY ./boilerplate/lyft/docker_build/docker_build.sh + +.PHONY: dockerhub_push +dockerhub_push: + IMAGE_NAME=lyft/$$REPOSITORY REGISTRY=docker.io ./boilerplate/lyft/docker_build/docker_build.sh diff --git a/flyteplugins/boilerplate/lyft/docker_build/Readme.rst b/flyteplugins/boilerplate/lyft/docker_build/Readme.rst new file mode 100644 index 0000000000..bb6af9b49e --- /dev/null +++ b/flyteplugins/boilerplate/lyft/docker_build/Readme.rst @@ -0,0 +1,23 @@ +Docker Build and Push +~~~~~~~~~~~~~~~~~~~~~ + +Provides a ``make docker_build`` target that builds your image locally. + +Provides a ``make dockerhub_push`` target that pushes your final image to Dockerhub. + +The Dockerhub image will tagged ``:`` + +If git head has a git tag, the Dockerhub image will also be tagged ``:``. + +**To Enable:** + +Add ``lyft/docker_build`` to your ``boilerplate/update.cfg`` file. + +Add ``include boilerplate/lyft/docker_build/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable + +:: + + REPOSITORY= + include boilerplate/lyft/docker_build/Makefile + +(this ensures the extra Make targets get included in your main Makefile) diff --git a/flyteplugins/boilerplate/lyft/docker_build/docker_build.sh b/flyteplugins/boilerplate/lyft/docker_build/docker_build.sh new file mode 100755 index 0000000000..f504c100c7 --- /dev/null +++ b/flyteplugins/boilerplate/lyft/docker_build/docker_build.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +set -e + +echo "" +echo "------------------------------------" +echo " DOCKER BUILD" +echo "------------------------------------" +echo "" + +if [ -n "$REGISTRY" ]; then + # Do not push if there are unstaged git changes + CHANGED=$(git status --porcelain) + if [ -n "$CHANGED" ]; then + echo "Please commit git changes before pushing to a registry" + exit 1 + fi +fi + + +GIT_SHA=$(git rev-parse HEAD) + +IMAGE_TAG_SUFFIX="" +# for intermediate build phases, append -$BUILD_PHASE to all image tags +if [ -n "$BUILD_PHASE" ]; then + IMAGE_TAG_SUFFIX="-${BUILD_PHASE}" +fi + +IMAGE_TAG_WITH_SHA="${IMAGE_NAME}:${GIT_SHA}${IMAGE_TAG_SUFFIX}" + +RELEASE_SEMVER=$(git describe --tags --exact-match "$GIT_SHA" 2>/dev/null) || true +if [ -n "$RELEASE_SEMVER" ]; then + IMAGE_TAG_WITH_SEMVER="${IMAGE_NAME}:${RELEASE_SEMVER}${IMAGE_TAG_SUFFIX}" +fi + +# build the image +# passing no build phase will build the final image +docker build -t "$IMAGE_TAG_WITH_SHA" --target=${BUILD_PHASE} . +echo "${IMAGE_TAG_WITH_SHA} built locally." + +# if REGISTRY specified, push the images to the remote registy +if [ -n "$REGISTRY" ]; then + + if [ -n "${DOCKER_REGISTRY_PASSWORD}" ]; then + docker login --username="$DOCKER_REGISTRY_USERNAME" --password="$DOCKER_REGISTRY_PASSWORD" + fi + + docker tag "$IMAGE_TAG_WITH_SHA" "${REGISTRY}/${IMAGE_TAG_WITH_SHA}" + + docker push "${REGISTRY}/${IMAGE_TAG_WITH_SHA}" + echo "${REGISTRY}/${IMAGE_TAG_WITH_SHA} pushed to remote." + + # If the current commit has a semver tag, also push the images with the semver tag + if [ -n "$RELEASE_SEMVER" ]; then + + docker tag "$IMAGE_TAG_WITH_SHA" "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER}" + + docker push "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER}" + echo "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER} pushed to remote." + + fi +fi diff --git a/flyteplugins/boilerplate/lyft/github_workflows/Readme.rst b/flyteplugins/boilerplate/lyft/github_workflows/Readme.rst new file mode 100644 index 0000000000..d32ba7cb41 --- /dev/null +++ b/flyteplugins/boilerplate/lyft/github_workflows/Readme.rst @@ -0,0 +1,20 @@ +Golang Github Actions +~~~~~~~~~~~~~~~~~ + +Provides a two github actions workflows. + +**To Enable:** + +Add ``lyft/github_workflows`` to your ``boilerplate/update.cfg`` file. + +Add a github secret ``flytegithub_repo`` with a the name of your fork (e.g. ``my_company/flytepropeller``). + +The actions will push to 2 repos: + + 1. ``docker.pkg.github.com/lyft//operator`` + 2. ``docker.pkg.github.com/lyft//operator-stages`` : this repo is used to cache build stages to speed up iterative builds after. + +There are two workflows that get deployed: + + 1. A workflow that runs on Pull Requests to build and push images to github registy tagged with the commit sha. + 2. A workflow that runs on master merges that bump the patch version of release tag, builds and pushes images to github registry tagged with the version, commit sha as well as "latest" diff --git a/flyteplugins/boilerplate/lyft/github_workflows/master.yml b/flyteplugins/boilerplate/lyft/github_workflows/master.yml new file mode 100644 index 0000000000..374fe8150b --- /dev/null +++ b/flyteplugins/boilerplate/lyft/github_workflows/master.yml @@ -0,0 +1,31 @@ +name: Master + +on: + push: + branches: + - master + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + fetch-depth: '0' + - name: Bump version and push tag + id: bump-version + uses: anothrNick/github-tag-action@1.17.2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WITH_V: true + DEFAULT_BUMP: patch + - name: Push Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 + with: + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" + image_name: ${{ secrets.flytegithub_repo }}/operator + image_tag: latest,${{ github.sha }},${{ steps.bump-version.outputs.tag }} + push_git_tag: true + registry: docker.pkg.github.com + build_extra_args: "--compress=true" diff --git a/flyteplugins/boilerplate/lyft/github_workflows/pull_request.yml b/flyteplugins/boilerplate/lyft/github_workflows/pull_request.yml new file mode 100644 index 0000000000..b87963a5f1 --- /dev/null +++ b/flyteplugins/boilerplate/lyft/github_workflows/pull_request.yml @@ -0,0 +1,19 @@ +name: Pull Request + +on: + pull_request + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Push Docker Image to Github Registry + uses: whoan/docker-build-with-cache-action@v5 + with: + username: "${{ github.actor }}" + password: "${{ secrets.GITHUB_TOKEN }}" + image_name: ${{ secrets.flytegithub_repo }}/operator + image_tag: ${{ github.sha }} + push_git_tag: true + registry: docker.pkg.github.com diff --git a/flyteplugins/boilerplate/lyft/github_workflows/update.sh b/flyteplugins/boilerplate/lyft/github_workflows/update.sh new file mode 100755 index 0000000000..1e3a099182 --- /dev/null +++ b/flyteplugins/boilerplate/lyft/github_workflows/update.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +mkdir -p ${DIR}/../../../.github/workflows + +echo " - generating github action workflows in root directory." +sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/master.yml > ${DIR}/../../../.github/workflows/master.yml +sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/pull_request.yml > ${DIR}/../../../.github/workflows/pull_request.yml diff --git a/flyteplugins/boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate b/flyteplugins/boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate new file mode 100644 index 0000000000..214d1e40bb --- /dev/null +++ b/flyteplugins/boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate @@ -0,0 +1,32 @@ +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +FROM golang:1.13.3-alpine3.10 as builder +RUN apk add git openssh-client make curl + +# COPY only the go mod files for efficient caching +COPY go.mod go.sum /go/src/github.com/lyft/{{REPOSITORY}}/ +WORKDIR /go/src/github.com/lyft/{{REPOSITORY}} + +# Pull dependencies +RUN go mod download + +# COPY the rest of the source code +COPY . /go/src/github.com/lyft/{{REPOSITORY}}/ + +# This 'linux_compile' target should compile binaries to the /artifacts directory +# The main entrypoint should be compiled to /artifacts/{{REPOSITORY}} +RUN make linux_compile + +# update the PATH to include the /artifacts directory +ENV PATH="/artifacts:${PATH}" + +# This will eventually move to centurylink/ca-certs:latest for minimum possible image size +FROM alpine:3.10 +COPY --from=builder /artifacts /bin + +RUN apk --update add ca-certificates + +CMD ["{{REPOSITORY}}"] diff --git a/flyteplugins/boilerplate/lyft/golang_dockerfile/Readme.rst b/flyteplugins/boilerplate/lyft/golang_dockerfile/Readme.rst new file mode 100644 index 0000000000..f801ef98d6 --- /dev/null +++ b/flyteplugins/boilerplate/lyft/golang_dockerfile/Readme.rst @@ -0,0 +1,16 @@ +Golang Dockerfile +~~~~~~~~~~~~~~~~~ + +Provides a Dockerfile that produces a small image. + +**To Enable:** + +Add ``lyft/golang_dockerfile`` to your ``boilerplate/update.cfg`` file. + +Create and configure a ``make linux_compile`` target that compiles your go binaries to the ``/artifacts`` directory :: + + .PHONY: linux_compile + linux_compile: + RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts {{ packages }} + +All binaries compiled to ``/artifacts`` will be available at ``/bin`` in your final image. diff --git a/flyteplugins/boilerplate/lyft/golang_dockerfile/update.sh b/flyteplugins/boilerplate/lyft/golang_dockerfile/update.sh new file mode 100755 index 0000000000..7d84663262 --- /dev/null +++ b/flyteplugins/boilerplate/lyft/golang_dockerfile/update.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +echo " - generating Dockerfile in root directory." +sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/Dockerfile.GoTemplate > ${DIR}/../../../Dockerfile diff --git a/flyteplugins/boilerplate/update.cfg b/flyteplugins/boilerplate/update.cfg index a1b1bff989..9f6e2eaade 100755 --- a/flyteplugins/boilerplate/update.cfg +++ b/flyteplugins/boilerplate/update.cfg @@ -2,3 +2,6 @@ lyft/golang_test_targets lyft/golangci_file lyft/golang_support_tools lyft/pull_request_template +lyft/github_workflows +lyft/golang_dockerfile +lyft/docker_build diff --git a/flyteplugins/copilot/README.md b/flyteplugins/copilot/README.md new file mode 100644 index 0000000000..688c09ae1c --- /dev/null +++ b/flyteplugins/copilot/README.md @@ -0,0 +1,52 @@ +#Flyte CoPilot + +## Overview +Flyte CoPilot provides a sidecar that understand Flyte Metadata Format as specified in FlyteIDL and make it possible to run arbitrary containers in Flyte. +This is achieved using `flyte-copilot` a binary that runs in 2 modes, + - *Downloader* - Downloads the metadata and any other data (if configured) to a provided path. In kubernetes this path could be a shared volume. + - *Sidecar* - Monitors the process and uploads any data that is generated by the process in a prescribed path/ + +## Mode: Downloader + +```bash +$ flyte-copilot downloader +``` + +In K8s `flyte-copilot downloader` can be run as part of the init containers with the download volume mounted. This guarantees that the metadata and any data (if configured) +is downloaded before the main container starts up. + +## Mode: Sidecar + As a sidecar process, that runs in parallel with the main container/process, the goal is to + 1. identify the main container + 2. Wait for the main container to start up + 3. Wait for the main container to exit + 4. Copy the data to remote store (especially the metadata) + 5. Exit + +```bash +$ flyte-copilot sidecar +``` + +### Raw notes + Solution 1: + poll Kubeapi. + - Works perfectly fine, but too much load on kubeapi + + Solution 2: + Create a protocol. Main container will exit and write a _SUCCESS file to a known location + - problem in the case of oom or random exits. Uploader will be stuck. We could use a timeout? and in the sidecar just kill the pod, when the main exits unhealthy? + + Solution 3: + Use shared process namespace. This allows all pids in a pod to share the namespace. Thus pids can see each other. + + Problems: + How to identify the main container? + - Container id is not known ahead of time and container name -> Pid mapping is not possible? + - How to wait for main container to start up. + One solution for both, call kubeapi and get pod info and find the container id + + Note: we can poll /proc/pid/cgroup file (it contains the container id) so we can create a blind container id to pid mapping. Then somehow get the main container id + + Once we know the main container, waiting for it to exit is simple and implemented + Copying data is simple and implemented + diff --git a/flyteplugins/copilot/cmd/containerwatcher/file_watcher.go b/flyteplugins/copilot/cmd/containerwatcher/file_watcher.go new file mode 100644 index 0000000000..c1d4073946 --- /dev/null +++ b/flyteplugins/copilot/cmd/containerwatcher/file_watcher.go @@ -0,0 +1,114 @@ +package containerwatcher + +import ( + "context" + "fmt" + "os" + "path" + + "github.com/fsnotify/fsnotify" + "github.com/lyft/flytestdlib/logger" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/sets" +) + +func FileExists(filePath string) (bool, error) { + _, err := os.Stat(filePath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + if os.IsPermission(err) { + return false, errors.Wrapf(err, "unable to read file [%s], Flyte does not have permissions", filePath) + } + return false, errors.Wrapf(err, "failed to read file") + } + return true, nil +} + +type fileWatcher struct { + startWatchDir string + startFile string + exitWatchDir string + successFile string + errorFile string +} + +func (k fileWatcher) WaitToStart(ctx context.Context) error { + return wait(ctx, k.startWatchDir, sets.NewString(k.startFile)) +} + +func (k fileWatcher) WaitToExit(ctx context.Context) error { + return wait(ctx, k.exitWatchDir, sets.NewString(k.successFile, k.errorFile)) +} + +func wait(ctx context.Context, watchDir string, s sets.String) error { + w, err := fsnotify.NewWatcher() + if err != nil { + return errors.Wrapf(err, "failed to create watcher") + } + defer func() { + err := w.Close() + if err != nil { + logger.Errorf(context.TODO(), "failed to close file watcher") + } + }() + + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + + done := make(chan error) + go func(ctx context.Context) { + for { + select { + case <-ctx.Done(): + done <- nil + return + case event, ok := <-w.Events: + if !ok { + done <- fmt.Errorf("failed to watch") + return + } + if event.Op == fsnotify.Create || event.Op == fsnotify.Write { + if s.Has(event.Name) { + logger.Infof(ctx, "%s file detected", event.Name) + done <- nil + return + } + } + case err, ok := <-w.Errors: + if !ok { + done <- fmt.Errorf("failed to watch") + return + } + done <- err + return + } + } + }(childCtx) + + if err := w.Add(watchDir); err != nil { + return errors.Wrapf(err, "failed to add dir: %s to watcher,", watchDir) + } + + for _, f := range s.List() { + if ok, err := FileExists(f); err != nil { + logger.Errorf(ctx, "Failed to check existence of file %s, err: %s", f, err) + return err + } else if ok { + logger.Infof(ctx, "File %s Already exists", f) + return nil + } + } + return <-done +} + +func NewSuccessFileWatcher(_ context.Context, watchDir, startFileName, successFileName, errorFileName string) (Watcher, error) { + return fileWatcher{ + startWatchDir: watchDir, + exitWatchDir: watchDir, + startFile: path.Join(watchDir, startFileName), + successFile: path.Join(watchDir, successFileName), + errorFile: path.Join(watchDir, errorFileName), + }, nil +} diff --git a/flyteplugins/copilot/cmd/containerwatcher/iface.go b/flyteplugins/copilot/cmd/containerwatcher/iface.go new file mode 100644 index 0000000000..081f2366a3 --- /dev/null +++ b/flyteplugins/copilot/cmd/containerwatcher/iface.go @@ -0,0 +1,35 @@ +package containerwatcher + +import ( + "context" + "fmt" +) + +var ErrTimeout = fmt.Errorf("timeout while waiting") + +type Watcher interface { + WaitToStart(ctx context.Context) error + WaitToExit(ctx context.Context) error +} + +type WatcherType = string + +const ( + // Uses KubeAPI to determine if the container is completed + WatcherTypeKubeAPI WatcherType = "kube-api" + // Uses a success file to determine if the container has completed. + // CAUTION: Does not work if the container exits because of OOM, etc + WatcherTypeFile WatcherType = "file" + // Uses Kube 1.17 feature - https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/ + // To look for pid in the shared namespace. + WatcherTypeSharedProcessNS WatcherType = "shared-process-ns" + // Dummy watcher. Exits immediately, assuming success + WatcherTypeNoop WatcherType = "noop" +) + +var AllWatcherTypes = []WatcherType{ + WatcherTypeKubeAPI, + WatcherTypeSharedProcessNS, + WatcherTypeFile, + WatcherTypeNoop, +} diff --git a/flyteplugins/copilot/cmd/containerwatcher/kubeapi_watcher.go b/flyteplugins/copilot/cmd/containerwatcher/kubeapi_watcher.go new file mode 100644 index 0000000000..ec3fad500a --- /dev/null +++ b/flyteplugins/copilot/cmd/containerwatcher/kubeapi_watcher.go @@ -0,0 +1,85 @@ +package containerwatcher + +import ( + "context" + "fmt" + + "github.com/lyft/flytestdlib/logger" + v13 "k8s.io/api/core/v1" + v12 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + v1 "k8s.io/client-go/kubernetes/typed/core/v1" +) + +type ContainerInformation struct { + Namespace string + PodName string + Name string +} + +type kubeAPIWatcher struct { + coreClient v1.CoreV1Interface + info ContainerInformation +} + +func (k kubeAPIWatcher) wait(ctx context.Context, info ContainerInformation, f func(ctx context.Context, pod *v13.Pod) (bool, error)) error { + s := fields.OneTermEqualSelector("metadata.name", info.PodName) + watcher, err := k.coreClient.Pods(info.Namespace).Watch(v12.ListOptions{ + Watch: true, + FieldSelector: s.String(), + }) + if err != nil { + return err + } + defer watcher.Stop() + for { + select { + case v := <-watcher.ResultChan(): + p := v.Object.(*v13.Pod) + if stop, err := f(ctx, p); err != nil { + return err + } else if stop { + return nil + } + case <-ctx.Done(): + logger.Infof(ctx, "Pod [%s/%s] watcher canceled", info.Namespace, info.PodName) + return ErrTimeout + } + } +} + +func (k kubeAPIWatcher) WaitToStart(ctx context.Context) error { + f := func(ctx context.Context, p *v13.Pod) (bool, error) { + for _, c := range p.Status.ContainerStatuses { + if c.Name == k.info.Name { + if c.Ready { + return true, nil + } + } + } + return false, nil + } + return k.wait(ctx, k.info, f) +} + +func (k kubeAPIWatcher) WaitToExit(ctx context.Context) error { + f := func(ctx context.Context, p *v13.Pod) (bool, error) { + for _, c := range p.Status.ContainerStatuses { + if c.Name == k.info.Name { + // TODO we may want to check failure reason and return that? + if c.State.Terminated != nil { + if c.State.Terminated.ExitCode != 0 { + return true, fmt.Errorf("Container exited with Exit code [%d]. Reason [%s]%s. ", c.State.Terminated.ExitCode, c.State.Terminated.Reason, c.State.Terminated.Message) + } + return true, nil + } + } + } + return false, nil + } + return k.wait(ctx, k.info, f) +} + +func NewKubeAPIWatcher(_ context.Context, coreClient v1.CoreV1Interface, info ContainerInformation) (Watcher, error) { + return kubeAPIWatcher{coreClient: coreClient, info: info}, nil +} diff --git a/flyteplugins/copilot/cmd/containerwatcher/noop_watcher.go b/flyteplugins/copilot/cmd/containerwatcher/noop_watcher.go new file mode 100644 index 0000000000..1fc7180017 --- /dev/null +++ b/flyteplugins/copilot/cmd/containerwatcher/noop_watcher.go @@ -0,0 +1,20 @@ +package containerwatcher + +import ( + "context" + + "github.com/lyft/flytestdlib/logger" +) + +type NoopWatcher struct { +} + +func (n NoopWatcher) WaitToStart(ctx context.Context) error { + logger.Warn(ctx, "noop container watcher setup. assuming container started.") + return nil +} + +func (n NoopWatcher) WaitToExit(ctx context.Context) error { + logger.Warn(ctx, "noop container watcher setup. assuming container exited.") + return nil +} diff --git a/flyteplugins/copilot/cmd/containerwatcher/spns_watcher.go b/flyteplugins/copilot/cmd/containerwatcher/spns_watcher.go new file mode 100644 index 0000000000..010eb63c1d --- /dev/null +++ b/flyteplugins/copilot/cmd/containerwatcher/spns_watcher.go @@ -0,0 +1,133 @@ +package containerwatcher + +import ( + "context" + "os" + "time" + + "github.com/lyft/flytestdlib/logger" + "github.com/mitchellh/go-ps" + "github.com/pkg/errors" + "k8s.io/apimachinery/pkg/util/sets" +) + +const ( + k8sPauseContainerPid = 1 + k8sAllowedParentPid = 0 +) + +// given list of processes returns a list of processes such that, +// the pid does not match any of the given filterPids, this is to filter the /pause and current process. +// and the parentPid is the allowedParentPid. The logic for this is because every process in the shared namespace +// always has a parent pid of 0 +func FilterProcessList(procs []ps.Process, filterPids sets.Int, allowedParentPid int) ([]ps.Process, error) { + var filteredProcs []ps.Process + for _, p := range procs { + proc := p + if proc.PPid() == allowedParentPid { + if !filterPids.Has(proc.Pid()) { + filteredProcs = append(filteredProcs, proc) + } + } + } + return filteredProcs, nil +} + +type SharedNamespaceProcessLister struct { + // PID for the current process + currentProcessPid int + pidsToFilter sets.Int +} + +func (s *SharedNamespaceProcessLister) AnyProcessRunning(ctx context.Context) (bool, error) { + procs, err := s.ListRunningProcesses(ctx) + if err != nil { + return false, err + } + return len(procs) > 0, nil +} + +// Polls all processes and returns a filtered set. Refer to FilterProcessList for understanding the process of filtering +func (s *SharedNamespaceProcessLister) ListRunningProcesses(ctx context.Context) ([]ps.Process, error) { + procs, err := ps.Processes() + if err != nil { + return nil, errors.Wrap(err, "Failed to list processes") + } + filteredProcs, err := FilterProcessList(procs, s.pidsToFilter, k8sAllowedParentPid) + if err != nil { + return nil, errors.Wrapf(err, "failed to filter processes") + } + return filteredProcs, nil +} + +// The best option for this is to use https://kubernetes.io/docs/tasks/configure-pod-container/share-process-namespace/ +// This is only available as Beta as of 1.16, so we will launch with this feature only as beta +// But this is the most efficient way to monitor the pod +type sharedProcessNSWatcher struct { + // Rate at which to poll the process list + pollInterval time.Duration + // Number of cycles to wait before finalizing exit of container + cyclesToWait int + + s SharedNamespaceProcessLister +} + +func (k sharedProcessNSWatcher) wait(ctx context.Context, cyclesToWait int, f func(ctx context.Context, otherProcessRunning bool) bool) error { + t := time.NewTicker(k.pollInterval) + defer t.Stop() + cyclesOfMissingProcesses := 0 + for { + select { + case <-ctx.Done(): + logger.Infof(ctx, "Context canceled") + return ErrTimeout + case <-t.C: + logger.Infof(ctx, "Checking processes to see if any process were started...") + yes, err := k.s.AnyProcessRunning(ctx) + if err != nil { + return err + } + if f(ctx, yes) { + cyclesOfMissingProcesses++ + if cyclesOfMissingProcesses >= cyclesToWait { + logger.Infof(ctx, "Exiting wait loop") + return nil + } + } + logger.Infof(ctx, "process not yet started") + } + } +} + +func (k sharedProcessNSWatcher) WaitToStart(ctx context.Context) error { + logger.Infof(ctx, "SNPS Watcher waiting for other processes to start") + defer logger.Infof(ctx, "SNPS Watcher detected process start") + return k.wait(ctx, 1, func(ctx context.Context, otherProcessRunning bool) bool { + return otherProcessRunning + }) +} + +func (k sharedProcessNSWatcher) WaitToExit(ctx context.Context) error { + logger.Infof(ctx, "SNPS Watcher waiting for other process to exit") + defer logger.Infof(ctx, "SNPS Watcher detected process exit") + return k.wait(ctx, k.cyclesToWait, func(ctx context.Context, otherProcessRunning bool) bool { + return !otherProcessRunning + }) +} + +// c -> clock.Clock allows for injecting a fake clock. The watcher uses a timer +// pollInterval -> time.Duration, wait for this amount of time between successive process checks +// waitNumIntervalsBeforeFinalize -> Number of successive poll intervals of missing processes for the container, before assuming process is complete. 0/1 indicate the first time a process is detected to be missing, the wait if finalized. +// containerStartupTimeout -> Duration for which to wait for the container to start up. If the container has not started up in this time, exit with error. +func NewSharedProcessNSWatcher(ctx context.Context, pollInterval time.Duration, waitNumIntervalsBeforeFinalize int) (Watcher, error) { + logger.Infof(ctx, "SNPS created with poll interval %s", pollInterval.String()) + currentPid := os.Getpid() + return sharedProcessNSWatcher{ + pollInterval: pollInterval, + cyclesToWait: waitNumIntervalsBeforeFinalize, + s: SharedNamespaceProcessLister{ + currentProcessPid: currentPid, + pidsToFilter: sets.NewInt(currentPid, k8sPauseContainerPid), + }, + }, nil +} diff --git a/flyteplugins/copilot/cmd/download.go b/flyteplugins/copilot/cmd/download.go new file mode 100644 index 0000000000..459261bef4 --- /dev/null +++ b/flyteplugins/copilot/cmd/download.go @@ -0,0 +1,121 @@ +package cmd + +import ( + "context" + "fmt" + "time" + + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/logger" + "github.com/lyft/flytestdlib/storage" + "github.com/spf13/cobra" + + "github.com/lyft/flyteplugins/copilot/data" +) + +type DownloadOptions struct { + *RootOptions + remoteInputsPath string + remoteOutputsPrefix string + localDirectoryPath string + inputInterface []byte + metadataFormat string + downloadMode string + timeout time.Duration +} + +func GetFormatVals() []string { + var vals []string + for k := range core.DataLoadingConfig_LiteralMapFormat_value { + vals = append(vals, k) + } + return vals +} + +func GetDownloadModeVals() []string { + var vals []string + for k := range core.IOStrategy_DownloadMode_value { + vals = append(vals, k) + } + return vals +} + +func GetUploadModeVals() []string { + var vals []string + for k := range core.IOStrategy_UploadMode_value { + vals = append(vals, k) + } + return vals +} + +func (d *DownloadOptions) Download(ctx context.Context) error { + if d.remoteOutputsPrefix == "" { + return fmt.Errorf("to-output-prefix is required") + } + + // We need remote outputs prefix to write and error file + err := func() error { + if d.localDirectoryPath == "" { + return fmt.Errorf("to-local-dir is required") + } + if d.remoteInputsPath == "" { + return fmt.Errorf("from-remote is required") + } + f, ok := core.DataLoadingConfig_LiteralMapFormat_value[d.metadataFormat] + if !ok { + return fmt.Errorf("incorrect input download format specified, given [%s], possible values [%+v]", d.metadataFormat, GetFormatVals()) + } + + m, ok := core.IOStrategy_DownloadMode_value[d.downloadMode] + if !ok { + return fmt.Errorf("incorrect input download mode specified, given [%s], possible values [%+v]", d.downloadMode, GetDownloadModeVals()) + } + dl := data.NewDownloader(ctx, d.Store, core.DataLoadingConfig_LiteralMapFormat(f), core.IOStrategy_DownloadMode(m)) + childCtx := ctx + cancelFn := func() {} + if d.timeout > 0 { + childCtx, cancelFn = context.WithTimeout(ctx, d.timeout) + } + defer cancelFn() + err := dl.DownloadInputs(childCtx, storage.DataReference(d.remoteInputsPath), d.localDirectoryPath) + if err != nil { + logger.Errorf(ctx, "Downloading failed, err %s", err) + return err + } + return nil + }() + + if err != nil { + if err2 := d.UploadError(ctx, "InputDownloadFailed", err, storage.DataReference(d.remoteOutputsPrefix)); err2 != nil { + logger.Errorf(ctx, "Failed to write error document, err :%s", err2) + return err2 + } + } + return nil +} + +func NewDownloadCommand(opts *RootOptions) *cobra.Command { + + downloadOpts := &DownloadOptions{ + RootOptions: opts, + } + + // deleteCmd represents the delete command + downloadCmd := &cobra.Command{ + Use: "download ", + Short: "downloads flytedata from the remotepath to a local directory.", + Long: `Currently it looks at the outputs.pb and creates one file per variable.`, + RunE: func(cmd *cobra.Command, args []string) error { + return downloadOpts.Download(context.Background()) + }, + } + + downloadCmd.Flags().StringVarP(&downloadOpts.remoteInputsPath, "from-remote", "f", "", "The remote path/key for inputs in stow store.") + downloadCmd.Flags().StringVarP(&downloadOpts.remoteOutputsPrefix, "to-output-prefix", "", "", "The remote path/key prefix for outputs in stow store. this is mostly used to write errors.pb.") + downloadCmd.Flags().StringVarP(&downloadOpts.localDirectoryPath, "to-local-dir", "o", "", "The local directory on disk where data should be downloaded.") + downloadCmd.Flags().StringVarP(&downloadOpts.metadataFormat, "format", "m", core.DataLoadingConfig_JSON.String(), fmt.Sprintf("What should be the output format for the primitive and structured types. Options [%v]", GetFormatVals())) + downloadCmd.Flags().StringVarP(&downloadOpts.downloadMode, "download-mode", "d", core.IOStrategy_DOWNLOAD_EAGER.String(), fmt.Sprintf("Download mode to use. Options [%v]", GetDownloadModeVals())) + downloadCmd.Flags().DurationVarP(&downloadOpts.timeout, "timeout", "t", time.Hour*1, "Max time to allow for downloads to complete, default is 1H") + downloadCmd.Flags().BytesBase64VarP(&downloadOpts.inputInterface, "input-interface", "i", nil, "Input interface proto message - core.VariableMap, base64 encoced string") + return downloadCmd +} diff --git a/flyteplugins/copilot/cmd/download_test.go b/flyteplugins/copilot/cmd/download_test.go new file mode 100644 index 0000000000..39aaecab7a --- /dev/null +++ b/flyteplugins/copilot/cmd/download_test.go @@ -0,0 +1,189 @@ +package cmd + +import ( + "bytes" + "context" + "io/ioutil" + "os" + "path/filepath" + "sort" + "strings" + "testing" + + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/promutils" + "github.com/lyft/flytestdlib/storage" + "github.com/stretchr/testify/assert" + + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/utils" +) + +func TestDownloadOptions_Download(t *testing.T) { + + tmpFolderLocation := "" + tmpPrefix := "download_test" + inputPath := "input/inputs.pb" + outputPath := "output" + + ctx := context.TODO() + dopts := DownloadOptions{ + remoteInputsPath: inputPath, + remoteOutputsPrefix: outputPath, + metadataFormat: core.DataLoadingConfig_JSON.String(), + downloadMode: core.IOStrategy_DOWNLOAD_EAGER.String(), + } + + collectFile := func(d string) []string { + var files []string + assert.NoError(t, filepath.Walk(d, func(path string, info os.FileInfo, err error) error { + if !strings.Contains(info.Name(), tmpPrefix) { + files = append(files, info.Name()) + } // Skip tmp folder + return nil + })) + sort.Strings(files) + return files + } + + t.Run("emptyInputs", func(t *testing.T) { + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + dopts.localDirectoryPath = tmpDir + + s := promutils.NewTestScope() + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage")) + assert.NoError(t, err) + dopts.RootOptions = &RootOptions{ + Scope: s, + Store: store, + } + + assert.NoError(t, store.WriteProtobuf(ctx, storage.DataReference(inputPath), storage.Options{}, &core.LiteralMap{})) + assert.NoError(t, dopts.Download(ctx)) + + assert.Len(t, collectFile(tmpDir), 0) + }) + + t.Run("primitiveInputs", func(t *testing.T) { + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + dopts.localDirectoryPath = tmpDir + + s := promutils.NewTestScope() + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage")) + assert.NoError(t, err) + dopts.RootOptions = &RootOptions{ + Scope: s, + Store: store, + } + + assert.NoError(t, store.WriteProtobuf(ctx, storage.DataReference(inputPath), storage.Options{}, &core.LiteralMap{ + Literals: map[string]*core.Literal{ + "x": utils.MustMakePrimitiveLiteral(1), + "y": utils.MustMakePrimitiveLiteral("hello"), + }, + })) + assert.NoError(t, dopts.Download(ctx), "Download Operation failed") + assert.Equal(t, []string{"inputs.json", "inputs.pb", "x", "y"}, collectFile(tmpDir)) + }) + + t.Run("primitiveAndBlobInputs", func(t *testing.T) { + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + dopts.localDirectoryPath = tmpDir + + s := promutils.NewTestScope() + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage")) + assert.NoError(t, err) + dopts.RootOptions = &RootOptions{ + Scope: s, + Store: store, + } + + blobLoc := storage.DataReference("blob-loc") + br := bytes.NewBuffer([]byte("Hello World!")) + assert.NoError(t, store.WriteRaw(ctx, blobLoc, int64(br.Len()), storage.Options{}, br)) + assert.NoError(t, store.WriteProtobuf(ctx, storage.DataReference(inputPath), storage.Options{}, &core.LiteralMap{ + Literals: map[string]*core.Literal{ + "x": utils.MustMakePrimitiveLiteral(1), + "y": utils.MustMakePrimitiveLiteral("hello"), + "blob": {Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Blob{ + Blob: &core.Blob{ + Uri: blobLoc.String(), + Metadata: &core.BlobMetadata{ + Type: &core.BlobType{ + Dimensionality: core.BlobType_SINGLE, + Format: ".xyz", + }, + }, + }, + }, + }, + }}, + }, + })) + assert.NoError(t, dopts.Download(ctx), "Download Operation failed") + assert.ElementsMatch(t, []string{"inputs.json", "inputs.pb", "x", "y", "blob"}, collectFile(tmpDir)) + }) + + t.Run("primitiveAndMissingBlobInputs", func(t *testing.T) { + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + dopts.localDirectoryPath = tmpDir + + s := promutils.NewTestScope() + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage")) + assert.NoError(t, err) + dopts.RootOptions = &RootOptions{ + Scope: s, + Store: store, + errorOutputName: "errors.pb", + } + + assert.NoError(t, store.WriteProtobuf(ctx, storage.DataReference(inputPath), storage.Options{}, &core.LiteralMap{ + Literals: map[string]*core.Literal{ + "x": utils.MustMakePrimitiveLiteral(1), + "y": utils.MustMakePrimitiveLiteral("hello"), + "blob": {Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Blob{ + Blob: &core.Blob{ + Uri: "blob", + Metadata: &core.BlobMetadata{ + Type: &core.BlobType{ + Dimensionality: core.BlobType_SINGLE, + Format: ".xyz", + }, + }, + }, + }, + }, + }}, + }, + })) + err = dopts.Download(ctx) + assert.NoError(t, err, "Download Operation failed") + errFile, err := store.ConstructReference(ctx, storage.DataReference(outputPath), "errors.pb") + assert.NoError(t, err) + errProto := &core.ErrorDocument{} + err = store.ReadProtobuf(ctx, errFile, errProto) + assert.NoError(t, err) + if assert.NotNil(t, errProto.Error) { + assert.Equal(t, core.ContainerError_RECOVERABLE, errProto.Error.Kind) + } + }) +} diff --git a/flyteplugins/copilot/cmd/root.go b/flyteplugins/copilot/cmd/root.go new file mode 100644 index 0000000000..fc21f6ab80 --- /dev/null +++ b/flyteplugins/copilot/cmd/root.go @@ -0,0 +1,156 @@ +package cmd + +import ( + "context" + "flag" + "fmt" + "os" + "runtime" + + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/config" + "github.com/lyft/flytestdlib/config/viper" + "github.com/lyft/flytestdlib/contextutils" + "github.com/lyft/flytestdlib/logger" + "github.com/lyft/flytestdlib/promutils" + "github.com/lyft/flytestdlib/promutils/labeled" + "github.com/lyft/flytestdlib/storage" + "github.com/lyft/flytestdlib/version" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/klog" +) + +type RootOptions struct { + *clientcmd.ConfigOverrides + showSource bool + clientConfig clientcmd.ClientConfig + restConfig *rest.Config + kubeClient kubernetes.Interface + Scope promutils.Scope + Store *storage.DataStore + configAccessor config.Accessor + cfgFile string + // The actual key name that should be created under the remote prefix where the error document is written of the form errors.pb + errorOutputName string +} + +func (r *RootOptions) executeRootCmd() error { + ctx := context.TODO() + logger.Infof(ctx, "Go Version: %s", runtime.Version()) + logger.Infof(ctx, "Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH) + version.LogBuildInformation("flytedata") + return fmt.Errorf("use one of the sub-commands") +} + +func (r RootOptions) UploadError(ctx context.Context, code string, recvErr error, prefix storage.DataReference) error { + if recvErr == nil { + recvErr = fmt.Errorf("unknown error") + } + errorPath, err := r.Store.ConstructReference(ctx, prefix, r.errorOutputName) + if err != nil { + logger.Errorf(ctx, "failed to create error file path err: %s", err) + return err + } + logger.Infof(ctx, "Uploading Error file to path [%s], errFile: %s", errorPath, r.errorOutputName) + return r.Store.WriteProtobuf(ctx, errorPath, storage.Options{}, &core.ErrorDocument{ + Error: &core.ContainerError{ + Code: code, + Message: recvErr.Error(), + Kind: core.ContainerError_RECOVERABLE, + }, + }) +} + +// NewCommand returns a new instance of the co-pilot root command +func NewDataCommand() *cobra.Command { + rootOpts := &RootOptions{} + command := &cobra.Command{ + Use: "flytedata", + Short: "flytedata is a simple go binary that can be used to retrieve and upload data from/to remote stow store to local disk.", + Long: `flytedata when used with conjunction with flytepropeller eliminates the need to have any flyte library installed inside the container`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + if err := rootOpts.initConfig(cmd, args); err != nil { + return err + } + rootOpts.Scope = promutils.NewScope("flyte:data") + cfg := storage.GetConfig() + store, err := storage.NewDataStore(cfg, rootOpts.Scope) + if err != nil { + return errors.Wrap(err, "failed to create datastore client") + } + rootOpts.Store = store + return rootOpts.ConfigureClient() + }, + RunE: func(cmd *cobra.Command, args []string) error { + return rootOpts.executeRootCmd() + }, + } + + command.AddCommand(NewDownloadCommand(rootOpts)) + command.AddCommand(NewUploadCommand(rootOpts)) + + loadingRules := clientcmd.NewDefaultClientConfigLoadingRules() + loadingRules.DefaultClientConfig = &clientcmd.DefaultClientConfig + rootOpts.ConfigOverrides = &clientcmd.ConfigOverrides{} + kflags := clientcmd.RecommendedConfigOverrideFlags("") + command.PersistentFlags().StringVar(&loadingRules.ExplicitPath, "kubeconfig", "", "Path to a kube config. Only required if out-of-cluster") + clientcmd.BindOverrideFlags(rootOpts.ConfigOverrides, command.PersistentFlags(), kflags) + rootOpts.clientConfig = clientcmd.NewInteractiveDeferredLoadingClientConfig(loadingRules, rootOpts.ConfigOverrides, os.Stdin) + + command.PersistentFlags().StringVar(&rootOpts.cfgFile, "config", "", "config file (default is $HOME/config.yaml)") + command.PersistentFlags().BoolVarP(&rootOpts.showSource, "show-source", "s", false, "Show line number for errors") + command.PersistentFlags().StringVar(&rootOpts.errorOutputName, "err-output-name", "errors.pb", "Actual key name under the prefix where the error protobuf should be written to") + + rootOpts.configAccessor = viper.NewAccessor(config.Options{StrictMode: true}) + // Here you will define your flags and configuration settings. Cobra supports persistent flags, which, if defined + // here, will be global for your application. + rootOpts.configAccessor.InitializePflags(command.PersistentFlags()) + + command.AddCommand(viper.GetConfigCommand()) + + return command +} + +func (r *RootOptions) initConfig(_ *cobra.Command, _ []string) error { + r.configAccessor = viper.NewAccessor(config.Options{ + StrictMode: true, + SearchPaths: []string{r.cfgFile}, + }) + + err := r.configAccessor.UpdateConfig(context.TODO()) + if err != nil { + return err + } + + return nil +} + +func (r *RootOptions) ConfigureClient() error { + restConfig, err := r.clientConfig.ClientConfig() + if err != nil { + return err + } + k, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + r.restConfig = restConfig + r.kubeClient = k + return nil +} + +func init() { + klog.InitFlags(flag.CommandLine) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) + err := flag.CommandLine.Parse([]string{}) + if err != nil { + logger.Error(context.TODO(), "Error in initializing: %v", err) + os.Exit(-1) + } + labeled.SetMetricKeys(contextutils.ProjectKey, contextutils.DomainKey, contextutils.WorkflowIDKey, contextutils.TaskIDKey) +} diff --git a/flyteplugins/copilot/cmd/sidecar.go b/flyteplugins/copilot/cmd/sidecar.go new file mode 100644 index 0000000000..a73e124134 --- /dev/null +++ b/flyteplugins/copilot/cmd/sidecar.go @@ -0,0 +1,186 @@ +package cmd + +import ( + "context" + "fmt" + "time" + + "github.com/golang/protobuf/proto" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/logger" + "github.com/lyft/flytestdlib/storage" + "github.com/pkg/errors" + "github.com/spf13/cobra" + + "github.com/lyft/flyteplugins/copilot/cmd/containerwatcher" + "github.com/lyft/flyteplugins/copilot/data" +) + +const ( + StartFile = "_START" + SuccessFile = "_SUCCESS" + ErrorFile = "_ERROR" +) + +type UploadOptions struct { + *RootOptions + // The remote prefix where all the meta outputs or error should be uploaded of the form s3://bucket/prefix + remoteOutputsPrefix string + // Name like outputs.pb under the remoteOutputsPrefix that should be created to upload the metaOutputs + metaOutputName string + // The remote prefix where all the raw outputs should be uploaded of the form s3://bucket/prefix/ + remoteOutputsRawPrefix string + // Local directory path where the sidecar should look for outputs. + localDirectoryPath string + // Non primitive types will be dumped in this output format + metadataFormat string + uploadMode string + timeout time.Duration + containerStartTimeout time.Duration + typedInterface []byte + startWatcherType containerwatcher.WatcherType + exitWatcherType containerwatcher.WatcherType + containerInfo containerwatcher.ContainerInformation +} + +func (u *UploadOptions) createWatcher(ctx context.Context, w containerwatcher.WatcherType) (containerwatcher.Watcher, error) { + switch w { + case containerwatcher.WatcherTypeKubeAPI: + // TODO, in this case container info should have namespace and podname and we can get it using downwardapi + // TODO https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/ + return containerwatcher.NewKubeAPIWatcher(ctx, u.RootOptions.kubeClient.CoreV1(), u.containerInfo) + case containerwatcher.WatcherTypeFile: + return containerwatcher.NewSuccessFileWatcher(ctx, u.localDirectoryPath, StartFile, SuccessFile, ErrorFile) + case containerwatcher.WatcherTypeSharedProcessNS: + return containerwatcher.NewSharedProcessNSWatcher(ctx, time.Second*2, 2) + case containerwatcher.WatcherTypeNoop: + return containerwatcher.NoopWatcher{}, nil + } + return nil, fmt.Errorf("unsupported watcher type") +} + +func (u *UploadOptions) uploader(ctx context.Context) error { + if u.typedInterface == nil { + logger.Infof(ctx, "No output interface provided. Assuming Void outputs.") + return nil + } + + iface := &core.TypedInterface{} + if err := proto.Unmarshal(u.typedInterface, iface); err != nil { + logger.Errorf(ctx, "Bad interface passed, failed to unmarshal err: %s", err) + return errors.Wrap(err, "Bad interface passed, failed to unmarshal, expected core.TypedInterface") + } + outputInterface := iface.Outputs + + if iface.Outputs == nil || iface.Outputs.Variables == nil || len(iface.Outputs.Variables) == 0 { + logger.Infof(ctx, "Empty output interface received. Assuming void outputs. Sidecar will exit immediately.") + return nil + } + + f, ok := core.DataLoadingConfig_LiteralMapFormat_value[u.metadataFormat] + if !ok { + return fmt.Errorf("incorrect input data format specified, given [%s], possible values [%+v]", u.metadataFormat, GetFormatVals()) + } + + m, ok := core.IOStrategy_UploadMode_value[u.uploadMode] + if !ok { + return fmt.Errorf("incorrect input upload mode specified, given [%s], possible values [%+v]", u.uploadMode, GetUploadModeVals()) + } + + logger.Infof(ctx, "Creating start watcher type: %s", u.startWatcherType) + w, err := u.createWatcher(ctx, u.startWatcherType) + if err != nil { + return err + } + + logger.Infof(ctx, "Waiting for Container to start with timeout %s.", u.containerStartTimeout) + childCtx, cancelFn := context.WithTimeout(ctx, u.containerStartTimeout) + defer cancelFn() + err = w.WaitToStart(childCtx) + if err != nil && err != containerwatcher.ErrTimeout { + return err + } + + if err != nil { + logger.Warnf(ctx, "Container start detection aborted, :%s", err.Error()) + } + + if u.startWatcherType != u.exitWatcherType { + logger.Infof(ctx, "Creating watcher type: %s", u.exitWatcherType) + w, err = u.createWatcher(ctx, u.exitWatcherType) + if err != nil { + return err + } + } + + logger.Infof(ctx, "Waiting for Container to exit.") + if err := w.WaitToExit(ctx); err != nil { + logger.Errorf(ctx, "Failed waiting for container to exit. Err: %s", err) + return err + } + + logger.Infof(ctx, "Container Exited! uploading data.") + + // TODO maybe we should just take the meta output path as an input argument + toOutputPath, err := u.Store.ConstructReference(ctx, storage.DataReference(u.remoteOutputsPrefix), u.metaOutputName) + if err != nil { + return err + } + + dl := data.NewUploader(ctx, u.Store, core.DataLoadingConfig_LiteralMapFormat(f), core.IOStrategy_UploadMode(m), ErrorFile) + + childCtx, cancelFn = context.WithTimeout(ctx, u.timeout) + defer cancelFn() + if err := dl.RecursiveUpload(childCtx, outputInterface, u.localDirectoryPath, toOutputPath, storage.DataReference(u.remoteOutputsRawPrefix)); err != nil { + logger.Errorf(ctx, "Uploading failed, err %s", err) + return err + } + + logger.Infof(ctx, "Uploader completed successfully!") + return nil +} + +func (u *UploadOptions) Sidecar(ctx context.Context) error { + + if err := u.uploader(ctx); err != nil { + logger.Errorf(ctx, "Uploading failed, err %s", err) + if err := u.UploadError(ctx, "OutputUploadFailed", err, storage.DataReference(u.remoteOutputsPrefix)); err != nil { + logger.Errorf(ctx, "Failed to write error document, err :%s", err) + return err + } + } + return nil +} + +func NewUploadCommand(opts *RootOptions) *cobra.Command { + + uploadOptions := &UploadOptions{ + RootOptions: opts, + } + + // deleteCmd represents the delete command + uploadCmd := &cobra.Command{ + Use: "sidecar ", + Short: "uploads flyteData from the localpath to a remote dir.", + Long: `Currently it looks at the outputs.pb and creates one file per variable.`, + RunE: func(cmd *cobra.Command, args []string) error { + return uploadOptions.Sidecar(context.Background()) + }, + } + + uploadCmd.Flags().StringVarP(&uploadOptions.remoteOutputsPrefix, "to-output-prefix", "o", "", "The remote path/key prefix for output metadata in stow store.") + uploadCmd.Flags().StringVarP(&uploadOptions.remoteOutputsRawPrefix, "to-raw-output", "x", "", "The remote path/key prefix for outputs in remote store. This is a sandbox directory and all data will be uploaded here.") + uploadCmd.Flags().StringVarP(&uploadOptions.localDirectoryPath, "from-local-dir", "f", "", "The local directory on disk where data will be available for upload.") + uploadCmd.Flags().StringVarP(&uploadOptions.metadataFormat, "format", "m", core.DataLoadingConfig_JSON.String(), fmt.Sprintf("What should be the output format for the primitive and structured types. Options [%v]", GetFormatVals())) + uploadCmd.Flags().StringVarP(&uploadOptions.uploadMode, "upload-mode", "u", core.IOStrategy_UPLOAD_ON_EXIT.String(), fmt.Sprintf("When should upload start/upload mode. Options [%v]", GetUploadModeVals())) + uploadCmd.Flags().StringVarP(&uploadOptions.metaOutputName, "meta-output-name", "", "outputs.pb", "The key name under the remoteOutputPrefix that should be return to provide meta information about the outputs on successful execution") + uploadCmd.Flags().DurationVarP(&uploadOptions.timeout, "timeout", "t", time.Hour*1, "Max time to allow for uploads to complete, default is 1H") + uploadCmd.Flags().BytesBase64VarP(&uploadOptions.typedInterface, "interface", "i", nil, "Typed Interface - core.TypedInterface, base64 encoded string of the serialized protobuf") + uploadCmd.Flags().DurationVarP(&uploadOptions.containerStartTimeout, "start-timeout", "", 0, "Max time to allow for container to startup. 0 indicates wait for ever.") + uploadCmd.Flags().StringVarP(&uploadOptions.startWatcherType, "start-watcher-type", "", containerwatcher.WatcherTypeSharedProcessNS, fmt.Sprintf("Sidecar will wait for container before starting upload process. Watcher type makes the type configurable. Available Type %+v", containerwatcher.AllWatcherTypes)) + uploadCmd.Flags().StringVarP(&uploadOptions.exitWatcherType, "exit-watcher-type", "", containerwatcher.WatcherTypeSharedProcessNS, fmt.Sprintf("Sidecar will wait for completion of the container before starting upload process. Watcher type makes the type configurable. Available Type %+v", containerwatcher.AllWatcherTypes)) + uploadCmd.Flags().StringVarP(&uploadOptions.containerInfo.Name, "watch-container", "", "", "For KubeAPI watcher, Wait for this container to exit.") + uploadCmd.Flags().StringVarP(&uploadOptions.containerInfo.Namespace, "namespace", "", "", "For KubeAPI watcher, Namespace of the pod [optional]") + uploadCmd.Flags().StringVarP(&uploadOptions.containerInfo.Name, "pod-name", "", "", "For KubeAPI watcher, Name of the pod [optional].") + return uploadCmd +} diff --git a/flyteplugins/copilot/cmd/sidecar_test.go b/flyteplugins/copilot/cmd/sidecar_test.go new file mode 100644 index 0000000000..526acf45ff --- /dev/null +++ b/flyteplugins/copilot/cmd/sidecar_test.go @@ -0,0 +1,99 @@ +package cmd + +import ( + "context" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/golang/protobuf/proto" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/promutils" + "github.com/lyft/flytestdlib/storage" + "github.com/stretchr/testify/assert" + + "github.com/lyft/flyteplugins/copilot/cmd/containerwatcher" +) + +func TestUploadOptions_Upload(t *testing.T) { + tmpFolderLocation := "" + tmpPrefix := "upload_test" + outputPath := "output" + + ctx := context.TODO() + + t.Run("uploadNoOutputs", func(t *testing.T) { + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + + s := promutils.NewTestScope() + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage")) + assert.NoError(t, err) + uopts := UploadOptions{ + RootOptions: &RootOptions{ + Scope: s, + Store: store, + }, + remoteOutputsPrefix: outputPath, + metadataFormat: core.DataLoadingConfig_JSON.String(), + uploadMode: core.IOStrategy_UPLOAD_ON_EXIT.String(), + startWatcherType: containerwatcher.WatcherTypeFile, + localDirectoryPath: tmpDir, + } + + assert.NoError(t, uopts.Sidecar(ctx)) + }) + + t.Run("uploadBlobType-FileNotFound", func(t *testing.T) { + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + s := promutils.NewTestScope() + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, s.NewSubScope("storage")) + assert.NoError(t, err) + + iface := &core.TypedInterface{ + Outputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": { + Type: &core.LiteralType{Type: &core.LiteralType_Blob{Blob: &core.BlobType{Dimensionality: core.BlobType_SINGLE}}}, + Description: "example", + }, + }, + }, + } + d, err := proto.Marshal(iface) + assert.NoError(t, err) + + uopts := UploadOptions{ + RootOptions: &RootOptions{ + Scope: s, + Store: store, + errorOutputName: "errors.pb", + }, + remoteOutputsPrefix: outputPath, + metadataFormat: core.DataLoadingConfig_JSON.String(), + uploadMode: core.IOStrategy_UPLOAD_ON_EXIT.String(), + startWatcherType: containerwatcher.WatcherTypeNoop, + exitWatcherType: containerwatcher.WatcherTypeFile, + typedInterface: d, + localDirectoryPath: tmpDir, + } + + success := path.Join(tmpDir, SuccessFile) + assert.NoError(t, ioutil.WriteFile(success, []byte("done"), os.ModePerm)) + ok, err := containerwatcher.FileExists(success) + assert.NoError(t, err) + assert.True(t, ok, "sucessfile not created") + assert.NoError(t, uopts.Sidecar(ctx)) + v, err := store.Head(ctx, "/output/errors.pb") + assert.NoError(t, err) + assert.True(t, v.Exists()) + }) +} diff --git a/flyteplugins/copilot/data/common.go b/flyteplugins/copilot/data/common.go new file mode 100644 index 0000000000..7433b44039 --- /dev/null +++ b/flyteplugins/copilot/data/common.go @@ -0,0 +1,19 @@ +// This module contains Flyte CoPilot related code. +// Currently it only has 2 utilities - downloader and an uploader. +// Usage Downloader: +// downloader := NewDownloader(...) +// downloader.DownloadInputs(...) // will recursively download all inputs +// +// Usage uploader: +// uploader := NewUploader(...) +// uploader.RecursiveUpload(...) // Will recursively upload all the data from the given path depending on the output interface +// All errors are bubbled up. +// +// Both the uploader and downloader accept context.Context variables. These should be used to control timeouts etc. +// TODO: Currently retries are not automatically handled. +package data + +import "github.com/lyft/flytestdlib/futures" + +type VarMap map[string]interface{} +type FutureMap map[string]futures.Future diff --git a/flyteplugins/copilot/data/download.go b/flyteplugins/copilot/data/download.go new file mode 100644 index 0000000000..3f47508913 --- /dev/null +++ b/flyteplugins/copilot/data/download.go @@ -0,0 +1,365 @@ +package data + +import ( + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "reflect" + "strconv" + + "github.com/gogo/protobuf/jsonpb" + "github.com/golang/protobuf/proto" + "github.com/golang/protobuf/ptypes" + structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/futures" + "github.com/lyft/flytestdlib/logger" + "github.com/lyft/flytestdlib/storage" + "github.com/pkg/errors" + "gopkg.in/yaml.v2" +) + +type Downloader struct { + format core.DataLoadingConfig_LiteralMapFormat + store *storage.DataStore + // TODO support download mode + mode core.IOStrategy_DownloadMode +} + +// TODO add support for multipart blobs +func (d Downloader) handleBlob(ctx context.Context, blob *core.Blob, toFilePath string) (interface{}, error) { + ref := storage.DataReference(blob.Uri) + scheme, _, _, err := ref.Split() + if err != nil { + return nil, errors.Wrapf(err, "Blob uri incorrectly formatted") + } + var reader io.ReadCloser + if scheme == "http" || scheme == "https" { + reader, err = DownloadFileFromHTTP(ctx, ref) + } else { + if blob.GetMetadata().GetType().Dimensionality == core.BlobType_MULTIPART { + logger.Warnf(ctx, "Currently only single part blobs are supported, we will force multipart to be 'path/00000'") + ref, err = d.store.ConstructReference(ctx, ref, "000000") + if err != nil { + return nil, err + } + } + reader, err = DownloadFileFromStorage(ctx, ref, d.store) + } + if err != nil { + logger.Errorf(ctx, "Failed to download from ref [%s]", ref) + return nil, err + } + defer func() { + err := reader.Close() + if err != nil { + logger.Errorf(ctx, "failed to close Blob read stream @ref [%s]. Error: %s", ref, err) + } + }() + + writer, err := os.Create(toFilePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to open file at path %s", toFilePath) + } + defer func() { + err := writer.Close() + if err != nil { + logger.Errorf(ctx, "failed to close File write stream. Error: %s", err) + } + }() + v, err := io.Copy(writer, reader) + if err != nil { + return nil, errors.Wrapf(err, "failed to write remote data to local filesystem") + } + logger.Infof(ctx, "Successfully copied [%d] bytes remote data from [%s] to local [%s]", v, ref, toFilePath) + return toFilePath, nil +} + +func (d Downloader) handleSchema(ctx context.Context, schema *core.Schema, toFilePath string) (interface{}, error) { + // TODO Handle schema type + return d.handleBlob(ctx, &core.Blob{Uri: schema.Uri, Metadata: &core.BlobMetadata{Type: &core.BlobType{Dimensionality: core.BlobType_MULTIPART}}}, toFilePath) +} + +func (d Downloader) handleBinary(_ context.Context, b *core.Binary, toFilePath string, writeToFile bool) (interface{}, error) { + // maybe we should return a map + v := b.GetValue() + if writeToFile { + return v, ioutil.WriteFile(toFilePath, v, os.ModePerm) + } + return v, nil +} + +func (d Downloader) handleError(_ context.Context, b *core.Error, toFilePath string, writeToFile bool) (interface{}, error) { + // maybe we should return a map + if writeToFile { + return b.Message, ioutil.WriteFile(toFilePath, []byte(b.Message), os.ModePerm) + } + return b.Message, nil +} + +func (d Downloader) handleGeneric(ctx context.Context, b *structpb.Struct, toFilePath string, writeToFile bool) (interface{}, error) { + if writeToFile && b != nil { + m := jsonpb.Marshaler{} + writer, err := os.Create(toFilePath) + if err != nil { + return nil, errors.Wrapf(err, "failed to open file at path %s", toFilePath) + } + defer func() { + err := writer.Close() + if err != nil { + logger.Errorf(ctx, "failed to close File write stream. Error: %s", err) + } + }() + return b, m.Marshal(writer, b) + } + return b, nil +} + +// Returns the primitive value in Golang native format and if the filePath is not empty, then writes the value to the given file path. +func (d Downloader) handlePrimitive(primitive *core.Primitive, toFilePath string, writeToFile bool) (interface{}, error) { + + var toByteArray func() ([]byte, error) + var v interface{} + var err error + + switch primitive.Value.(type) { + case *core.Primitive_StringValue: + v = primitive.GetStringValue() + toByteArray = func() ([]byte, error) { + return []byte(primitive.GetStringValue()), nil + } + case *core.Primitive_Boolean: + v = primitive.GetBoolean() + toByteArray = func() ([]byte, error) { + return []byte(strconv.FormatBool(primitive.GetBoolean())), nil + } + case *core.Primitive_Integer: + v = primitive.GetInteger() + toByteArray = func() ([]byte, error) { + return []byte(strconv.FormatInt(primitive.GetInteger(), 10)), nil + } + case *core.Primitive_FloatValue: + v = primitive.GetFloatValue() + toByteArray = func() ([]byte, error) { + return []byte(strconv.FormatFloat(primitive.GetFloatValue(), 'f', -1, 64)), nil + } + case *core.Primitive_Datetime: + v, err = ptypes.Timestamp(primitive.GetDatetime()) + if err != nil { + return nil, err + } + toByteArray = func() ([]byte, error) { + return []byte(ptypes.TimestampString(primitive.GetDatetime())), nil + } + case *core.Primitive_Duration: + v, err := ptypes.Duration(primitive.GetDuration()) + if err != nil { + return nil, err + } + toByteArray = func() ([]byte, error) { + return []byte(v.String()), nil + } + default: + v = nil + toByteArray = func() ([]byte, error) { + return []byte("null"), nil + } + } + if writeToFile { + b, err := toByteArray() + if err != nil { + return nil, err + } + return v, ioutil.WriteFile(toFilePath, b, os.ModePerm) + } + return v, nil +} + +func (d Downloader) handleScalar(ctx context.Context, scalar *core.Scalar, toFilePath string, writeToFile bool) (interface{}, *core.Scalar, error) { + switch scalar.GetValue().(type) { + case *core.Scalar_Primitive: + p := scalar.GetPrimitive() + i, err := d.handlePrimitive(p, toFilePath, writeToFile) + return i, scalar, err + case *core.Scalar_Blob: + b := scalar.GetBlob() + i, err := d.handleBlob(ctx, b, toFilePath) + return i, &core.Scalar{Value: &core.Scalar_Blob{Blob: &core.Blob{Metadata: b.Metadata, Uri: toFilePath}}}, err + case *core.Scalar_Schema: + b := scalar.GetSchema() + i, err := d.handleSchema(ctx, b, toFilePath) + return i, &core.Scalar{Value: &core.Scalar_Schema{Schema: &core.Schema{Type: b.Type, Uri: toFilePath}}}, err + case *core.Scalar_Binary: + b := scalar.GetBinary() + i, err := d.handleBinary(ctx, b, toFilePath, writeToFile) + return i, scalar, err + case *core.Scalar_Error: + b := scalar.GetError() + i, err := d.handleError(ctx, b, toFilePath, writeToFile) + return i, scalar, err + case *core.Scalar_Generic: + b := scalar.GetGeneric() + i, err := d.handleGeneric(ctx, b, toFilePath, writeToFile) + return i, scalar, err + case *core.Scalar_NoneType: + if writeToFile { + return nil, scalar, ioutil.WriteFile(toFilePath, []byte("null"), os.ModePerm) + } + return nil, scalar, nil + default: + return nil, nil, fmt.Errorf("unsupported scalar type [%v]", reflect.TypeOf(scalar.GetValue())) + } +} + +func (d Downloader) handleLiteral(ctx context.Context, lit *core.Literal, filePath string, writeToFile bool) (interface{}, *core.Literal, error) { + switch lit.GetValue().(type) { + case *core.Literal_Scalar: + v, s, err := d.handleScalar(ctx, lit.GetScalar(), filePath, writeToFile) + if err != nil { + return nil, nil, err + } + return v, &core.Literal{Value: &core.Literal_Scalar{ + Scalar: s, + }}, nil + case *core.Literal_Collection: + v, c2, err := d.handleCollection(ctx, lit.GetCollection(), filePath, writeToFile) + if err != nil { + return nil, nil, err + } + return v, &core.Literal{Value: &core.Literal_Collection{ + Collection: c2, + }}, nil + case *core.Literal_Map: + v, m, err := d.RecursiveDownload(ctx, lit.GetMap(), filePath, writeToFile) + if err != nil { + return nil, nil, err + } + return v, &core.Literal{Value: &core.Literal_Map{ + Map: m, + }}, nil + default: + return nil, nil, fmt.Errorf("unsupported literal type [%v]", reflect.TypeOf(lit.GetValue())) + } +} + +// Collection should be stored as a top level list file and may have accompanying files? +func (d Downloader) handleCollection(ctx context.Context, c *core.LiteralCollection, dir string, writePrimitiveToFile bool) ([]interface{}, *core.LiteralCollection, error) { + if c == nil || len(c.Literals) == 0 { + return []interface{}{}, c, nil + } + var collection []interface{} + litCollection := &core.LiteralCollection{} + for i, lit := range c.Literals { + filePath := path.Join(dir, strconv.Itoa(i)) + v, lit, err := d.handleLiteral(ctx, lit, filePath, writePrimitiveToFile) + if err != nil { + return nil, nil, err + } + collection = append(collection, v) + litCollection.Literals = append(litCollection.Literals, lit) + } + return collection, litCollection, nil +} + +type downloadedResult struct { + lit *core.Literal + v interface{} +} + +func (d Downloader) RecursiveDownload(ctx context.Context, inputs *core.LiteralMap, dir string, writePrimitiveToFile bool) (VarMap, *core.LiteralMap, error) { + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + if inputs == nil || len(inputs.Literals) == 0 { + return VarMap{}, nil, nil + } + f := make(FutureMap, len(inputs.Literals)) + for variable, literal := range inputs.Literals { + varPath := path.Join(dir, variable) + lit := literal + f[variable] = futures.NewAsyncFuture(childCtx, func(ctx2 context.Context) (interface{}, error) { + v, lit, err := d.handleLiteral(ctx2, lit, varPath, writePrimitiveToFile) + if err != nil { + return nil, err + } + return downloadedResult{lit: lit, v: v}, nil + }) + } + + m := &core.LiteralMap{ + Literals: make(map[string]*core.Literal), + } + vmap := make(VarMap, len(f)) + for variable, future := range f { + logger.Infof(ctx, "Waiting for [%s] to be persisted", variable) + v, err := future.Get(childCtx) + if err != nil { + logger.Errorf(ctx, "Failed to persist [%s], err %s", variable, err) + if err == futures.ErrAsyncFutureCanceled { + logger.Errorf(ctx, "Future was canceled, possibly Timeout!") + } + return nil, nil, errors.Wrapf(err, "variable [%s] download/store failed", variable) + } + dr := v.(downloadedResult) + vmap[variable] = dr.v + m.Literals[variable] = dr.lit + logger.Infof(ctx, "Completed persisting [%s]", variable) + } + + return vmap, m, nil +} + +func (d Downloader) DownloadInputs(ctx context.Context, inputRef storage.DataReference, outputDir string) error { + logger.Infof(ctx, "Downloading inputs from [%s]", inputRef) + defer logger.Infof(ctx, "Exited downloading inputs from [%s]", inputRef) + if err := os.MkdirAll(outputDir, os.ModePerm); err != nil { + logger.Errorf(ctx, "Failed to create output directories, err: %s", err) + return err + } + inputs := &core.LiteralMap{} + err := d.store.ReadProtobuf(ctx, inputRef, inputs) + if err != nil { + logger.Errorf(ctx, "Failed to download inputs from [%s], err [%s]", inputRef, err) + return errors.Wrapf(err, "failed to download input metadata message from remote store") + } + varMap, lMap, err := d.RecursiveDownload(ctx, inputs, outputDir, true) + if err != nil { + return errors.Wrapf(err, "failed to download input variable from remote store") + } + + // We will always write the protobuf + b, err := proto.Marshal(lMap) + if err != nil { + return err + } + if err := ioutil.WriteFile(path.Join(outputDir, "inputs.pb"), b, os.ModePerm); err != nil { + return err + } + + if d.format == core.DataLoadingConfig_JSON { + m, err := json.Marshal(varMap) + if err != nil { + return errors.Wrapf(err, "failed to marshal out inputs") + } + return ioutil.WriteFile(path.Join(outputDir, "inputs.json"), m, os.ModePerm) + } + if d.format == core.DataLoadingConfig_YAML { + m, err := yaml.Marshal(varMap) + if err != nil { + return errors.Wrapf(err, "failed to marshal out inputs") + } + return ioutil.WriteFile(path.Join(outputDir, "inputs.yaml"), m, os.ModePerm) + } + return nil +} + +func NewDownloader(_ context.Context, store *storage.DataStore, format core.DataLoadingConfig_LiteralMapFormat, mode core.IOStrategy_DownloadMode) Downloader { + return Downloader{ + format: format, + store: store, + mode: mode, + } +} diff --git a/flyteplugins/copilot/data/upload.go b/flyteplugins/copilot/data/upload.go new file mode 100644 index 0000000000..d2b6bf6325 --- /dev/null +++ b/flyteplugins/copilot/data/upload.go @@ -0,0 +1,200 @@ +package data + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "os" + "path" + "path/filepath" + "reflect" + + "github.com/golang/protobuf/proto" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/logger" + "github.com/lyft/flytestdlib/storage" + "github.com/lyft/flytestdlib/futures" + "github.com/pkg/errors" + + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/utils" +) + +const maxPrimitiveSize = 1024 + +type Unmarshal func(r io.Reader, msg proto.Message) error +type Uploader struct { + format core.DataLoadingConfig_LiteralMapFormat + mode core.IOStrategy_UploadMode + // TODO support multiple buckets + store *storage.DataStore + aggregateOutputFileName string + errorFileName string +} + +type dirFile struct { + path string + info os.FileInfo + ref storage.DataReference +} + +func (u Uploader) handleSimpleType(_ context.Context, t core.SimpleType, filePath string) (*core.Literal, error) { + fpath, info, err := IsFileReadable(filePath, true) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, fmt.Errorf("expected file for type [%s], found dir at path [%s]", t.String(), filePath) + } + if info.Size() > maxPrimitiveSize { + return nil, fmt.Errorf("maximum allowed filesize is [%d], but found [%d]", maxPrimitiveSize, info.Size()) + } + b, err := ioutil.ReadFile(fpath) + if err != nil { + return nil, err + } + return utils.MakeLiteralForSimpleType(t, string(b)) +} + +func (u Uploader) handleBlobType(ctx context.Context, localPath string, toPath storage.DataReference) (*core.Literal, error) { + fpath, info, err := IsFileReadable(localPath, true) + if err != nil { + return nil, err + } + if info.IsDir() { + var files []dirFile + err := filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + logger.Errorf(ctx, "encountered error when uploading multipart blob, %s", err) + return err + } + if info.IsDir() { + logger.Warnf(ctx, "Currently nested directories are not supported in multipart blob uploads, for directory @ %s", path) + } else { + ref, err := u.store.ConstructReference(ctx, toPath, info.Name()) + if err != nil { + return err + } + files = append(files, dirFile{ + path: path, + info: info, + ref: ref, + }) + } + return nil + }) + if err != nil { + return nil, err + } + + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + fileUploader := make([]futures.Future, 0, len(files)) + for _, f := range files { + pth := f.path + ref := f.ref + size := f.info.Size() + fileUploader = append(fileUploader, futures.NewAsyncFuture(childCtx, func(i2 context.Context) (i interface{}, e error) { + return nil, UploadFileToStorage(i2, pth, ref, size, u.store) + })) + } + + for _, f := range fileUploader { + // TODO maybe we should have timeouts, or we can have a global timeout at the top level + _, err := f.Get(ctx) + if err != nil { + return nil, err + } + } + + return utils.MakeLiteralForBlob(toPath, false, ""), nil + } + size := info.Size() + // Should we make this a go routine as well, so that we can introduce timeouts + return utils.MakeLiteralForBlob(toPath, false, ""), UploadFileToStorage(ctx, fpath, toPath, size, u.store) +} + +func (u Uploader) RecursiveUpload(ctx context.Context, vars *core.VariableMap, fromPath string, metaOutputPath, dataRawPath storage.DataReference) error { + childCtx, cancel := context.WithCancel(ctx) + defer cancel() + + errFile := path.Join(fromPath, u.errorFileName) + if info, err := os.Stat(errFile); err != nil { + if !os.IsNotExist(err) { + return err + } + } else if info.Size() > 1024*1024 { + return fmt.Errorf("error file too large %d", info.Size()) + } else if info.IsDir() { + return fmt.Errorf("error file is a directory") + } else { + b, err := ioutil.ReadFile(errFile) + if err != nil { + return err + } + return errors.Errorf("User Error: %s", string(b)) + } + + varFutures := make(map[string]futures.Future, len(vars.Variables)) + for varName, variable := range vars.Variables { + varPath := path.Join(fromPath, varName) + varType := variable.GetType() + switch varType.GetType().(type) { + case *core.LiteralType_Blob: + var varOutputPath storage.DataReference + var err error + if varName == u.aggregateOutputFileName { + varOutputPath, err = u.store.ConstructReference(ctx, dataRawPath, "_"+varName) + } else { + varOutputPath, err = u.store.ConstructReference(ctx, dataRawPath, varName) + } + if err != nil { + return err + } + varFutures[varName] = futures.NewAsyncFuture(childCtx, func(ctx2 context.Context) (interface{}, error) { + return u.handleBlobType(ctx2, varPath, varOutputPath) + }) + case *core.LiteralType_Simple: + varFutures[varName] = futures.NewAsyncFuture(childCtx, func(ctx2 context.Context) (interface{}, error) { + return u.handleSimpleType(ctx2, varType.GetSimple(), varPath) + }) + default: + return fmt.Errorf("currently CoPilot uploader does not support [%s], system error", varType) + } + } + + outputs := &core.LiteralMap{ + Literals: make(map[string]*core.Literal, len(varFutures)), + } + for k, f := range varFutures { + logger.Infof(ctx, "Waiting for [%s] to complete (it may have a background upload too)", k) + v, err := f.Get(ctx) + if err != nil { + logger.Errorf(ctx, "Failed to upload [%s], reason [%s]", k, err) + return err + } + l, ok := v.(*core.Literal) + if !ok { + return fmt.Errorf("IllegalState, expected core.Literal, received [%s]", reflect.TypeOf(v)) + } + outputs.Literals[k] = l + logger.Infof(ctx, "Var [%s] completed", k) + } + + logger.Infof(ctx, "Uploading final outputs to [%s]", metaOutputPath) + if err := u.store.WriteProtobuf(ctx, metaOutputPath, storage.Options{}, outputs); err != nil { + logger.Errorf(ctx, "Failed to upload final outputs file to [%s], err [%s]", metaOutputPath, err) + return err + } + logger.Infof(ctx, "Uploaded final outputs to [%s]", metaOutputPath) + return nil +} + +func NewUploader(_ context.Context, store *storage.DataStore, format core.DataLoadingConfig_LiteralMapFormat, mode core.IOStrategy_UploadMode, errorFileName string) Uploader { + return Uploader{ + format: format, + store: store, + errorFileName: errorFileName, + mode: mode, + } +} diff --git a/flyteplugins/copilot/data/upload_test.go b/flyteplugins/copilot/data/upload_test.go new file mode 100644 index 0000000000..c83faa3cbe --- /dev/null +++ b/flyteplugins/copilot/data/upload_test.go @@ -0,0 +1,63 @@ +package data + +import ( + "context" + "fmt" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/promutils" + "github.com/lyft/flytestdlib/storage" + "github.com/stretchr/testify/assert" +) + +func TestUploader_RecursiveUpload(t *testing.T) { + + tmpFolderLocation := "" + tmpPrefix := "upload_test" + + t.Run("upload-blob", func(t *testing.T) { + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + + vmap := &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": { + Type: &core.LiteralType{Type: &core.LiteralType_Blob{Blob: &core.BlobType{Dimensionality: core.BlobType_SINGLE}}}, + }, + }, + } + + data := []byte("data") + assert.NoError(t, ioutil.WriteFile(path.Join(tmpDir, "x"), data, os.ModePerm)) + fmt.Printf("Written to %s ", path.Join(tmpDir, "x")) + + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope()) + assert.NoError(t, err) + + outputRef := storage.DataReference("output") + rawRef := storage.DataReference("raw") + u := NewUploader(context.TODO(), store, core.DataLoadingConfig_JSON, core.IOStrategy_UPLOAD_ON_EXIT, "error") + assert.NoError(t, u.RecursiveUpload(context.TODO(), vmap, tmpDir, outputRef, rawRef)) + + outputs := &core.LiteralMap{} + assert.NoError(t, store.ReadProtobuf(context.TODO(), outputRef, outputs)) + assert.Len(t, outputs.Literals, 1) + assert.NotNil(t, outputs.Literals["x"]) + assert.NotNil(t, outputs.Literals["x"].GetScalar()) + assert.NotNil(t, outputs.Literals["x"].GetScalar().GetBlob()) + ref := storage.DataReference(outputs.Literals["x"].GetScalar().GetBlob().GetUri()) + r, err := store.ReadRaw(context.TODO(), ref) + assert.NoError(t, err, "%s does not exist", ref) + defer r.Close() + b, err := ioutil.ReadAll(r) + assert.NoError(t, err) + assert.Equal(t, string(data), string(b), "content dont match") + }) +} diff --git a/flyteplugins/copilot/data/utils.go b/flyteplugins/copilot/data/utils.go new file mode 100644 index 0000000000..2691eef1c7 --- /dev/null +++ b/flyteplugins/copilot/data/utils.go @@ -0,0 +1,90 @@ +package data + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + + "github.com/lyft/flytestdlib/logger" + "github.com/lyft/flytestdlib/storage" + "github.com/pkg/errors" +) + +// Checks if the given filepath is a valid and existing file path. If ignoreExtension is true, then the dir + basepath is checked for existence +// ignoring the extension. +// In the return the first return value is the actual path that exists (with the extension), second argument is the file info and finally the error +func IsFileReadable(fpath string, ignoreExtension bool) (string, os.FileInfo, error) { + info, err := os.Stat(fpath) + if err != nil { + if os.IsNotExist(err) { + if ignoreExtension { + logger.Infof(context.TODO(), "looking for any extensions") + matches, err := filepath.Glob(fpath + ".*") + if err == nil && len(matches) == 1 { + logger.Infof(context.TODO(), "Extension match found [%s]", matches[0]) + info, err = os.Stat(matches[0]) + if err == nil { + return matches[0], info, nil + } + } else { + logger.Errorf(context.TODO(), "Extension match not found [%v,%v]", err, matches) + } + } + return "", nil, errors.Wrapf(err, "file not found at path [%s]", fpath) + } + if os.IsPermission(err) { + return "", nil, errors.Wrapf(err, "unable to read file [%s], Flyte does not have permissions", fpath) + } + return "", nil, errors.Wrapf(err, "failed to read file") + } + return fpath, info, nil +} + +// Uploads a file to the data store. +func UploadFileToStorage(ctx context.Context, filePath string, toPath storage.DataReference, size int64, store *storage.DataStore) error { + f, err := os.Open(filePath) + if err != nil { + return err + } + defer func() { + err := f.Close() + if err != nil { + logger.Errorf(ctx, "failed to close blob file at path [%s]", filePath) + } + }() + return store.WriteRaw(ctx, toPath, size, storage.Options{}, f) +} + +func DownloadFileFromStorage(ctx context.Context, ref storage.DataReference, store *storage.DataStore) (io.ReadCloser, error) { + // We should probably directly use stow!?? + m, err := store.Head(ctx, ref) + if err != nil { + return nil, errors.Wrapf(err, "failed when looking up Blob") + } + if m.Exists() { + r, err := store.ReadRaw(ctx, ref) + if err != nil { + return nil, errors.Wrapf(err, "failed to read Blob from storage") + } + return r, err + + } + return nil, fmt.Errorf("incorrect blob reference, does not exist") +} + +// Downloads data from the given HTTP URL. If context is canceled then the request will be canceled. +func DownloadFileFromHTTP(ctx context.Context, ref storage.DataReference) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, ref.String(), nil) + if err != nil { + logger.Errorf(ctx, "failed to create new http request with context, %s", err) + return nil, err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, errors.Wrapf(err, "Failed to download from url :%s", ref) + } + return resp.Body, nil +} diff --git a/flyteplugins/copilot/data/utils_test.go b/flyteplugins/copilot/data/utils_test.go new file mode 100644 index 0000000000..ecf30bbbd2 --- /dev/null +++ b/flyteplugins/copilot/data/utils_test.go @@ -0,0 +1,119 @@ +package data + +import ( + "bytes" + "context" + "io/ioutil" + "os" + "path" + "testing" + + "github.com/lyft/flytestdlib/promutils" + "github.com/lyft/flytestdlib/promutils/labeled" + "github.com/lyft/flytestdlib/storage" + "github.com/stretchr/testify/assert" +) + +func TestIsFileReadable(t *testing.T) { + tmpFolderLocation := "" + tmpPrefix := "util_test" + + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + p := path.Join(tmpDir, "x") + f, i, err := IsFileReadable(p, false) + assert.Error(t, err) + assert.Empty(t, f) + assert.Nil(t, i) + + assert.NoError(t, ioutil.WriteFile(p, []byte("data"), os.ModePerm)) + f, i, err = IsFileReadable(p, false) + assert.NoError(t, err) + assert.Equal(t, p, f) + assert.NotNil(t, i) + assert.Equal(t, p, f) + + noExt := path.Join(tmpDir, "y") + p = path.Join(tmpDir, "y.png") + _, _, err = IsFileReadable(noExt, false) + assert.Error(t, err) + + assert.NoError(t, ioutil.WriteFile(p, []byte("data"), os.ModePerm)) + _, _, err = IsFileReadable(noExt, false) + assert.Error(t, err) + + f, i, err = IsFileReadable(noExt, true) + assert.NoError(t, err) + assert.Equal(t, p, f) + assert.NotNil(t, i) + assert.Equal(t, p, f) +} + +func TestUploadFile(t *testing.T) { + tmpFolderLocation := "" + tmpPrefix := "util_test" + + tmpDir, err := ioutil.TempDir(tmpFolderLocation, tmpPrefix) + assert.NoError(t, err) + defer func() { + assert.NoError(t, os.RemoveAll(tmpDir)) + }() + + exist := path.Join(tmpDir, "exist-file") + data := []byte("data") + l := int64(len(data)) + assert.NoError(t, ioutil.WriteFile(exist, data, os.ModePerm)) + nonExist := path.Join(tmpDir, "non-exist-file") + + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope()) + assert.NoError(t, err) + + ctx := context.TODO() + assert.NoError(t, UploadFileToStorage(ctx, exist, "exist", l, store)) + m, err := store.Head(ctx, "exist") + assert.True(t, m.Exists()) + assert.NoError(t, err) + + assert.Error(t, UploadFileToStorage(ctx, nonExist, "nonExist", l, store)) +} + +func TestDownloadFromHttp(t *testing.T) { + loc := storage.DataReference("https://raw.githubusercontent.com/lyft/flyte/master/README.md") + badLoc := storage.DataReference("https://no-exist") + f, err := DownloadFileFromHTTP(context.TODO(), loc) + if assert.NoError(t, err) { + if assert.NotNil(t, f) { + f.Close() + } + } + + _, err = DownloadFileFromHTTP(context.TODO(), badLoc) + assert.Error(t, err) +} + +func TestDownloadFromStorage(t *testing.T) { + store, err := storage.NewDataStore(&storage.Config{Type: storage.TypeMemory}, promutils.NewTestScope()) + assert.NoError(t, err) + ref := storage.DataReference("ref") + + f, err := DownloadFileFromStorage(context.TODO(), ref, store) + assert.Error(t, err) + assert.Nil(t, f) + + data := []byte("data") + l := int64(len(data)) + + assert.NoError(t, store.WriteRaw(context.TODO(), ref, l, storage.Options{}, bytes.NewReader(data))) + f, err = DownloadFileFromStorage(context.TODO(), ref, store) + if assert.NoError(t, err) { + assert.NotNil(t, f) + f.Close() + } +} + +func init() { + labeled.SetMetricKeys("test") +} diff --git a/flyteplugins/copilot/go.mod b/flyteplugins/copilot/go.mod new file mode 100644 index 0000000000..5e85990a3c --- /dev/null +++ b/flyteplugins/copilot/go.mod @@ -0,0 +1,33 @@ +module github.com/lyft/flyteplugins/copilot + +go 1.13 + +require ( + github.com/fsnotify/fsnotify v1.4.9 + github.com/gogo/protobuf v1.3.1 + github.com/golang/protobuf v1.4.2 + github.com/imdario/mergo v0.3.9 // indirect + github.com/lyft/flyteidl v0.17.32 + github.com/lyft/flyteplugins v0.3.29 + github.com/lyft/flytestdlib v0.3.9 + github.com/mitchellh/go-ps v1.0.0 + github.com/pkg/errors v0.9.1 + github.com/spf13/cobra v1.0.0 + github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.6.1 + gopkg.in/yaml.v2 v2.2.8 + k8s.io/api v0.18.3 + k8s.io/apimachinery v0.18.3 + k8s.io/client-go v11.0.0+incompatible + k8s.io/klog v1.0.0 + k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 // indirect +) + +replace ( + github.com/GoogleCloudPlatform/spark-on-k8s-operator => github.com/lyft/spark-on-k8s-operator v0.1.3 + github.com/googleapis/gnostic => github.com/googleapis/gnostic v0.3.1 + github.com/lyft/flyteplugins => ../ + k8s.io/api => github.com/lyft/api v0.0.0-20191031200350-b49a72c274e0 + k8s.io/apimachinery => github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f + k8s.io/client-go => k8s.io/client-go v0.0.0-20191016111102-bec269661e48 +) diff --git a/flyteplugins/copilot/go.sum b/flyteplugins/copilot/go.sum new file mode 100644 index 0000000000..6b6d36e828 --- /dev/null +++ b/flyteplugins/copilot/go.sum @@ -0,0 +1,735 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.52.0 h1:GGslhk/BU052LPlnI1vpp3fcbUs+hQ3E+Doti/3/vF8= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v32.5.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v38.2.0+incompatible h1:ZeCdp1E/V5lI8oLR/BjWQh0OW9aFBYlgXGKRVIWNPXY= +github.com/Azure/azure-sdk-for-go v38.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v39.0.0+incompatible h1:l2FVXqtd34UC7OZYkhcWY843CSFjRdrIRdBCTOanYwg= +github.com/Azure/azure-sdk-for-go v39.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.4 h1:1cM+NmKw91+8h5vfjgzK4ZGLuN72k87XVZBWyGwNjUM= +github.com/Azure/go-autorest/autorest v0.9.4/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk= +github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.23.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.28.9 h1:grIuBQc+p3dTRXerh5+2OxSuWFi0iXuxbFdTSg0jaW0= +github.com/aws/aws-sdk-go v1.28.9/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.28.11 h1:L2G5qI91s51cUP3hJli4mXRIZZ3alZHcwHWOJdMclKk= +github.com/aws/aws-sdk-go v1.28.11/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1/go.mod h1:jvdWlw8vowVGnZqSDC7yhPd7AifQeQbRDkZcQXV2nRg= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coocood/freecache v1.1.0 h1:ENiHOsWdj1BrrlPwblhbn4GdAsMymK3pZORJ+bJGAjA= +github.com/coocood/freecache v1.1.0/go.mod h1:ePwxCDzOYvARfHdr1pByNct1at3CoKnsipOHwKlNbzI= +github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/etcd v3.3.15+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180108230652-97fdf19511ea/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/docker v0.7.3-0.20190327010347-be7ac8be2ae0/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/ernesto-jimenez/gogen v0.0.0-20180125220232-d7d4131e6607/go.mod h1:Cg4fM0vhYWOZdgM7RIOSTRNIc8/VT7CXClC3Ni86lu4= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.8-0.20191012010759-4bf2d1fec783/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/zapr v0.1.0/go.mod h1:tabnROwaDl0UNxkVeFRbY8bwB37GwRv0P8lg6aAiEnk= +github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= +github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= +github.com/go-openapi/analysis v0.19.2/go.mod h1:3P1osvZa9jKjb8ed2TPng3f0i/UY9snX6gxi44djMjk= +github.com/go-openapi/errors v0.17.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.18.0/go.mod h1:LcZQpmvG4wyF5j4IhA73wkLFQg+QJXOQHVjmcZxhka0= +github.com/go-openapi/errors v0.19.2/go.mod h1:qX0BLWsyaKfvhluLejVpVNwNRdXZhEbTA4kxxpKBC94= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.17.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.18.0/go.mod h1:cOnomiV+CVVwFLk0A/MExoFMjwdsUdVpsRhURCKh+3M= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.17.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.18.0/go.mod h1:g4xxGn04lDIRh0GJb5QlpE3HfopLOL6uZrK/VgnsK9I= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/loads v0.17.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.18.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.0/go.mod h1:72tmFy5wsWx89uEVddd0RjRWPZm92WRLhf7AC+0+OOU= +github.com/go-openapi/loads v0.19.2/go.mod h1:QAskZPMX5V0C2gvfkGZzJlINuP7Hx/4+ix5jWFxsNPs= +github.com/go-openapi/runtime v0.0.0-20180920151709-4f900dc2ade9/go.mod h1:6v9a6LTXWQCdL8k1AO3cvqx5OtZY/Y9wKTgaoP6YRfA= +github.com/go-openapi/runtime v0.19.0/go.mod h1:OwNfisksmmaZse4+gpV3Ne9AyMOlP1lt4sK4FXt0O64= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.17.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.18.0/go.mod h1:XkF/MOi14NmjsfZ8VtAKf8pIlbZzyoTvZsdfssdxcBI= +github.com/go-openapi/spec v0.19.2/go.mod h1:sCxk3jxKgioEJikev4fgkNmwS+3kuYdJtcsZsD5zxMY= +github.com/go-openapi/strfmt v0.17.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.18.0/go.mod h1:P82hnJI0CXkErkXi8IKjPbNBM6lV6+5pLP5l494TcyU= +github.com/go-openapi/strfmt v0.19.0/go.mod h1:+uW+93UVvGGq2qGaZxdDeJqSAqBqBdl+ZPMF/cC8nDY= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.17.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.18.0/go.mod h1:AByQ+nYG6gQg71GINrmuDXCPWdL640yX49/kXLo40Tg= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+MYsct2VUrAJ4= +github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.5/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20180513044358-24b0969c4cb7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.0.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/readahead v0.0.0-20161222183148-eaceba169032/go.mod h1:qYysrqQXuV4tzsizt4oOQ6mrBZQ0xnQXP3ylXX8Jk5Y= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.1.0 h1:rVsPeBmXbYv4If/cumu1AzZPwV58q433hvONV1UEZoI= +github.com/googleapis/gnostic v0.1.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/googleapis/gnostic v0.3.1 h1:WeAefnSUHlBb0iJKwxFDZdbfGwkd7xRNuV+IpXMJhYk= +github.com/googleapis/gnostic v0.3.1/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= +github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/graymeta/stow v0.2.4 h1:qDGstknYXqcnmBQ5TRJtxD9Qv1MuRbYRhLoSMeUDs7U= +github.com/graymeta/stow v0.2.4/go.mod h1:+0vRL9oMECKjPMP7OeVWl8EIqRCpFwDlth3mrAeV2Kw= +github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v0.0.0-20190222133341-cfaf5686ec79/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.3.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.12.2/go.mod h1:8XEsbTttt/W+VvjtQhLACqCisSPWTxCZ7sBRjU6iH9c= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.9 h1:UauaLniWCFHWd+Jp9oCEkTBj8VO/9DKg3PV3VCNMDIg= +github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/influxdata/influxdb v1.7.9/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.8 h1:QiWkFLKq0T7mpzwOTu6BzNDbfTE8OLrYhVKYMLF46Ok= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kubeflow/pytorch-operator v0.6.0/go.mod h1:zHblV+yTwVG4PCgKTU2wPfOmQ6TJdfT87lDfHrP1a1Y= +github.com/kubeflow/tf-operator v0.5.3/go.mod h1:EBtz5LQoKaHUl/5fV5vD1qXVNVNyn3TrFaH6eVoQ8SY= +github.com/lyft/api v0.0.0-20191031200350-b49a72c274e0 h1:NGL46+1RYcCXb3sShp0nQq4W38fcgnpCD4+X02eeLL0= +github.com/lyft/api v0.0.0-20191031200350-b49a72c274e0/go.mod h1:/L5qH+AD540e7Cetbui1tuJeXdmNhO8jM6VkXeDdDhQ= +github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f h1:PGuAMDzAen0AulUfaEhNQMYmUpa41pAVo3zHI+GJsCM= +github.com/lyft/apimachinery v0.0.0-20191031200210-047e3ea32d7f/go.mod h1:llRdnznGEAqC3DcNm6yEj472xaFVfLM7hnYofMb12tQ= +github.com/lyft/flyteidl v0.17.32 h1:Iio3gYjTyPhAiOMWJ/H/4YtfWIZm5KZSlWMULT1Ef6U= +github.com/lyft/flyteidl v0.17.32/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20= +github.com/lyft/flyteplugins v0.3.29 h1:NN88yXv6sTouMVwQEgbP0A6k+uznGr00ZcKP6ZFUPrU= +github.com/lyft/flyteplugins v0.3.29/go.mod h1:HHO6KC/2z77n9o9KM697YvSP85IWDe6jl6tAIrMLqWU= +github.com/lyft/flytestdlib v0.3.0/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/flytestdlib v0.3.3/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/flytestdlib v0.3.7 h1:CBMcxEhti2IY491fO/z/31X4d/cxSeZTIt1CcRrs27I= +github.com/lyft/flytestdlib v0.3.7/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/flytestdlib v0.3.8 h1:tM91NdQELPC/D1vSlptuAlgk4ba9V1zPdiJCVQnEZPM= +github.com/lyft/flytestdlib v0.3.8/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/flytestdlib v0.3.9 h1:NaKp9xkeWWwhVvqTOcR/FqlASy1N2gu/kN7PVe4S7YI= +github.com/lyft/flytestdlib v0.3.9/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/spark-on-k8s-operator v0.1.3/go.mod h1:hkRqdqAsdNnxT/Zst6MNMRbTAoiCZ0JRw7svRgAYb0A= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/ncw/swift v1.0.49 h1:eQaKIjSt/PXLKfYgzg01nevmO+CMXfXGRhB1gOhDs7E= +github.com/ncw/swift v1.0.49/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.4.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.3.0/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4= +github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.0/go.mod h1:NxmoDg/QLVWluQDUYG7XBZTLUpKeFa8e3aMf1BfjyHk= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.3.0 h1:miYCvYqFXtl/J9FIy8eNpBfYthAEFg+Ys0XyUVEcDsc= +github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.4.0 h1:YVIb/fVcOTMSqtqZWSKnHpSLBxu8DKgxq8z6RuBZwqI= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/satori/uuid v1.2.0/go.mod h1:B8HLsPLik/YNn6KKWVMDJ8nzCL8RP5WyfsnmvnAEwIU= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.3/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= +github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.0.0 h1:6m/oheQuQ13N9ks4hubMG6BnvwOeaJrqSPLahSnczz8= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= +github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/xiang90/probing v0.0.0-20160813154853-07dd2e8dfe18/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.uber.org/atomic v0.0.0-20181018215023-8dc6146f7569/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v0.0.0-20180122172545-ddea229ff1df/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v0.0.0-20180814183419-67bc79d13d15/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad h1:Jh8cai0fqIK+f6nG0UgPW5wFk8wmiMhM3AyciDBdtQg= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 h1:+ELyKg6m8UBf0nPFSqD0mi7zUfwPyXo23HNjMnXPz7w= +golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180112015858-5ccada7d0a7b/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190812203447-cdfb69ac37fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191002035440-2ec189313ef0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180117170059-2c42eef0765b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190321052220-f7bb7a8bee54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20171227012246-e19ae1496984/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200124170513-3f4d10fc73b4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.16.0 h1:hhRbpE9nkabqMxGCewz2sikMYxm8yYWov7h2Eo4j3Is= +google.golang.org/api v0.16.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20190927181202-20e1ac93f88c/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150 h1:VPpdpQkGvFicX9yo4G5oxZPi9ALBnEOZblPSa/Wa2m4= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200205142000-a86caf926a67 h1:MBO9fkVSrTpJ8vgHLPi5gb+ZWXEy7/auJN8yqyu9EiE= +google.golang.org/genproto v0.0.0-20200205142000-a86caf926a67/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.0 h1:3zYtXIO92bvsdS3ggAdA8Gb4Azj0YU+TVY1uGYNFA8o= +gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.51.1 h1:GyboHr4UqMiLUybYjd22ZjQIKEJEpgtLXtuGbR21Oho= +gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/kothar/go-backblaze.v0 v0.0.0-20190520213052-702d4e7eb465/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.0.0/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.17.2/go.mod h1:BS9fjjLc4CMuqfSO8vgbHPKMt5+SF0ET6u/RVDihTo4= +k8s.io/api v0.18.3 h1:2AJaUQdgUZLoDZHrun21PW2Nx9+ll6cUzvn3IKhSIn0= +k8s.io/api v0.18.3/go.mod h1:UOaMwERbqJMfeeeHc8XJKawj4P9TgDRnViIqqBeH2QA= +k8s.io/apiextensions-apiserver v0.0.0-20190918161926-8f644eb6e783/go.mod h1:xvae1SZB3E17UpV59AWc271W/Ph25N+bjPyR63X6tPY= +k8s.io/apimachinery v0.17.2/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/apimachinery v0.18.3 h1:pOGcbVAhxADgUYnjS08EFXs9QMl8qaH5U4fr5LGUrSk= +k8s.io/apimachinery v0.18.3/go.mod h1:OaXp26zu/5J7p0f92ASynJa1pZo06YlV9fG7BoWbCko= +k8s.io/apiserver v0.0.0-20190918160949-bfa5e2e684ad/go.mod h1:XPCXEwhjaFN29a8NldXA901ElnKeKLrLtREO9ZhFyhg= +k8s.io/client-go v0.0.0-20191016111102-bec269661e48 h1:C2XVy2z0dV94q9hSSoCuTPp1KOG7IegvbdXuz9VGxoU= +k8s.io/client-go v0.0.0-20191016111102-bec269661e48/go.mod h1:hrwktSwYGI4JK+TJA3dMaFyyvHVi/aLarVHpbs8bgCU= +k8s.io/client-go v11.0.0+incompatible h1:LBbX2+lOwY9flffWlJM7f1Ct8V2SRNiMRDFeiwnJo9o= +k8s.io/client-go v11.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/code-generator v0.0.0-20190912054826-cd179ad6a269/go.mod h1:V5BD6M4CyaN5m+VthcclXWsVcT1Hu+glwa1bi3MIsyE= +k8s.io/component-base v0.0.0-20190918160511-547f6c5d7090/go.mod h1:933PBGtQFJky3TEwYx4aEPZ4IxqhWh3R6DCmzqIn1hA= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v0.4.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/kube-openapi v0.0.0-20190816220812-743ec37842bf/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +k8s.io/kube-openapi v0.0.0-20200410145947-61e04a5be9a6/go.mod h1:GRQhZsXIAJ1xR0C9bd8UpWHZ5plfAS9fzPjJuQ6JL3E= +k8s.io/utils v0.0.0-20190801114015-581e00157fb1/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200124190032-861946025e34/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= +k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 h1:v8ud2Up6QK1lNOKFgiIVrZdMg7MpmSnvtrOieolJKoE= +k8s.io/utils v0.0.0-20200603063816-c1c6865ac451/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/controller-runtime v0.4.0/go.mod h1:ApC79lpY3PHW9xj/w9pj+lYkLgwAAUZwfXkME1Lajns= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/structured-merge-diff v0.0.0-20190817042607-6149e4549fca/go.mod h1:IIgPezJWb76P0hotTxzDbWsMYB8APh18qZnxkomBpxA= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0-20200116222232-67a7b8c61874/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= +sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= +sigs.k8s.io/testing_frameworks v0.1.2/go.mod h1:ToQrwSC3s8Xf/lADdZp3Mktcql9CG0UAmdJG9th5i0w= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/flyteplugins/copilot/main.go b/flyteplugins/copilot/main.go new file mode 100644 index 0000000000..6fb7d89cc9 --- /dev/null +++ b/flyteplugins/copilot/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "fmt" + "os" + + "github.com/lyft/flyteplugins/copilot/cmd" +) + +func main() { + rootCmd := cmd.NewDataCommand() + if err := rootCmd.Execute(); err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/flyteplugins/go.mod b/flyteplugins/go.mod index 8ce37c3845..741077a22c 100644 --- a/flyteplugins/go.mod +++ b/flyteplugins/go.mod @@ -7,20 +7,24 @@ require ( github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.1.3 github.com/aws/aws-sdk-go v1.28.11 github.com/coocood/freecache v1.1.0 + github.com/fsnotify/fsnotify v1.4.8-0.20191012010759-4bf2d1fec783 github.com/go-test/deep v1.0.5 + github.com/gogo/protobuf v1.3.1 github.com/golang/protobuf v1.3.3 github.com/googleapis/gnostic v0.4.1 // indirect github.com/hashicorp/golang-lru v0.5.4 github.com/kubeflow/pytorch-operator v0.6.0 github.com/kubeflow/tf-operator v0.5.3 github.com/lyft/flyteidl v0.17.32 - github.com/lyft/flytestdlib v0.3.3 + github.com/lyft/flytestdlib v0.3.9 github.com/magiconair/properties v1.8.1 + github.com/mitchellh/go-ps v1.0.0 github.com/mitchellh/mapstructure v1.1.2 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.4.0 + github.com/spf13/cobra v0.0.5 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.6.1 go.opencensus.io v0.22.3 // indirect golang.org/x/crypto v0.0.0-20200204104054-c9f3fb736b72 // indirect golang.org/x/net v0.0.0-20200202094626-16171245cfb2 @@ -29,10 +33,13 @@ require ( google.golang.org/api v0.16.0 // indirect google.golang.org/genproto v0.0.0-20200205142000-a86caf926a67 // indirect google.golang.org/grpc v1.27.1 + gopkg.in/yaml.v2 v2.2.8 + gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c // indirect + gotest.tools v2.2.0+incompatible k8s.io/api v0.17.2 k8s.io/apimachinery v0.17.2 k8s.io/client-go v11.0.0+incompatible - k8s.io/klog v1.0.0 // indirect + k8s.io/klog v1.0.0 k8s.io/utils v0.0.0-20200124190032-861946025e34 // indirect sigs.k8s.io/controller-runtime v0.4.0 sigs.k8s.io/yaml v1.2.0 // indirect diff --git a/flyteplugins/go.sum b/flyteplugins/go.sum index e915e2c932..4c11fec80a 100644 --- a/flyteplugins/go.sum +++ b/flyteplugins/go.sum @@ -318,6 +318,14 @@ github.com/lyft/flytestdlib v0.3.2 h1:bY6Y+Fg6Jdc7zY4GAYuR7t2hjWwynIdmRvtLcRNaGn github.com/lyft/flytestdlib v0.3.2/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= github.com/lyft/flytestdlib v0.3.3 h1:MkWXPkwQinh6MR3Yf5siZhmRSt9r4YmsF+5kvVVVedE= github.com/lyft/flytestdlib v0.3.3/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/flytestdlib v0.3.7-0.20200604044950-5ebc3901b799 h1:2ke55PqVrGTQISuugbPxWtE2UbNnBsijuyNsy3i/9ws= +github.com/lyft/flytestdlib v0.3.7-0.20200604044950-5ebc3901b799/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/flytestdlib v0.3.7 h1:CBMcxEhti2IY491fO/z/31X4d/cxSeZTIt1CcRrs27I= +github.com/lyft/flytestdlib v0.3.7/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/flytestdlib v0.3.8 h1:tM91NdQELPC/D1vSlptuAlgk4ba9V1zPdiJCVQnEZPM= +github.com/lyft/flytestdlib v0.3.8/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= +github.com/lyft/flytestdlib v0.3.9 h1:NaKp9xkeWWwhVvqTOcR/FqlASy1N2gu/kN7PVe4S7YI= +github.com/lyft/flytestdlib v0.3.9/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU= github.com/lyft/spark-on-k8s-operator v0.1.3 h1:rmke8lR2Oy8mvKXRhloKuEu7fgGuXepDxiBNiorVUFI= github.com/lyft/spark-on-k8s-operator v0.1.3/go.mod h1:hkRqdqAsdNnxT/Zst6MNMRbTAoiCZ0JRw7svRgAYb0A= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -337,6 +345,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-ps v1.0.0 h1:i6ampVEEF4wQFF+bkYfwYgY+F/uYJDktmvLPf7qIgjc= +github.com/mitchellh/go-ps v1.0.0/go.mod h1:J4lOc8z8yJs6vUwklHw2XEIiT4z4C40KtWVN3nvg8Pg= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -459,6 +469,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -713,6 +725,11 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/async_client.go b/flyteplugins/go/tasks/pluginmachinery/catalog/async_client.go index d6c4604ace..62a0658c07 100644 --- a/flyteplugins/go/tasks/pluginmachinery/catalog/async_client.go +++ b/flyteplugins/go/tasks/pluginmachinery/catalog/async_client.go @@ -42,7 +42,7 @@ type Future interface { GetResponseError() error } -// Catalog Upload future to represent async process of uploading catalog artifacts. +// Catalog Sidecar future to represent async process of uploading catalog artifacts. type UploadFuture interface { Future } diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/async_client_impl_test.go b/flyteplugins/go/tasks/pluginmachinery/catalog/async_client_impl_test.go index fe3078c0a3..bc72feb872 100644 --- a/flyteplugins/go/tasks/pluginmachinery/catalog/async_client_impl_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/catalog/async_client_impl_test.go @@ -87,11 +87,11 @@ func TestAsyncClientImpl_Upload(t *testing.T) { } gotPutFuture, err := c.Upload(ctx, tt.requests...) if (err != nil) != tt.wantErr { - t.Errorf("AsyncClientImpl.Upload() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("AsyncClientImpl.Sidecar() error = %v, wantErr %v", err, tt.wantErr) return } if !reflect.DeepEqual(gotPutFuture, tt.wantPutFuture) { - t.Errorf("AsyncClientImpl.Upload() = %v, want %v", gotPutFuture, tt.wantPutFuture) + t.Errorf("AsyncClientImpl.Sidecar() = %v, want %v", gotPutFuture, tt.wantPutFuture) } }) } diff --git a/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/async_client.go b/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/async_client.go index 46fc8c5eca..8e9361f407 100644 --- a/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/async_client.go +++ b/flyteplugins/go/tasks/pluginmachinery/catalog/mocks/async_client.go @@ -72,16 +72,16 @@ func (_m AsyncClient_Upload) Return(putFuture catalog.UploadFuture, err error) * } func (_m *AsyncClient) OnUpload(ctx context.Context, requests ...catalog.UploadRequest) *AsyncClient_Upload { - c := _m.On("Upload", ctx, requests) + c := _m.On("Sidecar", ctx, requests) return &AsyncClient_Upload{Call: c} } func (_m *AsyncClient) OnUploadMatch(matchers ...interface{}) *AsyncClient_Upload { - c := _m.On("Upload", matchers...) + c := _m.On("Sidecar", matchers...) return &AsyncClient_Upload{Call: c} } -// Upload provides a mock function with given fields: ctx, requests +// Sidecar provides a mock function with given fields: ctx, requests func (_m *AsyncClient) Upload(ctx context.Context, requests ...catalog.UploadRequest) (catalog.UploadFuture, error) { _va := make([]interface{}, len(requests)) for _i := range requests { diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config.go index d13b850579..04ac491efa 100755 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config.go @@ -1,6 +1,14 @@ +// This package contains configuration for the flytek8s module. +// This config is under the subsection `k8s` and registered under the Plugin config +// All K8s based plugins can optionally use the flytek8s module and this configuration allows controlling the defaults +// For example if for every container execution if some default Environment Variables or Annotations should be used, then they can be configured here +// An important configuration is ResourceTolerations that are applied to every container execution that needs some resource on the cluster package config import ( + "time" + + config2 "github.com/lyft/flytestdlib/config" v1 "k8s.io/api/core/v1" "github.com/lyft/flyteplugins/go/tasks/config" @@ -17,6 +25,21 @@ var ( DefaultAnnotations: map[string]string{ "cluster-autoscaler.kubernetes.io/safe-to-evict": "false", }, + CoPilot: FlyteCoPilotConfig{ + NamePrefix: "flyte-copilot-", + Image: "docker.pkg.github.com/lyft/flyteplugins/operator:v0.4.0", + DefaultInputDataPath: "/var/flyte/inputs", + InputVolumeName: "flyte-inputs", + DefaultOutputPath: "/var/flyte/outputs", + OutputVolumeName: "flyte-outputs", + CPU: "500m", + Memory: "128Mi", + StartTimeout: config2.Duration{ + Duration: time.Second * 60, + }, + }, + DefaultCPURequest: defaultCPURequest, + DefaultMemoryRequest: defaultMemoryRequest, } // K8sPluginConfigSection provides a singular top level config section for all plugins. @@ -49,20 +72,37 @@ type K8sPluginConfig struct { // Node Selector Labels for interruptible pods: Similar to InterruptibleTolerations, these node selector labels are added for pods that can tolerate // eviction. InterruptibleNodeSelector map[string]string `json:"interruptible-node-selector" pflag:"-,Defines a set of node selector labels to add to the interruptible pods."` - // Scheduler name. + // Default scheduler that should be used for all pods or CRD that accept Scheduler name. SchedulerName string `json:"scheduler-name" pflag:",Defines scheduler name."` + // Flyte CoPilot Configuration + CoPilot FlyteCoPilotConfig `json:"co-pilot" pflag:",Co-Pilot Configuration"` +} + +type FlyteCoPilotConfig struct { + // Co-pilot sidecar container name + NamePrefix string `json:"name" pflag:",Flyte co-pilot sidecar container name prefix. (additional bits will be added after this)"` + // Docker image FQN where co-pilot binary is installed + Image string `json:"image" pflag:",Flyte co-pilot Docker Image FQN"` + // Default Input Path for every task execution that uses co-pilot. This is used only if a input path is not provided by the user and inputs are required for the task + DefaultInputDataPath string `json:"default-input-path" pflag:",Default path where the volume should be mounted"` + // Default Output Path for every task execution that uses co-pilot. This is used only if a output path is not provided by the user and outputs are required for the task + DefaultOutputPath string `json:"default-output-path" pflag:",Default path where the volume should be mounted"` + // Name of the input volume + InputVolumeName string `json:"input-vol-name" pflag:",Name of the data volume that is created for storing inputs"` + // Name of the output volume + OutputVolumeName string `json:"output-vol-name" pflag:",Name of the data volume that is created for storing outputs"` + // Time for which the sidecar container should wait after starting up, for the primary process to appear. If it does not show up in this time + // the process will be assumed to be dead or in a terminal condition and will trigger an abort. + StartTimeout config2.Duration `json:"start-timeout" pflag:",Time for which the sidecar should wait on startup before assuming the primary container to have failed startup."` + // Resources for CoPilot Containers + CPU string `json:"cpu" pflag:",Used to set cpu for co-pilot containers"` + Memory string `json:"memory" pflag:",Used to set memory for co-pilot containers"` + Storage string `json:"storage" pflag:",Default storage limit for individual inputs / outputs"` } // Retrieves the current k8s plugin config or default. func GetK8sPluginConfig() *K8sPluginConfig { - pluginsConfig := K8sPluginConfigSection.GetConfig().(*K8sPluginConfig) - if pluginsConfig.DefaultMemoryRequest == "" { - pluginsConfig.DefaultMemoryRequest = defaultMemoryRequest - } - if pluginsConfig.DefaultCPURequest == "" { - pluginsConfig.DefaultCPURequest = defaultCPURequest - } - return pluginsConfig + return K8sPluginConfigSection.GetConfig().(*K8sPluginConfig) } // [FOR TESTING ONLY] Sets current value for the config. diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config_test.go new file mode 100644 index 0000000000..eb3429774d --- /dev/null +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/config_test.go @@ -0,0 +1,12 @@ +package config + +import ( + "testing" + + "gotest.tools/assert" +) + +func TestGetK8sPluginConfig(t *testing.T) { + assert.Equal(t, GetK8sPluginConfig().DefaultCPURequest, defaultCPURequest) + assert.Equal(t, GetK8sPluginConfig().DefaultMemoryRequest, defaultMemoryRequest) +} diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags.go index 82726684ab..162c2bd86d 100755 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags.go @@ -45,5 +45,10 @@ func (cfg K8sPluginConfig) GetPFlagSet(prefix string) *pflag.FlagSet { cmdFlags.String(fmt.Sprintf("%v%v", prefix, "default-cpus"), *new(string), "Defines a default value for cpu for containers if not specified.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "default-memory"), *new(string), "Defines a default value for memory for containers if not specified.") cmdFlags.String(fmt.Sprintf("%v%v", prefix, "scheduler-name"), *new(string), "Defines scheduler name.") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "co-pilot.image"), *new(string), "Flyte co-pilot Docker Image FQN") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "co-pilot.default-input-path"), *new(string), "Default path where the volume should be mounted") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "co-pilot.default-output-path"), *new(string), "Default path where the volume should be mounted") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "co-pilot.input-vol-name"), *new(string), "Name of the data volume that is created for storing inputs") + cmdFlags.String(fmt.Sprintf("%v%v", prefix, "co-pilot.output-vol-name"), *new(string), "Name of the data volume that is created for storing outputs") return cmdFlags } diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags_test.go index 995b5d4c9e..103ba5bea2 100755 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags_test.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/config/k8spluginconfig_flags_test.go @@ -187,4 +187,114 @@ func TestK8sPluginConfig_SetFlags(t *testing.T) { } }) }) + t.Run("Test_co-pilot.image", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("co-pilot.image"); err == nil { + assert.Equal(t, string(*new(string)), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("co-pilot.image", testValue) + if vString, err := cmdFlags.GetString("co-pilot.image"); err == nil { + testDecodeJson_K8sPluginConfig(t, fmt.Sprintf("%v", vString), &actual.CoPilot.Image) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_co-pilot.default-input-path", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("co-pilot.default-input-path"); err == nil { + assert.Equal(t, string(*new(string)), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("co-pilot.default-input-path", testValue) + if vString, err := cmdFlags.GetString("co-pilot.default-input-path"); err == nil { + testDecodeJson_K8sPluginConfig(t, fmt.Sprintf("%v", vString), &actual.CoPilot.DefaultInputDataPath) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_co-pilot.default-output-path", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("co-pilot.default-output-path"); err == nil { + assert.Equal(t, string(*new(string)), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("co-pilot.default-output-path", testValue) + if vString, err := cmdFlags.GetString("co-pilot.default-output-path"); err == nil { + testDecodeJson_K8sPluginConfig(t, fmt.Sprintf("%v", vString), &actual.CoPilot.DefaultOutputPath) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_co-pilot.input-vol-name", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("co-pilot.input-vol-name"); err == nil { + assert.Equal(t, string(*new(string)), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("co-pilot.input-vol-name", testValue) + if vString, err := cmdFlags.GetString("co-pilot.input-vol-name"); err == nil { + testDecodeJson_K8sPluginConfig(t, fmt.Sprintf("%v", vString), &actual.CoPilot.InputVolumeName) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) + t.Run("Test_co-pilot.output-vol-name", func(t *testing.T) { + t.Run("DefaultValue", func(t *testing.T) { + // Test that default value is set properly + if vString, err := cmdFlags.GetString("co-pilot.output-vol-name"); err == nil { + assert.Equal(t, string(*new(string)), vString) + } else { + assert.FailNow(t, err.Error()) + } + }) + + t.Run("Override", func(t *testing.T) { + testValue := "1" + + cmdFlags.Set("co-pilot.output-vol-name", testValue) + if vString, err := cmdFlags.GetString("co-pilot.output-vol-name"); err == nil { + testDecodeJson_K8sPluginConfig(t, fmt.Sprintf("%v", vString), &actual.CoPilot.OutputVolumeName) + + } else { + assert.FailNow(t, err.Error()) + } + }) + }) } diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper.go index f6fc1ea954..f0583516a0 100755 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/container_helper.go @@ -85,7 +85,7 @@ func ApplyResourceOverrides(ctx context.Context, resources v1.ResourceRequiremen } // Returns a K8s Container for the execution -func ToK8sContainer(ctx context.Context, taskExecutionMetadata pluginsCore.TaskExecutionMetadata, taskContainer *core.Container, +func ToK8sContainer(ctx context.Context, taskExecutionMetadata pluginsCore.TaskExecutionMetadata, taskContainer *core.Container, iFace *core.TypedInterface, inputReader io.InputReader, outputPaths io.OutputFilePaths) (*v1.Container, error) { modifiedCommand, err := utils.ReplaceTemplateCommandArgs(ctx, taskContainer.GetCommand(), inputReader, outputPaths) if err != nil { @@ -130,5 +130,8 @@ func ToK8sContainer(ctx context.Context, taskExecutionMetadata pluginsCore.TaskE c.Resources = *res } + if err := AddCoPilotToContainer(ctx, config.GetK8sPluginConfig().CoPilot, c, iFace, taskContainer.DataConfig); err != nil { + return nil, err + } return c, nil } diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/copilot.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/copilot.go new file mode 100644 index 0000000000..d882ac85c7 --- /dev/null +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/copilot.go @@ -0,0 +1,269 @@ +package flytek8s + +import ( + "context" + "encoding/base64" + "fmt" + "time" + + "github.com/golang/protobuf/proto" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/logger" + "github.com/lyft/flytestdlib/storage" + "github.com/pkg/errors" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + core2 "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/io" +) + +const ( + flyteDataConfigVolume = "data-config-volume" + flyteDataConfigPath = "/etc/flyte/config-data" + flyteDataConfigMap = "flyte-data-config" + flyteSidecarContainerName = "sidecar" + flyteInitContainerName = "downloader" +) + +var pTraceCapability = v1.Capability("SYS_PTRACE") + +func FlyteCoPilotContainer(name string, cfg config.FlyteCoPilotConfig, args []string, volumeMounts ...v1.VolumeMount) (v1.Container, error) { + cpu, err := resource.ParseQuantity(cfg.CPU) + if err != nil { + return v1.Container{}, err + } + + mem, err := resource.ParseQuantity(cfg.Memory) + if err != nil { + return v1.Container{}, err + } + + return v1.Container{ + Name: cfg.NamePrefix + name, + Image: cfg.Image, + Command: []string{"/bin/flyte-copilot", "--config", "/etc/flyte/config**/*"}, + Args: args, + WorkingDir: "/", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: cpu, + v1.ResourceMemory: mem, + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: cpu, + v1.ResourceMemory: mem, + }, + }, + VolumeMounts: volumeMounts, + TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, + ImagePullPolicy: v1.PullIfNotPresent, + }, nil +} + +func SidecarCommandArgs(fromLocalPath string, outputPrefix, rawOutputPath storage.DataReference, startTimeout time.Duration, iface *core.TypedInterface) ([]string, error) { + if iface == nil { + return nil, fmt.Errorf("interface is required for CoPilot Sidecar") + } + b, err := proto.Marshal(iface) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal given core.TypedInterface") + } + return []string{ + "sidecar", + "--start-timeout", + startTimeout.String(), + "--to-raw-output", + rawOutputPath.String(), + "--to-output-prefix", + outputPrefix.String(), + "--from-local-dir", + fromLocalPath, + "--interface", + base64.StdEncoding.EncodeToString(b), + }, nil +} + +func DownloadCommandArgs(fromInputsPath, outputPrefix storage.DataReference, toLocalPath string, format core.DataLoadingConfig_LiteralMapFormat, inputInterface *core.VariableMap) ([]string, error) { + if inputInterface == nil { + return nil, fmt.Errorf("input Interface is required for CoPilot Downloader") + } + b, err := proto.Marshal(inputInterface) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal given input interface") + } + return []string{ + "download", + "--from-remote", + fromInputsPath.String(), + "--to-output-prefix", + outputPrefix.String(), + "--to-local-dir", + toLocalPath, + "--format", + format.String(), + "--input-interface", + base64.StdEncoding.EncodeToString(b), + }, nil +} + +func DataVolume(name string, size *resource.Quantity) v1.Volume { + return v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + Medium: v1.StorageMediumDefault, + SizeLimit: size, + }, + }, + } +} + +func CalculateStorageSize(requirements *v1.ResourceRequirements) *resource.Quantity { + if requirements == nil { + return nil + } + s, ok := requirements.Limits[v1.ResourceStorage] + if ok { + return &s + } + s, ok = requirements.Requests[v1.ResourceStorage] + if ok { + return &s + } + return nil +} + +func AddCoPilotToContainer(ctx context.Context, cfg config.FlyteCoPilotConfig, c *v1.Container, iFace *core.TypedInterface, pilot *core.DataLoadingConfig) error { + if pilot == nil || !pilot.Enabled { + return nil + } + logger.Infof(ctx, "Enabling CoPilot on main container [%s]", c.Name) + if c.SecurityContext == nil { + c.SecurityContext = &v1.SecurityContext{} + } + if c.SecurityContext.Capabilities == nil { + c.SecurityContext.Capabilities = &v1.Capabilities{} + } + c.SecurityContext.Capabilities.Add = append(c.SecurityContext.Capabilities.Add, pTraceCapability) + + if iFace != nil { + if iFace.Inputs != nil { + inPath := cfg.DefaultInputDataPath + if pilot.GetInputPath() != "" { + inPath = pilot.GetInputPath() + } + + c.VolumeMounts = append(c.VolumeMounts, v1.VolumeMount{ + Name: cfg.InputVolumeName, + MountPath: inPath, + }) + } + + if iFace.Outputs != nil { + outPath := cfg.DefaultOutputPath + if pilot.GetOutputPath() != "" { + outPath = pilot.GetOutputPath() + } + c.VolumeMounts = append(c.VolumeMounts, v1.VolumeMount{ + Name: cfg.OutputVolumeName, + MountPath: outPath, + }) + } + } + return nil +} + +func AddCoPilotToPod(ctx context.Context, cfg config.FlyteCoPilotConfig, coPilotPod *v1.PodSpec, iFace *core.TypedInterface, taskExecMetadata core2.TaskExecutionMetadata, inputPaths io.InputFilePaths, outputPaths io.OutputFilePaths, pilot *core.DataLoadingConfig) error { + if pilot == nil || !pilot.Enabled { + return nil + } + + logger.Infof(ctx, "CoPilot Enabled for task [%s]", taskExecMetadata.GetTaskExecutionID().GetID().TaskId.Name) + shareProcessNamespaceEnabled := true + coPilotPod.ShareProcessNamespace = &shareProcessNamespaceEnabled + if iFace != nil { + // TODO think about MountPropagationMode. Maybe we want to use that for acceleration in the future + if iFace.Inputs != nil || iFace.Outputs != nil { + // This is temporary. we have to mount the flyte data configuration into the pod + // TODO Remove the data configuration requirements + coPilotPod.Volumes = append(coPilotPod.Volumes, v1.Volume{ + Name: flyteDataConfigVolume, + VolumeSource: v1.VolumeSource{ + ConfigMap: &v1.ConfigMapVolumeSource{ + LocalObjectReference: v1.LocalObjectReference{ + Name: flyteDataConfigMap, + }, + }, + }, + }) + } + cfgVMount := v1.VolumeMount{ + Name: flyteDataConfigVolume, + MountPath: flyteDataConfigPath, + } + + if iFace.Inputs != nil { + inPath := cfg.DefaultInputDataPath + if pilot.GetInputPath() != "" { + inPath = pilot.GetInputPath() + } + + // TODO we should calculate input volume size based on the size of the inputs which is known ahead of time. We should store that as part of the metadata + size := CalculateStorageSize(taskExecMetadata.GetOverrides().GetResources()) + logger.Infof(ctx, "Adding Input path [%s] of Size [%d] for Task [%s]", size, inPath, taskExecMetadata.GetTaskExecutionID().GetID().TaskId.Name) + inputsVolumeMount := v1.VolumeMount{ + Name: cfg.InputVolumeName, + MountPath: inPath, + } + + format := pilot.Format + // Lets add the InputsVolume + coPilotPod.Volumes = append(coPilotPod.Volumes, DataVolume(cfg.InputVolumeName, size)) + + // Lets add the Inputs init container + args, err := DownloadCommandArgs(inputPaths.GetInputPath(), outputPaths.GetOutputPrefixPath(), inPath, format, iFace.Inputs) + if err != nil { + return err + } + downloader, err := FlyteCoPilotContainer(flyteInitContainerName, cfg, args, inputsVolumeMount, cfgVMount) + if err != nil { + return err + } + coPilotPod.InitContainers = append(coPilotPod.InitContainers, downloader) + } + + if iFace.Outputs != nil { + outPath := cfg.DefaultOutputPath + if pilot.GetOutputPath() != "" { + outPath = pilot.GetOutputPath() + } + + size := CalculateStorageSize(taskExecMetadata.GetOverrides().GetResources()) + logger.Infof(ctx, "Adding Output path [%s] of size [%d] for Task [%s]", size, outPath, taskExecMetadata.GetTaskExecutionID().GetID().TaskId.Name) + + outputsVolumeMount := v1.VolumeMount{ + Name: cfg.OutputVolumeName, + MountPath: outPath, + } + + // Lets add the InputsVolume + coPilotPod.Volumes = append(coPilotPod.Volumes, DataVolume(cfg.OutputVolumeName, size)) + + // Lets add the Inputs init container + args, err := SidecarCommandArgs(outPath, outputPaths.GetOutputPrefixPath(), outputPaths.GetRawOutputPrefix(), cfg.StartTimeout.Duration, iFace) + if err != nil { + return err + } + sidecar, err := FlyteCoPilotContainer(flyteSidecarContainerName, cfg, args, outputsVolumeMount, cfgVMount) + if err != nil { + return err + } + coPilotPod.Containers = append(coPilotPod.Containers, sidecar) + } + + } + + return nil +} diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/copilot_test.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/copilot_test.go new file mode 100644 index 0000000000..a100d955f1 --- /dev/null +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/copilot_test.go @@ -0,0 +1,584 @@ +package flytek8s + +import ( + "context" + "encoding/base64" + "reflect" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + config2 "github.com/lyft/flytestdlib/config" + "github.com/lyft/flytestdlib/storage" + "github.com/stretchr/testify/assert" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + pluginsCoreMock "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/core/mocks" + "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/flytek8s/config" + pluginsIOMock "github.com/lyft/flyteplugins/go/tasks/pluginmachinery/io/mocks" +) + +var resourceRequirements = &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("1024m"), + v1.ResourceStorage: resource.MustParse("100M"), + }, +} + +func TestFlyteCoPilotContainer(t *testing.T) { + cfg := config.FlyteCoPilotConfig{ + NamePrefix: "test-", + Image: "test", + DefaultInputDataPath: "/in", + DefaultOutputPath: "/out", + InputVolumeName: "inp", + OutputVolumeName: "out", + StartTimeout: config2.Duration{ + Duration: time.Second * 1, + }, + CPU: "1024m", + Memory: "1024Mi", + } + + t.Run("happy", func(t *testing.T) { + c, err := FlyteCoPilotContainer("x", cfg, []string{"hello"}) + assert.NoError(t, err) + assert.Equal(t, "test-x", c.Name) + assert.Equal(t, "test", c.Image) + assert.Equal(t, []string{"/bin/flyte-copilot", "--config", "/etc/flyte/config**/*"}, c.Command) + assert.Equal(t, []string{"hello"}, c.Args) + assert.Equal(t, 0, len(c.VolumeMounts)) + assert.Equal(t, "/", c.WorkingDir) + assert.Equal(t, 2, len(c.Resources.Limits)) + assert.Equal(t, 2, len(c.Resources.Requests)) + }) + + t.Run("happy-vols", func(t *testing.T) { + c, err := FlyteCoPilotContainer("x", cfg, []string{"hello"}, v1.VolumeMount{Name: "X", MountPath: "/"}) + assert.NoError(t, err) + assert.Equal(t, 1, len(c.VolumeMounts)) + }) + + t.Run("bad-res-cpu", func(t *testing.T) { + old := cfg.CPU + cfg.CPU = "x" + _, err := FlyteCoPilotContainer("x", cfg, []string{"hello"}, v1.VolumeMount{Name: "X", MountPath: "/"}) + assert.Error(t, err) + cfg.CPU = old + }) + + t.Run("bad-res-mem", func(t *testing.T) { + old := cfg.Memory + cfg.Memory = "x" + _, err := FlyteCoPilotContainer("x", cfg, []string{"hello"}, v1.VolumeMount{Name: "X", MountPath: "/"}) + assert.Error(t, err) + cfg.Memory = old + }) +} + +func TestDownloadCommandArgs(t *testing.T) { + _, err := DownloadCommandArgs("", "", "", core.DataLoadingConfig_YAML, nil) + assert.Error(t, err) + + iFace := &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + "y": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + } + d, err := DownloadCommandArgs("s3://from", "s3://output-meta", "/to", core.DataLoadingConfig_JSON, iFace) + assert.NoError(t, err) + expected := []string{"download", "--from-remote", "s3://from", "--to-output-prefix", "s3://output-meta", "--to-local-dir", "/to", "--format", "JSON", "--input-interface", ""} + if assert.Len(t, d, len(expected)) { + for i := 0; i < len(expected)-1; i++ { + assert.Equal(t, expected[i], d[i]) + } + // We cannot compare the last one, as the interface is a map the order is not guaranteed. + ifaceB64 := d[len(expected)-1] + serIFaceBytes, err := base64.StdEncoding.DecodeString(ifaceB64) + if assert.NoError(t, err) { + vm := &core.VariableMap{} + assert.NoError(t, proto.Unmarshal(serIFaceBytes, vm)) + assert.Len(t, vm.Variables, 2) + for k, v := range iFace.Variables { + v2, ok := vm.Variables[k] + assert.True(t, ok) + assert.Equal(t, v.Type.GetSimple(), v2.Type.GetSimple(), "for %s, types do not match", k) + } + } + } +} + +func TestSidecarCommandArgs(t *testing.T) { + _, err := SidecarCommandArgs("", "", "", time.Second*10, nil) + assert.Error(t, err) + + iFace := &core.TypedInterface{ + Outputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + "y": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + d, err := SidecarCommandArgs("/from", "s3://output-meta", "s3://raw-output", time.Second*10, iFace) + assert.NoError(t, err) + expected := []string{"sidecar", "--start-timeout", "10s", "--to-raw-output", "s3://raw-output", "--to-output-prefix", "s3://output-meta", "--from-local-dir", "/from", "--interface", ""} + if assert.Len(t, d, len(expected)) { + for i := 0; i < len(expected)-1; i++ { + assert.Equal(t, expected[i], d[i]) + } + // We cannot compare the last one, as the interface is a map the order is not guaranteed. + ifaceB64 := d[len(expected)-1] + serIFaceBytes, err := base64.StdEncoding.DecodeString(ifaceB64) + if assert.NoError(t, err) { + if2 := &core.TypedInterface{} + assert.NoError(t, proto.Unmarshal(serIFaceBytes, if2)) + assert.Len(t, if2.Outputs.Variables, 2) + for k, v := range iFace.Outputs.Variables { + v2, ok := if2.Outputs.Variables[k] + assert.True(t, ok) + assert.Equal(t, v.Type.GetSimple(), v2.Type.GetSimple(), "for %s, types do not match", k) + } + } + } +} + +func TestDataVolume(t *testing.T) { + v := DataVolume("x", nil) + assert.Equal(t, "x", v.Name) + assert.NotNil(t, v.EmptyDir) + assert.Nil(t, v.EmptyDir.SizeLimit) + assert.Equal(t, v1.StorageMediumDefault, v.EmptyDir.Medium) + + q := resource.MustParse("1024Mi") + v = DataVolume("x", &q) + assert.NotNil(t, v.EmptyDir.SizeLimit) + assert.Equal(t, q, *v.EmptyDir.SizeLimit) +} + +func assertContainerHasVolumeMounts(t *testing.T, cfg config.FlyteCoPilotConfig, pilot *core.DataLoadingConfig, iFace *core.TypedInterface, c *v1.Container) { + if iFace != nil { + vmap := map[string]v1.VolumeMount{} + for _, v := range c.VolumeMounts { + vmap[v.Name] = v + } + if iFace.Inputs != nil { + path := cfg.DefaultInputDataPath + if pilot.InputPath != "" { + path = pilot.InputPath + } + v, found := vmap[cfg.InputVolumeName] + assert.Equal(t, path, v.MountPath, "Input Path does not match") + assert.True(t, found, "Input volume mount expected but not found!") + } + + if iFace.Outputs != nil { + path := cfg.DefaultOutputPath + if pilot.OutputPath != "" { + path = pilot.OutputPath + } + v, found := vmap[cfg.OutputVolumeName] + assert.Equal(t, path, v.MountPath, "Output Path does not match") + assert.True(t, found, "Output volume mount expected but not found!") + } + } else { + assert.Len(t, c.VolumeMounts, 0) + } +} + +func assertContainerHasPTrace(t *testing.T, c *v1.Container) { + assert.NotNil(t, c.SecurityContext) + assert.NotNil(t, c.SecurityContext.Capabilities) + assert.NotNil(t, c.SecurityContext.Capabilities.Add) + capFound := false + for _, cap := range c.SecurityContext.Capabilities.Add { + if cap == pTraceCapability { + capFound = true + } + } + assert.True(t, capFound, "ptrace not found?") +} + +func assertPodHasSNPS(t *testing.T, pod *v1.PodSpec) { + assert.NotNil(t, pod.ShareProcessNamespace) + assert.True(t, *pod.ShareProcessNamespace) + + found := false + for _, c := range pod.Containers { + if c.Name == "test" { + found = true + assertContainerHasPTrace(t, &c) + } + } + assert.False(t, found, "user container absent?") +} + +func assertPodHasCoPilot(t *testing.T, cfg config.FlyteCoPilotConfig, pilot *core.DataLoadingConfig, iFace *core.TypedInterface, pod *v1.PodSpec) { + for _, c := range pod.Containers { + if c.Name == "test" { + assertContainerHasVolumeMounts(t, cfg, pilot, iFace, &c) + } else { + if c.Name == cfg.NamePrefix+flyteInitContainerName || c.Name == cfg.NamePrefix+flyteSidecarContainerName { + if iFace != nil { + vmap := map[string]v1.VolumeMount{} + for _, v := range c.VolumeMounts { + vmap[v.Name] = v + } + if iFace.Inputs != nil { + path := cfg.DefaultInputDataPath + if pilot != nil { + path = pilot.InputPath + } + v, found := vmap[cfg.InputVolumeName] + if c.Name == cfg.NamePrefix+flyteInitContainerName { + assert.Equal(t, path, v.MountPath, "Input Path does not match") + assert.True(t, found, "Input volume mount expected but not found!") + } else { + assert.False(t, found, "Input volume mount not expected but found!") + } + } + + if iFace.Outputs != nil { + path := cfg.DefaultOutputPath + if pilot != nil { + path = pilot.OutputPath + } + v, found := vmap[cfg.OutputVolumeName] + if c.Name == cfg.NamePrefix+flyteInitContainerName { + assert.False(t, found, "Output volume mount not expected but found on init container!") + } else { + assert.Equal(t, path, v.MountPath, "Output Path does not match") + assert.True(t, found, "Output volume mount expected but not found!") + } + } + + _, ok := vmap[flyteDataConfigVolume] + assert.True(t, ok) + } else { + assert.Len(t, c.VolumeMounts, 0) + } + } + } + } +} + +func TestCalculateStorageSize(t *testing.T) { + twoG := resource.MustParse("2048Mi") + oneG := resource.MustParse("1024Mi") + tests := []struct { + name string + args *v1.ResourceRequirements + want *resource.Quantity + }{ + {"nil", nil, nil}, + {"empty", &v1.ResourceRequirements{}, nil}, + {"limits", &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceStorage: twoG, + }}, &twoG}, + {"requests", &v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceStorage: oneG, + }}, &oneG}, + + {"max", &v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceStorage: twoG, + }, + Requests: v1.ResourceList{ + v1.ResourceStorage: oneG, + }}, &twoG}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := CalculateStorageSize(tt.args); !reflect.DeepEqual(got, tt.want) { + t.Errorf("CalculateStorageSize() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestAddCoPilotToContainer(t *testing.T) { + ctx := context.TODO() + cfg := config.FlyteCoPilotConfig{ + NamePrefix: "test-", + Image: "test", + DefaultInputDataPath: "/in", + DefaultOutputPath: "/out", + InputVolumeName: "inp", + OutputVolumeName: "out", + CPU: "1024m", + Memory: "1024Mi", + } + + t.Run("dataload-config-nil", func(t *testing.T) { + pilot := &core.DataLoadingConfig{} + assert.NoError(t, AddCoPilotToContainer(ctx, cfg, nil, nil, pilot)) + }) + + t.Run("disabled", func(t *testing.T) { + pilot := &core.DataLoadingConfig{} + assert.NoError(t, AddCoPilotToContainer(ctx, cfg, nil, nil, pilot)) + }) + + t.Run("nil-iface", func(t *testing.T) { + c := v1.Container{} + pilot := &core.DataLoadingConfig{Enabled: true} + assert.NoError(t, AddCoPilotToContainer(ctx, cfg, &c, nil, pilot)) + assertContainerHasVolumeMounts(t, cfg, pilot, nil, &c) + assertContainerHasPTrace(t, &c) + }) + + t.Run("happy-iface-empty-config", func(t *testing.T) { + + c := v1.Container{} + iface := &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + "y": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + Outputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "o": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + pilot := &core.DataLoadingConfig{Enabled: true} + assert.NoError(t, AddCoPilotToContainer(ctx, cfg, &c, iface, pilot)) + assertContainerHasPTrace(t, &c) + assertContainerHasVolumeMounts(t, cfg, pilot, iface, &c) + }) + + t.Run("happy-iface-set-config", func(t *testing.T) { + + c := v1.Container{} + iface := &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + "y": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + Outputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "o": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + pilot := &core.DataLoadingConfig{ + Enabled: true, + InputPath: "in", + OutputPath: "out", + } + assert.NoError(t, AddCoPilotToContainer(ctx, cfg, &c, iface, pilot)) + assertContainerHasPTrace(t, &c) + assertContainerHasVolumeMounts(t, cfg, pilot, iface, &c) + }) + + t.Run("happy-iface-inputs", func(t *testing.T) { + + c := v1.Container{} + iface := &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + "y": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + pilot := &core.DataLoadingConfig{ + Enabled: true, + InputPath: "in", + OutputPath: "out", + } + assert.NoError(t, AddCoPilotToContainer(ctx, cfg, &c, iface, pilot)) + assertContainerHasPTrace(t, &c) + assertContainerHasVolumeMounts(t, cfg, pilot, iface, &c) + }) + + t.Run("happy-iface-outputs", func(t *testing.T) { + + c := v1.Container{} + iface := &core.TypedInterface{ + Outputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "o": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + pilot := &core.DataLoadingConfig{ + Enabled: true, + InputPath: "in", + OutputPath: "out", + } + assert.NoError(t, AddCoPilotToContainer(ctx, cfg, &c, iface, pilot)) + assertContainerHasPTrace(t, &c) + assertContainerHasVolumeMounts(t, cfg, pilot, iface, &c) + }) +} + +func TestAddCoPilotToPod(t *testing.T) { + ctx := context.TODO() + cfg := config.FlyteCoPilotConfig{ + NamePrefix: "test-", + Image: "test", + DefaultInputDataPath: "/in", + DefaultOutputPath: "/out", + InputVolumeName: "inp", + OutputVolumeName: "out", + StartTimeout: config2.Duration{ + Duration: time.Second * 1, + }, + CPU: "1024m", + Memory: "1024Mi", + } + + taskMetadata := &pluginsCoreMock.TaskExecutionMetadata{} + taskMetadata.OnGetNamespace().Return("test-namespace") + taskMetadata.OnGetAnnotations().Return(map[string]string{"annotation-1": "val1"}) + taskMetadata.OnGetLabels().Return(map[string]string{"label-1": "val1"}) + taskMetadata.OnGetOwnerReference().Return(metav1.OwnerReference{ + Kind: "node", + Name: "blah", + }) + taskMetadata.OnGetK8sServiceAccount().Return("") + taskMetadata.OnGetOwnerID().Return(types.NamespacedName{ + Namespace: "test-namespace", + Name: "test-owner-name", + }) + taskMetadata.OnIsInterruptible().Return(false) + + tID := &pluginsCoreMock.TaskExecutionID{} + tID.OnGetID().Return(core.TaskExecutionIdentifier{ + TaskId: &core.Identifier{ + Name: "my-task", + }, + NodeExecutionId: &core.NodeExecutionIdentifier{ + ExecutionId: &core.WorkflowExecutionIdentifier{ + Name: "my_name", + Project: "my_project", + Domain: "my_domain", + }, + }, + }) + tID.OnGetGeneratedName().Return("name") + taskMetadata.OnGetTaskExecutionID().Return(tID) + + to := &pluginsCoreMock.TaskOverrides{} + to.OnGetResources().Return(resourceRequirements) + taskMetadata.OnGetOverrides().Return(to) + + inputPaths := &pluginsIOMock.InputFilePaths{} + inputs := "/base/inputs" + inputPaths.OnGetInputPrefixPath().Return(storage.DataReference(inputs)) + inputPaths.OnGetInputPath().Return(storage.DataReference(inputs + "/inputs.pb")) + + opath := &pluginsIOMock.OutputFilePaths{} + opath.OnGetRawOutputPrefix().Return("/raw") + opath.OnGetOutputPrefixPath().Return("/output") + + t.Run("happy", func(t *testing.T) { + pod := v1.PodSpec{} + iface := &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + "y": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + Outputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "o": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + pilot := &core.DataLoadingConfig{ + Enabled: true, + InputPath: "in", + OutputPath: "out", + } + assert.NoError(t, AddCoPilotToPod(ctx, cfg, &pod, iface, taskMetadata, inputPaths, opath, pilot)) + assertPodHasSNPS(t, &pod) + assertPodHasCoPilot(t, cfg, pilot, iface, &pod) + }) + + t.Run("happy-nil-iface", func(t *testing.T) { + pod := v1.PodSpec{} + pilot := &core.DataLoadingConfig{ + Enabled: true, + InputPath: "in", + OutputPath: "out", + } + assert.NoError(t, AddCoPilotToPod(ctx, cfg, &pod, nil, taskMetadata, inputPaths, opath, pilot)) + assertPodHasSNPS(t, &pod) + assertPodHasCoPilot(t, cfg, pilot, nil, &pod) + }) + + t.Run("happy-inputs-only", func(t *testing.T) { + pod := v1.PodSpec{} + iface := &core.TypedInterface{ + Inputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "x": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + "y": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + pilot := &core.DataLoadingConfig{ + Enabled: true, + InputPath: "in", + OutputPath: "out", + } + assert.NoError(t, AddCoPilotToPod(ctx, cfg, &pod, iface, taskMetadata, inputPaths, opath, pilot)) + assertPodHasSNPS(t, &pod) + assertPodHasCoPilot(t, cfg, pilot, iface, &pod) + }) + + t.Run("happy-outputs-only", func(t *testing.T) { + pod := v1.PodSpec{} + iface := &core.TypedInterface{ + Outputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "o": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + pilot := &core.DataLoadingConfig{ + Enabled: true, + InputPath: "in", + OutputPath: "out", + } + assert.NoError(t, AddCoPilotToPod(ctx, cfg, &pod, iface, taskMetadata, inputPaths, opath, pilot)) + assertPodHasSNPS(t, &pod) + assertPodHasCoPilot(t, cfg, pilot, iface, &pod) + }) + + t.Run("disabled", func(t *testing.T) { + pod := v1.PodSpec{} + iface := &core.TypedInterface{ + Outputs: &core.VariableMap{ + Variables: map[string]*core.Variable{ + "o": {Type: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}, + }, + }, + } + pilot := &core.DataLoadingConfig{ + Enabled: false, + InputPath: "in", + OutputPath: "out", + } + assert.NoError(t, AddCoPilotToPod(ctx, cfg, &pod, iface, taskMetadata, inputPaths, opath, pilot)) + assert.Len(t, pod.Volumes, 0) + }) + + t.Run("nil", func(t *testing.T) { + assert.NoError(t, AddCoPilotToPod(ctx, cfg, nil, nil, taskMetadata, inputPaths, opath, nil)) + }) +} diff --git a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go index 031ca9dfb5..7c17b889fd 100755 --- a/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go +++ b/flyteplugins/go/tasks/pluginmachinery/flytek8s/pod_helper.go @@ -27,7 +27,11 @@ func ToK8sPodSpec(ctx context.Context, taskExecutionMetadata pluginsCore.TaskExe logger.Warnf(ctx, "failed to read task information when trying to construct Pod, err: %s", err.Error()) return nil, err } - c, err := ToK8sContainer(ctx, taskExecutionMetadata, task.GetContainer(), inputs, outputPaths) + if task.GetContainer() == nil { + logger.Errorf(ctx, "Default Pod creation logic works for default container in the task template only.") + return nil, fmt.Errorf("container not specified in task template") + } + c, err := ToK8sContainer(ctx, taskExecutionMetadata, task.GetContainer(), task.Interface, inputs, outputPaths) if err != nil { return nil, err } @@ -46,15 +50,20 @@ func ToK8sPodSpec(ctx context.Context, taskExecutionMetadata pluginsCore.TaskExe SchedulerName: config.GetK8sPluginConfig().SchedulerName, }, nil } - return &v1.PodSpec{ + pod := &v1.PodSpec{ // We could specify Scheduler, Affinity, nodename etc RestartPolicy: v1.RestartPolicyNever, Containers: containers, Tolerations: GetPodTolerations(taskExecutionMetadata.IsInterruptible(), c.Resources), ServiceAccountName: taskExecutionMetadata.GetK8sServiceAccount(), SchedulerName: config.GetK8sPluginConfig().SchedulerName, - }, nil + } + + if err := AddCoPilotToPod(ctx, config.GetK8sPluginConfig().CoPilot, pod, task.GetInterface(), taskExecutionMetadata, inputs, outputPaths, task.GetContainer().GetDataConfig()); err != nil { + return nil, err + } + return pod, nil } func BuildPodWithSpec(podSpec *v1.PodSpec) *v1.Pod { @@ -202,7 +211,7 @@ func DemystifySuccess(status v1.PodStatus, info pluginsCore.TaskInfo) (pluginsCo func ConvertPodFailureToError(status v1.PodStatus) (code, message string) { code = "UnknownError" - message = "Container/Pod failed. No message received from kubernetes." + message = "Pod failed. No message received from kubernetes." if len(status.Reason) > 0 { code = status.Reason } @@ -228,11 +237,15 @@ func ConvertPodFailureToError(status v1.PodStatus) (code, message string) { code = Interrupted } - message += fmt.Sprintf("\r\nContainer [%v] terminated with exit code (%v). Reason [%v]. Message: [%v].", - c.Name, - containerState.Terminated.ExitCode, - containerState.Terminated.Reason, - containerState.Terminated.Message) + if containerState.Terminated.ExitCode == 0 { + message += fmt.Sprintf("\r\n[%v] terminated with ExitCode 0.", c.Name) + } else { + message += fmt.Sprintf("\r\n[%v] terminated with exit code (%v). Reason [%v]. Message: \n%v.", + c.Name, + containerState.Terminated.ExitCode, + containerState.Terminated.Reason, + containerState.Terminated.Message) + } } } return code, message diff --git a/flyteplugins/go/tasks/pluginmachinery/utils/literals.go b/flyteplugins/go/tasks/pluginmachinery/utils/literals.go new file mode 100644 index 0000000000..238a30916b --- /dev/null +++ b/flyteplugins/go/tasks/pluginmachinery/utils/literals.go @@ -0,0 +1,439 @@ +package utils + +import ( + "fmt" + "reflect" + "strconv" + "strings" + "time" + + "github.com/golang/protobuf/jsonpb" + "github.com/golang/protobuf/ptypes" + structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/storage" + "github.com/pkg/errors" +) + +func MakePrimitive(v interface{}) (*core.Primitive, error) { + switch p := v.(type) { + case int: + return &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: int64(p), + }, + }, nil + case int64: + return &core.Primitive{ + Value: &core.Primitive_Integer{ + Integer: p, + }, + }, nil + case float64: + return &core.Primitive{ + Value: &core.Primitive_FloatValue{ + FloatValue: p, + }, + }, nil + case time.Time: + t, err := ptypes.TimestampProto(p) + if err != nil { + return nil, err + } + return &core.Primitive{ + Value: &core.Primitive_Datetime{ + Datetime: t, + }, + }, nil + case time.Duration: + d := ptypes.DurationProto(p) + return &core.Primitive{ + Value: &core.Primitive_Duration{ + Duration: d, + }, + }, nil + case string: + return &core.Primitive{ + Value: &core.Primitive_StringValue{ + StringValue: p, + }, + }, nil + case bool: + return &core.Primitive{ + Value: &core.Primitive_Boolean{ + Boolean: p, + }, + }, nil + } + return nil, errors.Errorf("Failed to convert to a known primitive type. Input Type [%v] not supported", reflect.TypeOf(v).String()) +} + +func MustMakePrimitive(v interface{}) *core.Primitive { + f, err := MakePrimitive(v) + if err != nil { + panic(err) + } + return f +} + +func MakePrimitiveLiteral(v interface{}) (*core.Literal, error) { + p, err := MakePrimitive(v) + if err != nil { + return nil, err + } + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Primitive{ + Primitive: p, + }, + }, + }, + }, nil +} + +func MustMakePrimitiveLiteral(v interface{}) *core.Literal { + p, err := MakePrimitiveLiteral(v) + if err != nil { + panic(err) + } + return p +} + +func MakeLiteralMap(v map[string]interface{}) (*core.LiteralMap, error) { + + literals := make(map[string]*core.Literal, len(v)) + for key, val := range v { + l, err := MakeLiteral(val) + if err != nil { + return nil, err + } + + literals[key] = l + } + + return &core.LiteralMap{ + Literals: literals, + }, nil +} + +func MakeLiteralForMap(v map[string]interface{}) (*core.Literal, error) { + m, err := MakeLiteralMap(v) + if err != nil { + return nil, err + } + + return &core.Literal{ + Value: &core.Literal_Map{ + Map: m, + }, + }, nil +} + +func MakeLiteralForCollection(v []interface{}) (*core.Literal, error) { + literals := make([]*core.Literal, 0, len(v)) + for _, val := range v { + l, err := MakeLiteral(val) + if err != nil { + return nil, err + } + + literals = append(literals, l) + } + + return &core.Literal{ + Value: &core.Literal_Collection{ + Collection: &core.LiteralCollection{ + Literals: literals, + }, + }, + }, nil +} + +func MakeBinaryLiteral(v []byte) *core.Literal { + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Binary{ + Binary: &core.Binary{ + Value: v, + }, + }, + }, + }, + } +} + +func MakeGenericLiteral(v *structpb.Struct) *core.Literal { + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Generic{ + Generic: v, + }, + }, + }} +} + +func MakeLiteral(v interface{}) (*core.Literal, error) { + if v == nil { + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_NoneType{ + NoneType: &core.Void{}, + }, + }, + }, + }, nil + } + switch o := v.(type) { + case *core.Literal: + return o, nil + case []interface{}: + return MakeLiteralForCollection(o) + case map[string]interface{}: + return MakeLiteralForMap(o) + case []byte: + return MakeBinaryLiteral(v.([]byte)), nil + case *structpb.Struct: + return MakeGenericLiteral(v.(*structpb.Struct)), nil + case *core.Error: + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Error{ + Error: v.(*core.Error), + }, + }, + }, + }, nil + default: + return MakePrimitiveLiteral(o) + } +} + +func MustMakeLiteral(v interface{}) *core.Literal { + p, err := MakeLiteral(v) + if err != nil { + panic(err) + } + + return p +} + +func MustMakeDefaultLiteralForType(typ *core.LiteralType) *core.Literal { + if res, err := MakeDefaultLiteralForType(typ); err != nil { + panic(err) + } else { + return res + } +} + +func MakeDefaultLiteralForType(typ *core.LiteralType) (*core.Literal, error) { + if typ != nil { + switch t := typ.GetType().(type) { + case *core.LiteralType_Simple: + switch t.Simple { + case core.SimpleType_NONE: + return MakeLiteral(nil) + case core.SimpleType_INTEGER: + return MakeLiteral(int(0)) + case core.SimpleType_FLOAT: + return MakeLiteral(float64(0)) + case core.SimpleType_STRING: + return MakeLiteral("") + case core.SimpleType_BOOLEAN: + return MakeLiteral(false) + case core.SimpleType_DATETIME: + return MakeLiteral(time.Now()) + case core.SimpleType_DURATION: + return MakeLiteral(time.Second) + case core.SimpleType_BINARY: + return MakeLiteral([]byte{}) + case core.SimpleType_ERROR: + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Error{ + Error: &core.Error{ + Message: "Default Error message", + }, + }, + }, + }, + }, nil + case core.SimpleType_STRUCT: + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Generic{ + Generic: &structpb.Struct{}, + }, + }, + }, + }, nil + } + return nil, errors.Errorf("Not yet implemented. Default creation is not yet implemented for [%s] ", t.Simple.String()) + + case *core.LiteralType_Blob: + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Blob{ + Blob: &core.Blob{ + Metadata: &core.BlobMetadata{ + Type: t.Blob, + }, + Uri: "/tmp/somepath", + }, + }, + }, + }, + }, nil + case *core.LiteralType_CollectionType: + single, err := MakeDefaultLiteralForType(t.CollectionType) + if err != nil { + return nil, err + } + + return &core.Literal{ + Value: &core.Literal_Collection{ + Collection: &core.LiteralCollection{ + Literals: []*core.Literal{single}, + }, + }, + }, nil + case *core.LiteralType_MapValueType: + single, err := MakeDefaultLiteralForType(t.MapValueType) + if err != nil { + return nil, err + } + + return &core.Literal{ + Value: &core.Literal_Map{ + Map: &core.LiteralMap{ + Literals: map[string]*core.Literal{ + "itemKey": single, + }, + }, + }, + }, nil + // case *core.LiteralType_Schema: + } + } + return nil, errors.Errorf("Failed to convert to a known Literal. Input Type [%v] not supported", typ.String()) +} + +func MakePrimitiveForType(t core.SimpleType, s string) (*core.Primitive, error) { + p := &core.Primitive{} + switch t { + case core.SimpleType_INTEGER: + v, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return nil, errors.Wrap(err, "failed to parse integer value") + } + p.Value = &core.Primitive_Integer{Integer: v} + case core.SimpleType_FLOAT: + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Float value") + } + p.Value = &core.Primitive_FloatValue{FloatValue: v} + case core.SimpleType_BOOLEAN: + v, err := strconv.ParseBool(s) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Bool value") + } + p.Value = &core.Primitive_Boolean{Boolean: v} + case core.SimpleType_STRING: + p.Value = &core.Primitive_StringValue{StringValue: s} + case core.SimpleType_DURATION: + v, err := time.ParseDuration(s) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Duration, valid formats: e.g. 300ms, -1.5h, 2h45m") + } + p.Value = &core.Primitive_Duration{Duration: ptypes.DurationProto(v)} + case core.SimpleType_DATETIME: + v, err := time.Parse(time.RFC3339, s) + if err != nil { + return nil, errors.Wrap(err, "failed to parse Datetime in RFC3339 format") + } + ts, err := ptypes.TimestampProto(v) + if err != nil { + return nil, errors.Wrap(err, "failed to convert datetime to proto") + } + p.Value = &core.Primitive_Datetime{Datetime: ts} + default: + return nil, fmt.Errorf("unsupported type %s", t.String()) + } + return p, nil +} + +func MakeLiteralForSimpleType(t core.SimpleType, s string) (*core.Literal, error) { + s = strings.Trim(s, " \n\t") + scalar := &core.Scalar{} + switch t { + case core.SimpleType_STRUCT: + st := &structpb.Struct{} + err := jsonpb.UnmarshalString(s, st) + if err != nil { + return nil, errors.Wrapf(err, "failed to load generic type as json.") + } + scalar.Value = &core.Scalar_Generic{ + Generic: st, + } + case core.SimpleType_BINARY: + scalar.Value = &core.Scalar_Binary{ + Binary: &core.Binary{ + Value: []byte(s), + // TODO Tag not supported at the moment + }, + } + case core.SimpleType_ERROR: + scalar.Value = &core.Scalar_Error{ + Error: &core.Error{ + Message: s, + }, + } + case core.SimpleType_NONE: + scalar.Value = &core.Scalar_NoneType{ + NoneType: &core.Void{}, + } + default: + p, err := MakePrimitiveForType(t, s) + if err != nil { + return nil, err + } + scalar.Value = &core.Scalar_Primitive{Primitive: p} + } + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: scalar, + }, + }, nil +} + +func MakeLiteralForBlob(path storage.DataReference, isDir bool, format string) *core.Literal { + dim := core.BlobType_SINGLE + if isDir { + dim = core.BlobType_MULTIPART + } + return &core.Literal{ + Value: &core.Literal_Scalar{ + Scalar: &core.Scalar{ + Value: &core.Scalar_Blob{ + Blob: &core.Blob{ + Uri: path.String(), + Metadata: &core.BlobMetadata{ + Type: &core.BlobType{ + Dimensionality: dim, + Format: format, + }, + }, + }, + }, + }, + }, + } +} diff --git a/flyteplugins/go/tasks/pluginmachinery/utils/literals_test.go b/flyteplugins/go/tasks/pluginmachinery/utils/literals_test.go new file mode 100644 index 0000000000..245ef0225d --- /dev/null +++ b/flyteplugins/go/tasks/pluginmachinery/utils/literals_test.go @@ -0,0 +1,356 @@ +package utils + +import ( + "reflect" + "testing" + "time" + + "github.com/golang/protobuf/ptypes" + structpb "github.com/golang/protobuf/ptypes/struct" + "github.com/lyft/flyteidl/gen/pb-go/flyteidl/core" + "github.com/lyft/flytestdlib/storage" + "github.com/stretchr/testify/assert" +) + +func TestMakePrimitive(t *testing.T) { + { + v := 1 + p, err := MakePrimitive(v) + assert.NoError(t, err) + assert.Equal(t, "*core.Primitive_Integer", reflect.TypeOf(p.Value).String()) + assert.Equal(t, int64(v), p.GetInteger()) + } + { + v := int64(1) + p, err := MakePrimitive(v) + assert.NoError(t, err) + assert.Equal(t, "*core.Primitive_Integer", reflect.TypeOf(p.Value).String()) + assert.Equal(t, v, p.GetInteger()) + } + { + v := 1.0 + p, err := MakePrimitive(v) + assert.NoError(t, err) + assert.Equal(t, "*core.Primitive_FloatValue", reflect.TypeOf(p.Value).String()) + assert.Equal(t, v, p.GetFloatValue()) + } + { + v := "blah" + p, err := MakePrimitive(v) + assert.NoError(t, err) + assert.Equal(t, "*core.Primitive_StringValue", reflect.TypeOf(p.Value).String()) + assert.Equal(t, v, p.GetStringValue()) + } + { + v := true + p, err := MakePrimitive(v) + assert.NoError(t, err) + assert.Equal(t, "*core.Primitive_Boolean", reflect.TypeOf(p.Value).String()) + assert.Equal(t, v, p.GetBoolean()) + } + { + v := time.Now() + p, err := MakePrimitive(v) + assert.NoError(t, err) + assert.Equal(t, "*core.Primitive_Datetime", reflect.TypeOf(p.Value).String()) + j, err := ptypes.TimestampProto(v) + assert.NoError(t, err) + assert.Equal(t, j, p.GetDatetime()) + _, err = MakePrimitive(time.Date(0, 0, 0, 0, 0, 0, 0, time.UTC)) + assert.Error(t, err) + } + { + v := time.Second * 10 + p, err := MakePrimitive(v) + assert.NoError(t, err) + assert.Equal(t, "*core.Primitive_Duration", reflect.TypeOf(p.Value).String()) + assert.Equal(t, ptypes.DurationProto(v), p.GetDuration()) + } + { + v := struct { + }{} + _, err := MakePrimitive(v) + assert.Error(t, err) + } +} + +func TestMustMakePrimitive(t *testing.T) { + { + v := struct { + }{} + assert.Panics(t, func() { + MustMakePrimitive(v) + }) + } + { + v := time.Second * 10 + p := MustMakePrimitive(v) + assert.Equal(t, "*core.Primitive_Duration", reflect.TypeOf(p.Value).String()) + assert.Equal(t, ptypes.DurationProto(v), p.GetDuration()) + } +} + +func TestMakePrimitiveLiteral(t *testing.T) { + { + v := 1.0 + p, err := MakePrimitiveLiteral(v) + assert.NoError(t, err) + assert.NotNil(t, p.GetScalar()) + assert.Equal(t, "*core.Primitive_FloatValue", reflect.TypeOf(p.GetScalar().GetPrimitive().Value).String()) + assert.Equal(t, v, p.GetScalar().GetPrimitive().GetFloatValue()) + } + { + v := struct { + }{} + _, err := MakePrimitiveLiteral(v) + assert.Error(t, err) + } +} + +func TestMustMakePrimitiveLiteral(t *testing.T) { + t.Run("Panic", func(t *testing.T) { + v := struct { + }{} + assert.Panics(t, func() { + MustMakePrimitiveLiteral(v) + }) + }) + t.Run("FloatValue", func(t *testing.T) { + v := 1.0 + p := MustMakePrimitiveLiteral(v) + assert.NotNil(t, p.GetScalar()) + assert.Equal(t, "*core.Primitive_FloatValue", reflect.TypeOf(p.GetScalar().GetPrimitive().Value).String()) + assert.Equal(t, v, p.GetScalar().GetPrimitive().GetFloatValue()) + }) +} + +func TestMakeLiteral(t *testing.T) { + t.Run("Primitive", func(t *testing.T) { + lit, err := MakeLiteral("test_string") + assert.NoError(t, err) + assert.Equal(t, "*core.Primitive_StringValue", reflect.TypeOf(lit.GetScalar().GetPrimitive().Value).String()) + }) + + t.Run("Array", func(t *testing.T) { + lit, err := MakeLiteral([]interface{}{1, 2, 3}) + assert.NoError(t, err) + assert.Equal(t, "*core.Literal_Collection", reflect.TypeOf(lit.GetValue()).String()) + assert.Equal(t, "*core.Primitive_Integer", reflect.TypeOf(lit.GetCollection().Literals[0].GetScalar().GetPrimitive().Value).String()) + }) + + t.Run("Map", func(t *testing.T) { + lit, err := MakeLiteral(map[string]interface{}{ + "key1": []interface{}{1, 2, 3}, + "key2": []interface{}{5}, + }) + assert.NoError(t, err) + assert.Equal(t, "*core.Literal_Map", reflect.TypeOf(lit.GetValue()).String()) + assert.Equal(t, "*core.Literal_Collection", reflect.TypeOf(lit.GetMap().Literals["key1"].GetValue()).String()) + }) + + t.Run("Binary", func(t *testing.T) { + s := MakeBinaryLiteral([]byte{'h'}) + assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue()) + }) + + t.Run("NoneType", func(t *testing.T) { + p, err := MakeLiteral(nil) + assert.NoError(t, err) + assert.NotNil(t, p.GetScalar()) + assert.Equal(t, "*core.Scalar_NoneType", reflect.TypeOf(p.GetScalar().Value).String()) + }) +} + +func TestMustMakeLiteral(t *testing.T) { + v := "hello" + l := MustMakeLiteral(v) + assert.NotNil(t, l.GetScalar()) + assert.Equal(t, v, l.GetScalar().GetPrimitive().GetStringValue()) +} + +func TestMakeBinaryLiteral(t *testing.T) { + s := MakeBinaryLiteral([]byte{'h'}) + assert.Equal(t, []byte{'h'}, s.GetScalar().GetBinary().GetValue()) +} + +func TestMakeDefaultLiteralForType(t *testing.T) { + type args struct { + name string + ty core.SimpleType + tyName string + isPrimitive bool + } + tests := []args{ + {"None", core.SimpleType_NONE, "*core.Scalar_NoneType", false}, + {"Binary", core.SimpleType_BINARY, "*core.Scalar_Binary", false}, + {"Integer", core.SimpleType_INTEGER, "*core.Primitive_Integer", true}, + {"Float", core.SimpleType_FLOAT, "*core.Primitive_FloatValue", true}, + {"String", core.SimpleType_STRING, "*core.Primitive_StringValue", true}, + {"Boolean", core.SimpleType_BOOLEAN, "*core.Primitive_Boolean", true}, + {"Duration", core.SimpleType_DURATION, "*core.Primitive_Duration", true}, + {"Datetime", core.SimpleType_DATETIME, "*core.Primitive_Datetime", true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{Simple: test.ty}}) + assert.NoError(t, err) + if test.isPrimitive { + assert.Equal(t, test.tyName, reflect.TypeOf(l.GetScalar().GetPrimitive().Value).String()) + } else { + assert.Equal(t, test.tyName, reflect.TypeOf(l.GetScalar().Value).String()) + } + }) + } + + t.Run("Blob", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Blob{}}) + assert.NoError(t, err) + assert.Equal(t, "*core.Scalar_Blob", reflect.TypeOf(l.GetScalar().Value).String()) + }) + + t.Run("Collection", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_CollectionType{CollectionType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}}) + assert.NoError(t, err) + assert.Equal(t, "*core.LiteralCollection", reflect.TypeOf(l.GetCollection()).String()) + }) + + t.Run("Map", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_MapValueType{MapValueType: &core.LiteralType{Type: &core.LiteralType_Simple{Simple: core.SimpleType_INTEGER}}}}) + assert.NoError(t, err) + assert.Equal(t, "*core.LiteralMap", reflect.TypeOf(l.GetMap()).String()) + }) + + t.Run("error", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_ERROR, + }}) + assert.NoError(t, err) + assert.NotNil(t, l.GetScalar().GetError()) + }) + + t.Run("struct", func(t *testing.T) { + l, err := MakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Simple{ + Simple: core.SimpleType_STRUCT, + }}) + assert.NoError(t, err) + assert.NotNil(t, l.GetScalar().GetGeneric()) + }) +} + +func TestMustMakeDefaultLiteralForType(t *testing.T) { + t.Run("error", func(t *testing.T) { + assert.Panics(t, func() { + MustMakeDefaultLiteralForType(nil) + }) + }) + + t.Run("Blob", func(t *testing.T) { + l := MustMakeDefaultLiteralForType(&core.LiteralType{Type: &core.LiteralType_Blob{}}) + assert.Equal(t, "*core.Scalar_Blob", reflect.TypeOf(l.GetScalar().Value).String()) + }) +} + +func TestMakePrimitiveForType(t *testing.T) { + n := time.Now() + type args struct { + t core.SimpleType + s string + } + tests := []struct { + name string + args args + want *core.Primitive + wantErr bool + }{ + {"error-type", args{core.SimpleType_NONE, "x"}, nil, true}, + + {"error-int", args{core.SimpleType_INTEGER, "x"}, nil, true}, + {"int", args{core.SimpleType_INTEGER, "1"}, MustMakePrimitive(1), false}, + + {"error-bool", args{core.SimpleType_BOOLEAN, "x"}, nil, true}, + {"bool", args{core.SimpleType_BOOLEAN, "true"}, MustMakePrimitive(true), false}, + + {"error-float", args{core.SimpleType_FLOAT, "x"}, nil, true}, + {"float", args{core.SimpleType_FLOAT, "3.1416"}, MustMakePrimitive(3.1416), false}, + + {"string", args{core.SimpleType_STRING, "string"}, MustMakePrimitive("string"), false}, + + {"error-dt", args{core.SimpleType_DATETIME, "x"}, nil, true}, + {"dt", args{core.SimpleType_DATETIME, n.Format(time.RFC3339Nano)}, MustMakePrimitive(n), false}, + + {"error-dur", args{core.SimpleType_DURATION, "x"}, nil, true}, + {"dur", args{core.SimpleType_DURATION, time.Hour.String()}, MustMakePrimitive(time.Hour), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := MakePrimitiveForType(tt.args.t, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("MakePrimitiveForType() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("MakePrimitiveForType() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMakeLiteralForSimpleType(t *testing.T) { + type args struct { + t core.SimpleType + s string + } + tests := []struct { + name string + args args + want *core.Literal + wantErr bool + }{ + {"error-int", args{core.SimpleType_INTEGER, "x"}, nil, true}, + {"int", args{core.SimpleType_INTEGER, "1"}, MustMakeLiteral(1), false}, + + {"error-struct", args{core.SimpleType_STRUCT, "x"}, nil, true}, + {"struct", args{core.SimpleType_STRUCT, `{"x": 1}`}, MustMakeLiteral(&structpb.Struct{Fields: map[string]*structpb.Value{"x": {Kind: &structpb.Value_NumberValue{NumberValue: 1}}}}), false}, + + {"bin", args{core.SimpleType_BINARY, "x"}, MustMakeLiteral([]byte("x")), false}, + + {"error", args{core.SimpleType_ERROR, "err"}, MustMakeLiteral(&core.Error{Message: "err"}), false}, + + {"none", args{core.SimpleType_NONE, "null"}, MustMakeLiteral(nil), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := MakeLiteralForSimpleType(tt.args.t, tt.args.s) + if (err != nil) != tt.wantErr { + t.Errorf("MakeLiteralForSimpleType() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("MakeLiteralForSimpleType() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestMakeLiteralForBlob(t *testing.T) { + type args struct { + path storage.DataReference + isDir bool + format string + } + tests := []struct { + name string + args args + want *core.Blob + }{ + {"simple-key", args{path: "/key", isDir: false, format: "xyz"}, &core.Blob{Uri: "/key", Metadata: &core.BlobMetadata{Type: &core.BlobType{Format: "xyz", Dimensionality: core.BlobType_SINGLE}}}}, + {"simple-dir", args{path: "/key", isDir: true, format: "xyz"}, &core.Blob{Uri: "/key", Metadata: &core.BlobMetadata{Type: &core.BlobType{Format: "xyz", Dimensionality: core.BlobType_MULTIPART}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := MakeLiteralForBlob(tt.args.path, tt.args.isDir, tt.args.format); !reflect.DeepEqual(got.GetScalar().GetBlob(), tt.want) { + t.Errorf("MakeLiteralForBlob() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/flyteplugins/go/tasks/plugins/k8s/container/container.go b/flyteplugins/go/tasks/plugins/k8s/container/container.go index e487cf1efe..06646c50d9 100755 --- a/flyteplugins/go/tasks/plugins/k8s/container/container.go +++ b/flyteplugins/go/tasks/plugins/k8s/container/container.go @@ -17,10 +17,10 @@ const ( containerTaskType = "container" ) -type containerTaskExecutor struct { +type Plugin struct { } -func (containerTaskExecutor) GetTaskPhase(ctx context.Context, pluginContext k8s.PluginContext, r k8s.Resource) (pluginsCore.PhaseInfo, error) { +func (Plugin) GetTaskPhase(ctx context.Context, pluginContext k8s.PluginContext, r k8s.Resource) (pluginsCore.PhaseInfo, error) { pod := r.(*v1.Pod) @@ -53,7 +53,7 @@ func (containerTaskExecutor) GetTaskPhase(ctx context.Context, pluginContext k8s } // Creates a new Pod that will Exit on completion. The pods have no retries by design -func (containerTaskExecutor) BuildResource(ctx context.Context, taskCtx pluginsCore.TaskExecutionContext) (k8s.Resource, error) { +func (Plugin) BuildResource(ctx context.Context, taskCtx pluginsCore.TaskExecutionContext) (k8s.Resource, error) { podSpec, err := flytek8s.ToK8sPodSpec(ctx, taskCtx.TaskExecutionMetadata(), taskCtx.TaskReader(), taskCtx.InputReader(), taskCtx.OutputWriter()) if err != nil { @@ -68,7 +68,7 @@ func (containerTaskExecutor) BuildResource(ctx context.Context, taskCtx pluginsC return pod, nil } -func (containerTaskExecutor) BuildIdentityResource(_ context.Context, _ pluginsCore.TaskExecutionMetadata) (k8s.Resource, error) { +func (Plugin) BuildIdentityResource(_ context.Context, _ pluginsCore.TaskExecutionMetadata) (k8s.Resource, error) { return flytek8s.BuildIdentityPod(), nil } @@ -78,7 +78,7 @@ func init() { ID: containerTaskType, RegisteredTaskTypes: []pluginsCore.TaskType{containerTaskType}, ResourceToWatch: &v1.Pod{}, - Plugin: containerTaskExecutor{}, + Plugin: Plugin{}, IsDefault: true, }) } diff --git a/flyteplugins/go/tasks/plugins/k8s/container/container_test.go b/flyteplugins/go/tasks/plugins/k8s/container/container_test.go index 9f7288aadf..38301fd25c 100755 --- a/flyteplugins/go/tasks/plugins/k8s/container/container_test.go +++ b/flyteplugins/go/tasks/plugins/k8s/container/container_test.go @@ -97,7 +97,7 @@ func dummyContainerTaskContext(resources *v1.ResourceRequirements, command []str } func TestContainerTaskExecutor_BuildIdentityResource(t *testing.T) { - c := containerTaskExecutor{} + c := Plugin{} taskMetadata := &pluginsCoreMock.TaskExecutionMetadata{} r, err := c.BuildIdentityResource(context.TODO(), taskMetadata) assert.NoError(t, err) @@ -108,7 +108,7 @@ func TestContainerTaskExecutor_BuildIdentityResource(t *testing.T) { } func TestContainerTaskExecutor_BuildResource(t *testing.T) { - c := containerTaskExecutor{} + c := Plugin{} command := []string{"command"} args := []string{"{{.Input}}"} taskCtx := dummyContainerTaskContext(resourceRequirements, command, args) @@ -133,7 +133,7 @@ func TestContainerTaskExecutor_BuildResource(t *testing.T) { } func TestContainerTaskExecutor_GetTaskStatus(t *testing.T) { - c := containerTaskExecutor{} + c := Plugin{} j := &v1.Pod{ Status: v1.PodStatus{}, } diff --git a/flyteplugins/go/tasks/plugins/k8s/sidecar/sidecar_test.go b/flyteplugins/go/tasks/plugins/k8s/sidecar/sidecar_test.go index ff4ca9efad..8ddc2cccc4 100755 --- a/flyteplugins/go/tasks/plugins/k8s/sidecar/sidecar_test.go +++ b/flyteplugins/go/tasks/plugins/k8s/sidecar/sidecar_test.go @@ -148,6 +148,8 @@ func TestBuildSidecarResource(t *testing.T) { v1.ResourceStorage: {tolStorage}, ResourceNvidiaGPU: {tolGPU}, }, + DefaultCPURequest: "1024m", + DefaultMemoryRequest: "1024Mi", })) handler := &sidecarResourceHandler{} taskCtx := getDummySidecarTaskContext(&task, resourceRequirements)