Flyte CoPilot: Raw container support for all K8s containers (flyteorg#84

)

flyteplugins/.github/workflows/master.yml

@@ -0,0 +1,31 @@
+name: Master
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: '0'
+      - name: Bump version and push tag
+        id: bump-version
+        uses: anothrNick/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WITH_V: true
+          DEFAULT_BUMP: patch
+      - name: Push Docker Image to Github Registry
+        uses: whoan/docker-build-with-cache-action@v5
+        with:
+          username: "${{ github.actor }}"
+          password: "${{ secrets.GITHUB_TOKEN }}"
+          image_name: ${{ secrets.flytegithub_repo }}/flytecopilot
+          image_tag: latest,${{ github.sha }},${{ steps.bump-version.outputs.tag }}
+          push_git_tag: true
+          registry: docker.pkg.github.com
+          build_extra_args: "--compress=true"

flyteplugins/.github/workflows/pull_request.yml

@@ -0,0 +1,19 @@
+name: Pull Request
+
+on:
+  pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Push Docker Image to Github Registry
+        uses: whoan/docker-build-with-cache-action@v5
+        with:
+          username: "${{ github.actor }}"
+          password: "${{ secrets.GITHUB_TOKEN }}"
+          image_name: ${{ secrets.flytegithub_repo }}/flytecopilot
+          image_tag: ${{ github.sha }}
+          push_git_tag: true
+          registry: docker.pkg.github.com

flyteplugins/.gitignore

@@ -2,3 +2,4 @@
 .idea/
 vendor/
 *.swp
+artifacts/

flyteplugins/Dockerfile

@@ -0,0 +1,32 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
+
+FROM golang:1.13.3-alpine3.10 as builder
+RUN apk add git openssh-client make curl
+
+# COPY only the go mod files for efficient caching
+COPY go.mod go.sum /go/src/github.com/lyft/flyteplugins/
+WORKDIR /go/src/github.com/lyft/flyteplugins
+
+# Pull dependencies
+RUN go mod download
+
+# COPY the rest of the source code
+COPY . /go/src/github.com/lyft/flyteplugins/
+
+# This 'linux_compile' target should compile binaries to the /artifacts directory
+# The main entrypoint should be compiled to /artifacts/flyteplugins
+RUN make linux_compile
+
+# update the PATH to include the /artifacts directory
+ENV PATH="/artifacts:${PATH}"
+
+# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
+FROM alpine:3.10
+COPY --from=builder /artifacts /bin
+
+RUN apk --update add ca-certificates
+
+CMD ["flyte-copilot"]

flyteplugins/Makefile

@@ -1,4 +1,5 @@
 export REPOSITORY=flyteplugins
+include boilerplate/lyft/docker_build/Makefile
 include boilerplate/lyft/golang_test_targets/Makefile
 
 .PHONY: update_boilerplate
@@ -10,3 +11,17 @@ generate: download_tooling
 
 clean:
 	rm -rf bin
+
+.PHONY: linux_compile
+linux_compile:
+	cd copilot; GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts/flyte-copilot .; cd -
+
+.PHONY: compile
+compile:
+	mkdir -p ./artifacts
+	cd copilot; go build -o ../artifacts/flyte-copilot .; cd -
+
+cross_compile:
+	@glide install
+	@mkdir -p ./artifacts/cross
+	cd copilot; GOOS=linux GOARCH=amd64 go build -o ../artifacts/flyte-copilot .; cd -

flyteplugins/boilerplate/lyft/docker_build/Makefile

@@ -0,0 +1,12 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
+
+.PHONY: docker_build
+docker_build:
+	IMAGE_NAME=$$REPOSITORY ./boilerplate/lyft/docker_build/docker_build.sh
+
+.PHONY: dockerhub_push
+dockerhub_push:
+	IMAGE_NAME=lyft/$$REPOSITORY REGISTRY=docker.io ./boilerplate/lyft/docker_build/docker_build.sh

flyteplugins/boilerplate/lyft/docker_build/Readme.rst

@@ -0,0 +1,23 @@
+Docker Build and Push
+~~~~~~~~~~~~~~~~~~~~~
+
+Provides a ``make docker_build`` target that builds your image locally.
+
+Provides a ``make dockerhub_push`` target that pushes your final image to Dockerhub.
+
+The Dockerhub image will tagged ``<REPOSITORY>:<GIT COMMIT SHA>``
+
+If git head has a git tag, the Dockerhub image will also be tagged ``<IMAGE>:<GIT_TAG>``.
+
+**To Enable:**
+
+Add ``lyft/docker_build`` to your ``boilerplate/update.cfg`` file.
+
+Add ``include boilerplate/lyft/docker_build/Makefile`` in your main ``Makefile`` _after_ your REPOSITORY environment variable
+
+::
+
+    REPOSITORY=<myreponame>
+    include boilerplate/lyft/docker_build/Makefile
+
+(this ensures the extra Make targets get included in your main Makefile)

flyteplugins/boilerplate/lyft/docker_build/docker_build.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
+
+set -e
+
+echo ""
+echo "------------------------------------"
+echo "           DOCKER BUILD"
+echo "------------------------------------"
+echo ""
+
+if [ -n "$REGISTRY" ]; then
+  # Do not push if there are unstaged git changes
+  CHANGED=$(git status --porcelain)
+  if [ -n "$CHANGED" ]; then
+    echo "Please commit git changes before pushing to a registry"
+    exit 1
+  fi
+fi
+
+
+GIT_SHA=$(git rev-parse HEAD)
+
+IMAGE_TAG_SUFFIX=""
+# for intermediate build phases, append -$BUILD_PHASE to all image tags
+if [ -n "$BUILD_PHASE" ]; then
+  IMAGE_TAG_SUFFIX="-${BUILD_PHASE}"
+fi
+
+IMAGE_TAG_WITH_SHA="${IMAGE_NAME}:${GIT_SHA}${IMAGE_TAG_SUFFIX}"
+
+RELEASE_SEMVER=$(git describe --tags --exact-match "$GIT_SHA" 2>/dev/null) || true
+if [ -n "$RELEASE_SEMVER" ]; then
+  IMAGE_TAG_WITH_SEMVER="${IMAGE_NAME}:${RELEASE_SEMVER}${IMAGE_TAG_SUFFIX}"
+fi
+
+# build the image
+# passing no build phase will build the final image
+docker build -t "$IMAGE_TAG_WITH_SHA" --target=${BUILD_PHASE} .
+echo "${IMAGE_TAG_WITH_SHA} built locally."
+
+# if REGISTRY specified, push the images to the remote registy
+if [ -n "$REGISTRY" ]; then
+
+  if [ -n "${DOCKER_REGISTRY_PASSWORD}" ]; then
+    docker login --username="$DOCKER_REGISTRY_USERNAME" --password="$DOCKER_REGISTRY_PASSWORD"
+  fi
+
+  docker tag "$IMAGE_TAG_WITH_SHA" "${REGISTRY}/${IMAGE_TAG_WITH_SHA}"
+
+  docker push "${REGISTRY}/${IMAGE_TAG_WITH_SHA}"
+  echo "${REGISTRY}/${IMAGE_TAG_WITH_SHA} pushed to remote."
+
+  # If the current commit has a semver tag, also push the images with the semver tag
+  if [ -n "$RELEASE_SEMVER" ]; then
+
+    docker tag "$IMAGE_TAG_WITH_SHA" "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER}"
+
+    docker push "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER}"
+    echo "${REGISTRY}/${IMAGE_TAG_WITH_SEMVER} pushed to remote."
+
+  fi
+fi

flyteplugins/boilerplate/lyft/github_workflows/Readme.rst

@@ -0,0 +1,20 @@
+Golang Github Actions
+~~~~~~~~~~~~~~~~~
+
+Provides a two github actions workflows.
+
+**To Enable:**
+
+Add ``lyft/github_workflows`` to your ``boilerplate/update.cfg`` file.
+
+Add a github secret ``flytegithub_repo`` with a the name of your fork (e.g. ``my_company/flytepropeller``).
+
+The actions will push to 2 repos:
+
+	1. ``docker.pkg.github.com/lyft/<repo>/operator``
+	2. ``docker.pkg.github.com/lyft/<repo>/operator-stages`` : this repo is used to cache build stages to speed up iterative builds after.
+
+There are two workflows that get deployed:
+
+	1. A workflow that runs on Pull Requests to build and push images to github registy tagged with the commit sha.
+	2. A workflow that runs on master merges that bump the patch version of release tag, builds and pushes images to github registry tagged with the version, commit sha as well as "latest"

flyteplugins/boilerplate/lyft/github_workflows/master.yml

@@ -0,0 +1,31 @@
+name: Master
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+        with:
+          fetch-depth: '0'
+      - name: Bump version and push tag
+        id: bump-version
+        uses: anothrNick/[email protected]
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WITH_V: true
+          DEFAULT_BUMP: patch
+      - name: Push Docker Image to Github Registry
+        uses: whoan/docker-build-with-cache-action@v5
+        with:
+          username: "${{ github.actor }}"
+          password: "${{ secrets.GITHUB_TOKEN }}"
+          image_name: ${{ secrets.flytegithub_repo }}/operator
+          image_tag: latest,${{ github.sha }},${{ steps.bump-version.outputs.tag }}
+          push_git_tag: true
+          registry: docker.pkg.github.com
+          build_extra_args: "--compress=true"

flyteplugins/boilerplate/lyft/github_workflows/pull_request.yml

@@ -0,0 +1,19 @@
+name: Pull Request
+
+on:
+  pull_request
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Push Docker Image to Github Registry
+        uses: whoan/docker-build-with-cache-action@v5
+        with:
+          username: "${{ github.actor }}"
+          password: "${{ secrets.GITHUB_TOKEN }}"
+          image_name: ${{ secrets.flytegithub_repo }}/operator
+          image_tag: ${{ github.sha }}
+          push_git_tag: true
+          registry: docker.pkg.github.com

flyteplugins/boilerplate/lyft/github_workflows/update.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
+#
+# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+mkdir -p ${DIR}/../../../.github/workflows
+
+echo "     - generating github action workflows in root directory."
+sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/master.yml > ${DIR}/../../../.github/workflows/master.yml
+sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/pull_request.yml > ${DIR}/../../../.github/workflows/pull_request.yml

flyteplugins/boilerplate/lyft/golang_dockerfile/Dockerfile.GoTemplate

@@ -0,0 +1,32 @@
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
+
+FROM golang:1.13.3-alpine3.10 as builder
+RUN apk add git openssh-client make curl
+
+# COPY only the go mod files for efficient caching
+COPY go.mod go.sum /go/src/github.com/lyft/{{REPOSITORY}}/
+WORKDIR /go/src/github.com/lyft/{{REPOSITORY}}
+
+# Pull dependencies
+RUN go mod download
+
+# COPY the rest of the source code
+COPY . /go/src/github.com/lyft/{{REPOSITORY}}/
+
+# This 'linux_compile' target should compile binaries to the /artifacts directory
+# The main entrypoint should be compiled to /artifacts/{{REPOSITORY}}
+RUN make linux_compile
+
+# update the PATH to include the /artifacts directory
+ENV PATH="/artifacts:${PATH}"
+
+# This will eventually move to centurylink/ca-certs:latest for minimum possible image size
+FROM alpine:3.10
+COPY --from=builder /artifacts /bin
+
+RUN apk --update add ca-certificates
+
+CMD ["{{REPOSITORY}}"]

flyteplugins/boilerplate/lyft/golang_dockerfile/Readme.rst

@@ -0,0 +1,16 @@
+Golang Dockerfile
+~~~~~~~~~~~~~~~~~
+
+Provides a Dockerfile that produces a small image.
+
+**To Enable:**
+
+Add ``lyft/golang_dockerfile`` to your ``boilerplate/update.cfg`` file.
+
+Create and configure a ``make linux_compile`` target that compiles your go binaries to the ``/artifacts`` directory ::
+
+  .PHONY: linux_compile
+  linux_compile:
+    RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts {{ packages }}
+
+All binaries compiled to ``/artifacts`` will be available at ``/bin`` in your final image.

flyteplugins/boilerplate/lyft/golang_dockerfile/update.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES.
+# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY:
+# 
+# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst
+
+set -e
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
+
+echo "     - generating Dockerfile in root directory."
+sed -e "s/{{REPOSITORY}}/${REPOSITORY}/g" ${DIR}/Dockerfile.GoTemplate > ${DIR}/../../../Dockerfile

flyteplugins/boilerplate/update.cfg

@@ -2,3 +2,6 @@ lyft/golang_test_targets
 lyft/golangci_file
 lyft/golang_support_tools
 lyft/pull_request_template
+lyft/github_workflows
+lyft/golang_dockerfile
+lyft/docker_build