From fa9627fa86b5f16a1bfe5f78366f170c878a0db5 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sat, 5 Dec 2020 01:25:28 +0100 Subject: [PATCH] GitHub Actions for test Signed-off-by: CrazyMax --- .github/workflows/build.yml | 93 +++++++++++++++++++++++++++++++++++++ .github/workflows/godev.yml | 23 +++++++++ .gitignore | 3 +- README.md | 10 ++-- codecov.yml | 1 + hack/build_ci_first_pass | 38 +++++++++++++++ hack/test | 70 ++++++++++++---------------- hack/update-vendor | 49 ++++--------------- hack/util | 19 ++++++++ 9 files changed, 222 insertions(+), 84 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/godev.yml create mode 100644 codecov.yml create mode 100755 hack/build_ci_first_pass diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 000000000000..065c55fc92b9 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,93 @@ +name: build + +on: + schedule: + - cron: '0 10 * * *' # everyday at 10am + workflow_dispatch: + push: + branches: + - 'master' + tags: + - 'v*' + pull_request: + branches: + - 'master' + +env: + REPO_SLUG_ORIGIN: "moby/buildkit:master" + PLATFORMS: "linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le" + CACHEKEY_BINARIES: "binaries" + +jobs: + base: + runs-on: ubuntu-latest + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Cache ${{ env.CACHEKEY_BINARIES }} + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} + key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}- + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + - + name: Build ${{ env.CACHEKEY_BINARIES }} + run: | + ./hack/build_ci_first_pass binaries + env: + CACHEDIR_FROM: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} + CACHEDIR_TO: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}-new + - + # FIXME: Temp fix for https://github.com/moby/buildkit/issues/1850 + name: Move cache + run: | + rm -rf /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} + mv /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }}-new /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} + + test: + runs-on: ubuntu-latest + needs: [base] + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Cache ${{ env.CACHEKEY_BINARIES }} + uses: actions/cache@v2 + with: + path: /tmp/.buildx-cache/${{ env.CACHEKEY_BINARIES }} + key: ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-${{ env.CACHEKEY_BINARIES }}- + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + - + name: Test + run: | + make test + env: + TEST_COVERAGE: 1 + TESTFLAGS: -v --parallel=6 --timeout=20m + CACHEDIR_FROM: /tmp/.buildkit-cache/${{ env.CACHEKEY_BINARIES }} + - + name: Send to Codecov + uses: codecov/codecov-action@v1 + with: + file: ./coverage/coverage.txt diff --git a/.github/workflows/godev.yml b/.github/workflows/godev.yml new file mode 100644 index 000000000000..8d589099cfb7 --- /dev/null +++ b/.github/workflows/godev.yml @@ -0,0 +1,23 @@ +name: godev + +on: + push: + tags: + - 'v*' + +jobs: + update: + runs-on: ubuntu-latest + steps: + - + name: Set up Go + uses: actions/setup-go@v2 + with: + go-version: 1.13 + - + name: Call pkg.go.dev + run: | + go get github.com/${GITHUB_REPOSITORY}@${GITHUB_REF#refs/tags/} + env: + GO111MODULE: on + GOPROXY: https://proxy.golang.org diff --git a/.gitignore b/.gitignore index 87ecb1e2d6be..aeea52398c86 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ bin -cross-out \ No newline at end of file +coverage +cross-out diff --git a/README.md b/README.md index 5550fcfe1408..66a925443ecd 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,13 @@ # buildx -### Docker CLI plugin for extended build capabilities with BuildKit -_buildx is Tech Preview_ +[![PkgGoDev](https://img.shields.io/badge/go.dev-docs-007d9c?logo=go&logoColor=white)](https://pkg.go.dev/github.com/docker/buildx) +[![Build Status](https://github.com/docker/buildx/workflows/build/badge.svg)](https://github.com/docker/buildx/actions?query=workflow%3Abuild) +[![Go Report Card](https://goreportcard.com/badge/github.com/docker/buildx)](https://goreportcard.com/report/github.com/docker/buildx) +[![codecov](https://codecov.io/gh/docker/buildx/branch/master/graph/badge.svg)](https://codecov.io/gh/docker/buildx) -### TL;DR +`buildx` is a Docker CLI plugin for extended build capabilities with [BuildKit](https://github.com/moby/buildkit). + +Key features: - Familiar UI from `docker build` - Full BuildKit capabilities with container driver diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 000000000000..69cb76019a47 --- /dev/null +++ b/codecov.yml @@ -0,0 +1 @@ +comment: false diff --git a/hack/build_ci_first_pass b/hack/build_ci_first_pass new file mode 100755 index 000000000000..8a6f23b9abe9 --- /dev/null +++ b/hack/build_ci_first_pass @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +TYP=$1 + +. $(dirname $0)/util +set -e + +usage() { + echo "usage: ./hack/build_ci_first_pass " + exit 1 +} + +if [ -z "$TYP" ]; then + usage +fi + +importCacheFlags="" +exportCacheFlags="" +if [ "$GITHUB_ACTIONS" = "true" ]; then + if [ -n "$cacheRefFrom" ]; then + importCacheFlags="--cache-from=type=local,src=$cacheRefFrom" + fi + if [ -n "$cacheRefTo" ]; then + exportCacheFlags="--cache-to=type=local,dest=$cacheRefTo" + fi +fi + +case $TYP in + "binaries") + buildxCmd build $importCacheFlags $exportCacheFlags \ + --target "binaries" \ + $currentcontext + ;; + *) + echo >&2 "Unknown type $TYP" + exit 1 + ;; +esac diff --git a/hack/test b/hack/test index 2445ef8b7a6c..d4d0b30b0565 100755 --- a/hack/test +++ b/hack/test @@ -3,51 +3,39 @@ . $(dirname $0)/util set -eu -o pipefail -: ${CONTINUOUS_INTEGRATION=} -: ${BUILDX_NOCACHE=} - -progressFlag="" -if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi +: ${TEST_COVERAGE=} + +importCacheFlags="" +if [ -n "$cacheRefFrom" ]; then + if [ "$cacheType" = "local" ]; then + for ref in $cacheRefFrom; do + importCacheFlags="$importCacheFlags--cache-from=type=local,src=$ref " + done + else + importCacheFlags="--cache-from=type=registry,ref=$cacheRefFrom:integration-tests " + fi +fi iid="buildx-tests" iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) -set -x - -case $buildmode in -"buildkit") - tmpfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) - buildctl build $progressFlag --frontend=dockerfile.v0 \ - --local context=. --local dockerfile=. \ - --frontend-opt target=integration-tests \ - --output type=docker,name=$iid,dest=$tmpfile - docker load -i $tmpfile - rm $tmpfile - ;; -"docker-buildkit") - export DOCKER_BUILDKIT=1 - docker build --iidfile $iidfile --target integration-tests --force-rm . - iid=$(cat $iidfile) - ;; -*) - echo "docker with buildkit support is required" - exit 1 - ;; -esac - -cacheVolume="buildx-cache" -if ! docker inspect "$cacheVolume" > /dev/null 2>&1 ; then -cacheVolume=$(docker create --name=buildx-cache -v /root/.cache -v /go/pkg/mod alpine) + +coverageVol="" +coverageFlags="" +if [ "$TEST_COVERAGE" = "1" ]; then + covdir="$(pwd)/coverage" + mkdir -p "$covdir" + coverageVol="-v $covdir:/coverage" + coverageFlags="-coverprofile=/coverage/coverage.txt -covermode=atomic" fi -docker run --rm -v /tmp --volumes-from=$cacheVolume --privileged $iid go test ${TESTFLAGS:--v} ${TESTPKGS:-./...} +buildxCmd build $importCacheFlags \ + --target "integration-tests" \ + --output "type=docker,name=$iid" \ + $currentcontext -if [ -n "$BUILDX_NOCACHE" ]; then - docker rm -v $cacheVolume -fi +cacheVolume=$(docker create -v /root/.cache -v /root/.cache/registry -v /go/pkg/mod alpine) + +cid=$(docker create --rm -v /tmp $coverageVol --volumes-from=$cacheVolume -e BUILDKIT_REGISTRY_MIRROR_DIR=/root/.cache/registry --privileged $iid go test $coverageFlags ${TESTFLAGS:--v} ${TESTPKGS:-./...}) +docker start -a $cid -case $buildmode in -"docker-buildkit") - rm "$iidfile" - docker rmi $iid - ;; -esac +docker rm -v $cacheVolume diff --git a/hack/update-vendor b/hack/update-vendor index 29aae68715e0..18dd3237c608 100755 --- a/hack/update-vendor +++ b/hack/update-vendor @@ -1,45 +1,16 @@ #!/usr/bin/env bash . $(dirname $0)/util -set -eu -o pipefail -x +set -eu -: ${CONTINUOUS_INTEGRATION=} +output=$(mktemp -d -t buildx-output.XXXXXXXXXX) -progressFlag="" -if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi +buildxCmd build \ + --target "update" \ + --output "type=local,dest=$output" \ + --file "./hack/dockerfiles/vendor.Dockerfile" \ + . -case $buildmode in -"buildkit") - output=$(mktemp -d -t buildctl-output.XXXXXXXXXX) - buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \ - --frontend-opt target=update \ - --frontend-opt filename=./hack/dockerfiles/vendor.Dockerfile \ - --output type=local,dest=$output - rm -rf ./vendor - cp -R "$output/out/" . - rm -rf $output - ;; -*) - iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) - case $buildmode in - "docker-buildkit") - export DOCKER_BUILDKIT=1 - docker build --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --target update --force-rm . - ;; - *) - echo "buildctl or docker with buildkit support is required" - exit 1 - ;; - esac - iid=$(cat $iidfile) - cid=$(docker create $iid noop) - rm -rf ./vendor - - docker cp $cid:/out/go.mod . - docker cp $cid:/out/go.sum . - docker cp $cid:/out/vendor . - - docker rm $cid - rm -f $iidfile - ;; -esac +rm -rf ./vendor +cp -R "$output"/out/* . +rm -rf $output diff --git a/hack/util b/hack/util index 33a12e1bf0b2..f3f2ef1fe6c2 100755 --- a/hack/util +++ b/hack/util @@ -4,6 +4,9 @@ : ${PREFER_BUILDCTL=} : ${PREFER_LEGACY=} : ${CLI_PLATFORM=} +: ${GITHUB_ACTIONS=} +: ${CACHEDIR_FROM=} +: ${CACHEDIR_TO=} newerEqualThan() { # $1=minimum wanted version $2=actual-version [ "$1" = "$(printf "$1\n$2" | sort -V | head -n 1)" ] @@ -54,3 +57,19 @@ if [ -z "$CLI_PLATFORM" ]; then CLI_PLATFORM="windows/amd64" fi fi + +cacheType="" +cacheRefFrom="" +cacheRefTo="" +currentref="" +if [ "$GITHUB_ACTIONS" = "true" ]; then + currentref="git://github.com/$GITHUB_REPOSITORY#$GITHUB_REF" + cacheType="local" + cacheRefFrom="$CACHEDIR_FROM" + cacheRefTo="$CACHEDIR_TO" +fi + +currentcontext="." +if [ -n "$currentref" ]; then + currentcontext="--build-arg BUILDKIT_CONTEXT_KEEP_GIT_DIR=1 $currentref" +fi