From d1891c7605b4082684b0950bd62c2e611bf57f88 Mon Sep 17 00:00:00 2001 From: Prateek Rungta Date: Tue, 2 Oct 2018 14:32:01 -0400 Subject: [PATCH] BuildKite Setup (#1002) --- .buildkite/hooks/pre-command | 14 ++++++++++++ .buildkite/pipeline.yml | 29 +++++++++++++++++++++++++ .buildkite/project.json | 10 +++++++++ .buildkite/scripts/Dockerfile | 9 ++++++++ .ci | 2 +- .dockerignore | 4 ++++ .mergify.yml | 15 ------------- Makefile | 1 + docker-compose.yml | 19 ++++++++++++++++ scripts/proto-gen.sh | 7 +++++- src/dbnode/generated/thrift/generate.sh | 10 +++++++-- 11 files changed, 101 insertions(+), 19 deletions(-) create mode 100755 .buildkite/hooks/pre-command create mode 100644 .buildkite/pipeline.yml create mode 100644 .buildkite/project.json create mode 100644 .buildkite/scripts/Dockerfile create mode 100644 .dockerignore delete mode 100644 .mergify.yml create mode 100644 docker-compose.yml diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100755 index 0000000000..b8a6278e6e --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,14 @@ +#!/bin/bash + +set +u + +echo "--- :golang: Setting up Golang build environment" + +if [[ ! -z "${GIMME_GO_VERSION:-}" ]] && [[ "$GIMME_GO_VERSION" != "" ]]; then + eval "$(.ci/gimme.sh ${GIMME_GO_VERSION})" +else + echo "No \$GIMME_GO_VERSION set, skipping..." +fi + +echo "---- :buildkite: :codecov: environment variables" +export CI="true" # required by codecov.sh \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml new file mode 100644 index 0000000000..c29a928c36 --- /dev/null +++ b/.buildkite/pipeline.yml @@ -0,0 +1,29 @@ +steps: + - name: "Unit, Big Unit" + command: make clean install-vendor test-ci-unit test-ci-big-unit + plugins: + docker-compose#v2.5.1: + run: app + - name: "Integration (Docker), Services, Tools, Metalint" + command: make install-vendor metalint test-all-gen docker-integration-test tools services + env: + CGO_ENABLED: 0 + GIMME_GO_VERSION: 1.10.x + plugins: + gopath-checkout#v1.0.1: + import: github.com/m3db/m3 + - name: "Integration (All Metadata)" + command: make clean install-vendor test-ci-integration-dbnode cache_policy=all_metadata + plugins: + docker-compose#v2.5.1: + run: app + - name: "Integration (Recently Read)" + command: make clean install-vendor test-ci-integration-dbnode cache_policy=recently_read + plugins: + docker-compose#v2.5.1: + run: app + - name: "Integration (LRU)" + command: make clean install-vendor test-ci-integration-dbnode cache_policy=lru + plugins: + docker-compose#v2.5.1: + run: app diff --git a/.buildkite/project.json b/.buildkite/project.json new file mode 100644 index 0000000000..9f8c3e63c6 --- /dev/null +++ b/.buildkite/project.json @@ -0,0 +1,10 @@ +{ + "name": "M3 CI", + "steps": [ + { + "type": "script", + "name": ":pipeline:", + "command": "buildkite-agent pipeline upload" + } + ] +} diff --git a/.buildkite/scripts/Dockerfile b/.buildkite/scripts/Dockerfile new file mode 100644 index 0000000000..f3b7d07ce7 --- /dev/null +++ b/.buildkite/scripts/Dockerfile @@ -0,0 +1,9 @@ +# Built artifacts maintained externally at https://quay.io/repository/m3db/ci-base + +# Base image required for CI. +# We seal the version of Golang to get hermetic builds. +FROM golang:1.10.4-stretch +LABEL maintainer="The M3DB Authors " + +RUN mkdir -p /go/src/github.com/m3db/m3 +WORKDIR /go/src/github.com/m3db/m3 diff --git a/.ci b/.ci index c718f97bd8..792a7c7293 160000 --- a/.ci +++ b/.ci @@ -1 +1 @@ -Subproject commit c718f97bd8f1497671c280216b2fdd521d1c1a64 +Subproject commit 792a7c7293f1480477c2ab4dfe53c6b0dbc8cbbe diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..c1602a8706 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +vendor/ +bin/ +_tools/ +dist/ diff --git a/.mergify.yml b/.mergify.yml deleted file mode 100644 index 02096ee63f..0000000000 --- a/.mergify.yml +++ /dev/null @@ -1,15 +0,0 @@ -rules: - default: - protection: - required_pull_request_reviews: - required_approving_review_count: 1 - required_status_checks: - strict: true - contexts: - - deploy/netlify - - semaphoreci - - license/cla - - codecov/patch - - codecov/project - enabling_label: ready-to-be-merged - diff --git a/Makefile b/Makefile index f1627ea0df..7f3579ee39 100644 --- a/Makefile +++ b/Makefile @@ -322,5 +322,6 @@ test-all-gen: all-gen .PHONY: clean clean: @rm -f *.html *.xml *.out *.test + @rm -rf $(BUILD) .DEFAULT_GOAL := all diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000000..f51ad09415 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +app: + image: quay.io/m3db/ci-base:latest + volumes: + - .:/go/src/github.com/m3db/m3 + - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent + environment: + - CI + - BUILDKITE + - BUILDKITE_AGENT_ACCESS_TOKEN + - BUILDKITE_JOB_ID + - BUILDKITE_BUILD_ID + - BUILDKITE_BUILD_NUMBER + - BUILDKITE_BRANCH + - BUILDKITE_BUILD_URL + - BUILDKITE_PROJECT_SLUG + - BUILDKITE_COMMIT + - BUILDKITE_PULL_REQUEST + - BUILDKITE_TAG + - CODECOV_TOKEN diff --git a/scripts/proto-gen.sh b/scripts/proto-gen.sh index 0bb97f332e..14e339b12e 100755 --- a/scripts/proto-gen.sh +++ b/scripts/proto-gen.sh @@ -11,6 +11,11 @@ PROTOC_IMAGE_VERSION=${PROTOC_IMAGE_VERSION:-"znly/protoc:0.2.0"} # ensure docker is running docker run --rm hello-world >/dev/null +UID_FLAGS="-u $(id -u)" +if [[ -v BUILDKITE ]]; then + UID_FLAGS="-u root" +fi + PROTO_SRC=$1 for i in "${GOPATH}/src/${PROTO_SRC}"/*; do if ! [ -d $i ]; then @@ -23,7 +28,7 @@ for i in "${GOPATH}/src/${PROTO_SRC}"/*; do # need the additional m3db_path mount in docker because it's a symlink on the CI. m3db_path=$(realpath $GOPATH/src/github.com/m3db/m3) docker run --rm -w /src -v $GOPATH/src:/src -v ${m3db_path}:/src/github.com/m3db/m3 \ - -u $(id -u) $PROTOC_IMAGE_VERSION --gogofaster_out=plugins=grpc:/src \ + $UID_FLAGS $PROTOC_IMAGE_VERSION --gogofaster_out=plugins=grpc:/src \ -I/src -I/src/github.com/m3db/m3/vendor ${proto_files} fi done diff --git a/src/dbnode/generated/thrift/generate.sh b/src/dbnode/generated/thrift/generate.sh index e4928b6712..8ccdf93c84 100755 --- a/src/dbnode/generated/thrift/generate.sh +++ b/src/dbnode/generated/thrift/generate.sh @@ -8,8 +8,14 @@ docker run --rm hello-world >/dev/null # generate files using dockerized thrift-gen THRIFT_IMAGE_VERSION=${THRIFT_IMAGE_VERSION:-"quay.io/m3db/thrift-gen:0.1.0"} -docker run --rm -u $(id -u) -v "$(pwd):/data" \ - "$THRIFT_IMAGE_VERSION" --generateThrift \ + +UID_FLAGS="-u $(id -u)" +if [[ -v BUILDKITE ]]; then + UID_FLAGS="-u root" +fi + +docker run --rm -v "$(pwd):/data" $UID_FLAGS \ + "$THRIFT_IMAGE_VERSION" --generateThrift \ --inputFile /data/rpc.thrift --outputDir /data # ensure formatting is correct