BuildKite Setup (#1002)

.buildkite/hooks/pre-command

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set +u
+
+echo "--- :golang: Setting up Golang build environment"
+
+if [[ ! -z "${GIMME_GO_VERSION:-}" ]] && [[ "$GIMME_GO_VERSION" != "" ]]; then
+  eval "$(.ci/gimme.sh ${GIMME_GO_VERSION})"
+else
+  echo "No \$GIMME_GO_VERSION set, skipping..."
+fi
+
+echo "---- :buildkite: :codecov: environment variables"
+export CI="true" # required by codecov.sh

.buildkite/pipeline.yml

@@ -0,0 +1,29 @@
+steps:
+  - name: "Unit, Big Unit"
+    command: make clean install-vendor test-ci-unit test-ci-big-unit
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+  - name: "Integration (Docker), Services, Tools, Metalint"
+    command: make install-vendor metalint test-all-gen docker-integration-test tools services
+    env:
+      CGO_ENABLED: 0
+      GIMME_GO_VERSION: 1.10.x
+    plugins:
+        gopath-checkout#v1.0.1:
+          import: github.com/m3db/m3
+  - name: "Integration (All Metadata)"
+    command: make clean install-vendor test-ci-integration-dbnode cache_policy=all_metadata
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+  - name: "Integration (Recently Read)"
+    command: make clean install-vendor test-ci-integration-dbnode cache_policy=recently_read
+    plugins:
+      docker-compose#v2.5.1:
+        run: app
+  - name: "Integration (LRU)"
+    command: make clean install-vendor test-ci-integration-dbnode cache_policy=lru
+    plugins:
+      docker-compose#v2.5.1:
+        run: app

.buildkite/project.json

@@ -0,0 +1,10 @@
+{
+  "name": "M3 CI",
+  "steps": [
+    {
+      "type": "script",
+      "name": ":pipeline:",
+      "command": "buildkite-agent pipeline upload"
+    }
+  ]
+}

.buildkite/scripts/Dockerfile

@@ -0,0 +1,9 @@
+# Built artifacts maintained externally at https://quay.io/repository/m3db/ci-base
+
+# Base image required for CI.
+# We seal the version of Golang to get hermetic builds.
+FROM golang:1.10.4-stretch
+LABEL maintainer="The M3DB Authors <[email protected]>"
+
+RUN mkdir -p /go/src/github.com/m3db/m3
+WORKDIR /go/src/github.com/m3db/m3

.dockerignore

@@ -0,0 +1,4 @@
+vendor/
+bin/
+_tools/
+dist/

Makefile

@@ -322,5 +322,6 @@ test-all-gen: all-gen
 .PHONY: clean
 clean:
 	@rm -f *.html *.xml *.out *.test
+	@rm -rf $(BUILD)
 
 .DEFAULT_GOAL := all

docker-compose.yml

@@ -0,0 +1,19 @@
+app:
+  image: quay.io/m3db/ci-base:latest
+  volumes:
+    - .:/go/src/github.com/m3db/m3
+    - /usr/bin/buildkite-agent:/usr/bin/buildkite-agent
+  environment:
+    - CI
+    - BUILDKITE
+    - BUILDKITE_AGENT_ACCESS_TOKEN
+    - BUILDKITE_JOB_ID
+    - BUILDKITE_BUILD_ID
+    - BUILDKITE_BUILD_NUMBER
+    - BUILDKITE_BRANCH
+    - BUILDKITE_BUILD_URL
+    - BUILDKITE_PROJECT_SLUG
+    - BUILDKITE_COMMIT
+    - BUILDKITE_PULL_REQUEST
+    - BUILDKITE_TAG
+    - CODECOV_TOKEN

scripts/proto-gen.sh

@@ -11,6 +11,11 @@ PROTOC_IMAGE_VERSION=${PROTOC_IMAGE_VERSION:-"znly/protoc:0.2.0"}
 # ensure docker is running
 docker run --rm hello-world >/dev/null
 
+UID_FLAGS="-u $(id -u)"
+if [[ -v BUILDKITE ]]; then
+	UID_FLAGS="-u root"
+fi
+
 PROTO_SRC=$1
 for i in "${GOPATH}/src/${PROTO_SRC}"/*; do
 	if ! [ -d $i ]; then
@@ -23,7 +28,7 @@ for i in "${GOPATH}/src/${PROTO_SRC}"/*; do
 		# need the additional m3db_path mount in docker because it's a symlink on the CI.
 		m3db_path=$(realpath $GOPATH/src/github.com/m3db/m3)
 		docker run --rm -w /src -v $GOPATH/src:/src -v ${m3db_path}:/src/github.com/m3db/m3 \
-	   	-u $(id -u) $PROTOC_IMAGE_VERSION --gogofaster_out=plugins=grpc:/src                \
+	   	$UID_FLAGS $PROTOC_IMAGE_VERSION --gogofaster_out=plugins=grpc:/src           \
 			 -I/src -I/src/github.com/m3db/m3/vendor ${proto_files}
 	fi
 done

src/dbnode/generated/thrift/generate.sh

@@ -8,8 +8,14 @@ docker run --rm hello-world >/dev/null
 
 # generate files using dockerized thrift-gen
 THRIFT_IMAGE_VERSION=${THRIFT_IMAGE_VERSION:-"quay.io/m3db/thrift-gen:0.1.0"}
-docker run --rm -u $(id -u) -v "$(pwd):/data" \
-  "$THRIFT_IMAGE_VERSION" --generateThrift    \
+
+UID_FLAGS="-u $(id -u)"
+if [[ -v BUILDKITE ]]; then
+	UID_FLAGS="-u root"
+fi
+
+docker run --rm -v "$(pwd):/data" $UID_FLAGS    \
+  "$THRIFT_IMAGE_VERSION" --generateThrift      \
   --inputFile /data/rpc.thrift --outputDir /data
 
 # ensure formatting is correct