Merge pull request #249 from krancour/improve-releases

improve the build and release processes

Makefile

@@ -0,0 +1,102 @@
+SHELL ?= /bin/bash
+
+.DEFAULT_GOAL := build
+
+################################################################################
+# Version details                                                              #
+################################################################################
+
+# This will reliably return the short SHA1 of HEAD or, if the working directory
+# is dirty, will return that + "-dirty"
+GIT_VERSION = $(shell git describe --always --abbrev=7 --dirty --match=NeVeRmAtCh)
+
+################################################################################
+# Containerized development environment-- or lack thereof                      #
+################################################################################
+
+ifneq ($(SKIP_DOCKER),true)
+	DEV_IMAGE := deis/node-chrome:node8
+	DOCKER_CMD := docker run \
+		-it \
+		--rm \
+		-e SKIP_DOCKER=true \
+		-v $$(pwd):/code/kashti \
+		-w /code/kashti \
+		-p 4200:4200 \
+		$(DEV_IMAGE)
+endif
+
+################################################################################
+# Binaries and Docker images we build and publish                              #
+################################################################################
+
+ifdef DOCKER_REGISTRY
+	DOCKER_REGISTRY := $(DOCKER_REGISTRY)/
+endif
+
+ifdef DOCKER_ORG
+	DOCKER_ORG := $(DOCKER_ORG)/
+endif
+
+BASE_IMAGE_NAME := kashti
+
+ifdef VERSION
+	MUTABLE_DOCKER_TAG := latest
+else
+	VERSION            := $(GIT_VERSION)
+	MUTABLE_DOCKER_TAG := edge
+endif
+
+IMAGE_NAME         := $(DOCKER_REGISTRY)$(DOCKER_ORG)$(BASE_IMAGE_NAME):$(VERSION)
+MUTABLE_IMAGE_NAME := $(DOCKER_REGISTRY)$(DOCKER_ORG)$(BASE_IMAGE_NAME):$(MUTABLE_DOCKER_TAG)
+
+################################################################################
+# Utility targets                                                              #
+################################################################################
+
+.PHONY: bootstrap
+bootstrap: yarn-install
+
+.PHONY: yarn-install
+yarn-install:
+	$(DOCKER_CMD) yarn install
+
+.PHONY: serve
+serve: yarn-install
+	$(DOCKER_CMD) yarn serve
+
+################################################################################
+# Tests                                                                        #
+################################################################################
+
+.PHONY: lint
+lint: yarn-install
+	$(DOCKER_CMD) yarn lint
+
+.PHONY: test
+test: yarn-install
+	$(DOCKER_CMD) yarn test
+
+.PHONY: e2e
+e2e: yarn-install
+	$(DOCKER_CMD) yarn e2e
+
+################################################################################
+# Build / Publish                                                              #
+################################################################################
+
+.PHONY: build
+build: build-image
+
+.PHONY: build-image
+build-image:
+	docker build -t $(IMAGE_NAME) .
+	docker tag $(IMAGE_NAME) $(MUTABLE_IMAGE_NAME)
+
+.PHONY: push
+push: push-image
+
+.PHONY: push-image
+push-image: build-image
+	docker push $(IMAGE_NAME)
+	docker push $(MUTABLE_IMAGE_NAME)

brigade.js

@@ -1,101 +1,192 @@
 const { events, Job, Group } = require("brigadier");
 
 const projectName = "kashti";
+const projectOrg = "brigadecore";
 
-class TestJob extends Job {
-  constructor(name) {
-    super(name, "deis/node-chrome:node8");
-    this.tasks = [
-      "cd /src",
-      "yarn global add @angular/cli",
-      "yarn install",
-      "ng lint",
-      "ng test --browsers=ChromeHeadless",
-    ];
-  }
+const img = "deis/node-chrome:node8";
+
+const releaseTagRegex = /^refs\/tags\/(v[0-9]+(?:\.[0-9]+)*(?:\-.+)?)$/;
+
+function tests() {
+  // Create a new job to run tests
+  var job = new Job("tests", img);
+  // Set a few environment variables.
+  job.env = {
+    "SKIP_DOCKER": "true"
+  };
+  job.tasks = [
+    "cd /src",
+    "make lint test"
+  ];
+  return job;
 }
 
-class E2eJob extends Job {
-  constructor(name) {
-    super(name, "deis/node-chrome:node8");
-    this.tasks = [
-      "apt-get update -yq && apt-get install -yq --no-install-recommends udev ttf-freefont chromedriver chromium",
-      "cd /src",
-      "yarn global add @angular/cli",
-      "yarn install",
-      "ng e2e"
-    ];
-  }
+function e2e() {
+  // Create a new job to run e2e tests
+  var job = new Job("e2e", img);
+  // Set a few environment variables.
+  job.env = {
+    "SKIP_DOCKER": "true"
+  };
+  job.tasks = [
+    "cd /src",
+    "make e2e"
+  ];
+  return job;
 }
 
-class ACRBuildJob extends Job {
-  constructor(name, img, tag, dir, registry, username, token, tenant) {
-    super(name, "microsoft/azure-cli:latest");
-    let imgName = img + ":" + tag;
-    this.tasks = [
-      // service principal should have proper perms on the container registry.
-      `az login --service-principal -u ${username} -p ${token} --tenant ${tenant}`,
-      `cd ${dir}`,
-      `echo '========> building ${img}...'`,
-      `az acr build -r ${registry} -t ${imgName} .`,
-      `echo '<======== finished building ${img}.'`
-    ];
-  }
+function buildAndPublishImages(project, version) {
+  let dockerRegistry = project.secrets.dockerhubRegistry || "docker.io";
+  let dockerOrg = project.secrets.dockerhubOrg || "brigadecore";
+  var job = new Job("build-and-publish-images", "docker:stable-dind");
+  job.privileged = true;
+  job.tasks = [
+    "apk add --update --no-cache make git",
+    "dockerd-entrypoint.sh &",
+    "sleep 20",
+    "cd /src",
+    `docker login ${dockerRegistry} -u ${project.secrets.dockerhubUsername} -p ${project.secrets.dockerhubPassword}`,
+    `DOCKER_REGISTRY=${dockerRegistry} DOCKER_ORG=${dockerOrg} VERSION=${version} make build-image push-image`,
+    `docker logout ${dockerRegistry}`
+  ];
+  return job;
 }
 
-function ghNotify(state, msg, e, project) {
-  const gh = new Job(`notify-${state}`, "technosophos/github-notify:latest");
-  gh.env = {
-    GH_REPO: project.repo.name,
-    GH_STATE: state,
-    GH_DESCRIPTION: msg,
-    GH_CONTEXT: "brigade",
-    GH_TOKEN: project.secrets.ghToken,
-    GH_COMMIT: e.revision.commit
+function runSuite(e, p) {
+  // For the master branch, we build and publish images in response to the push
+  // event. We test as a precondition for doing that, so we DON'T test here
+  // for the master branch.
+  if (e.revision.ref != "master") {
+    // Important: To prevent Promise.all() from failing fast, we catch and
+    // return all errors. This ensures Promise.all() always resolves. We then
+    // iterate over all resolved values looking for errors. If we find one, we
+    // throw it so the whole build will fail.
+    //
+    // Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise/all#Promise.all_fail-fast_behaviour
+    //
+    // Note: as provided test type string is used in job naming, it must consist
+    // of lowercase letters and hyphens only (per Brigade/K8s restrictions)
+    return Promise.all([
+      runTests(e, p, tests).catch((err) => { return err }),
+      runTests(e, p, e2e).catch((err) => { return err }),
+    ])
+      .then((values) => {
+        values.forEach((value) => {
+          if (value instanceof Error) throw value;
+        });
+      });
   }
-  return gh
 }
 
-function test() {
-  const test = new TestJob(`${projectName}-test`)
-  const e2e = new E2eJob(`${projectName}-e2e`)
-  return Group.runAll([test, e2e]);
+// runTests is a Check Run that is run as part of a Checks Suite
+function runTests(e, p, jobFunc) {
+  console.log("Check requested");
+
+  let job = jobFunc();
+
+  // Create Notification object (which is just a Job to update GH using the Checks API)
+  var note = new Notification(job.name, e, p);
+  note.conclusion = "";
+  note.title = `Run ${job.name}`;
+  note.summary = `Running ${job.name} build/test targets for ${e.revision.commit}`;
+
+  // Send notification, then run, then send pass/fail notification
+  return notificationWrap(job, note);
 }
 
-function githubRelease(e, project) {
-  const gh = JSON.parse(e.payload);
-  if (gh.ref.startsWith("refs/tags/") || gh.ref == "refs/heads/master") {
-    const start = ghNotify("pending", `release started as ${e.buildID}`, e, project)
+// A GitHub Check Suite notification
+class Notification {
+  constructor(name, e, p) {
+    this.proj = p;
+    this.payload = e.payload;
+    this.name = name;
+    this.externalID = e.buildID;
+   this.detailsURL = `https://brigadecore.github.io/kashti/builds/${e.buildID}`;
+    this.title = "running check";
+    this.text = "";
+    this.summary = "";
 
-    let parts = gh.ref.split("/", 3);
-    let tag = parts[2];
-    var releaser = new ACRBuildJob(`${projectName}-release`, projectName, tag, "/src", project.secrets.acrName, project.secrets.acrUsername, project.secrets.acrToken, project.secrets.acrTenant);
-    var latestReleaser = new ACRBuildJob(`${projectName}-release-latest`, projectName, "latest", "/src", project.secrets.acrName, project.secrets.acrUsername, project.secrets.acrToken, project.secrets.acrTenant);
-    Group.runAll([start, releaser, latestReleaser])
-      .then(() => {
-        return ghNotify("success", `release ${e.buildID} finished successfully`, e, project).run()
-      })
-      .catch(err => {
-        return ghNotify("failure", `failed release ${e.buildID}`, e, project).run()
-      });
-  } else {
-    console.log('not a tag or a push to master; skipping')
+    // count allows us to send the notification multiple times, with a distinct pod name
+    // each time.
+    this.count = 0;
+
+    // One of: "success", "failure", "neutral", "cancelled", or "timed_out".
+    this.conclusion = "neutral";
+  }
+
+  // Send a new notification, and return a Promise<result>.
+  run() {
+    this.count++;
+    var job = new Job(`${this.name}-${this.count}`, "brigadecore/brigade-github-check-run:latest");
+    job.imageForcePull = true;
+    job.env = {
+      "CHECK_CONCLUSION": this.conclusion,
+      "CHECK_NAME": this.name,
+      "CHECK_TITLE": this.title,
+      "CHECK_PAYLOAD": this.payload,
+      "CHECK_SUMMARY": this.summary,
+      "CHECK_TEXT": this.text,
+      "CHECK_DETAILS_URL": this.detailsURL,
+      "CHECK_EXTERNAL_ID": this.externalID
+    };
+    return job.run();
   }
 }
 
-function githubTest(e, project) {
-  const start = ghNotify("pending", `build started as ${e.buildID}`, e, project)
-  const test = new TestJob(`${projectName}-test`)
-  const e2e = new E2eJob(`${projectName}-e2e`)
-  Group.runAll([start, test, e2e])
-    .then(() => {
-      return ghNotify("success", `build ${e.buildID} passed`, e, project).run()
-    })
-    .catch(err => {
-      return ghNotify("failure", `failed build ${e.buildID}`, e, project).run()
-    });
+// Helper to wrap a job execution between two notifications.
+async function notificationWrap(job, note) {
+  await note.run();
+  try {
+    let res = await job.run();
+    const logs = await job.logs();
+    note.conclusion = "success";
+    note.summary = `Task "${job.name}" passed`;
+    // These jobs produce too much output to store in an environment variable
+    // note.text = "```" + res.toString() + "```\nComplete";
+    return await note.run();
+  } catch (e) {
+    const logs = await job.logs();
+    note.conclusion = "failure";
+    note.summary = `Task "${job.name}" failed for ${e.buildID}`;
+    // These jobs produce too much output to store in an environment variable
+    // note.text = "```" + logs + "```\nFailed with error: " + e.toString();
+    try {
+      await note.run();
+    } catch (e2) {
+      console.error("failed to send notification: " + e2.toString());
+      console.error("original error: " + e.toString());
+    }
+    throw e;
+  }
 }
 
-events.on("exec", test);
-events.on("push", githubRelease);
-events.on("pull_request", githubTest);
+events.on("exec", (e, p) => {
+  return Group.runAll([
+    tests(),
+    e2e(),
+  ]);
+});
+
+events.on("push", (e, p) => {
+  let matchStr = e.revision.ref.match(releaseTagRegex);
+  if (matchStr) {
+    // This is an official release with a semantically versioned tag
+    let matchTokens = Array.from(matchStr);
+    let version = matchTokens[1];
+    return buildAndPublishImages(p, version).run();
+  }
+  if (e.revision.ref == "refs/heads/master") {
+    // This runs tests then builds and publishes "edge" images
+    return Group.runAll([
+      tests(),
+      e2e(),
+    ])
+      .then(() => {
+        buildAndPublishImages(p, "").run();
+      });
+  }
+});
+
+events.on("check_suite:requested", runSuite);
+events.on("check_suite:rerequested", runSuite);
+events.on("check_run:rerequested", runSuite);