Test the Kubernetes deployment

Requires 'k3d' available in the environment.

.github/workflows/master.yaml

@@ -15,6 +15,10 @@
                "name": "Installing Bazelisk",
                "run": "bazelisk_fingerprint=231ec5ca8115e94c75a1f4fbada1a062b48822ca04f21f26e4cb1cd8973cd458 &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) || (\n  mkdir -p ~/.cache/bazelisk &&\n  curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 > ~/.cache/bazelisk/bazelisk.tmp &&\n  chmod +x ~/.cache/bazelisk/bazelisk.tmp &&\n  mv ~/.cache/bazelisk/bazelisk.tmp ~/.cache/bazelisk/bazel\n) &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) &&\necho \"~/.cache/bazelisk\" >> ${GITHUB_PATH}\n"
             },
+            {
+               "name": "Install k3d",
+               "run": "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash"
+            },
             {
                "name": "Check out source code",
                "uses": "actions/checkout@v1"
@@ -59,6 +63,10 @@
                "name": "Test docker-compose deployment",
                "run": "tools/test-deployment-docker-compose.sh"
             },
+            {
+               "name": "Test Kubernetes deployment",
+               "run": "tools/test-deployment-kubernetes"
+            },
             {
                "name": "linux_amd64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."

.github/workflows/pull-requests.yaml

@@ -15,6 +15,10 @@
                "name": "Installing Bazelisk",
                "run": "bazelisk_fingerprint=231ec5ca8115e94c75a1f4fbada1a062b48822ca04f21f26e4cb1cd8973cd458 &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) || (\n  mkdir -p ~/.cache/bazelisk &&\n  curl -L https://github.com/bazelbuild/bazelisk/releases/download/v1.11.0/bazelisk-linux-amd64 > ~/.cache/bazelisk/bazelisk.tmp &&\n  chmod +x ~/.cache/bazelisk/bazelisk.tmp &&\n  mv ~/.cache/bazelisk/bazelisk.tmp ~/.cache/bazelisk/bazel\n) &&\n(echo \"${bazelisk_fingerprint} ${HOME}/.cache/bazelisk/bazel\" | sha256sum --check --quiet) &&\necho \"~/.cache/bazelisk\" >> ${GITHUB_PATH}\n"
             },
+            {
+               "name": "Install k3d",
+               "run": "curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash"
+            },
             {
                "name": "Check out source code",
                "uses": "actions/checkout@v1"
@@ -59,6 +63,10 @@
                "name": "Test docker-compose deployment",
                "run": "tools/test-deployment-docker-compose.sh"
             },
+            {
+               "name": "Test Kubernetes deployment",
+               "run": "tools/test-deployment-kubernetes"
+            },
             {
                "name": "linux_amd64: build and test",
                "run": "bazel build --platforms=@io_bazel_rules_go//go/toolchain:linux_amd64 //..."

tools/diff-docker-and-k8s-configs.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -eu -o pipefail
+set -eu -o pipefail -E
 
 function diff_config() {
     k8s_config="$1"

tools/github_workflows/workflows_template.libsonnet

@@ -75,6 +75,10 @@
             echo "~/.cache/bazelisk" >> ${GITHUB_PATH}
           |||,
         },
+        {
+          name: 'Install k3d',
+          run: 'curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash',
+        },
         {
           name: 'Check out source code',
           uses: 'actions/checkout@v1',
@@ -119,6 +123,10 @@
           name: 'Test docker-compose deployment',
           run: 'tools/test-deployment-docker-compose.sh',
         },
+        {
+          name: 'Test Kubernetes deployment',
+          run: 'tools/test-deployment-kubernetes',
+        },
       ] + std.flattenArrays([
         [{
           name: platform.name + ': build and test',

tools/remote-toolchains/extract-bazel-auto-toolchains-fixture/doit.sh

@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-set -eEuxo pipefail
+
+set -eux -o pipefail -E
 
 bazel_version="$1"
 

tools/remote-toolchains/extract-bazel-auto-toolchains.sh

@@ -7,7 +7,7 @@
 # `.bazelversion` in the repository root.
 #
 # Example usage: ./extract-bazel-auto-toolchains.sh ubuntu-act-22-04 ghcr.io/catthehacker/ubuntu:act-22.04
-set -eu -o pipefail
+set -eu -o pipefail -E
 
 script_dir="$(dirname "${BASH_SOURCE[0]}")"
 

tools/test-deployment-docker-compose.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 # Verifies that the docker-compose deployment works.
-set -eux -o pipefail
+set -eux -o pipefail -E
 
 script_dir=$(dirname "${BASH_SOURCE[0]}")
 cd "${script_dir}/../docker-compose"

tools/test-deployment-kubernetes

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Verifies that the Kuberenetes deployment works.
+
+set -eux -o pipefail -E
+
+script_dir=$(dirname "${BASH_SOURCE[0]}")
+root="$(realpath "$script_dir"/..)"
+cd "$root"
+
+# --- Launch Kubernetes deployment with k3d ---
+kubeconfig="$(realpath kubernetes/kubeconfig.yaml)"
+if test -e "$kubeconfig"; then
+    echo >&2 "ERROR: $kubeconfig already exists"
+    exit 1
+fi
+export KUBECONFIG="$kubeconfig"
+
+k3d cluster create bb-deployments-test
+
+cleanup() {
+    EXIT_STATUS=$?
+    # Ignore errors when tearing down.
+    set +e
+    k3d cluster delete bb-deployments-test
+    rm "$kubeconfig"
+    return "$EXIT_STATUS"
+}
+trap cleanup EXIT
+
+# Deploy Buildbarn.
+kubectl apply -k kubernetes
+kubectl rollout status --watch --namespace=buildbarn --timeout=300s statefulset
+kubectl rollout status --watch --namespace=buildbarn --timeout=300s deployment
+
+# Find the frontend.
+bb_frontend_ip=$(kubectl get service --namespace=buildbarn frontend -o json | jq --raw-output '.status.loadBalancer.ingress[0].ip')
+bb_frontend_address="grpc://$bb_frontend_ip:8980"
+ingress_ip=$(kubectl get service --namespace=kube-system traefik -o json | jq --raw-output '.status.loadBalancer.ingress[0].ip')
+
+# --- Test that the bb-browser and bb-scheduler UIs are responsive. ---
+curl --fail --header 'Host: bb-browser.example.com' "http://$ingress_ip/" | grep '<title>Buildbarn Browser</title>'
+curl --fail --header 'Host: bb-scheduler.example.com' "http://$ingress_ip/" | grep '<title>Buildbarn Scheduler</title>'
+
+# --- Run remote execution ---
+bazel_command_log="$(bazel info output_base)/command.log"
+bazel clean
+bazel test --color=no --curses=no --config=remote-ubuntu-22-04 --disk_cache= --remote_executor="$bb_frontend_address" @abseil-hello//:hello_test
+# Make sure there are remote executions but no cache hits.
+# INFO: 39 processes: 9 internal, 30 remote.
+grep -E '^INFO: [0-9]+ processes: .*[0-9]+ remote[.,]' \
+    "$bazel_command_log" \
+    | grep -v 'remote cache hit'
+
+# --- Check that the hardlinking workers are available ---
+bazel clean
+bazel test --color=no --curses=no --config=remote-ubuntu-22-04 --remote_instance_name=hardlinking --disk_cache= --remote_executor="$bb_frontend_address" @abseil-hello//:hello_test
+# Make sure there are remote executions but no cache hits.
+# INFO: 39 processes: 9 internal, 30 remote.
+grep -E '^INFO: [0-9]+ processes: .*[0-9]+ remote[.,]' \
+    "$bazel_command_log" \
+    | grep -v 'remote cache hit'

tools/update-container-image-versions.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
-set -eu -o pipefail
+
+set -eu -o pipefail -E
+
 # Updates the image version in the Docker Compose and Kubernetes deployments.
 #
 # Run this script as soon as go.mod has been updated.