From 5b3c08ada09dffc6202a5ccf66f1bcd2c634954d Mon Sep 17 00:00:00 2001
From: Yufeng Wang <yufengwang@google.com>
Date: Wed, 14 Dec 2022 10:11:03 -0800
Subject: [PATCH] Add cloud build config file for remote builder (#24018)

* Add cloud build config file for remote builder

* Address review comments
---
 .../cloudbuild/build-coverage-remote.yaml     | 18 +++++
 integrations/docker/images/build-all.sh       |  2 +-
 .../chip-build-remote-builder/Dockerfile      |  4 ++
 .../chip-build-remote-builder/README.md       | 24 +++++++
 .../images/chip-build-remote-builder/build.sh | 23 ++++++
 .../chip-build-remote-builder/cloudbuild.yaml |  6 ++
 .../images/chip-build-remote-builder/run.sh   | 70 +++++++++++++++++++
 7 files changed, 146 insertions(+), 1 deletion(-)
 create mode 100644 integrations/cloudbuild/build-coverage-remote.yaml
 create mode 100644 integrations/docker/images/chip-build-remote-builder/Dockerfile
 create mode 100644 integrations/docker/images/chip-build-remote-builder/README.md
 create mode 100755 integrations/docker/images/chip-build-remote-builder/build.sh
 create mode 100644 integrations/docker/images/chip-build-remote-builder/cloudbuild.yaml
 create mode 100755 integrations/docker/images/chip-build-remote-builder/run.sh

diff --git a/integrations/cloudbuild/build-coverage-remote.yaml b/integrations/cloudbuild/build-coverage-remote.yaml
new file mode 100644
index 00000000000000..1d78608cfe8a4d
--- /dev/null
+++ b/integrations/cloudbuild/build-coverage-remote.yaml
@@ -0,0 +1,18 @@
+steps:
+    - name: gcr.io/$PROJECT_ID/remote-builder
+      env:
+          - GIT_CLONE_CMD=git clone --recurse-submodules https://github.com/project-chip/connectedhomeip.git;
+          - RUN_COVERAGE_CMD=connectedhomeip/scripts/build_coverage.sh;
+          - CLEANUP_CMD=rm -rf connectedhomeip;
+          - PROJECT_ID=$PROJECT_ID
+
+logsBucket: matter-build-automation-coverage-logs
+
+# Global timeout for all steps
+timeout: 21600s
+queueTtl: 21600s
+
+artifacts:
+    objects:
+        location: "gs://matter-build-automation-coverage-logs/$PROJECT_ID/$COMMIT_SHA/"
+        paths: ["/workspace/coverage_html.tar.gz"]
\ No newline at end of file
diff --git a/integrations/docker/images/build-all.sh b/integrations/docker/images/build-all.sh
index 4aff2747ed571a..e2b942b0887a9f 100755
--- a/integrations/docker/images/build-all.sh
+++ b/integrations/docker/images/build-all.sh
@@ -21,7 +21,7 @@
 #  https://github.com/project-chip/connectedhomeip/issues/710
 #
 set -e
-find "$(git rev-parse --show-toplevel)"/integrations/docker/images/ -name Dockerfile ! -path "*chip-cert-bins/*" | while read -r dockerfile; do
+find "$(git rev-parse --show-toplevel)"/integrations/docker/images/ -name Dockerfile ! -path "*chip-cert-bins/*" ! -path "*chip-build-remote-builder/*" | while read -r dockerfile; do
     pushd "$(dirname "$dockerfile")" >/dev/null
     ./build.sh "$@"
     popd >/dev/null
diff --git a/integrations/docker/images/chip-build-remote-builder/Dockerfile b/integrations/docker/images/chip-build-remote-builder/Dockerfile
new file mode 100644
index 00000000000000..ebb6072113a3e6
--- /dev/null
+++ b/integrations/docker/images/chip-build-remote-builder/Dockerfile
@@ -0,0 +1,4 @@
+FROM google/cloud-sdk
+
+COPY run.sh /bin
+CMD ["bash", "-xe", "/bin/run.sh"]
diff --git a/integrations/docker/images/chip-build-remote-builder/README.md b/integrations/docker/images/chip-build-remote-builder/README.md
new file mode 100644
index 00000000000000..6c6e72ccd8221b
--- /dev/null
+++ b/integrations/docker/images/chip-build-remote-builder/README.md
@@ -0,0 +1,24 @@
+# Docker `gcr.io:\$PROJECT_ID:remote-builder`
+
+`gcr.io:\$PROJECT_ID:remote-builder` is the name of the Docker image used by
+Matter for continuous integration and coverage statistics using Google Cloud
+Platform.
+
+Contents of this directory:
+
+-   build.sh - utility for building (and optionally) tagging and pushing the
+    remote-builder Docker image
+-   cloudbuild.yaml - build config file contains instructions for Cloud Build to
+    build, package, and push the remote-builder Docker image.
+-   Dockerfile - description of the image
+
+In order to use remote-builder, you need to first build the builder:
+
+`./build.sh`
+
+When using the remote-builder image, the following will happen:
+
+1. A temporary SSH key will be created in your Container Builder workspace
+2. SSH into a virtual machine instance with your configured flags
+3. Your command will be run inside that compute engine instance's workspace
+4. The workspace will be copied back to your Container Builder workspace
diff --git a/integrations/docker/images/chip-build-remote-builder/build.sh b/integrations/docker/images/chip-build-remote-builder/build.sh
new file mode 100755
index 00000000000000..7e33a953862ce8
--- /dev/null
+++ b/integrations/docker/images/chip-build-remote-builder/build.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+#
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# build.sh   - utility for building (and optionally) tagging and pushing
+#               the a Docker image
+#
+
+gcloud builds submit --config=cloudbuild.yaml .
diff --git a/integrations/docker/images/chip-build-remote-builder/cloudbuild.yaml b/integrations/docker/images/chip-build-remote-builder/cloudbuild.yaml
new file mode 100644
index 00000000000000..23ca7f79627862
--- /dev/null
+++ b/integrations/docker/images/chip-build-remote-builder/cloudbuild.yaml
@@ -0,0 +1,6 @@
+steps:
+    - name: "gcr.io/cloud-builders/docker"
+      args: ["build", "-t", "gcr.io/$PROJECT_ID/remote-builder", "."]
+images:
+    - "gcr.io/$PROJECT_ID/remote-builder"
+tags: ["cloud-builders-community"]
diff --git a/integrations/docker/images/chip-build-remote-builder/run.sh b/integrations/docker/images/chip-build-remote-builder/run.sh
new file mode 100755
index 00000000000000..dd1a173d4e389e
--- /dev/null
+++ b/integrations/docker/images/chip-build-remote-builder/run.sh
@@ -0,0 +1,70 @@
+#!/bin/bash -xe
+
+#
+#    Copyright (c) 2022 Project CHIP Authors
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+# run.sh   - utility for running a Docker image
+#
+# This script expects to live in a directory named after the image
+#  with a version file next to it.  So: use symlinks
+#
+USERNAME=${USERNAME:-ubuntu}
+PROJECT=$(gcloud info --format='value(config.project)')
+INSTANCE_NAME=${INSTANCE_NAME:-matter-remote-builder}
+ZONE=${ZONE:-us-central1-c}
+GCLOUD=${GCLOUD:-gcloud}
+RETRIES=${RETRIES:-3}
+
+# Run command on the instance via ssh
+function ssh() {
+    "$GCLOUD" compute ssh --project="$PROJECT" --zone="$ZONE" "$USERNAME@$INSTANCE_NAME" -- "$1"
+}
+
+# Always delete workspace after attempting build
+function cleanup() {
+    ssh "$CLEANUP_CMD"
+}
+
+"$GCLOUD" config set project "$PROJECT_ID"
+"$GCLOUD" config set compute/zone "$ZONE"
+
+trap cleanup EXIT
+
+RETRY_COUNT=1
+while [ "$(ssh 'printf pass')" != "pass" ]; do
+    echo "[Try $RETRY_COUNT of $RETRIES] Waiting for instance to start accepting SSH connections..."
+    if [ "$RETRY_COUNT" == "$RETRIES" ]; then
+        echo "Retry limit reached, giving up!"
+        exit 1
+    fi
+    sleep 10
+    RETRY_COUNT=$(($RETRY_COUNT + 1))
+done
+
+# Cleanup workspace if there is leftover
+ssh "$CLEANUP_CMD"
+
+# Setup workspace with connectedhomeip
+ssh "$GIT_CLONE_CMD"
+
+# Run coverage tests
+ssh "$RUN_COVERAGE_CMD"
+
+"$GCLOUD" compute scp --project="$PROJECT" --zone="$ZONE" \
+    "$USERNAME@$INSTANCE_NAME:/home/ubuntu/connectedhomeip/out/coverage/coverage/coverage_html.tar.gz" "$PWD"
+
+# Always delete workspace after build
+ssh "$CLEANUP_CMD"