From 2596354d31370fc788807d0ea05437a16f58abd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szabolcs=20Gelencse=CC=81r?= Date: Mon, 25 Nov 2024 19:09:13 +0100 Subject: [PATCH 01/17] log remote actions --- .../lib/remote/RemoteExecutionService.java | 26 +++++++++++++++++++ .../lib/remote/merkletree/MerkleTree.java | 1 + .../build/lib/remote/util/DigestUtil.java | 1 + 3 files changed, 28 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 0da922fb789787..27ebca3c430745 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -137,6 +137,8 @@ import io.reactivex.rxjava3.schedulers.Schedulers; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.FileWriter; +import java.io.File; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -669,8 +671,32 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context Spawns.mayBeCachedRemotely(spawn), buildSalt(spawn, spawnScrubber)); + System.out.println("[DEBUG] RemoteExecutionService.buildRemoteAction"); ActionKey actionKey = digestUtil.computeActionKey(action); + String jsonPath = "/tmp/bazeldebug/" + actionKey.getDigest().getHash() + ".json"; + File file = new File(jsonPath); + try (FileWriter writer = new FileWriter(file)) { + writer.write("[Action]\n"); + writer.write(action.toString()); + writer.write("\n\n"); + + writer.write("[Command]\n"); + writer.write(command.toString()); + writer.write("\n\n"); + + writer.write("[Platform]\n"); + writer.write(platform.toString()); + writer.write("\n\n"); + + writer.write("[InputRoot]\n"); + writer.write(merkleTree.toString()); + + System.out.println(jsonPath + " written."); + } catch (IOException e) { + e.printStackTrace(); + } + RequestMetadata metadata = TracingMetadataUtils.buildMetadata( buildRequestId, commandId, actionKey.getDigest().getHash(), spawn.getResourceOwner()); diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java index 7cace2695d63b5..088f39612b2234 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java @@ -123,6 +123,7 @@ private MerkleTree( this.inputBytes = inputBytes; } + // TODO /** Returns the digest of the Merkle tree's root. */ @Nullable public Directory getRootProto() { diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java index a3332f8e5e73b2..59c4e0ecb6f3f8 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java +++ b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java @@ -92,6 +92,7 @@ public Digest compute(Path path, FileStatus status) throws IOException { } public Digest compute(VirtualActionInput input) throws IOException { + // TOGGLE: 9 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); input.writeTo(buffer); return compute(buffer.toByteArray()); From c0991bc74521a6186b31400c4f398518fad692e3 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Tue, 26 Nov 2024 09:05:24 +0100 Subject: [PATCH 02/17] Adding merkletree fields --- .../devtools/build/lib/remote/RemoteExecutionService.java | 6 +++++- .../devtools/build/lib/remote/merkletree/MerkleTree.java | 6 +++--- .../google/devtools/build/lib/remote/util/DigestUtil.java | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index 27ebca3c430745..e29a8b6dd8a769 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -690,7 +690,11 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context writer.write("\n\n"); writer.write("[InputRoot]\n"); - writer.write(merkleTree.toString()); + writer.write(merkleTree.rootDigest.toString()); + writer.write("[Root proto]\n"); + writer.write(merkleTree.rootProto.toString()); + writer.write("[Files]\n"); + writer.write(merkleTree.files.toString()); System.out.println(jsonPath + " written."); } catch (IOException e) { diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java index 088f39612b2234..f5c1a86179f5b5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java @@ -96,9 +96,9 @@ private interface MerkleTreeDirectoryVisitor { private Map digestDirectoryMap; private Map digestFileMap; - @Nullable private final Directory rootProto; - private final Digest rootDigest; - private final SortedSet files; + @Nullable public final Directory rootProto; + public final Digest rootDigest; + public final SortedSet files; private final SortedSet symlinks; private final SortedMap directories; private final long inputFiles; diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java index 59c4e0ecb6f3f8..d3be29bb053cea 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java +++ b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java @@ -92,7 +92,7 @@ public Digest compute(Path path, FileStatus status) throws IOException { } public Digest compute(VirtualActionInput input) throws IOException { - // TOGGLE: 9 + // TOGGLE: 18 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); input.writeTo(buffer); return compute(buffer.toByteArray()); From 092e38fa89d8419ba8f776c6ae7abdd6b1f32f28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Szabolcs=20Gelencse=CC=81r?= Date: Tue, 26 Nov 2024 09:52:26 +0100 Subject: [PATCH 03/17] done :tada: --- .../build/lib/remote/RemoteAction.java | 30 +++++++++++++++++++ .../lib/remote/RemoteExecutionService.java | 30 ------------------- .../build/lib/remote/RemoteSpawnCache.java | 2 ++ .../lib/remote/merkletree/MerkleTree.java | 2 +- 4 files changed, 33 insertions(+), 31 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java index e8230f56933194..f56e1b21f8d060 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java @@ -28,6 +28,7 @@ import com.google.devtools.build.lib.vfs.PathFragment; import java.util.SortedMap; import javax.annotation.Nullable; +import java.io.FileWriter; /** * A value class representing an action which can be executed remotely. @@ -73,6 +74,35 @@ public class RemoteAction { this.actionKey = actionKey; } + private void writeDirectoryToFile(FileWriter writer, String root, MerkleTree t) throws IOException { + writer.write("[Root proto of " + root + "]\n"); + writer.write(t.rootProto.toString()); + for (String dirName : t.directories.keySet()) { + MerkleTree dir = t.directories.get(dirName); + writeDirectoryToFile(writer, root + "/" + dirName, dir); + } + } + + public void writeToFile() { + String jsonPath = "/tmp/bazeldebug/" + actionKey.getDigest().getHash() + ".json"; + try (FileWriter writer = new FileWriter(jsonPath)) { + writer.write("[Action]\n"); + writer.write(action.toString()); + writer.write("\n\n"); + + writer.write("[Command]\n"); + writer.write(command.toString()); + writer.write("\n\n"); + + writer.write("[InputRoot]\n"); + writeDirectoryToFile(writer, ".", merkleTree); + + System.out.println(jsonPath + " written."); + } catch (IOException e) { + e.printStackTrace(); + } + } + public RemoteActionExecutionContext getRemoteActionExecutionContext() { return remoteActionExecutionContext; } diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java index e29a8b6dd8a769..aa2f64d06c7aa5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteExecutionService.java @@ -137,7 +137,6 @@ import io.reactivex.rxjava3.schedulers.Schedulers; import java.io.FileNotFoundException; import java.io.IOException; -import java.io.FileWriter; import java.io.File; import java.time.Instant; import java.util.ArrayList; @@ -671,36 +670,7 @@ public RemoteAction buildRemoteAction(Spawn spawn, SpawnExecutionContext context Spawns.mayBeCachedRemotely(spawn), buildSalt(spawn, spawnScrubber)); - System.out.println("[DEBUG] RemoteExecutionService.buildRemoteAction"); ActionKey actionKey = digestUtil.computeActionKey(action); - - String jsonPath = "/tmp/bazeldebug/" + actionKey.getDigest().getHash() + ".json"; - File file = new File(jsonPath); - try (FileWriter writer = new FileWriter(file)) { - writer.write("[Action]\n"); - writer.write(action.toString()); - writer.write("\n\n"); - - writer.write("[Command]\n"); - writer.write(command.toString()); - writer.write("\n\n"); - - writer.write("[Platform]\n"); - writer.write(platform.toString()); - writer.write("\n\n"); - - writer.write("[InputRoot]\n"); - writer.write(merkleTree.rootDigest.toString()); - writer.write("[Root proto]\n"); - writer.write(merkleTree.rootProto.toString()); - writer.write("[Files]\n"); - writer.write(merkleTree.files.toString()); - - System.out.println(jsonPath + " written."); - } catch (IOException e) { - e.printStackTrace(); - } - RequestMetadata metadata = TracingMetadataUtils.buildMetadata( buildRequestId, commandId, actionKey.getDigest().getHash(), spawn.getResourceOwner()); diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java index 06c97c5390db38..efc1123851fcd1 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnCache.java @@ -249,6 +249,8 @@ public void store(SpawnResult result) throws ExecException, InterruptedException return; } + action.writeToFile(); + if (options.experimentalGuardAgainstConcurrentChanges) { try (SilentCloseable c = prof.profile("checkForConcurrentModifications")) { checkForConcurrentModifications(); diff --git a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java index f5c1a86179f5b5..142fcbd135a961 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java +++ b/src/main/java/com/google/devtools/build/lib/remote/merkletree/MerkleTree.java @@ -100,7 +100,7 @@ private interface MerkleTreeDirectoryVisitor { public final Digest rootDigest; public final SortedSet files; private final SortedSet symlinks; - private final SortedMap directories; + public final SortedMap directories; private final long inputFiles; private final long inputBytes; From 3b6d138facb806fa0c355816d78c2641e08c4af5 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Tue, 26 Nov 2024 15:21:12 +0100 Subject: [PATCH 04/17] IOException + merkleTree nullcheck + ignoring remoteDiscardMerkleTrees --- .../devtools/build/lib/remote/RemoteAction.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java index f56e1b21f8d060..002f2566504cc0 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java @@ -29,6 +29,7 @@ import java.util.SortedMap; import javax.annotation.Nullable; import java.io.FileWriter; +import java.io.IOException; /** * A value class representing an action which can be executed remotely. @@ -43,6 +44,7 @@ public class RemoteAction { private final RemoteActionExecutionContext remoteActionExecutionContext; private final RemotePathResolver remotePathResolver; @Nullable private final MerkleTree merkleTree; + @Nullable private final MerkleTree merkleTreeForDebug; private final long inputBytes; private final long inputFiles; private final Digest commandHash; @@ -66,6 +68,7 @@ public class RemoteAction { this.remoteActionExecutionContext = remoteActionExecutionContext; this.remotePathResolver = remotePathResolver; this.merkleTree = remoteDiscardMerkleTrees ? null : merkleTree; + this.merkleTreeForDebug = merkleTree; this.inputBytes = merkleTree.getInputBytes(); this.inputFiles = merkleTree.getInputFiles(); this.commandHash = commandHash; @@ -76,10 +79,14 @@ public class RemoteAction { private void writeDirectoryToFile(FileWriter writer, String root, MerkleTree t) throws IOException { writer.write("[Root proto of " + root + "]\n"); - writer.write(t.rootProto.toString()); - for (String dirName : t.directories.keySet()) { - MerkleTree dir = t.directories.get(dirName); - writeDirectoryToFile(writer, root + "/" + dirName, dir); + if (t != null) { + writer.write(t.rootProto.toString()); + for (String dirName : t.directories.keySet()) { + MerkleTree dir = t.directories.get(dirName); + writeDirectoryToFile(writer, root + "/" + dirName, dir); + } + } else { + writer.write("[merkleTree is null]\n"); } } @@ -95,7 +102,7 @@ public void writeToFile() { writer.write("\n\n"); writer.write("[InputRoot]\n"); - writeDirectoryToFile(writer, ".", merkleTree); + writeDirectoryToFile(writer, ".", merkleTreeForDebug); System.out.println(jsonPath + " written."); } catch (IOException e) { From 68aed67a91044c4148c999737139612b01e51851 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Tue, 26 Nov 2024 16:19:35 +0100 Subject: [PATCH 05/17] Added a dummy change. --- .../java/com/google/devtools/build/lib/remote/RemoteAction.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java index 002f2566504cc0..5df5d90b0a6c5e 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java @@ -93,6 +93,7 @@ private void writeDirectoryToFile(FileWriter writer, String root, MerkleTree t) public void writeToFile() { String jsonPath = "/tmp/bazeldebug/" + actionKey.getDigest().getHash() + ".json"; try (FileWriter writer = new FileWriter(jsonPath)) { + writer.write("[** Generated by Bitrise for debug purposes.]\n"); writer.write("[Action]\n"); writer.write(action.toString()); writer.write("\n\n"); From 587cbffd8be8625eb48b61f7e336fce1cce3fdb8 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Tue, 26 Nov 2024 16:52:31 +0100 Subject: [PATCH 06/17] Cleanup --- .../java/com/google/devtools/build/lib/remote/RemoteAction.java | 1 - .../com/google/devtools/build/lib/remote/util/DigestUtil.java | 1 - 2 files changed, 2 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java index 5df5d90b0a6c5e..002f2566504cc0 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java @@ -93,7 +93,6 @@ private void writeDirectoryToFile(FileWriter writer, String root, MerkleTree t) public void writeToFile() { String jsonPath = "/tmp/bazeldebug/" + actionKey.getDigest().getHash() + ".json"; try (FileWriter writer = new FileWriter(jsonPath)) { - writer.write("[** Generated by Bitrise for debug purposes.]\n"); writer.write("[Action]\n"); writer.write(action.toString()); writer.write("\n\n"); diff --git a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java index d3be29bb053cea..a3332f8e5e73b2 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java +++ b/src/main/java/com/google/devtools/build/lib/remote/util/DigestUtil.java @@ -92,7 +92,6 @@ public Digest compute(Path path, FileStatus status) throws IOException { } public Digest compute(VirtualActionInput input) throws IOException { - // TOGGLE: 18 ByteArrayOutputStream buffer = new ByteArrayOutputStream(); input.writeTo(buffer); return compute(buffer.toByteArray()); From 7e10b1cfdd2d82e930529ac236829f32807c3b9c Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 09:11:39 +0100 Subject: [PATCH 07/17] yml wip --- .../devtools/build/lib/remote/bitrise.yml | 634 ++++++++++++++++++ 1 file changed, 634 insertions(+) create mode 100644 src/main/java/com/google/devtools/build/lib/remote/bitrise.yml diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml new file mode 100644 index 00000000000000..37893cf4037ba9 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -0,0 +1,634 @@ +--- +format_version: '13' +default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git +project_type: other +workflows: + _build-bazel-with-bazel: + steps: + - activate-ssh-key@4: + run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' + - git-clone@8: {} + - script: + title: Update GO -> 1.22 + inputs: + - script_file_path: + - content: | + #!/usr/bin/env bash + set -euxo pipefail + asdf install golang 1.22.0 + asdf global golang 1.22.0 + go version + - script@1: + title: Install bazelisk + inputs: + - script_file_path: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 + chmod +x /usr/local/bin/bazel + bazel --version + elif [[ "$OSTYPE" == "darwin"* ]]; then + go install github.com/bazelbuild/bazelisk@latest + sudo ln -s $(which bazelisk) /usr/local/bin/bazel + bazel --version + else + echo "Unknown OS" + fi + - activate-build-cache-for-bazel@1: {} + - script@1: + title: bazel fetch //src:bazel-dev + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + echo "common --show_timestamps" >> .bazelrc + echo "common --announce_rc" >> .bazelrc + echo "common --color=yes" >> .bazelrc + + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "startup --digest_function=blake3" >> .bazelrc + echo "build --remote_cache_compression" >> .bazelrc + fi + + bazel fetch //src:bazel-dev + - script@1: + title: bazel build //src:bazel-dev + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + START_TIME=$(date +%s) + bazel build --profile=./bazel-profile.gz --noslim_profile //src:bazel-dev + END_TIME=$(date +%s) + + ELAPSED_TIME=$((END_TIME - START_TIME)) + curl -X POST "https://api.datadoghq.com/api/v1/series" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: $DD_API_KEY" \ + -d "{ + \"series\" : [{ + \"metric\": \"build_cache.reference_build_duration\", + \"points\": [[$(date +%s), $ELAPSED_TIME]], + \"tags\": [\"datacenter:$BITRISE_DEN_VM_DATACENTER\", \"measurement_name:$BITRISE_GIT_MESSAGE\", \"app_name:$BITRISE_APP_TITLE\"], + \"type\": \"gauge\" + }] + }" + + curl -X POST "https://api.datadoghq.com/api/v1/distribution_points" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: $DD_API_KEY" \ + -d "{ + \"series\" : [{ + \"metric\": \"build_cache.reference_build_duration.dist\", + \"points\": [[$(date +%s), [$ELAPSED_TIME]]], + \"tags\": [\"datacenter:$BITRISE_DEN_VM_DATACENTER\", \"measurement_name:$BITRISE_GIT_MESSAGE\", \"app_name:$BITRISE_APP_TITLE\"], + \"type\": \"distribution\" + }] + }" + + cp ./bazel-profile.gz $BITRISE_DEPLOY_DIR + bazel analyze-profile ./bazel-profile.gz + - deploy-to-bitrise-io@2: {} + summary: Build bazel with bazel + _build-bazel-with-patched-bazel-wip: + steps: + - activate-ssh-key@4: + run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' + - script@1: + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + envman add --key BITRISE_GIT_BRANCH --value "szabi-log-action-master" + title: Fix branch to patched bazel (szabi-log-action-master) + - git-clone@8: + inputs: + - clone_depth: "-1" + - script: + title: Update GO -> 1.22 + inputs: + - script_file_path: + - content: | + #!/usr/bin/env bash + set -euxo pipefail + asdf install golang 1.22.0 + asdf global golang 1.22.0 + go version + - script@1: + title: Install bazelisk + inputs: + - script_file_path: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 + chmod +x /usr/local/bin/bazel + bazel --version + elif [[ "$OSTYPE" == "darwin"* ]]; then + go install github.com/bazelbuild/bazelisk@latest + sudo ln -s $(which bazelisk) /usr/local/bin/bazel + bazel --version + else + echo "Unknown OS" + fi + - activate-build-cache-for-bazel@1: {} + - script@1: + title: bazel fetch //src:bazel-dev + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + echo "common --show_timestamps" >> .bazelrc + echo "common --announce_rc" >> .bazelrc + echo "common --color=yes" >> .bazelrc + + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "startup --digest_function=blake3" >> .bazelrc + echo "build --remote_cache_compression" >> .bazelrc + fi + + bazel fetch //src:bazel-dev + - script@1: + title: build patched bazel + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + bazel build --profile=./bazel-profile.gz --noslim_profile --action_env=BITRISE_DEBUG_FLAG=BUILD_PATCHED_BAZEL --sandbox_debug --verbose_failures //src:bazel-dev + - set-java-version@1: + inputs: + - set_java_version: '21' + title: Set Java version to 21 + - script@1: + title: build bazel with patched bazel + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + mkdir /tmp/bazeldebug + bazel-bin/src/bazel-dev build --profile=./bazel-profile.gz --noslim_profile --action_env=--action_env=BITRISE_DEBUG_FLAG=BUILD_WITH_PATCHED_BAZEL //src:bazel-dev + + if [ -d "/tmp/bazeldebug" ] && [ "$(ls -A /tmp/bazeldebug)" ]; then + echo "debug action files found in /tmp/bazeldebug. Creating archive..." + + tar -czf "$BITRISE_DEPLOY_DIR/bazeldebug_archive.tar.gz" -C /tmp/bazeldebug . + + echo "archive created at $BITRISE_DEPLOY_DIR/bazeldebug_archive.tar.gz" + else + echo "no files found in /tmp/bazeldebug." + fi + - deploy-to-bitrise-io@2: {} + summary: Build bazel with bazel + _build-bazel-with-patched-bazel-wip-kv: + steps: + - activate-ssh-key@4: + run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' + - script@1: + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + envman add --key BITRISE_GIT_BRANCH --value "szabi-log-action-master" + title: Fix branch to patched bazel (szabi-log-action-master) + - git-clone@8: + inputs: + - clone_depth: "-1" + - script: + title: Update GO -> 1.22 + inputs: + - script_file_path: + - content: | + #!/usr/bin/env bash + set -euxo pipefail + asdf install golang 1.22.0 + asdf global golang 1.22.0 + go version + - restore-cache@2: + inputs: + - key: bazel-bin-{{ .Branch }}-{{ .CommitHash }} + - verbose: 'true' + title: Restore Bazel binary from KV cache + - activate-build-cache-for-bazel@1: {} + - set-java-version@1: + inputs: + - set_java_version: '21' + title: Set Java version to 21 + - script@1: + title: bazel fetch //src:bazel-dev + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + echo "common --show_timestamps" >> .bazelrc + echo "common --announce_rc" >> .bazelrc + echo "common --color=yes" >> .bazelrc + + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "startup --digest_function=blake3" >> .bazelrc + echo "build --remote_cache_compression" >> .bazelrc + fi + + bazel-bin/src/bazel-dev fetch //src:bazel-dev + - script@1: + title: build bazel with patched bazel + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + mkdir /tmp/bazeldebug + bazel-bin/src/bazel-dev build --profile=./bazel-profile.gz --noslim_profile //src:bazel-dev + + if [ -d "/tmp/bazeldebug" ] && [ "$(ls -A /tmp/bazeldebug)" ]; then + echo "debug action files found in /tmp/bazeldebug. Creating archive..." + + tar -czf "$BITRISE_DEPLOY_DIR/bazeldebug_archive.tar.gz" -C /tmp/bazeldebug . + + echo "archive created at $BITRISE_DEPLOY_DIR/bazeldebug_archive.tar.gz" + else + echo "no files found in /tmp/bazeldebug." + fi + - deploy-to-bitrise-io@2: {} + summary: Build bazel with bazel + _release_bazel_to_kv_cache: + steps: + - activate-ssh-key@4: + run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' + - script@1: + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + envman add --key BITRISE_GIT_BRANCH --value "szabi-log-action-master" + title: Fix branch to patched bazel (szabi-log-action-master) + - git-clone@8: + inputs: + - clone_depth: "-1" + - script: + title: Update GO -> 1.22 + inputs: + - script_file_path: + - content: | + #!/usr/bin/env bash + set -euxo pipefail + asdf install golang 1.22.0 + asdf global golang 1.22.0 + go version + - script@1: + title: Install bazelisk + inputs: + - script_file_path: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 + chmod +x /usr/local/bin/bazel + bazel --version + elif [[ "$OSTYPE" == "darwin"* ]]; then + go install github.com/bazelbuild/bazelisk@latest + sudo ln -s $(which bazelisk) /usr/local/bin/bazel + bazel --version + else + echo "Unknown OS" + fi + - script@1: + title: bazel fetch //src:bazel-dev + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + echo "common --show_timestamps" >> .bazelrc + echo "common --announce_rc" >> .bazelrc + echo "common --color=yes" >> .bazelrc + + if [[ "$OSTYPE" == "darwin"* ]]; then + echo "startup --digest_function=blake3" >> .bazelrc + echo "build --remote_cache_compression" >> .bazelrc + fi + + bazel fetch //src:bazel-dev + - script@1: + title: build patched bazel + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + bazel build //src:bazel-dev + - save-cache@1: + inputs: + - key: bazel-bin-{{ .Branch }}-{{ .CommitHash }} + - paths: bazel-bin/src + - verbose: 'true' + title: Save Bazel binary into KV cache + summary: Build bazel with bazel + build-bazel-with-bazel-buildbuddy: + steps: + - activate-ssh-key@4: + run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' + - git-clone@8: {} + - script: + title: Update GO -> 1.22 + inputs: + - script_file_path: + - content: | + #!/usr/bin/env bash + set -euxo pipefail + asdf install golang 1.22.0 + asdf global golang 1.22.0 + go version + - script@1: + title: Install bazelisk + inputs: + - script_file_path: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 + chmod +x /usr/local/bin/bazel + bazel --version + elif [[ "$OSTYPE" == "darwin"* ]]; then + go install github.com/bazelbuild/bazelisk@latest + sudo ln -s $(which bazelisk) /usr/local/bin/bazel + bazel --version + else + echo "Unknown OS" + fi + - script@1: + title: bazel setup for buildbuddy - no RBE + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + echo "build --bes_results_url=https://gogod3v-eval.buildbuddy.io/invocation/" >> .bazelrc + echo "build --bes_backend=grpcs://gogod3v-eval.buildbuddy.io" >> .bazelrc + echo "build --remote_cache=grpcs://gogod3v-eval.buildbuddy.io" >> .bazelrc + echo "build --remote_timeout=3600" >> .bazelrc + echo "build --remote_header=x-buildbuddy-api-key=$BB_KEY" >> .bazelrc + - script@1: + title: bazel fetch //src:bazel-dev + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + echo "common --show_timestamps" >> .bazelrc + echo "common --announce_rc" >> .bazelrc + echo "common --color=yes" >> .bazelrc + + #if [[ "$OSTYPE" == "darwin"* ]]; then + # echo "startup --digest_function=blake3" >> .bazelrc + # echo "build --remote_cache_compression" >> .bazelrc + #fi + + bazel fetch //src:bazel-dev + - script@1: + title: bazel build //src:bazel-dev + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + START_TIME=$(date +%s) + bazel build --profile=./bazel-profile.gz --noslim_profile --action_env=QWE=asd //src:bazel-dev + END_TIME=$(date +%s) + + ELAPSED_TIME=$((END_TIME - START_TIME)) + curl -X POST "https://api.datadoghq.com/api/v1/series" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: $DD_API_KEY" \ + -d "{ + \"series\" : [{ + \"metric\": \"build_cache.reference_build_duration_buildbuddy\", + \"points\": [[$(date +%s), $ELAPSED_TIME]], + \"tags\": [\"datacenter:$BITRISE_DEN_VM_DATACENTER\", \"measurement_name:$BITRISE_GIT_MESSAGE\", \"app_name:$BITRISE_APP_TITLE\"], + \"type\": \"gauge\" + }] + }" + + cp ./bazel-profile.gz $BITRISE_DEPLOY_DIR + bazel analyze-profile ./bazel-profile.gz + - script@1: + title: bazel clean + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + bazel clean + - script@1: + title: bazel setup for buildbuddy - RBE + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + echo "build --remote_executor=grpcs://remote.buildbuddy.io" >> .bazelrc + echo "build --config=remote_buildbuddy" >> .bazelrc + echo "build --noremote_upload_local_results" >> .bazelrc + - script@1: + title: bazel build //src:bazel-dev + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + START_TIME=$(date +%s) + bazel build --profile=./bazel-profile-rbe.gz --noslim_profile --action_env=QWE=asd //src:bazel-dev + END_TIME=$(date +%s) + + ELAPSED_TIME=$((END_TIME - START_TIME)) + curl -X POST "https://api.datadoghq.com/api/v1/series" \ + -H "Content-Type: application/json" \ + -H "DD-API-KEY: $DD_API_KEY" \ + -d "{ + \"series\" : [{ + \"metric\": \"build_cache.reference_build_duration_buildbuddy_rbe\", + \"points\": [[$(date +%s), $ELAPSED_TIME]], + \"tags\": [\"datacenter:$BITRISE_DEN_VM_DATACENTER\", \"measurement_name:$BITRISE_GIT_MESSAGE\", \"app_name:$BITRISE_APP_TITLE\"], + \"type\": \"gauge\" + }] + }" + + cp ./bazel-profile-rbe.gz $BITRISE_DEPLOY_DIR + bazel analyze-profile ./bazel-profile-rbe.gz + - deploy-to-bitrise-io@2: {} + summary: Build bazel with bazel @ BuildBuddy + meta: + bitrise.io: + stack: linux-docker-android-22.04 + machine_type_id: elite-xl + build-bazel-with-bazel-linux-p1: + after_run: + - _build-bazel-with-bazel + meta: + bitrise.io: + stack: linux-docker-android-22.04 + machine_type_id: elite-xl + build-bazel-with-bazel-p1: + after_run: + - _build-bazel-with-bazel + build-bazel-with-bazel-patched-wip: + after_run: + - _build-bazel-with-patched-bazel-wip + build-bazel-with-bazel-p1-staging: + after_run: + - build-bazel-with-bazel-p1 + steps: + - script@1: + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + export CACHE_ENDPOINT_LINUX_STAGING="grpcs://build-cache-api.services.bitrise.dev:443" + export CACHE_ENDPOINT_MAC_STAGING="grpc://build-cache-api-las.services.bitrise.dev:6666" + + if [[ "$OSTYPE" == "linux-gnu"* ]]; then + echo "Setting cache endpoint to $CACHE_ENDPOINT_LINUX_STAGING" + envman add --key BITRISE_BUILD_CACHE_ENDPOINT --value "$CACHE_ENDPOINT_LINUX_STAGING" + echo $BITRISE_BUILD_CACHE_ENDPOINT + elif [[ "$OSTYPE" == "darwin"* ]]; then + echo "Setting cache endpoint to $CACHE_ENDPOINT_MAC_STAGING" + envman add --key BITRISE_BUILD_CACHE_ENDPOINT --value "$CACHE_ENDPOINT_MAC_STAGING" + echo $BITRISE_BUILD_CACHE_ENDPOINT + else + echo "Unknown OS" + fi +meta: + bitrise.io: + stack: osx-xcode-15.3.x-las + machine_type_id: g2-m1.8core From d97f2a2b9581c101b64414f3900fd9fc8153e56b Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 09:15:59 +0100 Subject: [PATCH 08/17] wip --- src/main/java/com/google/devtools/build/lib/remote/bitrise.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml index 37893cf4037ba9..12d9e3b9164e1d 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -599,6 +599,9 @@ workflows: build-bazel-with-bazel-patched-wip: after_run: - _build-bazel-with-patched-bazel-wip + release-bazel-to-kv-cache-iad-ord-mac: + after_run: + - _release_bazel_to_kv_cache build-bazel-with-bazel-p1-staging: after_run: - build-bazel-with-bazel-p1 From 95c7eb49d9792745a8e153c43d5bb522a6c02303 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 09:16:59 +0100 Subject: [PATCH 09/17] wip --- .../java/com/google/devtools/build/lib/remote/bitrise.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml index 12d9e3b9164e1d..6a81a60b6e96b2 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -602,6 +602,10 @@ workflows: release-bazel-to-kv-cache-iad-ord-mac: after_run: - _release_bazel_to_kv_cache + meta: + bitrise.io: + stack: osx-xcode-15.4.x + machine_type_id: g2.mac.x-large build-bazel-with-bazel-p1-staging: after_run: - build-bazel-with-bazel-p1 From 5d8c0ad147ab65defec44a54d4e807c1599c746f Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 10:22:46 +0100 Subject: [PATCH 10/17] wip --- .../devtools/build/lib/remote/bitrise.yml | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml index 6a81a60b6e96b2..e2a0ecead552c6 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -229,7 +229,7 @@ workflows: fi - deploy-to-bitrise-io@2: {} summary: Build bazel with bazel - _build-bazel-with-patched-bazel-wip-kv: + build-bazel-with-patched-bazel-wip-kv: steps: - activate-ssh-key@4: run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' @@ -599,13 +599,27 @@ workflows: build-bazel-with-bazel-patched-wip: after_run: - _build-bazel-with-patched-bazel-wip - release-bazel-to-kv-cache-iad-ord-mac: + release-bazel-to-kv-cache-r2-darwin: after_run: - _release_bazel_to_kv_cache meta: bitrise.io: stack: osx-xcode-15.4.x machine_type_id: g2.mac.x-large + release-bazel-to-kv-cache-r2-linux: + after_run: + - _release_bazel_to_kv_cache + meta: + bitrise.io: + stack: linux-docker-android-22.04 + machine_type_id: g2.linux.x-large + release-bazel-to-kv-cache-gcp: + after_run: + - _release_bazel_to_kv_cache + meta: + bitrise.io: + stack: linux-docker-android-22.04 + machine_type_id: elite-xl build-bazel-with-bazel-p1-staging: after_run: - build-bazel-with-bazel-p1 @@ -639,3 +653,9 @@ meta: bitrise.io: stack: osx-xcode-15.3.x-las machine_type_id: g2-m1.8core +pipelines: + release-patched-bazel-to-kv: + workflows: + release-bazel-to-kv-cache-r2-darwin: {} + release-bazel-to-kv-cache-r2-linux: {} + release-bazel-to-kv-cache-gcp: {} From 0efc5b37ae8e33238414c65d1fcec7b55e62d03a Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 10:27:16 +0100 Subject: [PATCH 11/17] wip --- .../devtools/build/lib/remote/bitrise.yml | 261 +----------------- 1 file changed, 15 insertions(+), 246 deletions(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml index e2a0ecead552c6..5c1742c50a437d 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -18,33 +18,19 @@ workflows: asdf install golang 1.22.0 asdf global golang 1.22.0 go version - - script@1: - title: Install bazelisk + - restore-cache@2: inputs: - - script_file_path: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x /usr/local/bin/bazel - bazel --version - elif [[ "$OSTYPE" == "darwin"* ]]; then - go install github.com/bazelbuild/bazelisk@latest - sudo ln -s $(which bazelisk) /usr/local/bin/bazel - bazel --version - else - echo "Unknown OS" - fi + - key: bazel-bin-{{ .Branch }}-{{ .CommitHash }} + - verbose: 'true' + title: Restore Bazel binary from KV cache + - activate-build-cache-for-bazel@1: {} + - set-java-version@1: + inputs: + - set_java_version: '21' + title: Set Java version to 21 - activate-build-cache-for-bazel@1: {} - script@1: - title: bazel fetch //src:bazel-dev + title: bazel-bin/src/bazel-dev fetch //src:bazel-dev inputs: - content: |- #!/usr/bin/env bash @@ -64,9 +50,9 @@ workflows: echo "build --remote_cache_compression" >> .bazelrc fi - bazel fetch //src:bazel-dev + bazel-bin/src/bazel-dev fetch //src:bazel-dev - script@1: - title: bazel build //src:bazel-dev + title: bazel-bin/src/bazel-dev build //src:bazel-dev inputs: - content: |- #!/usr/bin/env bash @@ -78,22 +64,10 @@ workflows: set -x START_TIME=$(date +%s) - bazel build --profile=./bazel-profile.gz --noslim_profile //src:bazel-dev + bazel-bin/src/bazel-dev build --profile=./bazel-profile.gz --noslim_profile //src:bazel-dev END_TIME=$(date +%s) ELAPSED_TIME=$((END_TIME - START_TIME)) - curl -X POST "https://api.datadoghq.com/api/v1/series" \ - -H "Content-Type: application/json" \ - -H "DD-API-KEY: $DD_API_KEY" \ - -d "{ - \"series\" : [{ - \"metric\": \"build_cache.reference_build_duration\", - \"points\": [[$(date +%s), $ELAPSED_TIME]], - \"tags\": [\"datacenter:$BITRISE_DEN_VM_DATACENTER\", \"measurement_name:$BITRISE_GIT_MESSAGE\", \"app_name:$BITRISE_APP_TITLE\"], - \"type\": \"gauge\" - }] - }" - curl -X POST "https://api.datadoghq.com/api/v1/distribution_points" \ -H "Content-Type: application/json" \ -H "DD-API-KEY: $DD_API_KEY" \ @@ -110,212 +84,8 @@ workflows: bazel analyze-profile ./bazel-profile.gz - deploy-to-bitrise-io@2: {} summary: Build bazel with bazel - _build-bazel-with-patched-bazel-wip: - steps: - - activate-ssh-key@4: - run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' - - script@1: - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - envman add --key BITRISE_GIT_BRANCH --value "szabi-log-action-master" - title: Fix branch to patched bazel (szabi-log-action-master) - - git-clone@8: - inputs: - - clone_depth: "-1" - - script: - title: Update GO -> 1.22 - inputs: - - script_file_path: - - content: | - #!/usr/bin/env bash - set -euxo pipefail - asdf install golang 1.22.0 - asdf global golang 1.22.0 - go version - - script@1: - title: Install bazelisk - inputs: - - script_file_path: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x /usr/local/bin/bazel - bazel --version - elif [[ "$OSTYPE" == "darwin"* ]]; then - go install github.com/bazelbuild/bazelisk@latest - sudo ln -s $(which bazelisk) /usr/local/bin/bazel - bazel --version - else - echo "Unknown OS" - fi - - activate-build-cache-for-bazel@1: {} - - script@1: - title: bazel fetch //src:bazel-dev - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - echo "common --show_timestamps" >> .bazelrc - echo "common --announce_rc" >> .bazelrc - echo "common --color=yes" >> .bazelrc - - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "startup --digest_function=blake3" >> .bazelrc - echo "build --remote_cache_compression" >> .bazelrc - fi - - bazel fetch //src:bazel-dev - - script@1: - title: build patched bazel - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - bazel build --profile=./bazel-profile.gz --noslim_profile --action_env=BITRISE_DEBUG_FLAG=BUILD_PATCHED_BAZEL --sandbox_debug --verbose_failures //src:bazel-dev - - set-java-version@1: - inputs: - - set_java_version: '21' - title: Set Java version to 21 - - script@1: - title: build bazel with patched bazel - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - mkdir /tmp/bazeldebug - bazel-bin/src/bazel-dev build --profile=./bazel-profile.gz --noslim_profile --action_env=--action_env=BITRISE_DEBUG_FLAG=BUILD_WITH_PATCHED_BAZEL //src:bazel-dev - - if [ -d "/tmp/bazeldebug" ] && [ "$(ls -A /tmp/bazeldebug)" ]; then - echo "debug action files found in /tmp/bazeldebug. Creating archive..." - - tar -czf "$BITRISE_DEPLOY_DIR/bazeldebug_archive.tar.gz" -C /tmp/bazeldebug . - - echo "archive created at $BITRISE_DEPLOY_DIR/bazeldebug_archive.tar.gz" - else - echo "no files found in /tmp/bazeldebug." - fi - - deploy-to-bitrise-io@2: {} - summary: Build bazel with bazel - build-bazel-with-patched-bazel-wip-kv: - steps: - - activate-ssh-key@4: - run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' - - script@1: - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - envman add --key BITRISE_GIT_BRANCH --value "szabi-log-action-master" - title: Fix branch to patched bazel (szabi-log-action-master) - - git-clone@8: - inputs: - - clone_depth: "-1" - - script: - title: Update GO -> 1.22 - inputs: - - script_file_path: - - content: | - #!/usr/bin/env bash - set -euxo pipefail - asdf install golang 1.22.0 - asdf global golang 1.22.0 - go version - - restore-cache@2: - inputs: - - key: bazel-bin-{{ .Branch }}-{{ .CommitHash }} - - verbose: 'true' - title: Restore Bazel binary from KV cache - - activate-build-cache-for-bazel@1: {} - - set-java-version@1: - inputs: - - set_java_version: '21' - title: Set Java version to 21 - - script@1: - title: bazel fetch //src:bazel-dev - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - echo "common --show_timestamps" >> .bazelrc - echo "common --announce_rc" >> .bazelrc - echo "common --color=yes" >> .bazelrc - - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "startup --digest_function=blake3" >> .bazelrc - echo "build --remote_cache_compression" >> .bazelrc - fi - - bazel-bin/src/bazel-dev fetch //src:bazel-dev - - script@1: - title: build bazel with patched bazel - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - mkdir /tmp/bazeldebug - bazel-bin/src/bazel-dev build --profile=./bazel-profile.gz --noslim_profile //src:bazel-dev - - if [ -d "/tmp/bazeldebug" ] && [ "$(ls -A /tmp/bazeldebug)" ]; then - echo "debug action files found in /tmp/bazeldebug. Creating archive..." - - tar -czf "$BITRISE_DEPLOY_DIR/bazeldebug_archive.tar.gz" -C /tmp/bazeldebug . - - echo "archive created at $BITRISE_DEPLOY_DIR/bazeldebug_archive.tar.gz" - else - echo "no files found in /tmp/bazeldebug." - fi - - deploy-to-bitrise-io@2: {} - summary: Build bazel with bazel _release_bazel_to_kv_cache: steps: - activate-ssh-key@4: @@ -412,6 +182,8 @@ workflows: - verbose: 'true' title: Save Bazel binary into KV cache summary: Build bazel with bazel + + build-bazel-with-bazel-buildbuddy: steps: - activate-ssh-key@4: @@ -596,9 +368,6 @@ workflows: build-bazel-with-bazel-p1: after_run: - _build-bazel-with-bazel - build-bazel-with-bazel-patched-wip: - after_run: - - _build-bazel-with-patched-bazel-wip release-bazel-to-kv-cache-r2-darwin: after_run: - _release_bazel_to_kv_cache From cbb0d4ebebf698f90ef8d755417922bfead14a88 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 10:32:13 +0100 Subject: [PATCH 12/17] wip --- .../google/devtools/build/lib/remote/bitrise.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml index 5c1742c50a437d..20cfe6dd0b5204 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -7,6 +7,19 @@ workflows: steps: - activate-ssh-key@4: run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' + - script@1: + inputs: + - content: |- + #!/usr/bin/env bash + # fail if any commands fails + set -e + # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully + set -o pipefail + # debug log + set -x + + envman add --key BITRISE_GIT_BRANCH --value "szabi-log-action-master" + title: Fix branch to patched bazel (szabi-log-action-master) - git-clone@8: {} - script: title: Update GO -> 1.22 @@ -358,6 +371,8 @@ workflows: bitrise.io: stack: linux-docker-android-22.04 machine_type_id: elite-xl + + build-bazel-with-bazel-linux-p1: after_run: - _build-bazel-with-bazel From 7e01b692989c517f5ae396e2d5293eeeb7962724 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 10:50:01 +0100 Subject: [PATCH 13/17] wip --- src/main/java/com/google/devtools/build/lib/remote/bitrise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml index 20cfe6dd0b5204..69682b606ebd35 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -190,7 +190,7 @@ workflows: bazel build //src:bazel-dev - save-cache@1: inputs: - - key: bazel-bin-{{ .Branch }}-{{ .CommitHash }} + - key: bazel-bin-{{ .OS }}-{{ .Branch }}-{{ .CommitHash }} - paths: bazel-bin/src - verbose: 'true' title: Save Bazel binary into KV cache From ac711acbfc6434ea7c3e42b885af75578de7f8e6 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 10:50:28 +0100 Subject: [PATCH 14/17] wip --- src/main/java/com/google/devtools/build/lib/remote/bitrise.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml index 69682b606ebd35..fbb04995148846 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -33,7 +33,7 @@ workflows: go version - restore-cache@2: inputs: - - key: bazel-bin-{{ .Branch }}-{{ .CommitHash }} + - key: bazel-bin-{{ .OS }}-{{ .Branch }}-{{ .CommitHash }} - verbose: 'true' title: Restore Bazel binary from KV cache - activate-build-cache-for-bazel@1: {} From 8d0ee8cb85251bdc5d4b58d897d581bc9395cd02 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Wed, 27 Nov 2024 11:22:23 +0100 Subject: [PATCH 15/17] wip --- src/main/java/com/google/devtools/build/lib/remote/bitrise.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml index fbb04995148846..93e42bd4eb2601 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml @@ -76,6 +76,8 @@ workflows: # debug log set -x + mkdir /tmp/bazeldebug + START_TIME=$(date +%s) bazel-bin/src/bazel-dev build --profile=./bazel-profile.gz --noslim_profile //src:bazel-dev END_TIME=$(date +%s) From 3543bc2c8efb02cf9b56f43c4afcdd1991926ce6 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Thu, 28 Nov 2024 10:04:31 +0100 Subject: [PATCH 16/17] Debug files .json -> .txt --- .../java/com/google/devtools/build/lib/remote/RemoteAction.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java index 002f2566504cc0..77f399620cd1c5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java +++ b/src/main/java/com/google/devtools/build/lib/remote/RemoteAction.java @@ -91,7 +91,7 @@ private void writeDirectoryToFile(FileWriter writer, String root, MerkleTree t) } public void writeToFile() { - String jsonPath = "/tmp/bazeldebug/" + actionKey.getDigest().getHash() + ".json"; + String jsonPath = "/tmp/bazeldebug/" + actionKey.getDigest().getHash() + ".txt"; try (FileWriter writer = new FileWriter(jsonPath)) { writer.write("[Action]\n"); writer.write(action.toString()); From a4954ce10b8ee326f8025ec31cc324a2ed885347 Mon Sep 17 00:00:00 2001 From: bvatai-br Date: Thu, 28 Nov 2024 10:19:36 +0100 Subject: [PATCH 17/17] Delete bitrise.yml --- .../devtools/build/lib/remote/bitrise.yml | 447 ------------------ 1 file changed, 447 deletions(-) delete mode 100644 src/main/java/com/google/devtools/build/lib/remote/bitrise.yml diff --git a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml b/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml deleted file mode 100644 index 93e42bd4eb2601..00000000000000 --- a/src/main/java/com/google/devtools/build/lib/remote/bitrise.yml +++ /dev/null @@ -1,447 +0,0 @@ ---- -format_version: '13' -default_step_lib_source: https://github.com/bitrise-io/bitrise-steplib.git -project_type: other -workflows: - _build-bazel-with-bazel: - steps: - - activate-ssh-key@4: - run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' - - script@1: - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - envman add --key BITRISE_GIT_BRANCH --value "szabi-log-action-master" - title: Fix branch to patched bazel (szabi-log-action-master) - - git-clone@8: {} - - script: - title: Update GO -> 1.22 - inputs: - - script_file_path: - - content: | - #!/usr/bin/env bash - set -euxo pipefail - asdf install golang 1.22.0 - asdf global golang 1.22.0 - go version - - restore-cache@2: - inputs: - - key: bazel-bin-{{ .OS }}-{{ .Branch }}-{{ .CommitHash }} - - verbose: 'true' - title: Restore Bazel binary from KV cache - - activate-build-cache-for-bazel@1: {} - - set-java-version@1: - inputs: - - set_java_version: '21' - title: Set Java version to 21 - - activate-build-cache-for-bazel@1: {} - - script@1: - title: bazel-bin/src/bazel-dev fetch //src:bazel-dev - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - echo "common --show_timestamps" >> .bazelrc - echo "common --announce_rc" >> .bazelrc - echo "common --color=yes" >> .bazelrc - - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "startup --digest_function=blake3" >> .bazelrc - echo "build --remote_cache_compression" >> .bazelrc - fi - - bazel-bin/src/bazel-dev fetch //src:bazel-dev - - script@1: - title: bazel-bin/src/bazel-dev build //src:bazel-dev - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - mkdir /tmp/bazeldebug - - START_TIME=$(date +%s) - bazel-bin/src/bazel-dev build --profile=./bazel-profile.gz --noslim_profile //src:bazel-dev - END_TIME=$(date +%s) - - ELAPSED_TIME=$((END_TIME - START_TIME)) - curl -X POST "https://api.datadoghq.com/api/v1/distribution_points" \ - -H "Content-Type: application/json" \ - -H "DD-API-KEY: $DD_API_KEY" \ - -d "{ - \"series\" : [{ - \"metric\": \"build_cache.reference_build_duration.dist\", - \"points\": [[$(date +%s), [$ELAPSED_TIME]]], - \"tags\": [\"datacenter:$BITRISE_DEN_VM_DATACENTER\", \"measurement_name:$BITRISE_GIT_MESSAGE\", \"app_name:$BITRISE_APP_TITLE\"], - \"type\": \"distribution\" - }] - }" - - cp ./bazel-profile.gz $BITRISE_DEPLOY_DIR - bazel analyze-profile ./bazel-profile.gz - - deploy-to-bitrise-io@2: {} - summary: Build bazel with bazel - - - _release_bazel_to_kv_cache: - steps: - - activate-ssh-key@4: - run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' - - script@1: - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - envman add --key BITRISE_GIT_BRANCH --value "szabi-log-action-master" - title: Fix branch to patched bazel (szabi-log-action-master) - - git-clone@8: - inputs: - - clone_depth: "-1" - - script: - title: Update GO -> 1.22 - inputs: - - script_file_path: - - content: | - #!/usr/bin/env bash - set -euxo pipefail - asdf install golang 1.22.0 - asdf global golang 1.22.0 - go version - - script@1: - title: Install bazelisk - inputs: - - script_file_path: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x /usr/local/bin/bazel - bazel --version - elif [[ "$OSTYPE" == "darwin"* ]]; then - go install github.com/bazelbuild/bazelisk@latest - sudo ln -s $(which bazelisk) /usr/local/bin/bazel - bazel --version - else - echo "Unknown OS" - fi - - script@1: - title: bazel fetch //src:bazel-dev - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - echo "common --show_timestamps" >> .bazelrc - echo "common --announce_rc" >> .bazelrc - echo "common --color=yes" >> .bazelrc - - if [[ "$OSTYPE" == "darwin"* ]]; then - echo "startup --digest_function=blake3" >> .bazelrc - echo "build --remote_cache_compression" >> .bazelrc - fi - - bazel fetch //src:bazel-dev - - script@1: - title: build patched bazel - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - bazel build //src:bazel-dev - - save-cache@1: - inputs: - - key: bazel-bin-{{ .OS }}-{{ .Branch }}-{{ .CommitHash }} - - paths: bazel-bin/src - - verbose: 'true' - title: Save Bazel binary into KV cache - summary: Build bazel with bazel - - - build-bazel-with-bazel-buildbuddy: - steps: - - activate-ssh-key@4: - run_if: '{{getenv "SSH_RSA_PRIVATE_KEY" | ne ""}}' - - git-clone@8: {} - - script: - title: Update GO -> 1.22 - inputs: - - script_file_path: - - content: | - #!/usr/bin/env bash - set -euxo pipefail - asdf install golang 1.22.0 - asdf global golang 1.22.0 - go version - - script@1: - title: Install bazelisk - inputs: - - script_file_path: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - curl -L -o /usr/local/bin/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64 - chmod +x /usr/local/bin/bazel - bazel --version - elif [[ "$OSTYPE" == "darwin"* ]]; then - go install github.com/bazelbuild/bazelisk@latest - sudo ln -s $(which bazelisk) /usr/local/bin/bazel - bazel --version - else - echo "Unknown OS" - fi - - script@1: - title: bazel setup for buildbuddy - no RBE - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - echo "build --bes_results_url=https://gogod3v-eval.buildbuddy.io/invocation/" >> .bazelrc - echo "build --bes_backend=grpcs://gogod3v-eval.buildbuddy.io" >> .bazelrc - echo "build --remote_cache=grpcs://gogod3v-eval.buildbuddy.io" >> .bazelrc - echo "build --remote_timeout=3600" >> .bazelrc - echo "build --remote_header=x-buildbuddy-api-key=$BB_KEY" >> .bazelrc - - script@1: - title: bazel fetch //src:bazel-dev - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - echo "common --show_timestamps" >> .bazelrc - echo "common --announce_rc" >> .bazelrc - echo "common --color=yes" >> .bazelrc - - #if [[ "$OSTYPE" == "darwin"* ]]; then - # echo "startup --digest_function=blake3" >> .bazelrc - # echo "build --remote_cache_compression" >> .bazelrc - #fi - - bazel fetch //src:bazel-dev - - script@1: - title: bazel build //src:bazel-dev - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - START_TIME=$(date +%s) - bazel build --profile=./bazel-profile.gz --noslim_profile --action_env=QWE=asd //src:bazel-dev - END_TIME=$(date +%s) - - ELAPSED_TIME=$((END_TIME - START_TIME)) - curl -X POST "https://api.datadoghq.com/api/v1/series" \ - -H "Content-Type: application/json" \ - -H "DD-API-KEY: $DD_API_KEY" \ - -d "{ - \"series\" : [{ - \"metric\": \"build_cache.reference_build_duration_buildbuddy\", - \"points\": [[$(date +%s), $ELAPSED_TIME]], - \"tags\": [\"datacenter:$BITRISE_DEN_VM_DATACENTER\", \"measurement_name:$BITRISE_GIT_MESSAGE\", \"app_name:$BITRISE_APP_TITLE\"], - \"type\": \"gauge\" - }] - }" - - cp ./bazel-profile.gz $BITRISE_DEPLOY_DIR - bazel analyze-profile ./bazel-profile.gz - - script@1: - title: bazel clean - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - bazel clean - - script@1: - title: bazel setup for buildbuddy - RBE - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - echo "build --remote_executor=grpcs://remote.buildbuddy.io" >> .bazelrc - echo "build --config=remote_buildbuddy" >> .bazelrc - echo "build --noremote_upload_local_results" >> .bazelrc - - script@1: - title: bazel build //src:bazel-dev - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - START_TIME=$(date +%s) - bazel build --profile=./bazel-profile-rbe.gz --noslim_profile --action_env=QWE=asd //src:bazel-dev - END_TIME=$(date +%s) - - ELAPSED_TIME=$((END_TIME - START_TIME)) - curl -X POST "https://api.datadoghq.com/api/v1/series" \ - -H "Content-Type: application/json" \ - -H "DD-API-KEY: $DD_API_KEY" \ - -d "{ - \"series\" : [{ - \"metric\": \"build_cache.reference_build_duration_buildbuddy_rbe\", - \"points\": [[$(date +%s), $ELAPSED_TIME]], - \"tags\": [\"datacenter:$BITRISE_DEN_VM_DATACENTER\", \"measurement_name:$BITRISE_GIT_MESSAGE\", \"app_name:$BITRISE_APP_TITLE\"], - \"type\": \"gauge\" - }] - }" - - cp ./bazel-profile-rbe.gz $BITRISE_DEPLOY_DIR - bazel analyze-profile ./bazel-profile-rbe.gz - - deploy-to-bitrise-io@2: {} - summary: Build bazel with bazel @ BuildBuddy - meta: - bitrise.io: - stack: linux-docker-android-22.04 - machine_type_id: elite-xl - - - build-bazel-with-bazel-linux-p1: - after_run: - - _build-bazel-with-bazel - meta: - bitrise.io: - stack: linux-docker-android-22.04 - machine_type_id: elite-xl - build-bazel-with-bazel-p1: - after_run: - - _build-bazel-with-bazel - release-bazel-to-kv-cache-r2-darwin: - after_run: - - _release_bazel_to_kv_cache - meta: - bitrise.io: - stack: osx-xcode-15.4.x - machine_type_id: g2.mac.x-large - release-bazel-to-kv-cache-r2-linux: - after_run: - - _release_bazel_to_kv_cache - meta: - bitrise.io: - stack: linux-docker-android-22.04 - machine_type_id: g2.linux.x-large - release-bazel-to-kv-cache-gcp: - after_run: - - _release_bazel_to_kv_cache - meta: - bitrise.io: - stack: linux-docker-android-22.04 - machine_type_id: elite-xl - build-bazel-with-bazel-p1-staging: - after_run: - - build-bazel-with-bazel-p1 - steps: - - script@1: - inputs: - - content: |- - #!/usr/bin/env bash - # fail if any commands fails - set -e - # make pipelines' return status equal the last command to exit with a non-zero status, or zero if all commands exit successfully - set -o pipefail - # debug log - set -x - - export CACHE_ENDPOINT_LINUX_STAGING="grpcs://build-cache-api.services.bitrise.dev:443" - export CACHE_ENDPOINT_MAC_STAGING="grpc://build-cache-api-las.services.bitrise.dev:6666" - - if [[ "$OSTYPE" == "linux-gnu"* ]]; then - echo "Setting cache endpoint to $CACHE_ENDPOINT_LINUX_STAGING" - envman add --key BITRISE_BUILD_CACHE_ENDPOINT --value "$CACHE_ENDPOINT_LINUX_STAGING" - echo $BITRISE_BUILD_CACHE_ENDPOINT - elif [[ "$OSTYPE" == "darwin"* ]]; then - echo "Setting cache endpoint to $CACHE_ENDPOINT_MAC_STAGING" - envman add --key BITRISE_BUILD_CACHE_ENDPOINT --value "$CACHE_ENDPOINT_MAC_STAGING" - echo $BITRISE_BUILD_CACHE_ENDPOINT - else - echo "Unknown OS" - fi -meta: - bitrise.io: - stack: osx-xcode-15.3.x-las - machine_type_id: g2-m1.8core -pipelines: - release-patched-bazel-to-kv: - workflows: - release-bazel-to-kv-cache-r2-darwin: {} - release-bazel-to-kv-cache-r2-linux: {} - release-bazel-to-kv-cache-gcp: {}