Skip to content

Commit

Permalink
Add no_prelude remote execution to github actions
Browse files Browse the repository at this point in the history
Enabling testing of remote execution on github actions for `no_prelude`

* Create github action that generates .buckconfig.local using `NATIVELINK_HEADER_RW_KEY` from gha secrets.
* Set `container-image` used for remote execution specific to setup of buck2.
* Update .gitignore to include .buckconfig.local.
* Include a `platforms` configuration for the `no_prelude` example.
* Update `go_toolchain.bzl` to be remote enabled friendly,
  no need to download a toolchain when remote worker has one.

Open question:
* Was the downloading of go toolchain to be remote execution
  compatible prior to this change?
  • Loading branch information
adam-singer committed Aug 7, 2024
1 parent e0b062e commit 3dd7d4c
Show file tree
Hide file tree
Showing 7 changed files with 89 additions and 13 deletions.
27 changes: 27 additions & 0 deletions .github/actions/build_example_nativelink_no_prelude/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
name: build_example_nativelink_no_prelude
inputs:
NATIVELINK_HEADER_RW_KEY_SECRET:
description: ''
required: true
runs:
using: composite
steps:
- name: Build example/no_prelude directory using remote execution
run: |-
{
echo "[buck2_re_client]
engine_address = grpc://scheduler-buck2.build-faster.nativelink.net:443
action_cache_address = grpc://cas-buck2.build-faster.nativelink.net:443
cas_address = grpc://cas-buck2.build-faster.nativelink.net:443
http_headers = x-nativelink-api-key:${NATIVELINK_HEADER_RW_KEY}
tls = true
instance_name = main
enabled = true
[build]
execution_platforms = root//platforms:platforms"
} > examples/no_prelude/.buckconfig.local
cd examples/no_prelude
$RUNNER_TEMP/artifacts/buck2 build //... -v 4
env:
NATIVELINK_HEADER_RW_KEY: ${{ inputs.NATIVELINK_HEADER_RW_KEY_SECRET }}
shell: bash
3 changes: 3 additions & 0 deletions .github/workflows/build-and-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ jobs:
$RUNNER_TEMP/artifacts/buck2 build //... -v 2
$RUNNER_TEMP/artifacts/buck2 test //... -v 2
- uses: ./.github/actions/build_example_no_prelude
- uses: ./.github/actions/build_example_nativelink_no_prelude
with:
NATIVELINK_HEADER_RW_KEY_SECRET: ${{ secrets.NATIVELINK_HEADER_RW_KEY_SECRET }}
- uses: ./.github/actions/setup_reindeer
- uses: ./.github/actions/build_bootstrap
linux-build-examples:
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
Cargo.lock
buck-out
/.direnv
.buckconfig.local

# symlinks
/examples/with_prelude/prelude
Expand Down
2 changes: 1 addition & 1 deletion examples/no_prelude/go/rules.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def _go_binary_impl(ctx: AnalysisContext) -> list[Provider]:

cmd = cmd_args([ctx.attrs.toolchain[GoCompilerInfo].compiler_path, "build", "-o", out.as_output()] + sources)

ctx.actions.run(cmd, category = "compile")
ctx.actions.run(cmd, env = {"GOCACHE":ctx.attrs.toolchain[GoCompilerInfo].GOCACHE}, category = "compile")

return [
DefaultInfo(default_output = out),
Expand Down
3 changes: 3 additions & 0 deletions examples/no_prelude/platforms/BUCK
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
load(":defs.bzl", "platforms")

platforms(name = "platforms")
33 changes: 33 additions & 0 deletions examples/no_prelude/platforms/defs.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under both the MIT license found in the
# LICENSE-MIT file in the root directory of this source tree and the Apache
# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
# of this source tree.

def _platforms(ctx):
configuration = ConfigurationInfo(
constraints = {},
values = {},
)

platform = ExecutionPlatformInfo(
label = ctx.label.raw_target(),
configuration = configuration,
executor_config = CommandExecutorConfig(
local_enabled = True,
remote_enabled = True,
use_limited_hybrid = True,
# Set those up based on what workers you've registered with NativeLink.
remote_execution_properties = {
"OSFamily": "linux",
"container-image": "docker://buck2-github:latest",
},
remote_execution_use_case = "buck2-default",
remote_output_paths = "output_paths",
),
)

return [DefaultInfo(), ExecutionPlatformRegistrationInfo(platforms = [platform])]

platforms = rule(attrs = {}, impl = _platforms)
33 changes: 21 additions & 12 deletions examples/no_prelude/toolchains/go_toolchain.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -7,24 +7,33 @@

GoCompilerInfo = provider(
doc = "Information about how to invoke the go compiler.",
fields = ["compiler_path", "GOROOT"],
fields = ["compiler_path", "GOROOT", "GOCACHE"],
)

def is_remote_enabled() -> bool:
re_enabled = read_root_config("buck2_re_client", "enabled", "false")
return re_enabled == "true"

def _go_toolchain_impl(ctx):
download = _download_toolchain(ctx)
is_re_enabled = is_remote_enabled()
gocache="/tmp/gocache"
if is_re_enabled:
return [DefaultInfo(), GoCompilerInfo(compiler_path = "go", GOROOT = "", GOCACHE=gocache)]
else:
download = _download_toolchain(ctx)

compiler_dst = ctx.actions.declare_output("compiler.exe" if host_info().os.is_windows else "compiler")
compiler_dst = ctx.actions.declare_output("compiler.exe" if host_info().os.is_windows else "compiler")

cmd = cmd_args()
if host_info().os.is_windows:
compiler_src = cmd_args(download, format = "{}\\go\\bin\\go.exe", relative_to = (compiler_dst, 1))
cmd.add([ctx.attrs._symlink_bat, compiler_dst.as_output(), compiler_src])
else:
compiler_src = cmd_args(download, format = "{}/go/bin/go", relative_to = (compiler_dst, 1))
cmd.add(["ln", "-sf", compiler_src, compiler_dst.as_output()])
cmd = cmd_args()
if host_info().os.is_windows:
compiler_src = cmd_args(download, format = "{}\\go\\bin\\go.exe", relative_to = (compiler_dst, 1))
cmd.add([ctx.attrs._symlink_bat, compiler_dst.as_output(), compiler_src])
else:
compiler_src = cmd_args(download, format = "{}/go/bin/go", relative_to = (compiler_dst, 1))
cmd.add(["ln", "-sf", compiler_src, compiler_dst.as_output()])

ctx.actions.run(cmd, category = "cp_compiler")
return [DefaultInfo(default_output = download), GoCompilerInfo(compiler_path = compiler_dst, GOROOT = "")]
ctx.actions.run(cmd, category = "cp_compiler")
return [DefaultInfo(default_output = download), GoCompilerInfo(compiler_path = compiler_dst, GOROOT = "", GOCACHE=gocache)]

go_toolchain = rule(
impl = _go_toolchain_impl,
Expand Down

0 comments on commit 3dd7d4c

Please sign in to comment.