diff --git a/.bazelrc b/.bazelrc index 2e94a7cd69c37..b696ac4783cce 100644 --- a/.bazelrc +++ b/.bazelrc @@ -28,7 +28,7 @@ build --copt=-Wno-error=deprecated-declarations # Required to stamp our development builds with the current git hash. # # This script gets run before every build, see the script for more info. -build --workspace_status_command "python3 misc/bazel/build-info/workspace_status.py" +build:release-stamp --stamp --workspace_status_command "python3 misc/bazel/build-info/workspace_status.py" # Output all test output by default, this makes it most like cargo. # diff --git a/Cargo.toml b/Cargo.toml index 03b5637e2557a..97eba141615e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -333,7 +333,7 @@ store = { path = "misc/cargo-vet" } # DO NOT EDIT. Automatically generated by bin/gen-lints. [workspace.lints.rust] unknown_lints = "allow" -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bazel, coverage, nightly_doc_features, release, tokio_unstable)'] } +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bazel, stamped, coverage, nightly_doc_features, release, tokio_unstable)'] } [workspace.lints.rustdoc] unportable_markdown = "allow" diff --git a/misc/bazel/build-info/BUILD.bazel b/misc/bazel/build-info/BUILD.bazel index 245db2be71f0b..71d334cd5d116 100644 --- a/misc/bazel/build-info/BUILD.bazel +++ b/misc/bazel/build-info/BUILD.bazel @@ -38,3 +38,8 @@ gen_build_info( rust_file = "build_info.rs", visibility = ["//visibility:public"], ) + +config_setting( + name = "stamped", + values = {"stamp": "true"}, +) diff --git a/misc/python/materialize/cli/bazel.py b/misc/python/materialize/cli/bazel.py index 1f0c863acaa88..58a25a96a6181 100644 --- a/misc/python/materialize/cli/bazel.py +++ b/misc/python/materialize/cli/bazel.py @@ -17,6 +17,10 @@ from materialize import MZ_ROOT, ui from materialize.bazel.utils import output_paths as bazel_output_paths +# Path where we put the current revision of the repo that we can side channel +# into Bazel. +MZ_GIT_HASH_FILE = "/tmp/mz_git_hash.txt" + def main() -> int: parser = argparse.ArgumentParser( @@ -27,6 +31,9 @@ def main() -> int: (args, sub_args) = parser.parse_known_args() + # Always update our side-channel git hash incase some command needs it. + write_git_hash() + if args.action == "gen": gen_cmd(sub_args) elif args.action == "fmt": @@ -149,5 +156,33 @@ def output_path(target) -> pathlib.Path: return pathlib.Path(path) +def write_git_hash(): + """ + Temporary file where we write the current git hash, so we can side channel + it into Bazel. + + For production releases we stamp builds with the `workspace_status_command` + but this workflow is not friendly to remote caching. Specifically, the + "volatile status" of a workspace is not supposed to cause builds to get + invalidated, and it doesn't when the result is cached locally, but it does + when it's cached remotely. + + See: + + """ + + repo = MZ_ROOT / ".git" + cmd_args = ["git", f"--git-dir={repo}", "rev-parse", "HEAD"] + result = subprocess.run( + cmd_args, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + + if result.returncode == 0: + with open(MZ_GIT_HASH_FILE, "w") as f: + f.write(result.stdout.strip()) + else: + ui.warn(f"Failed to get current revision of {MZ_ROOT}, falling back to all 0s") + + if __name__ == "__main__": main() diff --git a/misc/python/materialize/cli/gen-lints.py b/misc/python/materialize/cli/gen-lints.py index c6d3088cd2f16..32e216223693a 100644 --- a/misc/python/materialize/cli/gen-lints.py +++ b/misc/python/materialize/cli/gen-lints.py @@ -178,7 +178,7 @@ EXCLUDE_CRATES = ["workspace-hack"] -CHECK_CFGS = "bazel, coverage, nightly_doc_features, release, tokio_unstable" +CHECK_CFGS = "bazel, stamped, coverage, nightly_doc_features, release, tokio_unstable" def main() -> None: diff --git a/misc/wasm/Cargo.toml b/misc/wasm/Cargo.toml index 5dfd422e19532..45534e53a317d 100644 --- a/misc/wasm/Cargo.toml +++ b/misc/wasm/Cargo.toml @@ -16,7 +16,7 @@ store = { path = "../cargo-vet" } # DO NOT EDIT. Automatically generated by bin/gen-lints. [workspace.lints.rust] unknown_lints = "allow" -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bazel, coverage, nightly_doc_features, release, tokio_unstable)'] } +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(bazel, stamped, coverage, nightly_doc_features, release, tokio_unstable)'] } [workspace.lints.rustdoc] unportable_markdown = "allow" diff --git a/src/build-info/BUILD.bazel b/src/build-info/BUILD.bazel index c9146509f175d..a009420289ee8 100644 --- a/src/build-info/BUILD.bazel +++ b/src/build-info/BUILD.bazel @@ -22,15 +22,24 @@ rust_library( normal = True, proc_macro = True, ), - compile_data = ["@//misc/bazel/build-info:gen_build_info"], + compile_data = select({ + "@//misc/bazel/build-info:stamped": ["@//misc/bazel/build-info:gen_build_info"], + "//conditions:default": [], + }), crate_features = [ "default", "semver", ], data = [], proc_macro_deps = [] + all_crate_deps(proc_macro = True), - rustc_env = {"BAZEL_GEN_BUILD_INFO": "$(execpath @//misc/bazel/build-info:gen_build_info)"}, - rustc_flags = ["--cfg=bazel"], + rustc_env = select({ + "@//misc/bazel/build-info:stamped": {"BAZEL_GEN_BUILD_INFO": "$(execpath @//misc/bazel/build-info:gen_build_info)"}, + "//conditions:default": {}, + }), + rustc_flags = ["--cfg=bazel"] + select({ + "@//misc/bazel/build-info:stamped": ["--cfg=stamped"], + "//conditions:default": [], + }), version = "0.0.0", deps = [":mz_build_info_build_script"] + all_crate_deps(normal = True), ) @@ -57,7 +66,10 @@ rust_test( proc_macro_dev = True, ), rustc_env = {}, - rustc_flags = ["--cfg=bazel"], + rustc_flags = ["--cfg=bazel"] + select({ + "@//misc/bazel/build-info:stamped": ["--cfg=stamped"], + "//conditions:default": [], + }), version = "0.0.0", deps = [] + all_crate_deps( normal = True, diff --git a/src/build-info/Cargo.toml b/src/build-info/Cargo.toml index 778be2cc80a96..02bfcf5ae5ecf 100644 --- a/src/build-info/Cargo.toml +++ b/src/build-info/Cargo.toml @@ -20,17 +20,7 @@ default = ["semver", "workspace-hack"] [package.metadata.cargo-udeps.ignore] normal = ["workspace-hack"] -[package.metadata.cargo-gazelle.lib] -compile_data = ["@//misc/bazel/build-info:gen_build_info"] -rustc_flags = ["--cfg=bazel"] - -# Skip generating doc tests because there isn't a way to set the rustc flags -# used for the test, so we can't set the `--cfg=bazel` flag. -[package.metadata.cargo-gazelle.test.doc] -skip = true - -[package.metadata.cargo-gazelle.lib.rustc_env] -BAZEL_GEN_BUILD_INFO = "$(execpath @//misc/bazel/build-info:gen_build_info)" - -[package.metadata.cargo-gazelle.test.lib] -rustc_flags = ["--cfg=bazel"] +# This crate requires some pretty specific configuration, so we manually +# maintain the BUILD file instead of having cargo-gazelle generate it for us. +[package.metadata.cargo-gazelle] +skip_generating = true diff --git a/src/build-info/src/lib.rs b/src/build-info/src/lib.rs index 6f2f2734d7093..836e85cb82743 100644 --- a/src/build-info/src/lib.rs +++ b/src/build-info/src/lib.rs @@ -90,7 +90,7 @@ macro_rules! build_info { }; } -#[cfg(bazel)] +#[cfg(all(bazel, stamped))] #[macro_export] macro_rules! __git_sha_internal { () => { @@ -98,6 +98,26 @@ macro_rules! __git_sha_internal { }; } +// To improve the effectiveness of remote caching we only stamp "release" builds +// and otherwise side-channel git status through a known file. +// +// See: +#[cfg(all(bazel, not(stamped)))] +#[macro_export] +macro_rules! __git_sha_internal { + () => { + $crate::private::run_command_str!( + "sh", + "-c", + r#"if [ -f /tmp/mz_git_hash.txt ]; then + cat /tmp/mz_git_hash.txt + else + echo "0000000000000000000000000000000000000000" + fi"# + ) + }; +} + #[cfg(not(bazel))] #[macro_export] macro_rules! __git_sha_internal { @@ -125,15 +145,6 @@ macro_rules! __git_sha_internal { } } -#[cfg(bazel)] -#[macro_export] -macro_rules! __build_time_internal { - () => { - $crate::private::bazel_variables::MZ_BUILD_TIME - }; -} - -#[cfg(not(bazel))] #[macro_export] macro_rules! __build_time_internal { () => { @@ -148,7 +159,7 @@ pub mod private { // Bazel has a "workspace status" feature that allows us to collect info from // the workspace (e.g. git hash) at build time. These values get incleded via // a generated file. - #[cfg(bazel)] + #[cfg(all(bazel, stamped))] #[allow(unused)] pub mod bazel_variables { include!(std::env!("BAZEL_GEN_BUILD_INFO"));