From ce35f8ec56ec7e89d351cd95001f4d819c97e07d Mon Sep 17 00:00:00 2001 From: danakj Date: Thu, 22 Jul 2021 14:52:45 -0400 Subject: [PATCH 1/4] remap-cwd-prefix --- compiler/rustc_interface/src/tests.rs | 1 + compiler/rustc_session/src/config.rs | 15 +++++-- compiler/rustc_session/src/options.rs | 2 + src/doc/rustc/src/command-line-arguments.md | 13 ++++++ .../reproducible-build/Makefile | 42 +++++++++++++++++++ 5 files changed, 70 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index afab919bc3c2c..94166715f74f3 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -753,6 +753,7 @@ fn test_debugging_options_tracking_hash() { tracked!(profiler_runtime, "abc".to_string()); tracked!(relax_elf_relocations, Some(true)); tracked!(relro_level, Some(RelroLevel::Full)); + tracked!(remap_cwd_prefix, Some(PathBuf::from("abc"))); tracked!(simulate_remapped_rust_src_base, Some(PathBuf::from("/rustc/abc"))); tracked!(report_delayed_bugs, true); tracked!(sanitizer, SanitizerSet::ADDRESS); diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index fdedb7e6a4afe..32aa035e1cdec 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -1920,9 +1920,10 @@ fn parse_extern_dep_specs( fn parse_remap_path_prefix( matches: &getopts::Matches, + debugging_opts: &DebuggingOptions, error_format: ErrorOutputType, ) -> Vec<(PathBuf, PathBuf)> { - matches + let mut mapping: Vec<(PathBuf, PathBuf)> = matches .opt_strs("remap-path-prefix") .into_iter() .map(|remap| match remap.rsplit_once('=') { @@ -1932,7 +1933,15 @@ fn parse_remap_path_prefix( ), Some((from, to)) => (PathBuf::from(from), PathBuf::from(to)), }) - .collect() + .collect(); + match &debugging_opts.remap_cwd_prefix { + Some(to) => match std::env::current_dir() { + Ok(cwd) => mapping.push((cwd, to.clone())), + Err(_) => (), + }, + None => (), + }; + mapping } pub fn build_session_options(matches: &getopts::Matches) -> Options { @@ -2077,7 +2086,7 @@ pub fn build_session_options(matches: &getopts::Matches) -> Options { let crate_name = matches.opt_str("crate-name"); - let remap_path_prefix = parse_remap_path_prefix(matches, error_format); + let remap_path_prefix = parse_remap_path_prefix(matches, &debugging_opts, error_format); let pretty = parse_pretty(&debugging_opts, error_format); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 9a1be40558ccb..293cf1d0ba365 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1236,6 +1236,8 @@ options! { "whether ELF relocations can be relaxed"), relro_level: Option = (None, parse_relro_level, [TRACKED], "choose which RELRO level to use"), + remap_cwd_prefix: Option = (None, parse_opt_pathbuf, [TRACKED], + "remap paths under the current working directory to this path prefix"), simulate_remapped_rust_src_base: Option = (None, parse_opt_pathbuf, [TRACKED], "simulate the effect of remap-debuginfo = true at bootstrapping by remapping path \ to rust's source base directory. only meant for testing purposes"), diff --git a/src/doc/rustc/src/command-line-arguments.md b/src/doc/rustc/src/command-line-arguments.md index 7f482f0f2b1a0..f38b4ca744f74 100644 --- a/src/doc/rustc/src/command-line-arguments.md +++ b/src/doc/rustc/src/command-line-arguments.md @@ -345,6 +345,19 @@ replacement is purely textual, with no consideration of the current system's pathname syntax. For example `--remap-path-prefix foo=bar` will match `foo/lib.rs` but not `./foo/lib.rs`. + +## `--remap-cwd-prefix`: remap paths under the cwd in output + +Remap all absolute paths that are rooted under the current working directory to +be under the given value instead. The given value may be absolute or relative, +or empty. This switch takes precidence over `--remap-path-prefix` in case they +would both match a given path. + +This flag allows the command line to be universally reproducible, such that the +same execution will work on all machines, regardless of build environment. + +This is an unstable option. Use `-Z remap-cwd-prefix=val` to specify a value. + ## `--json`: configure json messages printed by the compiler diff --git a/src/test/run-make-fulldeps/reproducible-build/Makefile b/src/test/run-make-fulldeps/reproducible-build/Makefile index a17ec212cfd58..762cf5ed2ea71 100644 --- a/src/test/run-make-fulldeps/reproducible-build/Makefile +++ b/src/test/run-make-fulldeps/reproducible-build/Makefile @@ -10,6 +10,9 @@ all: \ link_paths \ remap_paths \ different_source_dirs \ + remap_cwd_bin \ + remap_cwd_rlib \ + remap_cwd_to_empty \ extern_flags smoke: @@ -64,6 +67,45 @@ different_source_dirs: --crate-type rlib) cmp "$(TMPDIR)/libreproducible_build.rlib" "$(TMPDIR)/libfoo.rlib" || exit 1 +remap_cwd_bin: + rm -rf $(TMPDIR) && mkdir $(TMPDIR) + $(RUSTC) reproducible-build-aux.rs + mkdir $(TMPDIR)/test + cp reproducible-build.rs $(TMPDIR)/test + $(RUSTC) reproducible-build.rs --crate-type bin -C debuginfo=2 \ + -Z remap-cwd-prefix=. + cp $(TMPDIR)/reproducible-build $(TMPDIR)/first + (cd $(TMPDIR)/test && \ + $(RUSTC) reproducible-build.rs --crate-type bin -C debuginfo=2 \ + -Z remap-cwd-prefix=.) + cmp "$(TMPDIR)/first" "$(TMPDIR)/reproducible-build" || exit 1 + +remap_cwd_rlib: + rm -rf $(TMPDIR) && mkdir $(TMPDIR) + $(RUSTC) reproducible-build-aux.rs + mkdir $(TMPDIR)/test + cp reproducible-build.rs $(TMPDIR)/test + $(RUSTC) reproducible-build.rs --crate-type rlib -C debuginfo=2 \ + -Z remap-cwd-prefix=. + cp $(TMPDIR)/libreproducible_build.rlib $(TMPDIR)/libfirst.rlib + (cd $(TMPDIR)/test && \ + $(RUSTC) reproducible-build.rs --crate-type rlib -C debuginfo=2 \ + -Z remap-cwd-prefix=.) + cmp "$(TMPDIR)/libfirst.rlib" "$(TMPDIR)/libreproducible_build.rlib" || exit 1 + +remap_cwd_to_empty: + rm -rf $(TMPDIR) && mkdir $(TMPDIR) + $(RUSTC) reproducible-build-aux.rs + mkdir $(TMPDIR)/test + cp reproducible-build.rs $(TMPDIR)/test + $(RUSTC) reproducible-build.rs --crate-type rlib -C debuginfo=2 \ + -Z remap-cwd-prefix= + cp $(TMPDIR)/libreproducible_build.rlib $(TMPDIR)/libfirst.rlib + (cd $(TMPDIR)/test && \ + $(RUSTC) reproducible-build.rs --crate-type rlib -C debuginfo=2 \ + -Z remap-cwd-prefix=) + cmp "$(TMPDIR)/libfirst.rlib" "$(TMPDIR)/libreproducible_build.rlib" || exit 1 + extern_flags: rm -rf $(TMPDIR) && mkdir $(TMPDIR) $(RUSTC) reproducible-build-aux.rs From 2a687deee872917b09519519290521fa8dd472b7 Mon Sep 17 00:00:00 2001 From: danakj Date: Tue, 7 Sep 2021 15:38:29 -0400 Subject: [PATCH 2/4] Move documentation to the unstable book --- src/doc/rustc/src/command-line-arguments.md | 13 ---------- .../src/compiler-flags/remap-cwd-prefix.md | 24 +++++++++++++++++++ 2 files changed, 24 insertions(+), 13 deletions(-) create mode 100644 src/doc/unstable-book/src/compiler-flags/remap-cwd-prefix.md diff --git a/src/doc/rustc/src/command-line-arguments.md b/src/doc/rustc/src/command-line-arguments.md index f38b4ca744f74..7f482f0f2b1a0 100644 --- a/src/doc/rustc/src/command-line-arguments.md +++ b/src/doc/rustc/src/command-line-arguments.md @@ -345,19 +345,6 @@ replacement is purely textual, with no consideration of the current system's pathname syntax. For example `--remap-path-prefix foo=bar` will match `foo/lib.rs` but not `./foo/lib.rs`. - -## `--remap-cwd-prefix`: remap paths under the cwd in output - -Remap all absolute paths that are rooted under the current working directory to -be under the given value instead. The given value may be absolute or relative, -or empty. This switch takes precidence over `--remap-path-prefix` in case they -would both match a given path. - -This flag allows the command line to be universally reproducible, such that the -same execution will work on all machines, regardless of build environment. - -This is an unstable option. Use `-Z remap-cwd-prefix=val` to specify a value. - ## `--json`: configure json messages printed by the compiler diff --git a/src/doc/unstable-book/src/compiler-flags/remap-cwd-prefix.md b/src/doc/unstable-book/src/compiler-flags/remap-cwd-prefix.md new file mode 100644 index 0000000000000..977d258529f8c --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/remap-cwd-prefix.md @@ -0,0 +1,24 @@ +# `remap-cwd-prefix` + +The tracking issue for this feature is: [#87325](https://github.com/rust-lang/rust/issues/87325). + +------------------------ + +This flag will rewrite absolute paths under the current working directory, +replacing the current working directory prefix with a specified value. + +The given value may be absolute or relative, or empty. This switch takes +precidence over `--remap-path-prefix` in case they would both match a given +path. + +This flag helps to produce deterministic output, by removing the current working +directory from build output, while allowing the command line to be universally +reproducible, such that the same execution will work on all machines, regardless +of build environment. + +## Example +```sh +# This would produce an absolute path to main.rs in build outputs of +# "./main.rs". +rustc -Z remap-cwd-prefix=. main.rs +``` From 4933be9e2f7bec805980342f882c235c92755a7c Mon Sep 17 00:00:00 2001 From: danakj Date: Wed, 15 Sep 2021 11:49:12 -0400 Subject: [PATCH 3/4] Verify bin crates are not deterministic on Windows This disables the remap_cwd_bin test which is failing on windows, and adds a test for --remap-path-prefix making a bin crate instead, to see if it will fail the same way. --- .github/workflows/ci.yml | 5 ++++ src/ci/github-actions/ci.yml | 5 ++++ .../reproducible-build/Makefile | 24 ++++++++++++++++--- 3 files changed, 31 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff4fa1527e93a..76318ab7cb922 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -534,6 +534,11 @@ jobs: strategy: matrix: include: + - name: x86_64-msvc-1 + env: + RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" + SCRIPT: make ci-subset-1 + os: windows-latest-xl - name: dist-x86_64-linux os: ubuntu-latest-xl env: {} diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 6417f5a984ad5..e4c402b417e6b 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -668,6 +668,11 @@ jobs: strategy: matrix: include: + - name: x86_64-msvc-1 + env: + RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler + SCRIPT: make ci-subset-1 + <<: *job-windows-xl - *dist-x86_64-linux master: diff --git a/src/test/run-make-fulldeps/reproducible-build/Makefile b/src/test/run-make-fulldeps/reproducible-build/Makefile index 762cf5ed2ea71..46885fdb7df84 100644 --- a/src/test/run-make-fulldeps/reproducible-build/Makefile +++ b/src/test/run-make-fulldeps/reproducible-build/Makefile @@ -9,12 +9,18 @@ all: \ opt \ link_paths \ remap_paths \ - different_source_dirs \ - remap_cwd_bin \ + different_source_dirs_bin \ + different_source_dirs_rlib \ remap_cwd_rlib \ remap_cwd_to_empty \ extern_flags +# TODO: Deterministic builds of `bin` crate types are not deterministic with +# debuginfo=2 on Windows. +# See https://github.com/rust-lang/rust/pull/87320#issuecomment-920105533 +# different_source_dirs_bin \ +# remap_cwd_bin \ + smoke: rm -rf $(TMPDIR) && mkdir $(TMPDIR) $(RUSTC) linker.rs -O @@ -55,7 +61,19 @@ remap_paths: $(RUSTC) reproducible-build.rs --crate-type rlib --remap-path-prefix=/b=/c cmp "$(TMPDIR)/libreproducible_build.rlib" "$(TMPDIR)/libfoo.rlib" || exit 1 -different_source_dirs: +different_source_dirs_bin: + rm -rf $(TMPDIR) && mkdir $(TMPDIR) + $(RUSTC) reproducible-build-aux.rs + mkdir $(TMPDIR)/test + cp reproducible-build.rs $(TMPDIR)/test + $(RUSTC) reproducible-build.rs --crate-type bin --remap-path-prefix=$$PWD=/b + cp $(TMPDIR)/reproducible-build $(TMPDIR)/foo + (cd $(TMPDIR)/test && $(RUSTC) reproducible-build.rs \ + --remap-path-prefix=$(TMPDIR)/test=/b \ + --crate-type bin) + cmp "$(TMPDIR)/reproducible-build" "$(TMPDIR)/foo" || exit 1 + +different_source_dirs_rlib: rm -rf $(TMPDIR) && mkdir $(TMPDIR) $(RUSTC) reproducible-build-aux.rs mkdir $(TMPDIR)/test From c0118289efbd3d0c61ae8912426a211e53abe208 Mon Sep 17 00:00:00 2001 From: danakj Date: Wed, 15 Sep 2021 13:37:22 -0400 Subject: [PATCH 4/4] Disable both reproducible-build tests for crate-type=bin These tests fail on Windows, as the build is not deterministic there for bin targets. Issue https://github.com/rust-lang/rust/issues/88982 is filed for this problem. --- .github/workflows/ci.yml | 5 ----- src/ci/github-actions/ci.yml | 5 ----- src/test/run-make-fulldeps/reproducible-build/Makefile | 9 +++++---- 3 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 76318ab7cb922..ff4fa1527e93a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -534,11 +534,6 @@ jobs: strategy: matrix: include: - - name: x86_64-msvc-1 - env: - RUST_CONFIGURE_ARGS: "--build=x86_64-pc-windows-msvc --enable-profiler" - SCRIPT: make ci-subset-1 - os: windows-latest-xl - name: dist-x86_64-linux os: ubuntu-latest-xl env: {} diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index e4c402b417e6b..6417f5a984ad5 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -668,11 +668,6 @@ jobs: strategy: matrix: include: - - name: x86_64-msvc-1 - env: - RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-profiler - SCRIPT: make ci-subset-1 - <<: *job-windows-xl - *dist-x86_64-linux master: diff --git a/src/test/run-make-fulldeps/reproducible-build/Makefile b/src/test/run-make-fulldeps/reproducible-build/Makefile index 46885fdb7df84..adccc15356848 100644 --- a/src/test/run-make-fulldeps/reproducible-build/Makefile +++ b/src/test/run-make-fulldeps/reproducible-build/Makefile @@ -9,15 +9,16 @@ all: \ opt \ link_paths \ remap_paths \ - different_source_dirs_bin \ different_source_dirs_rlib \ remap_cwd_rlib \ remap_cwd_to_empty \ extern_flags -# TODO: Deterministic builds of `bin` crate types are not deterministic with -# debuginfo=2 on Windows. -# See https://github.com/rust-lang/rust/pull/87320#issuecomment-920105533 +# TODO: Builds of `bin` crate types are not deterministic with debuginfo=2 on +# Windows. +# See: https://github.com/rust-lang/rust/pull/87320#issuecomment-920105533 +# Issue: https://github.com/rust-lang/rust/issues/88982 +# # different_source_dirs_bin \ # remap_cwd_bin \