From e0b366c465bd3fbd44e22ec5de8d600804f8a32c Mon Sep 17 00:00:00 2001 From: Jarkko Sakkinen Date: Wed, 4 Dec 2024 18:55:33 +0100 Subject: [PATCH] wasm-builder: Use riscv32emac-unknown-none-polkavm.json target (#6419) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Description Closes #6335. ## Integration N/A ## Review Notes `RuntimeTarget` is converted to return path to the custom target JSON file --------- Signed-off-by: Jarkko Sakkinen Co-authored-by: Alexander Theißen Co-authored-by: Koute --- Cargo.lock | 12 +- Cargo.toml | 4 +- prdoc/pr_6419.prdoc | 12 ++ substrate/utils/wasm-builder/src/builder.rs | 3 +- substrate/utils/wasm-builder/src/lib.rs | 105 +++++++----------- .../utils/wasm-builder/src/prerequisites.rs | 7 +- .../utils/wasm-builder/src/wasm_project.rs | 28 +++-- 7 files changed, 87 insertions(+), 84 deletions(-) create mode 100644 prdoc/pr_6419.prdoc diff --git a/Cargo.lock b/Cargo.lock index eee12dc5bc40..dad578ba0c1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14752,7 +14752,7 @@ dependencies = [ "anyhow", "frame-system 28.0.0", "log", - "polkavm-linker 0.17.0", + "polkavm-linker 0.17.1", "sp-core 28.0.0", "sp-io 30.0.0", "sp-runtime 31.0.1", @@ -19936,9 +19936,9 @@ dependencies = [ [[package]] name = "polkavm-linker" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d359dc721d2cc9b555ebb3558c305112ddc5bdac09d26f95f2f7b49c1f2db7e9" +checksum = "0422ead3030d5cde69e2206dbc7d65da872b121876507cd5363f6c6e6aa45157" dependencies = [ "dirs", "gimli 0.31.1", @@ -26495,7 +26495,7 @@ dependencies = [ "libsecp256k1", "log", "parity-scale-codec", - "polkavm-derive 0.9.1", + "polkavm-derive 0.17.0", "rustversion", "secp256k1 0.28.2", "sp-core 28.0.0", @@ -26979,7 +26979,7 @@ dependencies = [ "bytes", "impl-trait-for-tuples", "parity-scale-codec", - "polkavm-derive 0.9.1", + "polkavm-derive 0.17.0", "primitive-types 0.13.1", "rustversion", "sp-core 28.0.0", @@ -28623,7 +28623,7 @@ dependencies = [ "merkleized-metadata", "parity-scale-codec", "parity-wasm", - "polkavm-linker 0.9.2", + "polkavm-linker 0.17.1", "sc-executor 0.32.0", "shlex", "sp-core 28.0.0", diff --git a/Cargo.toml b/Cargo.toml index 49fdc198fe33..383fc46c4e76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1090,8 +1090,8 @@ polkadot-test-client = { path = "polkadot/node/test/client" } polkadot-test-runtime = { path = "polkadot/runtime/test-runtime" } polkadot-test-service = { path = "polkadot/node/test/service" } polkavm = { version = "0.9.3", default-features = false } -polkavm-derive = "0.9.1" -polkavm-linker = "0.9.2" +polkavm-derive = "0.17.0" +polkavm-linker = "0.17.1" portpicker = { version = "0.1.1" } pretty_assertions = { version = "1.3.0" } primitive-types = { version = "0.13.1", default-features = false, features = [ diff --git a/prdoc/pr_6419.prdoc b/prdoc/pr_6419.prdoc new file mode 100644 index 000000000000..6cc155d64b91 --- /dev/null +++ b/prdoc/pr_6419.prdoc @@ -0,0 +1,12 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Use the custom target riscv32emac-unknown-none-polkavm +doc: + - audience: Runtime Dev + description: | + Closes: https://github.com/paritytech/polkadot-sdk/issues/6335 + +crates: +- name: substrate-wasm-builder + bump: patch diff --git a/substrate/utils/wasm-builder/src/builder.rs b/substrate/utils/wasm-builder/src/builder.rs index a40aafe1d812..5bdc743eac31 100644 --- a/substrate/utils/wasm-builder/src/builder.rs +++ b/substrate/utils/wasm-builder/src/builder.rs @@ -235,7 +235,8 @@ impl WasmBuilder { /// Build the WASM binary. pub fn build(mut self) { - let target = crate::runtime_target(); + let target = RuntimeTarget::new(); + if target == RuntimeTarget::Wasm { if self.export_heap_base { self.rust_flags.push("-Clink-arg=--export=__heap_base".into()); diff --git a/substrate/utils/wasm-builder/src/lib.rs b/substrate/utils/wasm-builder/src/lib.rs index 420ecd63e1dc..ce90f492e08f 100644 --- a/substrate/utils/wasm-builder/src/lib.rs +++ b/substrate/utils/wasm-builder/src/lib.rs @@ -112,7 +112,6 @@ //! wasm32-unknown-unknown --toolchain nightly-2020-02-20`. use std::{ - collections::BTreeSet, env, fs, io::BufRead, path::{Path, PathBuf}, @@ -254,26 +253,22 @@ struct CargoCommand { program: String, args: Vec, version: Option, - target_list: Option>, } impl CargoCommand { fn new(program: &str) -> Self { let version = Self::extract_version(program, &[]); - let target_list = Self::extract_target_list(program, &[]); - CargoCommand { program: program.into(), args: Vec::new(), version, target_list } + CargoCommand { program: program.into(), args: Vec::new(), version } } fn new_with_args(program: &str, args: &[&str]) -> Self { let version = Self::extract_version(program, args); - let target_list = Self::extract_target_list(program, args); CargoCommand { program: program.into(), args: args.iter().map(ToString::to_string).collect(), version, - target_list, } } @@ -294,23 +289,6 @@ impl CargoCommand { Version::extract(&version) } - fn extract_target_list(program: &str, args: &[&str]) -> Option> { - // This is technically an unstable option, but we don't care because we only need this - // to build RISC-V runtimes, and those currently require a specific nightly toolchain - // anyway, so it's totally fine for this to fail in other cases. - let list = Command::new(program) - .args(args) - .args(&["rustc", "-Z", "unstable-options", "--print", "target-list"]) - // Make sure if we're called from within a `build.rs` the host toolchain won't override - // a rustup toolchain we've picked. - .env_remove("RUSTC") - .output() - .ok() - .and_then(|o| String::from_utf8(o.stdout).ok())?; - - Some(list.trim().split("\n").map(ToString::to_string).collect()) - } - /// Returns the version of this cargo command or `None` if it failed to extract the version. fn version(&self) -> Option { self.version @@ -326,19 +304,10 @@ impl CargoCommand { fn supports_substrate_runtime_env(&self, target: RuntimeTarget) -> bool { match target { RuntimeTarget::Wasm => self.supports_substrate_runtime_env_wasm(), - RuntimeTarget::Riscv => self.supports_substrate_runtime_env_riscv(), + RuntimeTarget::Riscv => true, } } - /// Check if the supplied cargo command supports our RISC-V runtime environment. - fn supports_substrate_runtime_env_riscv(&self) -> bool { - let Some(target_list) = self.target_list.as_ref() else { return false }; - // This is our custom target which currently doesn't exist on any upstream toolchain, - // so if it exists it's guaranteed to be our custom toolchain and have have everything - // we need, so any further version checks are unnecessary at this point. - target_list.contains("riscv32ema-unknown-none-elf") - } - /// Check if the supplied cargo command supports our Substrate wasm environment. /// /// This means that either the cargo version is at minimum 1.68.0 or this is a nightly cargo. @@ -409,13 +378,6 @@ fn get_bool_environment_variable(name: &str) -> Option { } } -/// Returns whether we need to also compile the standard library when compiling the runtime. -fn build_std_required() -> bool { - let default = runtime_target() == RuntimeTarget::Wasm; - - crate::get_bool_environment_variable(crate::WASM_BUILD_STD).unwrap_or(default) -} - #[derive(Copy, Clone, PartialEq, Eq)] enum RuntimeTarget { Wasm, @@ -423,36 +385,55 @@ enum RuntimeTarget { } impl RuntimeTarget { - fn rustc_target(self) -> &'static str { + /// Creates a new instance. + fn new() -> Self { + let Some(value) = env::var_os(RUNTIME_TARGET) else { + return Self::Wasm; + }; + + if value == "wasm" { + Self::Wasm + } else if value == "riscv" { + Self::Riscv + } else { + build_helper::warning!( + "RUNTIME_TARGET environment variable must be set to either \"wasm\" or \"riscv\"" + ); + std::process::exit(1); + } + } + + /// Figures out the target parameter value for rustc. + fn rustc_target(self) -> String { match self { - RuntimeTarget::Wasm => "wasm32-unknown-unknown", - RuntimeTarget::Riscv => "riscv32ema-unknown-none-elf", + RuntimeTarget::Wasm => "wasm32-unknown-unknown".to_string(), + RuntimeTarget::Riscv => { + let path = polkavm_linker::target_json_32_path().expect("riscv not found"); + path.into_os_string().into_string().unwrap() + }, } } - fn build_subdirectory(self) -> &'static str { - // Keep the build directories separate so that when switching between - // the targets we won't trigger unnecessary rebuilds. + /// Figures out the target directory name used by cargo. + fn rustc_target_dir(self) -> &'static str { match self { - RuntimeTarget::Wasm => "wbuild", - RuntimeTarget::Riscv => "rbuild", + RuntimeTarget::Wasm => "wasm32-unknown-unknown", + RuntimeTarget::Riscv => "riscv32emac-unknown-none-polkavm", } } -} -fn runtime_target() -> RuntimeTarget { - let Some(value) = env::var_os(RUNTIME_TARGET) else { - return RuntimeTarget::Wasm; - }; + /// Figures out the build-std argument. + fn rustc_target_build_std(self) -> Option<&'static str> { + if !crate::get_bool_environment_variable(crate::WASM_BUILD_STD).unwrap_or(true) { + return None; + } - if value == "wasm" { - RuntimeTarget::Wasm - } else if value == "riscv" { - RuntimeTarget::Riscv - } else { - build_helper::warning!( - "the '{RUNTIME_TARGET}' environment variable has an invalid value; it must be either 'wasm' or 'riscv'" - ); - std::process::exit(1); + // This is a nightly-only flag. + let arg = match self { + RuntimeTarget::Wasm => "build-std", + RuntimeTarget::Riscv => "build-std=core,alloc", + }; + + Some(arg) } } diff --git a/substrate/utils/wasm-builder/src/prerequisites.rs b/substrate/utils/wasm-builder/src/prerequisites.rs index 4de6b87f618d..9abfd1725237 100644 --- a/substrate/utils/wasm-builder/src/prerequisites.rs +++ b/substrate/utils/wasm-builder/src/prerequisites.rs @@ -196,11 +196,14 @@ fn check_wasm_toolchain_installed( error, colorize_aux_message(&"-".repeat(60)), )) - } + }; } let version = dummy_crate.get_rustc_version(); - if crate::build_std_required() { + + let target = RuntimeTarget::new(); + assert!(target == RuntimeTarget::Wasm); + if target.rustc_target_build_std().is_some() { if let Some(sysroot) = dummy_crate.get_sysroot() { let src_path = Path::new(sysroot.trim()).join("lib").join("rustlib").join("src").join("rust"); diff --git a/substrate/utils/wasm-builder/src/wasm_project.rs b/substrate/utils/wasm-builder/src/wasm_project.rs index 26edd2ea1f22..6530e4c22fb9 100644 --- a/substrate/utils/wasm-builder/src/wasm_project.rs +++ b/substrate/utils/wasm-builder/src/wasm_project.rs @@ -109,6 +109,15 @@ fn crate_metadata(cargo_manifest: &Path) -> Metadata { crate_metadata } +/// Keep the build directories separate so that when switching between the +/// targets we won't trigger unnecessary rebuilds. +fn build_subdirectory(target: RuntimeTarget) -> &'static str { + match target { + RuntimeTarget::Wasm => "wbuild", + RuntimeTarget::Riscv => "rbuild", + } +} + /// Creates the WASM project, compiles the WASM binary and compacts the WASM binary. /// /// # Returns @@ -125,7 +134,7 @@ pub(crate) fn create_and_compile( #[cfg(feature = "metadata-hash")] enable_metadata_hash: Option, ) -> (Option, WasmBinaryBloaty) { let runtime_workspace_root = get_wasm_workspace_root(); - let runtime_workspace = runtime_workspace_root.join(target.build_subdirectory()); + let runtime_workspace = runtime_workspace_root.join(build_subdirectory(target)); let crate_metadata = crate_metadata(orig_project_cargo_toml); @@ -770,7 +779,7 @@ impl BuildConfiguration { .collect::>() .iter() .rev() - .take_while(|c| c.as_os_str() != target.build_subdirectory()) + .take_while(|c| c.as_os_str() != build_subdirectory(target)) .last() .expect("We put the runtime project within a `target/.../[rw]build` path; qed") .as_os_str() @@ -841,9 +850,7 @@ fn build_bloaty_blob( "-C target-cpu=mvp -C target-feature=-sign-ext -C link-arg=--export-table ", ); }, - RuntimeTarget::Riscv => { - rustflags.push_str("-C target-feature=+lui-addi-fusion -C relocation-model=pie -C link-arg=--emit-relocs -C link-arg=--unique "); - }, + RuntimeTarget::Riscv => (), } rustflags.push_str(default_rustflags); @@ -907,10 +914,9 @@ fn build_bloaty_blob( // // So here we force the compiler to also compile the standard library crates for us // to make sure that they also only use the MVP features. - if crate::build_std_required() { - // Unfortunately this is still a nightly-only flag, but FWIW it is pretty widely used - // so it's unlikely to break without a replacement. - build_cmd.arg("-Z").arg("build-std"); + if let Some(arg) = target.rustc_target_build_std() { + build_cmd.arg("-Z").arg(arg); + if !cargo_cmd.supports_nightly_features() { build_cmd.env("RUSTC_BOOTSTRAP", "1"); } @@ -934,7 +940,7 @@ fn build_bloaty_blob( let blob_name = get_blob_name(target, &manifest_path); let target_directory = project .join("target") - .join(target.rustc_target()) + .join(target.rustc_target_dir()) .join(blob_build_profile.directory()); match target { RuntimeTarget::Riscv => { @@ -968,7 +974,7 @@ fn build_bloaty_blob( }, }; - std::fs::write(&polkavm_path, program.as_bytes()) + std::fs::write(&polkavm_path, program) .expect("writing the blob to a file always works"); }