From 62ec727e9174ec1536d6336e21b2aaf2b9014c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Tue, 9 Apr 2024 21:05:11 +0200 Subject: [PATCH 1/3] ci: specify hermit features directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- .github/workflows/ci.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90492ffd5f..5c82cd6d08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,25 +190,25 @@ jobs: if: ${{ matrix.arch != 'aarch64' }} - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package rftrace-example --virtiofsd if: ${{ matrix.arch == 'x86_64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package httpd --features ci,dhcpv4 --netdev virtio-net-pci + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package httpd --features ci,hermit/dhcpv4 --netdev virtio-net-pci if: ${{ matrix.arch != 'riscv64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package httpd --features ci,dhcpv4 --netdev rtl8139 --features rtl8139 + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package httpd --features ci,hermit/dhcpv4 --netdev rtl8139 --features hermit/rtl8139 if: ${{ matrix.arch == 'x86_64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package testudp --features udp,dhcpv4 --netdev virtio-net-pci + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package testudp --features hermit/udp,hermit/dhcpv4 --netdev virtio-net-pci if: ${{ matrix.arch != 'riscv64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package testudp --features udp,dhcpv4 --netdev rtl8139 --features rtl8139 + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package testudp --features hermit/udp,hermit/dhcpv4 --netdev rtl8139 --features hermit/rtl8139 if: ${{ matrix.arch == 'x86_64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package miotcp --features dhcpv4 --netdev virtio-net-pci + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package miotcp --features hermit/dhcpv4 --netdev virtio-net-pci if: ${{ matrix.arch != 'riscv64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package miotcp --features dhcpv4 --netdev rtl8139 --features rtl8139 + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package miotcp --features hermit/dhcpv4 --netdev rtl8139 --features hermit/rtl8139 if: ${{ matrix.arch == 'x86_64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package poll --features dhcpv4 --netdev virtio-net-pci + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package poll --features hermit/dhcpv4 --netdev virtio-net-pci if: ${{ matrix.arch != 'riscv64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package poll --features dhcpv4 --netdev rtl8139 --features rtl8139 + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package poll --features hermit/dhcpv4 --netdev rtl8139 --features hermit/rtl8139 if: ${{ matrix.arch == 'x86_64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package mioudp --features udp,dhcpv4 --netdev virtio-net-pci + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package mioudp --features hermit/udp,hermit/dhcpv4 --netdev virtio-net-pci if: ${{ matrix.arch != 'riscv64' }} - - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package mioudp --features udp,dhcpv4 --netdev rtl8139 --features rtl8139 + - run: cargo xtask ci qemu --arch ${{ matrix.arch }} --profile ${{ matrix.profile }} ${{ matrix.flags }} --package mioudp --features hermit/udp,hermit/dhcpv4 --netdev rtl8139 --features hermit/rtl8139 if: ${{ matrix.arch == 'x86_64' }} run-x86_64-kvm: From 9b89f2c3a36687e5f0d12bd3f59e4486e4b9709f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Tue, 9 Apr 2024 16:09:27 +0200 Subject: [PATCH 2/3] refactor(xtask): make cargo_build_args independent of xshell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- xtask/src/build.rs | 6 ++--- xtask/src/cargo_build.rs | 48 ++++++++++++++++++---------------------- xtask/src/ci/build.rs | 4 ++-- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/xtask/src/build.rs b/xtask/src/build.rs index 24c3b22a2e..95e4c09889 100644 --- a/xtask/src/build.rs +++ b/xtask/src/build.rs @@ -5,7 +5,7 @@ use anyhow::Result; use clap::Args; use xshell::cmd; -use crate::cargo_build::{CargoBuild, CmdExt}; +use crate::cargo_build::CargoBuild; /// Build the kernel. #[derive(Args)] @@ -32,7 +32,7 @@ impl Build { cmd!(sh, "cargo build") .env("CARGO_ENCODED_RUSTFLAGS", self.cargo_encoded_rustflags()?) .args(self.cargo_build.artifact.arch.cargo_args()) - .cargo_build_args(&self.cargo_build) + .args(self.cargo_build.cargo_build_args()) .run()?; let build_archive = self.cargo_build.artifact.build_archive(); @@ -52,7 +52,7 @@ impl Build { cmd!(sh, "cargo build --release") .arg("--manifest-path=hermit-builtins/Cargo.toml") .args(self.cargo_build.artifact.arch.builtins_cargo_args()) - .target_dir_args(&self.cargo_build) + .args(self.cargo_build.target_dir_args()) .run()?; eprintln!("Exporting hermit-builtins symbols"); diff --git a/xtask/src/cargo_build.rs b/xtask/src/cargo_build.rs index 585b8294f5..54f36b2ae4 100644 --- a/xtask/src/cargo_build.rs +++ b/xtask/src/cargo_build.rs @@ -1,7 +1,4 @@ -use std::ffi::OsStr; - use clap::Args; -use xshell::Cmd; use crate::artifact::Artifact; @@ -19,36 +16,35 @@ pub struct CargoBuild { pub features: Vec, } -pub trait CmdExt { - fn cargo_build_args(self, cargo_build: &CargoBuild) -> Self; - fn target_dir_args(self, cargo_build: &CargoBuild) -> Self; -} - -impl CmdExt for Cmd<'_> { - fn cargo_build_args(self, cargo_build: &CargoBuild) -> Self { - let cmd = self - .target_dir_args(cargo_build) - .args(cargo_build.no_default_features_args()) - .args(cargo_build.features_args()) - .args(cargo_build.release_args()); - - if let Some(profile) = &cargo_build.artifact.profile { - cmd.args(&["--profile", profile]) - } else { - cmd +impl CargoBuild { + pub fn cargo_build_args(&self) -> Vec { + let mut args = vec![]; + args.extend(self.target_dir_args()); + args.extend( + self.no_default_features_args() + .iter() + .map(|s| s.to_string()), + ); + args.extend(self.features_args().map(|s| s.to_string())); + args.extend(self.release_args().iter().map(|s| s.to_string())); + if let Some(profile) = &self.artifact.profile { + args.push("--profile".to_string()); + args.push(profile.to_string()); } + args } - fn target_dir_args(self, cargo_build: &CargoBuild) -> Self { - if let Some(target_dir) = &cargo_build.artifact.target_dir { - self.args::<&[&OsStr]>(&["--target-dir".as_ref(), target_dir.as_ref()]) + pub fn target_dir_args(&self) -> Vec { + if let Some(target_dir) = &self.artifact.target_dir { + vec![ + "--target-dir".to_string(), + target_dir.to_str().unwrap().to_string(), + ] } else { - self + vec![] } } -} -impl CargoBuild { fn release_args(&self) -> &'static [&'static str] { if self.artifact.release { &["--release"] diff --git a/xtask/src/ci/build.rs b/xtask/src/ci/build.rs index 58d25b54de..5e69ec8c1a 100644 --- a/xtask/src/ci/build.rs +++ b/xtask/src/ci/build.rs @@ -4,7 +4,7 @@ use anyhow::Result; use clap::Args; use xshell::cmd; -use crate::cargo_build::{CargoBuild, CmdExt}; +use crate::cargo_build::CargoBuild; /// Build hermit-rs images. #[derive(Args)] @@ -38,7 +38,7 @@ impl Build { sh.change_dir(".."); cmd!(sh, "cargo build") .args(self.cargo_build.artifact.arch.ci_cargo_args()) - .cargo_build_args(&self.cargo_build) + .args(self.cargo_build.cargo_build_args()) .args(&["--package", self.package.as_str()]) .run()?; From 223962408987f0062fc4279252b2368e9740e9d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Kr=C3=B6ning?= Date: Tue, 9 Apr 2024 16:33:51 +0200 Subject: [PATCH 3/3] fix(xtask): properly use cargo proxy for foreign toolchains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Martin Kröning --- .github/workflows/ci.yml | 2 ++ xtask/src/ci/build.rs | 53 +++++++++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c82cd6d08..3c4b828ec8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,6 +161,8 @@ jobs: with: components: rust-src - uses: mkroening/rust-toolchain-toml@main + - run: rustup component add llvm-tools + working-directory: . - uses: mkroening/rust-toolchain-toml@main with: toolchain-file: 'kernel/rust-toolchain.toml' diff --git a/xtask/src/ci/build.rs b/xtask/src/ci/build.rs index 5e69ec8c1a..cc82db4dfb 100644 --- a/xtask/src/ci/build.rs +++ b/xtask/src/ci/build.rs @@ -1,8 +1,9 @@ +use std::env; use std::path::PathBuf; +use std::process::Command; use anyhow::Result; use clap::Args; -use xshell::cmd; use crate::cargo_build::CargoBuild; @@ -24,23 +25,25 @@ impl Build { eprintln!("::group::cargo build") } - let sh = crate::sh()?; + let mut cargo = cargo(); - let _push_env = if self.package.contains("rftrace") { - Some(sh.push_env( + if self.package.contains("rftrace") { + cargo.env( "RUSTFLAGS", "-Zinstrument-mcount -Cpasses=ee-instrument", - )) - } else { - None + ); }; - sh.change_dir(".."); - cmd!(sh, "cargo build") + cargo + .current_dir("..") + .arg("build") .args(self.cargo_build.artifact.arch.ci_cargo_args()) .args(self.cargo_build.cargo_build_args()) - .args(&["--package", self.package.as_str()]) - .run()?; + .args(["--package", self.package.as_str()]); + + eprintln!("$ {cargo:?}"); + let status = cargo.status()?; + assert!(status.success()); if super::in_ci() { eprintln!("::endgroup::") @@ -53,3 +56,31 @@ impl Build { self.cargo_build.artifact.ci_image(&self.package) } } + +fn cargo() -> Command { + let cargo = { + let exe = format!("cargo{}", env::consts::EXE_SUFFIX); + // On windows, the userspace toolchain ends up in front of the rustup proxy in $PATH. + // To reach the rustup proxy nonetheless, we explicitly query $CARGO_HOME. + let mut cargo_home = PathBuf::from(env::var_os("CARGO_HOME").unwrap()); + cargo_home.push("bin"); + cargo_home.push(&exe); + if cargo_home.exists() { + cargo_home + } else { + PathBuf::from(exe) + } + }; + + let mut cargo = Command::new(cargo); + + // Remove rust-toolchain-specific environment variables from kernel cargo + cargo.env_remove("LD_LIBRARY_PATH"); + env::vars() + .filter(|(key, _value)| key.starts_with("CARGO") || key.starts_with("RUST")) + .for_each(|(key, _value)| { + cargo.env_remove(&key); + }); + + cargo +}