From 92eaa2f85befc54a6ad18e352bb24870bb9647a8 Mon Sep 17 00:00:00 2001 From: Vinnie Magro Date: Wed, 20 Sep 2023 09:51:10 -0700 Subject: [PATCH] [antlir2][features] pass feature on stdin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Some features are too large to be passed on the cli. This is just a stop-gap until D48046541 is fixed with btrfsutil. Test Plan: ``` ❯ buck2 test fbcode//antlir/antlir2/test_images/install: Buck UI: https://www.internalfb.com/buck2/921806c5-e9ae-4ae5-bfd7-f35589e36c89 Test UI: https://www.internalfb.com/intern/testinfra/testrun/4503599827564391 Tests finished: Pass 33. Fail 0. Fatal 0. Skip 0. Build failure 0 ``` Reviewed By: sergeyfd Differential Revision: D49388701 fbshipit-source-id: 93c8b61e90a7fc962200e831df5b75eb712f66de --- antlir/antlir2/antlir2_compile/BUCK | 1 + antlir/antlir2/antlir2_compile/src/lib.rs | 15 +++++++++++++-- antlir/antlir2/antlir2_features/BUCK | 1 + antlir/antlir2/antlir2_features/src/lib.rs | 12 +++++++++--- antlir/antlir2/dnf/driver.py | 7 ++----- antlir/antlir2/features/BUCK | 1 + antlir/antlir2/features/defs.bzl | 1 + antlir/antlir2/features/main.rs | 13 ++++++++----- antlir/antlir2/features/rpm.rs | 14 ++++++++++---- 9 files changed, 46 insertions(+), 19 deletions(-) diff --git a/antlir/antlir2/antlir2_compile/BUCK b/antlir/antlir2/antlir2_compile/BUCK index bca388e13f5..5e034663fb0 100644 --- a/antlir/antlir2/antlir2_compile/BUCK +++ b/antlir/antlir2/antlir2_compile/BUCK @@ -10,6 +10,7 @@ rust_library( ]), deps = [ "anyhow", + "memfd", "serde", "serde_json", "thiserror", diff --git a/antlir/antlir2/antlir2_compile/src/lib.rs b/antlir/antlir2/antlir2_compile/src/lib.rs index 89de7d5798f..78a754ec2db 100644 --- a/antlir/antlir2/antlir2_compile/src/lib.rs +++ b/antlir/antlir2/antlir2_compile/src/lib.rs @@ -13,7 +13,10 @@ use std::collections::BTreeMap; use std::collections::BTreeSet; use std::fmt::Display; +use std::fs::File; +use std::io::Seek; use std::io::Write; +use std::os::fd::IntoRawFd; use std::path::Path; use std::path::PathBuf; use std::process::Command; @@ -260,13 +263,21 @@ pub trait CompileFeature { } } +fn ctx_memfd(ctx: &CompilerContext) -> Result { + let opts = memfd::MemfdOptions::default().close_on_exec(false); + let mfd = opts.create("ctx").context("while creating memfd")?; + serde_json::to_writer(&mut mfd.as_file(), &ctx).context("while serializing CompilerContext")?; + mfd.as_file().rewind()?; + Ok(mfd.into_file()) +} + impl CompileFeature for Feature { fn base_compileish_cmd(&self, sub: &'static str, ctx: &CompilerContext) -> Result { - let ctx_json = serde_json::to_string(ctx).context("while serializing CompilerContext")?; + let ctx_file = ctx_memfd(ctx).context("while serializing CompilerContext")?; let mut cmd = Feature::base_cmd(self); cmd.arg(sub) .arg("--ctx") - .arg(ctx_json) + .arg(Path::new("/proc/self/fd").join(ctx_file.into_raw_fd().to_string())) .env("RUST_LOG", "trace"); Ok(cmd) } diff --git a/antlir/antlir2/antlir2_features/BUCK b/antlir/antlir2/antlir2_features/BUCK index 58fbb34264b..f10780c9d58 100644 --- a/antlir/antlir2/antlir2_features/BUCK +++ b/antlir/antlir2/antlir2_features/BUCK @@ -6,6 +6,7 @@ rust_library( name = "antlir2_features", srcs = glob(["src/**/*.rs"]), deps = [ + "memfd", "serde", "serde_json", "//antlir/buck/buck_label:buck_label", diff --git a/antlir/antlir2/antlir2_features/src/lib.rs b/antlir/antlir2/antlir2_features/src/lib.rs index 7174b8eb878..9a5c0a47e76 100644 --- a/antlir/antlir2/antlir2_features/src/lib.rs +++ b/antlir/antlir2/antlir2_features/src/lib.rs @@ -6,6 +6,7 @@ */ use std::cmp::Ordering; +use std::io::Seek; use std::process::Command; use buck_label::Label; @@ -29,14 +30,19 @@ impl Feature { /// Create a Command that will run this feature implementation process with the data passed as a cli arg pub fn base_cmd(&self) -> Command { let mut run_info = self.run_info.iter(); - let feature_json = serde_json::to_string(&self.data) - .expect("serde_json::Value reserialization will never fail"); let mut cmd = Command::new( run_info .next() .expect("run_info will always have >=1 element"), ); - cmd.args(run_info).arg(feature_json); + let opts = memfd::MemfdOptions::default().close_on_exec(false); + let mfd = opts + .create("stdin") + .expect("failed to create memfd for stdin"); + serde_json::to_writer(&mut mfd.as_file(), &self.data) + .expect("serde_json::Value reserialization will never fail"); + mfd.as_file().rewind().expect("failed to rewind memfd"); + cmd.args(run_info).stdin(mfd.into_file()); cmd } } diff --git a/antlir/antlir2/dnf/driver.py b/antlir/antlir2/dnf/driver.py index a455b17c9fa..17148b412f6 100755 --- a/antlir/antlir2/dnf/driver.py +++ b/antlir/antlir2/dnf/driver.py @@ -445,11 +445,8 @@ def driver(spec) -> None: def main(): - parser = argparse.ArgumentParser() - parser.add_argument("spec", type=json.loads) - - args = parser.parse_args() - driver(args.spec) + spec = json.load(sys.stdin) + driver(spec) if __name__ == "__main__": diff --git a/antlir/antlir2/features/BUCK b/antlir/antlir2/features/BUCK index 6b1f0e064ff..3f296efc9a6 100644 --- a/antlir/antlir2/features/BUCK +++ b/antlir/antlir2/features/BUCK @@ -97,6 +97,7 @@ feature_impl( feature_impl( name = "rpm", deps = [ + "memfd", "nix", "serde_json", "//antlir/buck/buck_label:buck_label", diff --git a/antlir/antlir2/features/defs.bzl b/antlir/antlir2/features/defs.bzl index 0e7e99dcb06..b052343b399 100644 --- a/antlir/antlir2/features/defs.bzl +++ b/antlir/antlir2/features/defs.bzl @@ -47,6 +47,7 @@ def feature_impl( deps = [ "anyhow", "clap", + "serde", "serde_json", "tracing-glog", "tracing-subscriber", diff --git a/antlir/antlir2/features/main.rs b/antlir/antlir2/features/main.rs index b9dadea7837..8e752ebb91c 100644 --- a/antlir/antlir2/features/main.rs +++ b/antlir/antlir2/features/main.rs @@ -7,15 +7,16 @@ use antlir2_compile::CompilerContext; use antlir2_feature_impl::Feature as _; +use anyhow::Context; use anyhow::Result; use clap::Parser; use r#impl::Feature; -use json_arg::Json; +use json_arg::JsonFile; +use serde::de::Deserialize; use tracing_subscriber::prelude::*; #[derive(Parser)] struct Args { - feature: Json, #[clap(subcommand)] cmd: Cmd, } @@ -26,11 +27,11 @@ enum Cmd { Requires, Compile { #[clap(long)] - ctx: Json, + ctx: JsonFile, }, Plan { #[clap(long)] - ctx: Json, + ctx: JsonFile, }, } @@ -50,7 +51,9 @@ fn main() -> Result<()> { .init(); let args = Args::parse(); - let feature = args.feature.into_inner(); + let mut deser = serde_json::Deserializer::from_reader(std::io::stdin()); + let feature = + r#impl::Feature::deserialize(&mut deser).context("while deserializing feature data")?; match args.cmd { Cmd::Provides => { serde_json::to_writer_pretty(std::io::stdout(), &Feature::provides(&feature)?)?; diff --git a/antlir/antlir2/features/rpm.rs b/antlir/antlir2/features/rpm.rs index 425c38904dd..1d2bdf753d7 100644 --- a/antlir/antlir2/features/rpm.rs +++ b/antlir/antlir2/features/rpm.rs @@ -8,6 +8,7 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; +use std::io::Seek; use std::path::Path; use std::process::Command; use std::process::Stdio; @@ -318,7 +319,7 @@ fn run_dnf_driver( .into_iter() .flatten() .collect::>(); - let input = serde_json::to_string(&DriverSpec { + let spec = DriverSpec { repos: Some(ctx.dnf().repos()), install_root: ctx.root(), items: &items, @@ -329,8 +330,7 @@ fn run_dnf_driver( resolved_transaction, ignore_postin_script_error: internal_only_options.ignore_postin_script_error, layer_label: ctx.label().clone(), - }) - .context("while serializing dnf-driver input")?; + }; std::fs::create_dir_all(ctx.dst_path("/dev")).context("while ensuring /dev exists")?; nix::mount::mount( @@ -342,8 +342,14 @@ fn run_dnf_driver( ) .context("while mounting /dev in installroot")?; + let opts = memfd::MemfdOptions::default().close_on_exec(false); + let mfd = opts.create("input").context("while creating memfd")?; + serde_json::to_writer(&mut mfd.as_file(), &spec) + .context("while serializing dnf-driver input")?; + mfd.as_file().rewind()?; + let mut child = Command::new("/__antlir2__/dnf/driver") - .arg(&input) + .stdin(mfd.into_file()) .stdout(Stdio::piped()) .spawn() .context("while spawning dnf-driver")?;