[antlir2][features] pass feature on stdin

Summary:
Some features are too large to be passed on the cli. This is just a stop-gap
until D48046541 is fixed with btrfsutil.

Test Plan:
```
❯ buck2 test fbcode//antlir/antlir2/test_images/install:
Buck UI: https://www.internalfb.com/buck2/921806c5-e9ae-4ae5-bfd7-f35589e36c89
Test UI: https://www.internalfb.com/intern/testinfra/testrun/4503599827564391
Tests finished: Pass 33. Fail 0. Fatal 0. Skip 0. Build failure 0
```

Reviewed By: sergeyfd

Differential Revision: D49388701

fbshipit-source-id: 93c8b61e90a7fc962200e831df5b75eb712f66de

antlir/antlir2/antlir2_compile/BUCK

@@ -10,6 +10,7 @@ rust_library(
     ]),
     deps = [
         "anyhow",
+        "memfd",
         "serde",
         "serde_json",
         "thiserror",

antlir/antlir2/antlir2_compile/src/lib.rs

@@ -13,7 +13,10 @@
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
 use std::fmt::Display;
+use std::fs::File;
+use std::io::Seek;
 use std::io::Write;
+use std::os::fd::IntoRawFd;
 use std::path::Path;
 use std::path::PathBuf;
 use std::process::Command;
@@ -260,13 +263,21 @@ pub trait CompileFeature {
     }
 }
 
+fn ctx_memfd(ctx: &CompilerContext) -> Result<File> {
+    let opts = memfd::MemfdOptions::default().close_on_exec(false);
+    let mfd = opts.create("ctx").context("while creating memfd")?;
+    serde_json::to_writer(&mut mfd.as_file(), &ctx).context("while serializing CompilerContext")?;
+    mfd.as_file().rewind()?;
+    Ok(mfd.into_file())
+}
+
 impl CompileFeature for Feature {
     fn base_compileish_cmd(&self, sub: &'static str, ctx: &CompilerContext) -> Result<Command> {
-        let ctx_json = serde_json::to_string(ctx).context("while serializing CompilerContext")?;
+        let ctx_file = ctx_memfd(ctx).context("while serializing CompilerContext")?;
         let mut cmd = Feature::base_cmd(self);
         cmd.arg(sub)
             .arg("--ctx")
-            .arg(ctx_json)
+            .arg(Path::new("/proc/self/fd").join(ctx_file.into_raw_fd().to_string()))
             .env("RUST_LOG", "trace");
         Ok(cmd)
     }

antlir/antlir2/antlir2_features/BUCK

@@ -6,6 +6,7 @@ rust_library(
     name = "antlir2_features",
     srcs = glob(["src/**/*.rs"]),
     deps = [
+        "memfd",
         "serde",
         "serde_json",
         "//antlir/buck/buck_label:buck_label",

antlir/antlir2/antlir2_features/src/lib.rs

@@ -6,6 +6,7 @@
  */
 
 use std::cmp::Ordering;
+use std::io::Seek;
 use std::process::Command;
 
 use buck_label::Label;
@@ -29,14 +30,19 @@ impl Feature {
     /// Create a Command that will run this feature implementation process with the data passed as a cli arg
     pub fn base_cmd(&self) -> Command {
         let mut run_info = self.run_info.iter();
-        let feature_json = serde_json::to_string(&self.data)
-            .expect("serde_json::Value reserialization will never fail");
         let mut cmd = Command::new(
             run_info
                 .next()
                 .expect("run_info will always have >=1 element"),
         );
-        cmd.args(run_info).arg(feature_json);
+        let opts = memfd::MemfdOptions::default().close_on_exec(false);
+        let mfd = opts
+            .create("stdin")
+            .expect("failed to create memfd for stdin");
+        serde_json::to_writer(&mut mfd.as_file(), &self.data)
+            .expect("serde_json::Value reserialization will never fail");
+        mfd.as_file().rewind().expect("failed to rewind memfd");
+        cmd.args(run_info).stdin(mfd.into_file());
         cmd
     }
 }

antlir/antlir2/dnf/driver.py

@@ -445,11 +445,8 @@ def driver(spec) -> None:
 
 
 def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("spec", type=json.loads)
-
-    args = parser.parse_args()
-    driver(args.spec)
+    spec = json.load(sys.stdin)
+    driver(spec)
 
 
 if __name__ == "__main__":

antlir/antlir2/features/BUCK

@@ -97,6 +97,7 @@ feature_impl(
 feature_impl(
     name = "rpm",
     deps = [
+        "memfd",
         "nix",
         "serde_json",
         "//antlir/buck/buck_label:buck_label",

antlir/antlir2/features/defs.bzl

@@ -47,6 +47,7 @@ def feature_impl(
         deps = [
             "anyhow",
             "clap",
+            "serde",
             "serde_json",
             "tracing-glog",
             "tracing-subscriber",

antlir/antlir2/features/main.rs

@@ -7,15 +7,16 @@
 
 use antlir2_compile::CompilerContext;
 use antlir2_feature_impl::Feature as _;
+use anyhow::Context;
 use anyhow::Result;
 use clap::Parser;
 use r#impl::Feature;
-use json_arg::Json;
+use json_arg::JsonFile;
+use serde::de::Deserialize;
 use tracing_subscriber::prelude::*;
 
 #[derive(Parser)]
 struct Args {
-    feature: Json<r#impl::Feature>,
     #[clap(subcommand)]
     cmd: Cmd,
 }
@@ -26,11 +27,11 @@ enum Cmd {
     Requires,
     Compile {
         #[clap(long)]
-        ctx: Json<CompilerContext>,
+        ctx: JsonFile<CompilerContext>,
     },
     Plan {
         #[clap(long)]
-        ctx: Json<CompilerContext>,
+        ctx: JsonFile<CompilerContext>,
     },
 }
 
@@ -50,7 +51,9 @@ fn main() -> Result<()> {
         .init();
 
     let args = Args::parse();
-    let feature = args.feature.into_inner();
+    let mut deser = serde_json::Deserializer::from_reader(std::io::stdin());
+    let feature =
+        r#impl::Feature::deserialize(&mut deser).context("while deserializing feature data")?;
     match args.cmd {
         Cmd::Provides => {
             serde_json::to_writer_pretty(std::io::stdout(), &Feature::provides(&feature)?)?;

antlir/antlir2/features/rpm.rs

@@ -8,6 +8,7 @@
 use std::borrow::Cow;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
+use std::io::Seek;
 use std::path::Path;
 use std::process::Command;
 use std::process::Stdio;
@@ -318,7 +319,7 @@ fn run_dnf_driver(
         .into_iter()
         .flatten()
         .collect::<Vec<_>>();
-    let input = serde_json::to_string(&DriverSpec {
+    let spec = DriverSpec {
         repos: Some(ctx.dnf().repos()),
         install_root: ctx.root(),
         items: &items,
@@ -329,8 +330,7 @@ fn run_dnf_driver(
         resolved_transaction,
         ignore_postin_script_error: internal_only_options.ignore_postin_script_error,
         layer_label: ctx.label().clone(),
-    })
-    .context("while serializing dnf-driver input")?;
+    };
 
     std::fs::create_dir_all(ctx.dst_path("/dev")).context("while ensuring /dev exists")?;
     nix::mount::mount(
@@ -342,8 +342,14 @@ fn run_dnf_driver(
     )
     .context("while mounting /dev in installroot")?;
 
+    let opts = memfd::MemfdOptions::default().close_on_exec(false);
+    let mfd = opts.create("input").context("while creating memfd")?;
+    serde_json::to_writer(&mut mfd.as_file(), &spec)
+        .context("while serializing dnf-driver input")?;
+    mfd.as_file().rewind()?;
+
     let mut child = Command::new("/__antlir2__/dnf/driver")
-        .arg(&input)
+        .stdin(mfd.into_file())
         .stdout(Stdio::piped())
         .spawn()
         .context("while spawning dnf-driver")?;