Add API and CLI to update detached metadata

lib/src/cli.rs

@@ -221,6 +221,23 @@ enum ContainerImageOpts {
         imgref: OstreeImageReference,
     },
 
+    /// Replace the detached metadata (e.g. to add a signature)
+    ReplaceDetachedMetadata {
+        /// Path to the source repository
+        #[structopt(long)]
+        #[structopt(parse(try_from_str = parse_base_imgref))]
+        src: ImageReference,
+
+        /// Target image
+        #[structopt(long)]
+        #[structopt(parse(try_from_str = parse_base_imgref))]
+        dest: ImageReference,
+
+        /// Path to file containing new detached metadata; if not provided,
+        /// any existing detached metadata will be deleted.
+        contents: Option<Utf8PathBuf>,
+    },
+
     /// Unreference one or more pulled container images and perform a garbage collection.
     Remove {
         /// Path to the repository
@@ -711,6 +728,21 @@ where
                     dest_repo,
                     imgref,
                 } => crate::container::store::copy(&src_repo, &dest_repo, &imgref).await,
+                ContainerImageOpts::ReplaceDetachedMetadata {
+                    src,
+                    dest,
+                    contents,
+                } => {
+                    let contents = contents.map(std::fs::read).transpose()?;
+                    let digest = crate::container::update_detached_metadata(
+                        &src,
+                        &dest,
+                        contents.as_deref(),
+                    )
+                    .await?;
+                    println!("Pushed: {}", digest);
+                    Ok(())
+                }
                 ContainerImageOpts::Deploy {
                     sysroot,
                     stateroot,

lib/src/container/mod.rs

@@ -243,6 +243,8 @@ pub use unencapsulate::*;
 pub(crate) mod ocidir;
 mod skopeo;
 pub mod store;
+mod update_detachedmeta;
+pub use update_detachedmeta::*;
 
 #[cfg(test)]
 mod tests {

lib/src/container/update_detachedmeta.rs

@@ -0,0 +1,126 @@
+use super::ImageReference;
+use crate::container::{ocidir, skopeo};
+use crate::container::{store as container_store, Transport};
+use anyhow::{anyhow, Context, Result};
+use camino::Utf8Path;
+use std::io::{BufReader, BufWriter};
+use std::rc::Rc;
+
+/// Given an OSTree container image reference, update the detached metadata (e.g. GPG signature)
+/// while preserving all other container image metadata.
+///
+/// The return value is the manifest digest of (e.g. `@sha256:`) the image.
+pub async fn update_detached_metadata(
+    src: &ImageReference,
+    dest: &ImageReference,
+    detached_buf: Option<&[u8]>,
+) -> Result<String> {
+    // For now, convert the source to a temporary OCI directory, so we can directly
+    // parse and manipulate it.  In the future this will be replaced by https://github.com/ostreedev/ostree-rs-ext/issues/153
+    // and other work to directly use the containers/image API via containers-image-proxy.
+    let tempdir = tempfile::tempdir_in("/var/tmp")?;
+    let tempsrc = tempdir.path().join("src");
+    let tempsrc_utf8 = Utf8Path::from_path(&tempsrc).ok_or_else(|| anyhow!("Invalid tempdir"))?;
+    let tempsrc_ref = ImageReference {
+        transport: Transport::OciDir,
+        name: tempsrc_utf8.to_string(),
+    };
+
+    // Full copy of the source image
+    let pulled_digest: String = skopeo::copy(src, &tempsrc_ref)
+        .await
+        .context("Creating temporary copy to OCI dir")?;
+
+    // Copy to the thread
+    let detached_buf = detached_buf.map(Vec::from);
+    let tempsrc_ref_path = tempsrc_ref.name.clone();
+    // Fork a thread to do the heavy lifting of filtering the tar stream, rewriting the manifest/config.
+    crate::tokio_util::spawn_blocking_cancellable_flatten(move |cancellable| {
+        // Open the temporary OCI directory.
+        let tempsrc = Rc::new(openat::Dir::open(tempsrc_ref_path).context("Opening src")?);
+        let tempsrc = ocidir::OciDir::open(tempsrc)?;
+
+        // Load the manifest, platform, and config
+        let (mut manifest, manifest_descriptor) = tempsrc
+            .read_manifest_and_descriptor()
+            .context("Reading manifest from source")?;
+        anyhow::ensure!(manifest_descriptor.digest().as_str() == pulled_digest.as_str());
+        let platform = manifest_descriptor
+            .platform()
+            .as_ref()
+            .cloned()
+            .unwrap_or_default();
+        let mut config: oci_spec::image::ImageConfiguration =
+            tempsrc.read_json_blob(manifest.config())?;
+        let mut ctrcfg = config
+            .config()
+            .as_ref()
+            .cloned()
+            .ok_or_else(|| anyhow!("Image is missing container configuration"))?;
+
+        // Find the OSTree commit layer we want to replace
+        let commit_layer = container_store::ostree_layer(&manifest, &config)?;
+        let commit_layer_idx = manifest
+            .layers()
+            .iter()
+            .position(|x| x == commit_layer)
+            .unwrap();
+
+        // Create a new layer
+        let out_layer = {
+            // Create tar streams for source and destination
+            let src_layer = BufReader::new(tempsrc.read_blob(commit_layer)?);
+            let mut src_layer = flate2::read::GzDecoder::new(src_layer);
+            let mut out_layer = BufWriter::new(tempsrc.create_raw_layer(None)?);
+
+            // Process the tar stream and inject our new detached metadata
+            crate::tar::update_detached_metadata(
+                &mut src_layer,
+                &mut out_layer,
+                detached_buf.as_deref(),
+                Some(cancellable),
+            )?;
+
+            // Flush all wrappers, and finalize the layer
+            out_layer
+                .into_inner()
+                .map_err(|_| anyhow!("Failed to flush buffer"))?
+                .complete()?
+        };
+        // Get the diffid and descriptor for our new tar layer
+        let out_layer_diffid = format!("sha256:{}", out_layer.uncompressed_sha256);
+        let out_layer_descriptor = out_layer
+            .descriptor()
+            .media_type(oci_spec::image::MediaType::ImageLayerGzip)
+            .build()
+            .unwrap(); // SAFETY: We pass all required fields
+
+        // Splice it into both the manifest and config
+        manifest.layers_mut()[commit_layer_idx] = out_layer_descriptor;
+        config.rootfs_mut().diff_ids_mut()[commit_layer_idx] = out_layer_diffid.clone();
+
+        let labels = ctrcfg.labels_mut().get_or_insert_with(Default::default);
+        labels.insert(
+            crate::container::OSTREE_DIFFID_LABEL.into(),
+            out_layer_diffid,
+        );
+        config.set_config(Some(ctrcfg));
+
+        // Write the config and manifest
+        let new_config_descriptor = tempsrc.write_config(config)?;
+        manifest.set_config(new_config_descriptor);
+        // This entirely replaces the single entry in the OCI directory, which skopeo will find by default.
+        tempsrc
+            .write_manifest(manifest, platform)
+            .context("Writing manifest")?;
+        Ok(())
+    })
+    .await
+    .context("Regenerating commit layer")?;
+
+    // Finally, copy the mutated image back to the target.  For chunked images,
+    // because we only changed one layer, skopeo should know not to re-upload shared blobs.
+    crate::container::skopeo::copy(&tempsrc_ref, dest)
+        .await
+        .context("Copying to destination")
+}

lib/src/fixture.rs

@@ -402,6 +402,15 @@ impl Fixture {
         &self.destrepo
     }
 
+    // Delete all objects in the destrepo
+    pub fn clear_destrepo(&self) -> Result<()> {
+        self.destrepo()
+            .set_ref_immediate(None, self.testref(), None, gio::NONE_CANCELLABLE)?;
+        self.destrepo()
+            .prune(ostree::RepoPruneFlags::REFS_ONLY, 0, gio::NONE_CANCELLABLE)?;
+        Ok(())
+    }
+
     pub fn write_filedef(&self, root: &ostree::MutableTree, def: &FileDef) -> Result<()> {
         let parent_path = def.path.parent();
         let parent = if let Some(parent_path) = parent_path {

lib/src/tar/export.rs

@@ -14,6 +14,7 @@ use ostree::gio;
 use std::borrow::Borrow;
 use std::borrow::Cow;
 use std::collections::HashSet;
+use std::convert::TryInto;
 use std::io::BufReader;
 
 /// The repository mode generated by a tar export stream.
@@ -631,6 +632,84 @@ pub(crate) fn export_final_chunk<W: std::io::Write>(
     write_chunk(writer, &chunking.remainder)
 }
 
+/// Process an exported tar stream, and update the detached metadata.
+#[allow(clippy::while_let_on_iterator)]
+#[context("Replacing detached metadata")]
+pub(crate) fn reinject_detached_metadata<C: IsA<gio::Cancellable>>(
+    src: &mut tar::Archive<impl std::io::Read>,
+    dest: &mut tar::Builder<impl std::io::Write>,
+    detached_buf: Option<&[u8]>,
+    cancellable: Option<&C>,
+) -> Result<()> {
+    let mut entries = src.entries()?;
+    let mut commit_ent = None;
+    // Loop through the tar stream until we find the commit object; copy all prior entries
+    // such as the baseline directory structure.
+    while let Some(entry) = entries.next() {
+        if let Some(c) = cancellable {
+            c.set_error_if_cancelled()?;
+        }
+        let entry = entry?;
+        let header = entry.header();
+        let path = entry.path()?;
+        let path: &Utf8Path = (&*path).try_into()?;
+        if !(header.entry_type() == tar::EntryType::Regular && path.as_str().ends_with(".commit")) {
+            crate::tar::write::copy_entry(entry, dest, None)?;
+        } else {
+            commit_ent = Some(entry);
+            break;
+        }
+    }
+    let commit_ent = commit_ent.ok_or_else(|| anyhow!("Missing commit object"))?;
+    let commit_path = commit_ent.path()?;
+    let commit_path = Utf8Path::from_path(&*commit_path)
+        .ok_or_else(|| anyhow!("Invalid non-utf8 path {:?}", commit_path))?;
+    let (checksum, objtype) = crate::tar::import::Importer::parse_metadata_entry(commit_path)?;
+    assert_eq!(objtype, ostree::ObjectType::Commit); // Should have been verified above
+    crate::tar::write::copy_entry(commit_ent, dest, None)?;
+
+    // If provided, inject our new detached metadata object
+    if let Some(detached_buf) = detached_buf {
+        let detached_path = object_path(ostree::ObjectType::CommitMeta, &checksum);
+        tar_append_default_data(dest, &detached_path, detached_buf)?;
+    }
+
+    // If the next entry is detached metadata, then drop it since we wrote a new one
+    let next_ent = entries
+        .next()
+        .ok_or_else(|| anyhow!("Expected metadata object after commit"))??;
+    let next_ent_path = next_ent.path()?;
+    let next_ent_path: &Utf8Path = (&*next_ent_path).try_into()?;
+    let objtype = crate::tar::import::Importer::parse_metadata_entry(next_ent_path)?.1;
+    if objtype != ostree::ObjectType::CommitMeta {
+        dbg!(objtype);
+        crate::tar::write::copy_entry(next_ent, dest, None)?;
+    }
+
+    // Finally, copy all remaining entries.
+    while let Some(entry) = entries.next() {
+        if let Some(c) = cancellable {
+            c.set_error_if_cancelled()?;
+        }
+        crate::tar::write::copy_entry(entry?, dest, None)?;
+    }
+
+    Ok(())
+}
+
+/// Replace the detached metadata in an tar stream which is an export of an OSTree commit.
+pub fn update_detached_metadata<D: std::io::Write, C: IsA<gio::Cancellable>>(
+    src: impl std::io::Read,
+    dest: D,
+    detached_buf: Option<&[u8]>,
+    cancellable: Option<&C>,
+) -> Result<D> {
+    let mut src = tar::Archive::new(src);
+    let mut dest = tar::Builder::new(dest);
+    reinject_detached_metadata(&mut src, &mut dest, detached_buf, cancellable)?;
+    dest.into_inner().map_err(Into::into)
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

lib/src/tar/import.rs

@@ -214,7 +214,7 @@ impl Importer {
         }
     }
 
-    fn parse_metadata_entry(path: &Utf8Path) -> Result<(String, ostree::ObjectType)> {
+    pub(crate) fn parse_metadata_entry(path: &Utf8Path) -> Result<(String, ostree::ObjectType)> {
         let (parentname, name, objtype) = parse_object_entry_path(path)?;
         let checksum = parse_checksum(parentname, name)?;
         let objtype = objtype_from_string(objtype)