WIP: add ostree-layers and ostree-overrride-layers

The use case for `ostree-layers` is to support injecting non-RPM
content in a more flexible way than can be done with `add-files`,
and also without dropping all the way to split composes.

This starts with support on the `compose tree` side but down the
line I'd like to make it more convenient to do *client* side too.

For `ostree-override-layers` this is mainly a development thing
for tools like coreos-assembler.  Rather than building an RPM
we just `make install DESTDIR` then commit and add to
`ostree-override-layers`.

rust/src/treefile.rs

@@ -338,6 +338,8 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
     merge_vecs!(
         repos,
         packages,
+        ostree_layers,
+        ostree_override_layers,
         install_langs,
         initramfs_args,
         units,
@@ -593,6 +595,12 @@ struct TreeComposeConfig {
     // Deprecated option
     #[serde(skip_serializing_if = "Option::is_none")]
     bootstrap_packages: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "ostree-layers")]
+    ostree_layers: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "ostree-override-layers")]
+    ostree_override_layers: Option<Vec<String>>,
 
     // Content installation opts
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -994,6 +1002,7 @@ packages:
 mod ffi {
     use super::*;
     use glib_sys;
+    use glib::translate::*;
     use libc;
     use std::io::Seek;
     use std::os::unix::io::{AsRawFd, RawFd};
@@ -1077,6 +1086,40 @@ mod ffi {
         ref_from_raw_ptr(tf).serialized.as_ptr()
     }
 
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char  {
+        let tf = ref_from_raw_ptr(tf);
+        if let Some(ref layers) = tf.parsed.ostree_layers {
+            layers.to_glib_full()
+        } else {
+            ptr::null_mut()
+        }
+    }
+
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_ostree_override_layers(tf: *mut Treefile) -> *mut *mut libc::c_char  {
+        let tf = ref_from_raw_ptr(tf);
+        if let Some(ref layers) = tf.parsed.ostree_override_layers {
+            layers.to_glib_full()
+        } else {
+            ptr::null_mut()
+        }
+    }
+
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_all_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char  {
+        let tf = ref_from_raw_ptr(tf);
+        let mut ret : Vec<String> = Vec::new();
+        if let Some(ref layers) = tf.parsed.ostree_layers {
+            ret.extend(layers.iter().cloned())
+        }
+        if let Some(ref layers) = tf.parsed.ostree_override_layers {
+            ret.extend(layers.iter().cloned())
+        }
+        ret.to_glib_full()
+    }
+
+
     #[no_mangle]
     pub extern "C" fn ror_treefile_get_rojig_spec_path(tf: *mut Treefile) -> *const libc::c_char {
         let tf = ref_from_raw_ptr(tf);

src/app/rpmostree-compose-builtin-rojig.c

@@ -166,6 +166,7 @@ install_packages (RpmOstreeRojigCompose  *self,
 
   g_autofree char *ret_new_inputhash = NULL;
   if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx),
+                                       self->repo,
                                        self->treefile_rs, self->treefile,
                                        &ret_new_inputhash, error))
     return FALSE;

src/app/rpmostree-compose-builtin-tree.c

@@ -51,6 +51,13 @@
 
 #include "libglnx.h"
 
+static gboolean
+pull_local_into_target_repo (OstreeRepo   *src_repo,
+                             OstreeRepo   *dest_repo,
+                             const char   *checksum,
+                             GCancellable *cancellable,
+                             GError      **error);
+
 static char *opt_workdir;
 static gboolean opt_workdir_tmpfs;
 static char *opt_cachedir;
@@ -336,7 +343,7 @@ install_packages (RpmOstreeTreeComposeContext  *self,
 
   /* FIXME - just do a depsolve here before we compute download requirements */
   g_autofree char *ret_new_inputhash = NULL;
-  if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx),
+  if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx), self->repo,
                                        self->treefile_rs, self->treefile,
                                        &ret_new_inputhash, error))
     return FALSE;
@@ -689,6 +696,18 @@ rpm_ostree_compose_context_new (const char    *treefile_pathstr,
                                error))
     return FALSE;
 
+  if (self->build_repo != self->repo)
+    {
+      g_auto(GStrv) layers = ror_treefile_get_all_ostree_layers (self->treefile_rs);
+      for (char **iter = layers; iter && *iter; iter++)
+        {
+          const char *layer = *iter;
+          if (!pull_local_into_target_repo (self->repo, self->build_repo, layer,
+                                            cancellable, error))
+            return FALSE;
+        }
+    }
+
   self->treefile_rootval = json_parser_get_root (self->treefile_parser);
   if (!JSON_NODE_HOLDS_OBJECT (self->treefile_rootval))
     return glnx_throw (error, "Treefile root is not an object");

src/app/rpmostree-composeutil.c

@@ -50,6 +50,7 @@
 
 gboolean
 rpmostree_composeutil_checksum (HyGoal             goal,
+                                OstreeRepo        *repo,
                                 RORTreefile       *tf,
                                 JsonObject        *treefile,
                                 char             **out_checksum,
@@ -92,6 +93,24 @@ rpmostree_composeutil_checksum (HyGoal             goal,
   if (!rpmostree_dnf_add_checksum_goal (checksum, goal, NULL, error))
     return FALSE;
 
+  if (tf)
+    {
+      GLNX_AUTO_PREFIX_ERROR ("Resolving ostree-layers", error);
+      g_auto(GStrv) layers = ror_treefile_get_all_ostree_layers (tf);
+      for (char **iter = layers; iter && *iter; iter++)
+        {
+          const char *layer = *iter;
+          g_autofree char *rev = NULL;
+          if (!ostree_repo_resolve_rev (repo, layer, FALSE, &rev, error))
+            return FALSE;
+          g_autoptr(GVariant) commit = NULL;
+          if (!ostree_repo_load_commit (repo, rev, &commit, NULL, error))
+            return FALSE;
+          const char *content_checksum = ostree_commit_get_content_checksum (commit);
+          g_checksum_update (checksum, (const guint8*) content_checksum, strlen (content_checksum));
+        }
+    }
+
   *out_checksum = g_strdup (g_checksum_get_string (checksum));
   return TRUE;
 }

src/app/rpmostree-composeutil.h

@@ -29,6 +29,7 @@ G_BEGIN_DECLS
 
 gboolean
 rpmostree_composeutil_checksum (HyGoal             goal,
+                                OstreeRepo        *repo,
                                 RORTreefile       *tf,
                                 JsonObject        *treefile,
                                 char             **out_checksum,

src/libpriv/rpmostree-core.c

@@ -3683,6 +3683,81 @@ rpmostree_context_get_kernel_changed (RpmOstreeContext *self)
   return self->kernel_changed;
 }
 
+static gboolean
+process_one_ostree_layer (RpmOstreeContext *self,
+                          int               rootfs_dfd,
+                          const char       *ref,
+                          OstreeRepoCheckoutOverwriteMode ovw_mode,
+                          GCancellable     *cancellable,
+                          GError          **error)
+{
+  OstreeRepo *repo = self->ostreerepo;
+  OstreeRepoCheckoutAtOptions opts = { OSTREE_REPO_CHECKOUT_MODE_USER,
+                                       ovw_mode };
+
+  /* We want the checkout to match the repo type so that we get hardlinks. */
+  if (ostree_repo_get_mode (repo) == OSTREE_REPO_MODE_BARE)
+    opts.mode = OSTREE_REPO_CHECKOUT_MODE_NONE;
+
+  /* Explicitly don't provide a devino cache here.  We can't do it reliably because
+   * of SELinux.  The ostree layering infrastructure doesn't have the relabeling
+   * that we do for the pkgcache repo because we can't assume that we can mutate
+   * the input commits right now.
+   * opts.devino_to_csum_cache = self->devino_cache;
+   */
+  /* Always want hardlinks */
+  opts.no_copy_fallback = TRUE;
+
+  g_autofree char *rev = NULL;
+  if (!ostree_repo_resolve_rev (repo, ref, FALSE, &rev, error))
+    return FALSE;
+
+  return ostree_repo_checkout_at (repo, &opts, rootfs_dfd, ".",
+                                  rev, cancellable, error);
+}
+
+static gboolean
+process_ostree_layers (RpmOstreeContext *self,
+                       int               rootfs_dfd,
+                       GCancellable     *cancellable,
+                       GError          **error)
+{
+  if (!self->treefile_rs)
+    return TRUE;
+
+  g_auto(GStrv) layers = ror_treefile_get_ostree_layers (self->treefile_rs);
+  g_auto(GStrv) override_layers = ror_treefile_get_ostree_override_layers (self->treefile_rs);
+  const size_t n = (layers ? g_strv_length (layers) : 0) + (override_layers ? g_strv_length (override_layers) : 0);
+  if (n == 0)
+    return TRUE;
+
+  g_auto(RpmOstreeProgress) checkout_progress = { 0, };
+  rpmostree_output_progress_nitems_begin (&checkout_progress, n, "Checking out ostree layers");
+  size_t i = 0;
+  for (char **iter = layers; iter && *iter; iter++)
+    {
+      const char *ref = *iter;
+      if (!process_one_ostree_layer (self, rootfs_dfd, ref,
+                                     OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL,
+                                     cancellable, error))
+        return FALSE;
+      i++;
+      rpmostree_output_progress_n_items (i);
+    }
+  for (char **iter = override_layers; iter && *iter; iter++)
+    {
+      const char *ref = *iter;
+      if (!process_one_ostree_layer (self, rootfs_dfd, ref,
+                                     OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES,
+                                     cancellable, error))
+        return FALSE;
+      i++;
+      rpmostree_output_progress_n_items (i);
+    }
+  rpmostree_output_progress_end (&checkout_progress);
+  return TRUE;
+}
+
 gboolean
 rpmostree_context_assemble (RpmOstreeContext      *self,
                             GCancellable          *cancellable,
@@ -4106,6 +4181,7 @@ rpmostree_context_assemble (RpmOstreeContext      *self,
             }
         }
 
+      {
       g_auto(RpmOstreeProgress) task = { 0, };
       rpmostree_output_task_begin (&task, "Running post scripts");
       guint n_post_scripts_run = 0;
@@ -4131,6 +4207,16 @@ rpmostree_context_assemble (RpmOstreeContext      *self,
                                 &n_post_scripts_run, cancellable, error))
             return FALSE;
         }
+      }
+
+      /* Any ostree refs to overlay */
+      if (!process_ostree_layers (self, tmprootfs_dfd, cancellable, error))
+        return FALSE;
+
+      {
+      g_auto(RpmOstreeProgress) task = { 0, };
+      rpmostree_output_task_begin (&task, "Running posttrans scripts");
+      guint n_posttrans_scripts_run = 0;
 
       /* %posttrans */
       for (guint i = 0; i < n_rpmts_elements; i++)
@@ -4145,16 +4231,17 @@ rpmostree_context_assemble (RpmOstreeContext      *self,
           rpmostree_output_set_sub_message (dnf_package_get_name (pkg));
           if (!run_script_sync (self, tmprootfs_dfd, &var_lib_rpm_statedir,
                                 pkg, RPMOSTREE_SCRIPT_POSTTRANS,
-                                &n_post_scripts_run, cancellable, error))
+                                &n_posttrans_scripts_run, cancellable, error))
             return FALSE;
         }
 
       /* file triggers */
       if (!run_all_transfiletriggers (self, ordering_ts, tmprootfs_dfd,
-                                      &n_post_scripts_run, cancellable, error))
+                                      &n_posttrans_scripts_run, cancellable, error))
         return FALSE;
 
-      rpmostree_output_progress_end_msg (&task, "%u done", n_post_scripts_run);
+      rpmostree_output_progress_end_msg (&task, "%u done", n_posttrans_scripts_run);
+      }
 
       /* We want this to be the first error message if something went wrong
        * with a script; see https://github.com/projectatomic/rpm-ostree/pull/888