From e72c5de34316af4bd64e3604c67200d63b6ee176 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 7 May 2019 12:20:56 +0000 Subject: [PATCH] WIP: add ostree-layers and ostree-overrride-layers The use case for `ostree-layers` is to support injecting non-RPM content in a more flexible way than can be done with `add-files`, and also without dropping all the way to split composes. This starts with support on the `compose tree` side but down the line I'd like to make it more convenient to do *client* side too. For `ostree-override-layers` this is mainly a development thing for tools like coreos-assembler. Rather than building an RPM we just `make install DESTDIR` then commit and add to `ostree-override-layers`. --- rust/src/treefile.rs | 43 +++++++++++ src/app/rpmostree-compose-builtin-rojig.c | 1 + src/app/rpmostree-compose-builtin-tree.c | 21 ++++- src/app/rpmostree-composeutil.c | 19 +++++ src/app/rpmostree-composeutil.h | 1 + src/libpriv/rpmostree-core.c | 93 ++++++++++++++++++++++- 6 files changed, 174 insertions(+), 4 deletions(-) diff --git a/rust/src/treefile.rs b/rust/src/treefile.rs index 926d4c4670..e9edd1f3a0 100644 --- a/rust/src/treefile.rs +++ b/rust/src/treefile.rs @@ -338,6 +338,8 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) { merge_vecs!( repos, packages, + ostree_layers, + ostree_override_layers, install_langs, initramfs_args, units, @@ -593,6 +595,12 @@ struct TreeComposeConfig { // Deprecated option #[serde(skip_serializing_if = "Option::is_none")] bootstrap_packages: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "ostree-layers")] + ostree_layers: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + #[serde(rename = "ostree-override-layers")] + ostree_override_layers: Option>, // Content installation opts #[serde(skip_serializing_if = "Option::is_none")] @@ -994,6 +1002,7 @@ packages: mod ffi { use super::*; use glib_sys; + use glib::translate::*; use libc; use std::io::Seek; use std::os::unix::io::{AsRawFd, RawFd}; @@ -1077,6 +1086,40 @@ mod ffi { ref_from_raw_ptr(tf).serialized.as_ptr() } + #[no_mangle] + pub extern "C" fn ror_treefile_get_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char { + let tf = ref_from_raw_ptr(tf); + if let Some(ref layers) = tf.parsed.ostree_layers { + layers.to_glib_full() + } else { + ptr::null_mut() + } + } + + #[no_mangle] + pub extern "C" fn ror_treefile_get_ostree_override_layers(tf: *mut Treefile) -> *mut *mut libc::c_char { + let tf = ref_from_raw_ptr(tf); + if let Some(ref layers) = tf.parsed.ostree_override_layers { + layers.to_glib_full() + } else { + ptr::null_mut() + } + } + + #[no_mangle] + pub extern "C" fn ror_treefile_get_all_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char { + let tf = ref_from_raw_ptr(tf); + let mut ret : Vec = Vec::new(); + if let Some(ref layers) = tf.parsed.ostree_layers { + ret.extend(layers.iter().cloned()) + } + if let Some(ref layers) = tf.parsed.ostree_override_layers { + ret.extend(layers.iter().cloned()) + } + ret.to_glib_full() + } + + #[no_mangle] pub extern "C" fn ror_treefile_get_rojig_spec_path(tf: *mut Treefile) -> *const libc::c_char { let tf = ref_from_raw_ptr(tf); diff --git a/src/app/rpmostree-compose-builtin-rojig.c b/src/app/rpmostree-compose-builtin-rojig.c index 02f0e03e32..42ca89f5db 100644 --- a/src/app/rpmostree-compose-builtin-rojig.c +++ b/src/app/rpmostree-compose-builtin-rojig.c @@ -166,6 +166,7 @@ install_packages (RpmOstreeRojigCompose *self, g_autofree char *ret_new_inputhash = NULL; if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx), + self->repo, self->treefile_rs, self->treefile, &ret_new_inputhash, error)) return FALSE; diff --git a/src/app/rpmostree-compose-builtin-tree.c b/src/app/rpmostree-compose-builtin-tree.c index fd27a0dedb..62a668ae44 100644 --- a/src/app/rpmostree-compose-builtin-tree.c +++ b/src/app/rpmostree-compose-builtin-tree.c @@ -51,6 +51,13 @@ #include "libglnx.h" +static gboolean +pull_local_into_target_repo (OstreeRepo *src_repo, + OstreeRepo *dest_repo, + const char *checksum, + GCancellable *cancellable, + GError **error); + static char *opt_workdir; static gboolean opt_workdir_tmpfs; static char *opt_cachedir; @@ -336,7 +343,7 @@ install_packages (RpmOstreeTreeComposeContext *self, /* FIXME - just do a depsolve here before we compute download requirements */ g_autofree char *ret_new_inputhash = NULL; - if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx), + if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx), self->repo, self->treefile_rs, self->treefile, &ret_new_inputhash, error)) return FALSE; @@ -689,6 +696,18 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr, error)) return FALSE; + if (self->build_repo != self->repo) + { + g_auto(GStrv) layers = ror_treefile_get_all_ostree_layers (self->treefile_rs); + for (char **iter = layers; iter && *iter; iter++) + { + const char *layer = *iter; + if (!pull_local_into_target_repo (self->repo, self->build_repo, layer, + cancellable, error)) + return FALSE; + } + } + self->treefile_rootval = json_parser_get_root (self->treefile_parser); if (!JSON_NODE_HOLDS_OBJECT (self->treefile_rootval)) return glnx_throw (error, "Treefile root is not an object"); diff --git a/src/app/rpmostree-composeutil.c b/src/app/rpmostree-composeutil.c index e6dd971c36..2f90b11039 100644 --- a/src/app/rpmostree-composeutil.c +++ b/src/app/rpmostree-composeutil.c @@ -50,6 +50,7 @@ gboolean rpmostree_composeutil_checksum (HyGoal goal, + OstreeRepo *repo, RORTreefile *tf, JsonObject *treefile, char **out_checksum, @@ -92,6 +93,24 @@ rpmostree_composeutil_checksum (HyGoal goal, if (!rpmostree_dnf_add_checksum_goal (checksum, goal, NULL, error)) return FALSE; + if (tf) + { + GLNX_AUTO_PREFIX_ERROR ("Resolving ostree-layers", error); + g_auto(GStrv) layers = ror_treefile_get_all_ostree_layers (tf); + for (char **iter = layers; iter && *iter; iter++) + { + const char *layer = *iter; + g_autofree char *rev = NULL; + if (!ostree_repo_resolve_rev (repo, layer, FALSE, &rev, error)) + return FALSE; + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_commit (repo, rev, &commit, NULL, error)) + return FALSE; + const char *content_checksum = ostree_commit_get_content_checksum (commit); + g_checksum_update (checksum, (const guint8*) content_checksum, strlen (content_checksum)); + } + } + *out_checksum = g_strdup (g_checksum_get_string (checksum)); return TRUE; } diff --git a/src/app/rpmostree-composeutil.h b/src/app/rpmostree-composeutil.h index 11a19328fb..0b08a84ebd 100644 --- a/src/app/rpmostree-composeutil.h +++ b/src/app/rpmostree-composeutil.h @@ -29,6 +29,7 @@ G_BEGIN_DECLS gboolean rpmostree_composeutil_checksum (HyGoal goal, + OstreeRepo *repo, RORTreefile *tf, JsonObject *treefile, char **out_checksum, diff --git a/src/libpriv/rpmostree-core.c b/src/libpriv/rpmostree-core.c index dc74e2d13e..fca295af83 100644 --- a/src/libpriv/rpmostree-core.c +++ b/src/libpriv/rpmostree-core.c @@ -3683,6 +3683,81 @@ rpmostree_context_get_kernel_changed (RpmOstreeContext *self) return self->kernel_changed; } +static gboolean +process_one_ostree_layer (RpmOstreeContext *self, + int rootfs_dfd, + const char *ref, + OstreeRepoCheckoutOverwriteMode ovw_mode, + GCancellable *cancellable, + GError **error) +{ + OstreeRepo *repo = self->ostreerepo; + OstreeRepoCheckoutAtOptions opts = { OSTREE_REPO_CHECKOUT_MODE_USER, + ovw_mode }; + + /* We want the checkout to match the repo type so that we get hardlinks. */ + if (ostree_repo_get_mode (repo) == OSTREE_REPO_MODE_BARE) + opts.mode = OSTREE_REPO_CHECKOUT_MODE_NONE; + + /* Explicitly don't provide a devino cache here. We can't do it reliably because + * of SELinux. The ostree layering infrastructure doesn't have the relabeling + * that we do for the pkgcache repo because we can't assume that we can mutate + * the input commits right now. + * opts.devino_to_csum_cache = self->devino_cache; + */ + /* Always want hardlinks */ + opts.no_copy_fallback = TRUE; + + g_autofree char *rev = NULL; + if (!ostree_repo_resolve_rev (repo, ref, FALSE, &rev, error)) + return FALSE; + + return ostree_repo_checkout_at (repo, &opts, rootfs_dfd, ".", + rev, cancellable, error); +} + +static gboolean +process_ostree_layers (RpmOstreeContext *self, + int rootfs_dfd, + GCancellable *cancellable, + GError **error) +{ + if (!self->treefile_rs) + return TRUE; + + g_auto(GStrv) layers = ror_treefile_get_ostree_layers (self->treefile_rs); + g_auto(GStrv) override_layers = ror_treefile_get_ostree_override_layers (self->treefile_rs); + const size_t n = (layers ? g_strv_length (layers) : 0) + (override_layers ? g_strv_length (override_layers) : 0); + if (n == 0) + return TRUE; + + g_auto(RpmOstreeProgress) checkout_progress = { 0, }; + rpmostree_output_progress_nitems_begin (&checkout_progress, n, "Checking out ostree layers"); + size_t i = 0; + for (char **iter = layers; iter && *iter; iter++) + { + const char *ref = *iter; + if (!process_one_ostree_layer (self, rootfs_dfd, ref, + OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL, + cancellable, error)) + return FALSE; + i++; + rpmostree_output_progress_n_items (i); + } + for (char **iter = override_layers; iter && *iter; iter++) + { + const char *ref = *iter; + if (!process_one_ostree_layer (self, rootfs_dfd, ref, + OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES, + cancellable, error)) + return FALSE; + i++; + rpmostree_output_progress_n_items (i); + } + rpmostree_output_progress_end (&checkout_progress); + return TRUE; +} + gboolean rpmostree_context_assemble (RpmOstreeContext *self, GCancellable *cancellable, @@ -4106,6 +4181,7 @@ rpmostree_context_assemble (RpmOstreeContext *self, } } + { g_auto(RpmOstreeProgress) task = { 0, }; rpmostree_output_task_begin (&task, "Running post scripts"); guint n_post_scripts_run = 0; @@ -4131,6 +4207,16 @@ rpmostree_context_assemble (RpmOstreeContext *self, &n_post_scripts_run, cancellable, error)) return FALSE; } + } + + /* Any ostree refs to overlay */ + if (!process_ostree_layers (self, tmprootfs_dfd, cancellable, error)) + return FALSE; + + { + g_auto(RpmOstreeProgress) task = { 0, }; + rpmostree_output_task_begin (&task, "Running posttrans scripts"); + guint n_posttrans_scripts_run = 0; /* %posttrans */ for (guint i = 0; i < n_rpmts_elements; i++) @@ -4145,16 +4231,17 @@ rpmostree_context_assemble (RpmOstreeContext *self, rpmostree_output_set_sub_message (dnf_package_get_name (pkg)); if (!run_script_sync (self, tmprootfs_dfd, &var_lib_rpm_statedir, pkg, RPMOSTREE_SCRIPT_POSTTRANS, - &n_post_scripts_run, cancellable, error)) + &n_posttrans_scripts_run, cancellable, error)) return FALSE; } /* file triggers */ if (!run_all_transfiletriggers (self, ordering_ts, tmprootfs_dfd, - &n_post_scripts_run, cancellable, error)) + &n_posttrans_scripts_run, cancellable, error)) return FALSE; - rpmostree_output_progress_end_msg (&task, "%u done", n_post_scripts_run); + rpmostree_output_progress_end_msg (&task, "%u done", n_posttrans_scripts_run); + } /* We want this to be the first error message if something went wrong * with a script; see https://github.com/projectatomic/rpm-ostree/pull/888