From a7c2fc9bd5ceb66d5c76871324b1dca1213eb0c1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Nov 2017 16:55:41 -0500 Subject: [PATCH 1/8] tests/ex-container: Disable parallelism for now Adding the 2nd test revealed this didn't actually work, will fix at some point. --- tests/ex-container | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/ex-container b/tests/ex-container index 1b93f8df1e..47bf46b3ba 100755 --- a/tests/ex-container +++ b/tests/ex-container @@ -34,5 +34,8 @@ mkdir -p ${LOGDIR} # Ideally pass $(cwd) down into parallel somehow export test_tmpdir=${tmpdir} +# We use -j 1 since we can't actually be parallel at the moment due +# to locking issues with metadata downloads at least. Down the line +# we should try to do a dry run pull first like the compose tests do. ls ${dn}/ex-container-tests/test-*.sh | sort | - parallel --tag --halt soon,fail=1 --joblog joblog --results ${LOGDIR} --line-buffer {} + parallel -j 1 --tag --halt soon,fail=1 --joblog joblog --results ${LOGDIR} --line-buffer {} From bff371216caddd56e2db5bc670115d1d83a5da21 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Nov 2017 14:25:57 -0500 Subject: [PATCH 2/8] unpacker: Handle uid/gid when running unprivileged (non-root) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It seems that libarchive ends up returning `getuid()` actually, possibly because the cpio doesn't actually have ownership information? Anyways, what we really want here is to set `0/0`, which is what happens for the `ex container` path via `OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS`. Prep for unified core 🌐. --- src/libpriv/rpmostree-unpacker.c | 41 +++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/src/libpriv/rpmostree-unpacker.c b/src/libpriv/rpmostree-unpacker.c index c3a3191ccd..44e6719c9e 100644 --- a/src/libpriv/rpmostree-unpacker.c +++ b/src/libpriv/rpmostree-unpacker.c @@ -61,6 +61,7 @@ struct RpmOstreeUnpacker GHashTable *doc_files; GString *tmpfiles_d; RpmOstreeUnpackerFlags flags; + gboolean unpacking_as_nonroot; DnfPackage *pkg; char *hdr_sha256; @@ -314,6 +315,7 @@ rpmostree_unpacker_new_fd (int fd, ret->fi = g_steal_pointer (&fi); ret->archive = g_steal_pointer (&archive); ret->flags = flags; + ret->unpacking_as_nonroot = (getuid () != 0); ret->hdr = g_steal_pointer (&hdr); ret->cpio_offset = cpio_offset; ret->pkg = pkg ? g_object_ref (pkg) : NULL; @@ -650,9 +652,6 @@ compose_filter_cb (OstreeRepo *repo, const char *user = NULL; const char *group = NULL; - guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid"); - guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid"); - gboolean error_was_set = (error && *error != NULL); /* Are we filtering out docs? Let's check that first */ @@ -662,6 +661,33 @@ compose_filter_cb (OstreeRepo *repo, /* Lookup any rpmfi overrides (was parsed from the header) */ get_rpmfi_override (self, path, &user, &group, NULL); + if (self->unpacking_as_nonroot) + { + /* In the unprivileged case, libarchive returns our own uid by default. + * Let's ensure the object is always owned by 0/0, since we apply rpm + * header uid/gid at checkout time anyways. + * + * Note that for `ex container` we use + * OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS, which forces + * this, and that path also doesn't use this function, it uses + * unprivileged_filter_cb. + */ + g_file_info_set_attribute_uint32 (file_info, "unix::uid", 0); + g_file_info_set_attribute_uint32 (file_info, "unix::gid", 0); + } + else + { + /* sanity check that RPM isn't using CPIO id fields */ + const guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid"); + const guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid"); + if (uid != 0 || gid != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, + "RPM had unexpected non-root owned path \"%s\", marked as %u:%u)", path, uid, gid); + return OSTREE_REPO_COMMIT_FILTER_SKIP; + } + } + /* convert /run and /var entries to tmpfiles.d */ if (g_str_has_prefix (path, "/" VAR_SELINUX_TARGETED_PATH)) ; /* Handled by pathname translation */ @@ -674,15 +700,8 @@ compose_filter_cb (OstreeRepo *repo, } else if (!error_was_set) { - /* sanity check that RPM isn't using CPIO id fields */ - if (uid != 0 || gid != 0) - { - g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, - "RPM had unexpected non-root owned path \"%s\", marked as %u:%u)", path, uid, gid); - return OSTREE_REPO_COMMIT_FILTER_SKIP; - } /* And ensure the RPM installs into supported paths */ - else if (!path_is_ostree_compliant (path)) + if (!path_is_ostree_compliant (path)) { if ((self->flags & RPMOSTREE_UNPACKER_FLAGS_SKIP_EXTRANEOUS) == 0) g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED, From eee69395a2d3bf15b9b14aaa3200c4be95544c8e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Nov 2017 14:46:40 -0500 Subject: [PATCH 3/8] core: Don't try to apply non-root uid/gid when run as non-root In an unprivileged case, we can't do this on the real filesystem. For `ex container`, we want to completely ignore uid/gid. I added a test installing `httpd` which failed previously. TODO: For non-root `--ex-unified-core` we need to do it as a commit modifier. --- src/libpriv/rpmostree-core.c | 8 ++++++++ tests/ex-container-tests/test-httpd.sh | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100755 tests/ex-container-tests/test-httpd.sh diff --git a/src/libpriv/rpmostree-core.c b/src/libpriv/rpmostree-core.c index e3b3e2a22b..8d5736e66c 100644 --- a/src/libpriv/rpmostree-core.c +++ b/src/libpriv/rpmostree-core.c @@ -2700,6 +2700,14 @@ apply_rpmfi_overrides (RpmOstreeContext *self, GCancellable *cancellable, GError **error) { + /* In an unprivileged case, we can't do this on the real filesystem. For `ex + * container`, we want to completely ignore uid/gid. + * + * TODO: For non-root `--ex-unified-core` we need to do it as a commit modifier. + */ + if (getuid () != 0) + return TRUE; /* 🔚 Early return */ + int i; g_auto(rpmfi) fi = NULL; gboolean emitted_nonusr_warning = FALSE; diff --git a/tests/ex-container-tests/test-httpd.sh b/tests/ex-container-tests/test-httpd.sh new file mode 100755 index 0000000000..0becbf9519 --- /dev/null +++ b/tests/ex-container-tests/test-httpd.sh @@ -0,0 +1,19 @@ +#!/usr/bin/bash +set -xeuo pipefail + +cd ${test_tmpdir} + +dn=$(cd $(dirname $0) && pwd) +. ${dn}/../common/libtest-core.sh + +cat >httpd.conf < Date: Wed, 15 Nov 2017 21:28:03 -0500 Subject: [PATCH 4/8] compose: Add --ex-unified-core The "--ex" prefix here means it's an experimental option. A tremendous change here is that start to support non-uid 0, but there are various things to fix there; the unpacker for example needs to learn to set imported objects fully based on the rpmfi information (i.e. default to uid 0, since libarchive gives the current uid by default). And even when run as uid 0, there are some bugs, though I'm not sure of any showstoppers yet. For example, dracut's `dracut-install` calls `cp --preserve=xattrs` which fails to copy the `user.ostreemeta` xattrs from a checkout (it shouldn't be copying that anyways...) Nevertheless, the infrastructure behind this really helps (is almost a hard requirement for) the [jigdo effort](https://github.com/projectatomic/rpm-ostree/issues/1081). Which is really only true due to SELinux - we need to import the packages, then generate the final tree to get the final policy, then use that policy to relabel all of the packages. --- ci/build.sh | 2 +- src/app/rpmostree-builtin-compose.c | 2 +- src/app/rpmostree-compose-builtin-tree.c | 187 ++++++++++++++++------ src/libpriv/rpmostree-passwd-util.c | 10 +- src/libpriv/rpmostree-passwd-util.h | 3 +- src/libpriv/rpmostree-postprocess.c | 19 ++- src/libpriv/rpmostree-postprocess.h | 2 + src/libpriv/rpmostree-script-gperf.gperf | 3 + tests/compose-tests/libbasic-test.sh | 41 +++++ tests/compose-tests/test-basic-unified.sh | 32 ++++ tests/compose-tests/test-basic.sh | 40 +---- 11 files changed, 238 insertions(+), 103 deletions(-) create mode 100644 tests/compose-tests/libbasic-test.sh create mode 100755 tests/compose-tests/test-basic-unified.sh diff --git a/ci/build.sh b/ci/build.sh index c523540cd3..7124fcd158 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -22,7 +22,7 @@ pkg_install_builddeps rpm-ostree # Mostly dependencies for tests pkg_install ostree{,-devel,-grub2} createrepo_c /usr/bin/jq PyYAML \ libubsan libasan libtsan elfutils fuse sudo python-gobject-base \ - selinux-policy-devel + selinux-policy-devel selinux-policy-targeted # For ex-container tests and clang build pkg_install_if_os fedora parallel clang diff --git a/src/app/rpmostree-builtin-compose.c b/src/app/rpmostree-builtin-compose.c index ecca6b8479..896452af50 100644 --- a/src/app/rpmostree-builtin-compose.c +++ b/src/app/rpmostree-builtin-compose.c @@ -29,7 +29,7 @@ #include static RpmOstreeCommand compose_subcommands[] = { - { "tree", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT, + { "tree", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD, "Process a \"treefile\"; install packages and commit the result to an OSTree repository", rpmostree_compose_builtin_tree }, { "install", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT, diff --git a/src/app/rpmostree-compose-builtin-tree.c b/src/app/rpmostree-compose-builtin-tree.c index 366ab8b588..45c15adba8 100644 --- a/src/app/rpmostree-compose-builtin-tree.c +++ b/src/app/rpmostree-compose-builtin-tree.c @@ -48,6 +48,7 @@ static gboolean opt_workdir_tmpfs; static char *opt_cachedir; static gboolean opt_force_nocache; static gboolean opt_cache_only; +static gboolean opt_ex_unified_core; static char *opt_proxy; static char *opt_output_repodata_dir; static char **opt_metadata_strings; @@ -68,6 +69,7 @@ static GOptionEntry install_option_entries[] = { { "force-nocache", 0, 0, G_OPTION_ARG_NONE, &opt_force_nocache, "Always create a new OSTree commit, even if nothing appears to have changed", NULL }, { "cache-only", 0, 0, G_OPTION_ARG_NONE, &opt_cache_only, "Assume cache is present, do not attempt to update it", NULL }, { "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" }, + { "ex-unified-core", 0, 0, G_OPTION_ARG_NONE, &opt_ex_unified_core, "Use new \"unified core\" codepath", NULL }, { "proxy", 0, 0, G_OPTION_ARG_STRING, &opt_proxy, "HTTP proxy", "PROXY" }, { "dry-run", 0, 0, G_OPTION_ARG_NONE, &opt_dry_run, "Just print the transaction and exit", NULL }, { "output-repodata-dir", 0, 0, G_OPTION_ARG_STRING, &opt_output_repodata_dir, "Save downloaded repodata in DIR", "DIR" }, @@ -101,6 +103,8 @@ typedef struct { int rootfs_dfd; int cachedir_dfd; OstreeRepo *repo; + OstreeRepo *pkgcache_repo; + OstreeRepoDevInoCache *devino_cache; char *ref; char *previous_checksum; @@ -126,6 +130,8 @@ rpm_ostree_tree_compose_context_free (RpmOstreeTreeComposeContext *ctx) glnx_close_fd (&ctx->rootfs_dfd); glnx_close_fd (&ctx->cachedir_dfd); g_clear_object (&ctx->repo); + g_clear_object (&ctx->pkgcache_repo); + g_clear_pointer (&ctx->devino_cache, (GDestroyNotify)ostree_repo_devino_cache_unref); g_free (ctx->ref); g_free (ctx->previous_checksum); g_clear_object (&ctx->treefile_parser); @@ -319,16 +325,6 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, GCancellable *cancellable, GError **error) { - /* TODO - uncomment this once we have SELinux working */ -#if 0 - g_autoptr(OstreeRepo) cache_repo = - ostree_repo_create_at (self->cachedir_dfd, "repo", - OSTREE_REPO_MODE_BARE_USER, NULL, - cancellable, error); - if (!cache_repo) - return FALSE; -#endif - DnfContext *dnfctx = rpmostree_context_get_dnf (self->corectx); if (opt_proxy) dnf_context_set_http_proxy (dnfctx, opt_proxy); @@ -347,8 +343,10 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, GFile *contextdir = self->treefile_context_dirs->pdata[0]; dnf_context_set_repo_dir (dnfctx, gs_file_get_path_cached (contextdir)); - /* By default, retain packages in addition to metadata with --cachedir */ - if (opt_cachedir) + /* By default, retain packages in addition to metadata with --cachedir, unless + * we're doing unified core, in which case the pkgcache repo is the cache. + */ + if (opt_cachedir && !opt_ex_unified_core) dnf_context_set_keep_cache (dnfctx, TRUE); /* For compose, always try to refresh metadata; we're used in build servers * where fetching should be cheap. Otherwise, if --cache-only is set, it's @@ -412,6 +410,35 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, return FALSE; } + /* For unified core, we have a pkgcache repo. This may be auto-created under + * the workdir, or live explicitly in the dir for --cache. + */ + glnx_autofd int host_rootfs_dfd = -1; + if (opt_ex_unified_core) + { + int pkgcache_dfd = self->cachedir_dfd != -1 ? self->cachedir_dfd : self->workdir_dfd; + self->pkgcache_repo = ostree_repo_create_at (pkgcache_dfd, "pkgcache-repo", + OSTREE_REPO_MODE_BARE_USER, NULL, + cancellable, error); + if (!self->pkgcache_repo) + return FALSE; + rpmostree_context_set_repos (self->corectx, self->repo, self->pkgcache_repo); + self->devino_cache = ostree_repo_devino_cache_new (); + + /* Ensure that the imported packages are labeled with *a* policy if + * possible, even if it's not the final one. This helps avoid duplicating + * all of the content. + */ + if (!glnx_opendirat (AT_FDCWD, "/", TRUE, &host_rootfs_dfd, error)) + return FALSE; + g_autoptr(OstreeSePolicy) sepolicy = ostree_sepolicy_new_at (host_rootfs_dfd, cancellable, error); + if (!sepolicy) + return FALSE; + if (ostree_sepolicy_get_name (sepolicy) == NULL) + return glnx_throw (error, "Unable to load SELinux policy from /"); + rpmostree_context_set_sepolicy (self->corectx, sepolicy); + } + if (!rpmostree_context_prepare (self->corectx, cancellable, error)) return FALSE; @@ -475,52 +502,77 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, if (generate_from_previous) { - if (!rpmostree_generate_passwd_from_previous (self->repo, rootfs_dfd, + const char *dest = opt_ex_unified_core ? "usr/etc/" : "etc/"; + if (!rpmostree_generate_passwd_from_previous (self->repo, rootfs_dfd, dest, treefile_dirpath, self->previous_root, treedata, cancellable, error)) return FALSE; } - /* Before we install packages, drop a file to suppress the kernel.rpm dracut run. - * */ - const char *kernel_installd_path = "usr/lib/kernel/install.d"; - if (!glnx_shutil_mkdir_p_at (rootfs_dfd, kernel_installd_path, 0755, cancellable, error)) - return FALSE; - const char skip_kernel_install_data[] = "#!/usr/bin/sh\nexit 77\n"; - const char *kernel_skip_path = glnx_strjoina (kernel_installd_path, "/00-rpmostree-skip.install"); - if (!glnx_file_replace_contents_with_perms_at (rootfs_dfd, kernel_skip_path, - (guint8*)skip_kernel_install_data, - strlen (skip_kernel_install_data), - 0755, 0, 0, - GLNX_FILE_REPLACE_NODATASYNC, - cancellable, error)) - return FALSE; + if (opt_ex_unified_core) + { + if (!rpmostree_context_import (self->corectx, cancellable, error)) + return FALSE; + if (!rpmostree_context_relabel (self->corectx, cancellable, error)) + return FALSE; + rpmostree_context_set_tmprootfs_dfd (self->corectx, rootfs_dfd); + if (!rpmostree_context_assemble (self->corectx, cancellable, error)) + return FALSE; - /* Now actually run through librpm to install the packages. Note this bit - * will be replaced in the future with a unified core: - * https://github.com/projectatomic/rpm-ostree/issues/729 - */ - { g_auto(GLnxConsoleRef) console = { 0, }; - g_autoptr(DnfState) hifstate = dnf_state_new (); + /* Now reload the policy from the tmproot, and relabel the pkgcache - this + * is the same thing done in rpmostree_context_commit(). But here we want + * to ensure our pkgcache labels are accurate, since that will + * be important for the ostree-jigdo work. + */ + g_autoptr(OstreeSePolicy) sepolicy = ostree_sepolicy_new_at (rootfs_dfd, cancellable, error); + rpmostree_context_set_sepolicy (self->corectx, sepolicy); - guint progress_sigid = g_signal_connect (hifstate, "percentage-changed", - G_CALLBACK (on_hifstate_percentage_changed), - "Installing packages:"); + if (!rpmostree_context_relabel (self->corectx, cancellable, error)) + return FALSE; + } + else + { + /* The non-unified core path */ - glnx_console_lock (&console); + /* Before we install packages, drop a file to suppress the kernel.rpm dracut run. + * */ + const char *kernel_installd_path = "usr/lib/kernel/install.d"; + if (!glnx_shutil_mkdir_p_at (rootfs_dfd, kernel_installd_path, 0755, cancellable, error)) + return FALSE; + const char skip_kernel_install_data[] = "#!/usr/bin/sh\nexit 77\n"; + const char *kernel_skip_path = glnx_strjoina (kernel_installd_path, "/00-rpmostree-skip.install"); + if (!glnx_file_replace_contents_with_perms_at (rootfs_dfd, kernel_skip_path, + (guint8*)skip_kernel_install_data, + strlen (skip_kernel_install_data), + 0755, 0, 0, + GLNX_FILE_REPLACE_NODATASYNC, + cancellable, error)) + return FALSE; - if (!libcontainer_prep_dev (rootfs_dfd, error)) - return FALSE; + /* Now actually run through librpm to install the packages. Note this bit + * will be replaced in the future with a unified core: + * https://github.com/projectatomic/rpm-ostree/issues/729 + */ + g_auto(GLnxConsoleRef) console = { 0, }; + g_autoptr(DnfState) hifstate = dnf_state_new (); - if (!dnf_transaction_commit (dnf_context_get_transaction (dnfctx), - dnf_context_get_goal (dnfctx), - hifstate, - error)) - return FALSE; + guint progress_sigid = g_signal_connect (hifstate, "percentage-changed", + G_CALLBACK (on_hifstate_percentage_changed), + "Installing packages:"); - g_signal_handler_disconnect (hifstate, progress_sigid); - } + glnx_console_lock (&console); + + if (!libcontainer_prep_dev (rootfs_dfd, error)) + return FALSE; + + if (!dnf_transaction_commit (dnf_context_get_transaction (dnfctx), + dnf_context_get_goal (dnfctx), + hifstate, error)) + return FALSE; + + g_signal_handler_disconnect (hifstate, progress_sigid); + } if (out_unmodified) *out_unmodified = FALSE; @@ -682,9 +734,32 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr, if (!rpmostree_bwrap_selftest (error)) return FALSE; + self->repo = ostree_repo_open_at (AT_FDCWD, opt_repo, cancellable, error); + if (!self->repo) + return FALSE; + if (opt_workdir_tmpfs) g_print ("note: --workdir-tmpfs is deprecated and will be ignored\n"); - if (!opt_workdir) + if (opt_ex_unified_core) + { + if (opt_workdir) + g_printerr ("note: --workdir is ignored for --ex-unified-core\n"); + + /* For unified core, our workdir must be underneath the repo tmp/ + * in order to use hardlinks. We also really want a bare-user repo. + * We hard require that for now, but down the line we may automatically + * do a pull-local from the bare-user repo to the archive. + */ + if (ostree_repo_get_mode (self->repo) != OSTREE_REPO_MODE_BARE_USER) + return glnx_throw (error, "--ex-unified-core requires a bare-user repository"); + if (!glnx_mkdtempat (ostree_repo_get_dfd (self->repo), "tmp/rpm-ostree-compose.XXXXXX", 0700, + &self->workdir_tmp, error)) + return FALSE; + /* Note special handling of this aliasing in _finalize() */ + self->workdir_dfd = self->workdir_tmp.fd; + + } + else if (!opt_workdir) { if (!glnx_mkdtempat (AT_FDCWD, "/var/tmp/rpm-ostree.XXXXXX", 0700, &self->workdir_tmp, error)) return FALSE; @@ -698,9 +773,6 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr, } self->treefile_context_dirs = g_ptr_array_new_with_free_func ((GDestroyNotify)g_object_unref); - self->repo = ostree_repo_open_at (AT_FDCWD, opt_repo, cancellable, error); - if (!self->repo) - return FALSE; self->treefile_path = g_file_new_for_path (treefile_pathstr); @@ -787,6 +859,14 @@ impl_install_tree (RpmOstreeTreeComposeContext *self, GCancellable *cancellable, GError **error) { + if (getuid () != 0) + { + if (!opt_ex_unified_core) + return glnx_throw (error, "This command requires root privileges"); + g_printerr ("NOTICE: Running this command as non-root is currently known not to work completely.\n"); + g_printerr ("NOTICE: Proceeding anyways.\n"); + } + /* FIXME - is this still necessary? */ if (fchdir (self->workdir_dfd) != 0) return glnx_throw_errno_prefix (error, "fchdir"); @@ -954,7 +1034,8 @@ impl_install_tree (RpmOstreeTreeComposeContext *self, /* Start postprocessing */ if (!rpmostree_treefile_postprocessing (self->rootfs_dfd, self->treefile_context_dirs->pdata[0], self->serialized_treefile, self->treefile, - next_version, cancellable, error)) + next_version, opt_ex_unified_core, + cancellable, error)) return glnx_prefix_error (error, "Postprocessing"); /* Until here, we targeted "rootfs.tmp" in the working directory. Most @@ -1031,7 +1112,7 @@ impl_commit_tree (RpmOstreeTreeComposeContext *self, if (!rpmostree_rootfs_postprocess_common (self->rootfs_dfd, cancellable, error)) return EXIT_FAILURE; - if (!rpmostree_postprocess_final (self->rootfs_dfd, self->treefile, + if (!rpmostree_postprocess_final (self->rootfs_dfd, self->treefile, opt_ex_unified_core, cancellable, error)) return EXIT_FAILURE; @@ -1164,7 +1245,7 @@ rpmostree_compose_builtin_postprocess (int argc, return EXIT_FAILURE; if (!rpmostree_rootfs_postprocess_common (rootfs_dfd, cancellable, error)) return EXIT_FAILURE; - if (!rpmostree_postprocess_final (rootfs_dfd, treefile, + if (!rpmostree_postprocess_final (rootfs_dfd, treefile, opt_ex_unified_core, cancellable, error)) return EXIT_FAILURE; return EXIT_SUCCESS; diff --git a/src/libpriv/rpmostree-passwd-util.c b/src/libpriv/rpmostree-passwd-util.c index e7f80b28af..fd10491986 100644 --- a/src/libpriv/rpmostree-passwd-util.c +++ b/src/libpriv/rpmostree-passwd-util.c @@ -873,6 +873,7 @@ concat_passwd_file (int rootfs_fd, static gboolean _data_from_json (int rootfs_dfd, + const char *dest, GFile *treefile_dirpath, JsonObject *treedata, RpmOstreePasswdMigrateKind kind, @@ -921,7 +922,7 @@ _data_from_json (int rootfs_dfd, *out_found = TRUE; const char *filebasename = passwd ? "passwd" : "group"; - const char *target_etc_filename = glnx_strjoina ("etc/", filebasename); + const char *target_etc_filename = glnx_strjoina (dest, filebasename); g_autoptr(FILE) dest_stream = open_file_stream_write_at (rootfs_dfd, target_etc_filename, "w", error); if (!dest_stream) return FALSE; @@ -937,6 +938,7 @@ _data_from_json (int rootfs_dfd, gboolean rpmostree_generate_passwd_from_previous (OstreeRepo *repo, int rootfs_dfd, + const char *dest, GFile *treefile_dirpath, GFile *previous_root, JsonObject *treedata, @@ -952,10 +954,10 @@ rpmostree_generate_passwd_from_previous (OstreeRepo *repo, * is really hard because filesystem depends on setup which installs * the files... */ - if (!glnx_ensure_dir (rootfs_dfd, "etc", 0755, error)) + if (!glnx_shutil_mkdir_p_at (rootfs_dfd, dest, 0755, cancellable, error)) return FALSE; - if (!_data_from_json (rootfs_dfd, treefile_dirpath, + if (!_data_from_json (rootfs_dfd, dest, treefile_dirpath, treedata, RPM_OSTREE_PASSWD_MIGRATE_PASSWD, &found_passwd_data, cancellable, error)) return FALSE; @@ -969,7 +971,7 @@ rpmostree_generate_passwd_from_previous (OstreeRepo *repo, cancellable, error)) return FALSE; - if (!_data_from_json (rootfs_dfd, treefile_dirpath, + if (!_data_from_json (rootfs_dfd, dest, treefile_dirpath, treedata, RPM_OSTREE_PASSWD_MIGRATE_GROUP, &found_groups_data, cancellable, error)) return FALSE; diff --git a/src/libpriv/rpmostree-passwd-util.h b/src/libpriv/rpmostree-passwd-util.h index bcca7c4c6e..39c100cf5b 100644 --- a/src/libpriv/rpmostree-passwd-util.h +++ b/src/libpriv/rpmostree-passwd-util.h @@ -57,6 +57,7 @@ rpmostree_passwd_migrate_except_root (int rootfs_dfd, gboolean rpmostree_generate_passwd_from_previous (OstreeRepo *repo, int rootfs_dfd, + const char *dest, GFile *treefile_dirpath, GFile *previous_root, JsonObject *treedata, @@ -80,7 +81,7 @@ rpmostree_passwd_cleanup (int rootfs_dfd, GCancellable *cancellable, GError **er gboolean rpmostree_passwd_prepare_rpm_layering (int rootfs_dfd, const char *merge_passwd_dir, - gboolean *out_have_passwd, + gboolean *out_have_usrlib_passwd, GCancellable *cancellable, GError **error); diff --git a/src/libpriv/rpmostree-postprocess.c b/src/libpriv/rpmostree-postprocess.c index bc47b1d554..03a22173f3 100644 --- a/src/libpriv/rpmostree-postprocess.c +++ b/src/libpriv/rpmostree-postprocess.c @@ -51,13 +51,14 @@ typedef enum { RPMOSTREE_POSTPROCESS_BOOT_LOCATION_NEW } RpmOstreePostprocessBootLocation; -/* This bwrap case is for treecompose which isn't yet operating on - * hardlinks, so we just bind mount things mutably. +/* The "unified_core_mode" flag controls whether or not we use rofiles-fuse, + * just like pkg layering. */ static gboolean run_bwrap_mutably (int rootfs_fd, const char *binpath, char **child_argv, + gboolean unified_core_mode, GCancellable *cancellable, GError **error) { @@ -74,7 +75,10 @@ run_bwrap_mutably (int rootfs_fd, etc_bind = "etc"; g_autoptr(RpmOstreeBwrap) bwrap = - rpmostree_bwrap_new (rootfs_fd, RPMOSTREE_BWRAP_MUTATE_FREELY, error, + rpmostree_bwrap_new (rootfs_fd, + unified_core_mode ? RPMOSTREE_BWRAP_MUTATE_ROFILES : + RPMOSTREE_BWRAP_MUTATE_FREELY, + error, "--bind", "var", "/var", "--bind", etc_bind, "/etc", NULL); @@ -253,6 +257,7 @@ hardlink_recurse (int src_dfd, static gboolean process_kernel_and_initramfs (int rootfs_dfd, JsonObject *treefile, + gboolean unified_core_mode, GCancellable *cancellable, GError **error) { @@ -329,7 +334,7 @@ process_kernel_and_initramfs (int rootfs_dfd, */ { char *child_argv[] = { "depmod", (char*)kver, NULL }; - if (!run_bwrap_mutably (rootfs_dfd, "depmod", child_argv, cancellable, error)) + if (!run_bwrap_mutably (rootfs_dfd, "depmod", child_argv, unified_core_mode, cancellable, error)) return FALSE; } @@ -891,6 +896,7 @@ postprocess_selinux_policy_store_location (int rootfs_dfd, gboolean rpmostree_postprocess_final (int rootfs_dfd, JsonObject *treefile, + gboolean unified_core_mode, GCancellable *cancellable, GError **error) { @@ -984,7 +990,7 @@ rpmostree_postprocess_final (int rootfs_dfd, if (!glnx_shutil_rm_rf_at (rootfs_dfd, "boot/loader", cancellable, error)) return FALSE; - if (!process_kernel_and_initramfs (rootfs_dfd, treefile, + if (!process_kernel_and_initramfs (rootfs_dfd, treefile, unified_core_mode, cancellable, error)) return glnx_prefix_error (error, "During kernel processing"); } @@ -1377,6 +1383,7 @@ rpmostree_treefile_postprocessing (int rootfs_fd, GBytes *serialized_treefile, JsonObject *treefile, const char *next_version, + gboolean unified_core_mode, GCancellable *cancellable, GError **error) { @@ -1632,7 +1639,7 @@ rpmostree_treefile_postprocessing (int rootfs_fd, { char *child_argv[] = { binpath, NULL }; - if (!run_bwrap_mutably (rootfs_fd, binpath, child_argv, cancellable, error)) + if (!run_bwrap_mutably (rootfs_fd, binpath, child_argv, unified_core_mode, cancellable, error)) return glnx_prefix_error (error, "While executing postprocessing script '%s'", bn); } diff --git a/src/libpriv/rpmostree-postprocess.h b/src/libpriv/rpmostree-postprocess.h index 45de3f23bf..e6fb55dcee 100644 --- a/src/libpriv/rpmostree-postprocess.h +++ b/src/libpriv/rpmostree-postprocess.h @@ -34,6 +34,7 @@ rpmostree_treefile_postprocessing (int rootfs_fd, GBytes *serialized_treefile, JsonObject *treefile, const char *next_version, + gboolean unified_core_mode, GCancellable *cancellable, GError **error); @@ -73,6 +74,7 @@ rpmostree_prepare_rootfs_for_commit (int src_rootfs_dfd, gboolean rpmostree_postprocess_final (int rootfs_dfd, JsonObject *treefile, + gboolean unified_core_mode, GCancellable *cancellable, GError **error); diff --git a/src/libpriv/rpmostree-script-gperf.gperf b/src/libpriv/rpmostree-script-gperf.gperf index 07b82fdec5..a2b957f05c 100644 --- a/src/libpriv/rpmostree-script-gperf.gperf +++ b/src/libpriv/rpmostree-script-gperf.gperf @@ -13,6 +13,9 @@ struct RpmOstreePackageScriptHandler; %includes %% glibc.prein, RPMOSTREE_SCRIPT_ACTION_IGNORE +# We take over depmod/dracut etc. It's `kernel` in C7 and kernel-core in F25+ +kernel.posttrans, RPMOSTREE_SCRIPT_ACTION_IGNORE +kernel-core.posttrans, RPMOSTREE_SCRIPT_ACTION_IGNORE # Legacy workaround glibc-headers.prein, RPMOSTREE_SCRIPT_ACTION_IGNORE # workaround for old bug? diff --git a/tests/compose-tests/libbasic-test.sh b/tests/compose-tests/libbasic-test.sh new file mode 100644 index 0000000000..051c70133b --- /dev/null +++ b/tests/compose-tests/libbasic-test.sh @@ -0,0 +1,41 @@ +# This used to live in test-basic.sh, but it's now shared with test-basic-unified.sh +basic_test() { +ostree --repo=${repobuild} ls -R ${treeref} /usr/lib/ostree-boot > bootls.txt +if ostree --repo=${repobuild} ls -R ${treeref} /usr/etc/passwd-; then + assert_not_reached "Found /usr/etc/passwd- backup file in tree" +fi +echo "ok compose" + +ostree --repo=${repobuild} show --print-metadata-key exampleos.gitrepo ${treeref} > meta.txt +assert_file_has_content meta.txt 'rev.*97ec21c614689e533d294cdae464df607b526ab9' +assert_file_has_content meta.txt 'src.*https://gitlab.com/exampleos/custom-atomic-host' +ostree --repo=${repobuild} show --print-metadata-key exampleos.tests ${treeref} > meta.txt +assert_file_has_content meta.txt 'smoketested.*e2e' +ostree --repo=${repobuild} show --print-metadata-key rpmostree.rpmmd-repos ${treeref} > meta.txt +assert_file_has_content meta.txt 'id.*fedora.*timestamp' +echo "ok metadata" + +for path in /boot /usr/lib/ostree-boot; do + ostree --repo=${repobuild} ls -R ${treeref} ${path} > bootls.txt + assert_file_has_content bootls.txt vmlinuz- + assert_file_has_content bootls.txt initramfs- + echo "ok boot files" +done +kver=$(grep /vmlinuz bootls.txt | sed -e 's,.*/vmlinuz-\(.*\)-[0-9a-e].*$,\1,') +ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/{vmlinuz,initramfs.img} >/dev/null + +ostree --repo=${repobuild} ls -R ${treeref} /usr/share/man > manpages.txt +assert_file_has_content manpages.txt man5/ostree.repo.5 +echo "ok manpages" + +# https://github.com/projectatomic/rpm-ostree/issues/669 +ostree --repo=${repobuild} ls ${treeref} /tmp > ls.txt +assert_file_has_content ls.txt 'l00777 0 0 0 /tmp -> sysroot/tmp' +echo "ok /tmp" + +ostree --repo=${repobuild} ls ${treeref} /usr/share/rpm > ls.txt +assert_not_file_has_content ls.txt '__db' 'lock' +ostree --repo=${repobuild} ls -R ${treeref} /usr/etc/selinux > ls.txt +assert_not_file_has_content ls.txt 'LOCK' +echo "ok no leftover files" +} diff --git a/tests/compose-tests/test-basic-unified.sh b/tests/compose-tests/test-basic-unified.sh new file mode 100755 index 0000000000..1a515459b9 --- /dev/null +++ b/tests/compose-tests/test-basic-unified.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +set -xeuo pipefail + +dn=$(cd $(dirname $0) && pwd) +. ${dn}/libcomposetest.sh + +prepare_compose_test "basic" +# Test metadata json with objects, arrays, numbers +cat > metadata.json < autovar.txt +# Picked this one at random as an example of something that won't likely be +# converted to tmpfiles.d upstream. But if it is, we can change this test. +assert_file_has_content_literal autovar.txt 'd /var/cache 0755 root root - -' +ostree --repo=${repobuild} cat ${treeref} /usr/lib/tmpfiles.d/pkg-chrony.conf > autovar.txt +# And this one has a non-root uid +assert_file_has_content_literal autovar.txt 'd /var/log/chrony 0755 chrony chrony - -' +echo "ok autovar" diff --git a/tests/compose-tests/test-basic.sh b/tests/compose-tests/test-basic.sh index bb2f27a3a6..15d11edda4 100755 --- a/tests/compose-tests/test-basic.sh +++ b/tests/compose-tests/test-basic.sh @@ -17,45 +17,11 @@ cat > metadata.json < bootls.txt -if ostree --repo=${repobuild} ls -R ${treeref} /usr/etc/passwd-; then - assert_not_reached "Found /usr/etc/passwd- backup file in tree" -fi -echo "ok compose" -ostree --repo=${repobuild} show --print-metadata-key exampleos.gitrepo ${treeref} > meta.txt -assert_file_has_content meta.txt 'rev.*97ec21c614689e533d294cdae464df607b526ab9' -assert_file_has_content meta.txt 'src.*https://gitlab.com/exampleos/custom-atomic-host' -ostree --repo=${repobuild} show --print-metadata-key exampleos.tests ${treeref} > meta.txt -assert_file_has_content meta.txt 'smoketested.*e2e' -ostree --repo=${repobuild} show --print-metadata-key rpmostree.rpmmd-repos ${treeref} > meta.txt -assert_file_has_content meta.txt 'id.*fedora.*timestamp' -echo "ok metadata" - -for path in /boot /usr/lib/ostree-boot; do - ostree --repo=${repobuild} ls -R ${treeref} ${path} > bootls.txt - assert_file_has_content bootls.txt vmlinuz- - assert_file_has_content bootls.txt initramfs- - echo "ok boot files" -done -kver=$(grep /vmlinuz bootls.txt | sed -e 's,.*/vmlinuz-\(.*\)-[0-9a-e].*$,\1,') -ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/{vmlinuz,initramfs.img} >/dev/null - -ostree --repo=${repobuild} ls -R ${treeref} /usr/share/man > manpages.txt -assert_file_has_content manpages.txt man5/ostree.repo.5 -echo "ok manpages" - -# https://github.com/projectatomic/rpm-ostree/issues/669 -ostree --repo=${repobuild} ls ${treeref} /tmp > ls.txt -assert_file_has_content ls.txt 'l00777 0 0 0 /tmp -> sysroot/tmp' -echo "ok /tmp" - -ostree --repo=${repobuild} ls ${treeref} /usr/share/rpm > ls.txt -assert_not_file_has_content ls.txt '__db' 'lock' -ostree --repo=${repobuild} ls -R ${treeref} /usr/etc/selinux > ls.txt -assert_not_file_has_content ls.txt 'LOCK' -echo "ok no leftover files" +. ${dn}/libbasic-test.sh +basic_test +# This one is done by postprocessing /var ostree --repo=${repobuild} cat ${treeref} /usr/lib/tmpfiles.d/rpm-ostree-1-autovar.conf > autovar.txt # Picked this one at random as an example of something that won't likely be # converted to tmpfiles.d upstream. But if it is, we can change this test. From af03a024e7789ab0c4f015ed0a62e378c1bb907e Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Thu, 16 Nov 2017 17:14:45 -0500 Subject: [PATCH 5/8] fixup! compose: Add --ex-unified-core --- src/app/rpmostree-compose-builtin-tree.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/app/rpmostree-compose-builtin-tree.c b/src/app/rpmostree-compose-builtin-tree.c index 45c15adba8..cf2b7e0ba2 100644 --- a/src/app/rpmostree-compose-builtin-tree.c +++ b/src/app/rpmostree-compose-builtin-tree.c @@ -514,8 +514,6 @@ install_packages_in_root (RpmOstreeTreeComposeContext *self, { if (!rpmostree_context_import (self->corectx, cancellable, error)) return FALSE; - if (!rpmostree_context_relabel (self->corectx, cancellable, error)) - return FALSE; rpmostree_context_set_tmprootfs_dfd (self->corectx, rootfs_dfd); if (!rpmostree_context_assemble (self->corectx, cancellable, error)) return FALSE; From 281ea4672892deeeb650cb731f7f886d0d5e86e3 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Nov 2017 10:11:29 -0500 Subject: [PATCH 6/8] fixup! compose: Add --ex-unified-core --- tests/compose-tests/test-basic-unified.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/compose-tests/test-basic-unified.sh b/tests/compose-tests/test-basic-unified.sh index 1a515459b9..7a88d1c95d 100755 --- a/tests/compose-tests/test-basic-unified.sh +++ b/tests/compose-tests/test-basic-unified.sh @@ -5,7 +5,7 @@ set -xeuo pipefail dn=$(cd $(dirname $0) && pwd) . ${dn}/libcomposetest.sh -prepare_compose_test "basic" +prepare_compose_test "basic-unified" # Test metadata json with objects, arrays, numbers cat > metadata.json < Date: Fri, 17 Nov 2017 12:31:36 -0500 Subject: [PATCH 7/8] fixup! compose: Add --ex-unified-core --- tests/compose-tests/libbasic-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/compose-tests/libbasic-test.sh b/tests/compose-tests/libbasic-test.sh index 051c70133b..5497b6d7e0 100644 --- a/tests/compose-tests/libbasic-test.sh +++ b/tests/compose-tests/libbasic-test.sh @@ -21,7 +21,8 @@ for path in /boot /usr/lib/ostree-boot; do assert_file_has_content bootls.txt initramfs- echo "ok boot files" done -kver=$(grep /vmlinuz bootls.txt | sed -e 's,.*/vmlinuz-\(.*\)-[0-9a-e].*$,\1,') +vmlinuz_line=$(grep -o '/vmlinuz.*$') +kver=$(echo ${vmlinuz_line} | sed -e 's,^/,,' -e 's,-[0-9a-f]*$,,') ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/{vmlinuz,initramfs.img} >/dev/null ostree --repo=${repobuild} ls -R ${treeref} /usr/share/man > manpages.txt From cb761dc8b2ac39ed0ad82d148627029cb3112160 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 17 Nov 2017 12:44:44 -0500 Subject: [PATCH 8/8] fixup! compose: Add --ex-unified-core --- tests/compose-tests/libbasic-test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/compose-tests/libbasic-test.sh b/tests/compose-tests/libbasic-test.sh index 5497b6d7e0..61d3250361 100644 --- a/tests/compose-tests/libbasic-test.sh +++ b/tests/compose-tests/libbasic-test.sh @@ -21,8 +21,8 @@ for path in /boot /usr/lib/ostree-boot; do assert_file_has_content bootls.txt initramfs- echo "ok boot files" done -vmlinuz_line=$(grep -o '/vmlinuz.*$') -kver=$(echo ${vmlinuz_line} | sed -e 's,^/,,' -e 's,-[0-9a-f]*$,,') +vmlinuz_line=$(grep -o '/vmlinuz.*$' bootls.txt) +kver=$(echo ${vmlinuz_line} | sed -e 's,^/vmlinuz-,,' -e 's,-[0-9a-f]*$,,') ostree --repo=${repobuild} ls ${treeref} /usr/lib/modules/${kver}/{vmlinuz,initramfs.img} >/dev/null ostree --repo=${repobuild} ls -R ${treeref} /usr/share/man > manpages.txt