diff --git a/src/daemon/rpmostree-sysroot-core.c b/src/daemon/rpmostree-sysroot-core.c index 6bdf15e170..57316f141e 100644 --- a/src/daemon/rpmostree-sysroot-core.c +++ b/src/daemon/rpmostree-sysroot-core.c @@ -181,10 +181,6 @@ clean_pkgcache_orphans (OstreeSysroot *sysroot, g_autoptr(GHashTable) referenced_pkgs = /* cache refs of packages we want to keep */ g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL); - g_autoptr(OstreeRepo) pkgcache_repo = NULL; - if (!rpmostree_syscore_get_pkgcache_repo (repo, &pkgcache_repo, cancellable, error)) - return FALSE; - g_autoptr(GPtrArray) deployments = ostree_sysroot_get_deployments (sysroot); for (guint i = 0; i < deployments->len; i++) { @@ -220,7 +216,7 @@ clean_pkgcache_orphans (OstreeSysroot *sysroot, GLNX_HASH_TABLE_FOREACH (local_replace, const char*, nevra) { g_autofree char *cachebranch = NULL; - if (!rpmostree_find_cache_branch_by_nevra (pkgcache_repo, nevra, &cachebranch, + if (!rpmostree_find_cache_branch_by_nevra (repo, nevra, &cachebranch, cancellable, error)) return FALSE; @@ -229,7 +225,7 @@ clean_pkgcache_orphans (OstreeSysroot *sysroot, } g_autoptr(GHashTable) current_refs = NULL; - if (!ostree_repo_list_refs_ext (pkgcache_repo, "rpmostree/pkg", ¤t_refs, + if (!ostree_repo_list_refs_ext (repo, "rpmostree/pkg", ¤t_refs, OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error)) return FALSE; @@ -239,15 +235,17 @@ clean_pkgcache_orphans (OstreeSysroot *sysroot, if (g_hash_table_contains (referenced_pkgs, ref)) continue; - if (!ostree_repo_set_ref_immediate (pkgcache_repo, NULL, ref, NULL, + if (!ostree_repo_set_ref_immediate (repo, NULL, ref, NULL, cancellable, error)) return FALSE; n_freed++; } + /* note that we're called right after an ostree_sysroot_cleanup(), so the stats reported + * accurately reflect pkgcache branches only */ guint64 freed_space; gint n_objects_total, n_objects_pruned; - if (!ostree_repo_prune (pkgcache_repo, OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, 0, + if (!ostree_repo_prune (repo, OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, 0, &n_objects_total, &n_objects_pruned, &freed_space, cancellable, error)) return FALSE; diff --git a/src/daemon/rpmostree-sysroot-upgrader.c b/src/daemon/rpmostree-sysroot-upgrader.c index c76385d5fc..3e923f967a 100644 --- a/src/daemon/rpmostree-sysroot-upgrader.c +++ b/src/daemon/rpmostree-sysroot-upgrader.c @@ -612,15 +612,12 @@ finalize_replacement_overrides (RpmOstreeSysrootUpgrader *self, g_ptr_array_new_with_free_func (g_free); g_autoptr(GPtrArray) inactive_replacements = g_ptr_array_new (); - g_autoptr(OstreeRepo) pkgcache_repo = NULL; - if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error)) - return FALSE; GLNX_HASH_TABLE_FOREACH_KV (local_replacements, const char*, nevra, const char*, sha256) { /* use the pkgcache because there's no safe way to go from nevra --> pkgname */ g_autofree char *pkgname = NULL; - if (!rpmostree_get_nevra_from_pkgcache (pkgcache_repo, nevra, &pkgname, NULL, NULL, + if (!rpmostree_get_nevra_from_pkgcache (self->repo, nevra, &pkgname, NULL, NULL, NULL, NULL, cancellable, error)) return FALSE; @@ -687,17 +684,13 @@ finalize_overlays (RpmOstreeSysrootUpgrader *self, if (!initialize_metatmpdir (self, error)) return FALSE; - g_autoptr(OstreeRepo) pkgcache_repo = NULL; - if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error)) - return FALSE; - GLNX_HASH_TABLE_FOREACH_KV (local_pkgs, const char*, nevra, const char*, sha256) { g_autoptr(GVariant) header = NULL; g_autofree char *path = g_strdup_printf ("%s/%s.rpm", self->metatmpdir.path, nevra); - if (!rpmostree_pkgcache_find_pkg_header (pkgcache_repo, nevra, sha256, + if (!rpmostree_pkgcache_find_pkg_header (self->repo, nevra, sha256, &header, cancellable, error)) return FALSE; @@ -802,7 +795,7 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self, GError **error) { g_assert (!self->ctx); - self->ctx = rpmostree_context_new_system (cancellable, error); + self->ctx = rpmostree_context_new_system (self->repo, cancellable, error); g_autofree char *tmprootfs_abspath = glnx_fdrel_abspath (self->tmprootfs_dfd, "."); if (!prepare_context_for_assembly (self, tmprootfs_abspath, cancellable, error)) @@ -821,12 +814,6 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self, cancellable, error)) return FALSE; - g_autoptr(OstreeRepo) pkgcache_repo = NULL; - if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error)) - return FALSE; - - rpmostree_context_set_repos (self->ctx, self->repo, pkgcache_repo); - const gboolean have_packages = (self->overlay_packages->len > 0 || g_hash_table_size (local_pkgs) > 0 || self->override_remove_packages->len > 0 || diff --git a/src/daemon/rpmostreed-sysroot.c b/src/daemon/rpmostreed-sysroot.c index d490f979b7..c476608b0b 100644 --- a/src/daemon/rpmostreed-sysroot.c +++ b/src/daemon/rpmostreed-sysroot.c @@ -727,6 +727,11 @@ rpmostreed_sysroot_populate (RpmostreedSysroot *self, if (!ostree_sysroot_get_repo (self->ot_sysroot, &self->repo, cancellable, error)) return FALSE; + /* Migrate legacy pkgcache repo into system repo. After the first time, this boils down to + * one stat() call. */ + if (!rpmostree_migrate_pkgcache_repo (self->repo, cancellable, error)) + return FALSE; + if (!sysroot_populate_deployments_unlocked (self, NULL, error)) return FALSE; diff --git a/src/daemon/rpmostreed-transaction-types.c b/src/daemon/rpmostreed-transaction-types.c index 44e5c67a8b..a7b7e1dda5 100644 --- a/src/daemon/rpmostreed-transaction-types.c +++ b/src/daemon/rpmostreed-transaction-types.c @@ -467,24 +467,17 @@ deploy_transaction_finalize (GObject *object) } static gboolean -import_local_rpm (OstreeRepo *parent, +import_local_rpm (OstreeRepo *repo, int fd, char **sha256_nevra, GCancellable *cancellable, GError **error) { - g_autoptr(OstreeRepo) pkgcache_repo = NULL; - - /* It might seem risky to rely on the cache as the source of truth for local - * RPMs. However, the core will never re-import the same NEVRA if it's already - * present. To be safe, we do also record the SHA-256 of the RPM header in the - * origin. We don't record the checksum of the branch itself, because it may - * need relabeling and that's OK. + /* Note that we record the SHA-256 of the RPM header in the origin to make sure that e.g. + * if we somehow re-import the same NEVRA with different content, we error out. We don't + * record the checksum of the branch itself, because it may need relabeling and that's OK. * */ - if (!rpmostree_syscore_get_pkgcache_repo (parent, &pkgcache_repo, cancellable, error)) - return FALSE; - /* let's just use the current sepolicy -- we'll just relabel it if the new * base turns out to have a different one */ glnx_autofd int rootfs_dfd = -1; @@ -498,8 +491,7 @@ import_local_rpm (OstreeRepo *parent, if (unpacker == NULL) return FALSE; - if (!rpmostree_importer_run (unpacker, pkgcache_repo, policy, - NULL, cancellable, error)) + if (!rpmostree_importer_run (unpacker, repo, policy, NULL, cancellable, error)) return FALSE; g_autofree char *nevra = rpmostree_importer_get_nevra (unpacker); @@ -510,7 +502,7 @@ import_local_rpm (OstreeRepo *parent, } static gboolean -import_many_local_rpms (OstreeRepo *parent, +import_many_local_rpms (OstreeRepo *repo, GUnixFDList *fdl, GPtrArray **out_pkgs, GCancellable *cancellable, @@ -523,7 +515,7 @@ import_many_local_rpms (OstreeRepo *parent, for (guint i = 0; i < nfds; i++) { g_autofree char *sha256_nevra = NULL; - if (!import_local_rpm (parent, fds[i], &sha256_nevra, cancellable, error)) + if (!import_local_rpm (repo, fds[i], &sha256_nevra, cancellable, error)) return FALSE; g_ptr_array_add (pkgs, g_steal_pointer (&sha256_nevra)); @@ -1348,7 +1340,8 @@ refresh_md_transaction_execute (RpmostreedTransaction *transaction, g_autofree char *origin_deployment_root = g_build_filename (sysroot_path, origin_deployment_dirpath, NULL); - g_autoptr(RpmOstreeContext) ctx = rpmostree_context_new_system (cancellable, error); + OstreeRepo *repo = ostree_sysroot_repo (sysroot); + g_autoptr(RpmOstreeContext) ctx = rpmostree_context_new_system (repo, cancellable, error); /* We could bypass rpmostree_context_setup() here and call dnf_context_setup() ourselves * since we're not actually going to perform any installation. Though it does provide us diff --git a/src/libpriv/rpmostree-core.c b/src/libpriv/rpmostree-core.c index b2613a40ff..a8a9bbbda6 100644 --- a/src/libpriv/rpmostree-core.c +++ b/src/libpriv/rpmostree-core.c @@ -321,10 +321,14 @@ set_rpm_macro_define (const char *key, const char *value) } RpmOstreeContext * -rpmostree_context_new_system (GCancellable *cancellable, +rpmostree_context_new_system (OstreeRepo *repo, + GCancellable *cancellable, GError **error) { + g_return_val_if_fail (repo != NULL, FALSE); + RpmOstreeContext *self = g_object_new (RPMOSTREE_TYPE_CONTEXT, NULL); + self->ostreerepo = g_object_ref (repo); /* We can always be control-c'd at any time; this is new API, * otherwise we keep calling _rpmostree_reset_rpm_sighandlers() in @@ -366,8 +370,7 @@ rpmostree_context_new_tree (int userroot_dfd, GCancellable *cancellable, GError **error) { - g_autoptr(RpmOstreeContext) ret = - rpmostree_context_new_system (cancellable, error); + g_autoptr(RpmOstreeContext) ret = rpmostree_context_new_system (repo, cancellable, error); if (!ret) return NULL; @@ -387,9 +390,6 @@ rpmostree_context_new_tree (int userroot_dfd, dnf_context_set_lock_dir (ret->dnfctx, lockdir); } - /* open user root repo if exists (container path) */ - ret->ostreerepo = repo ? g_object_ref (repo) : NULL; - return g_steal_pointer (&ret); } @@ -927,7 +927,7 @@ checkout_pkg_metadata_by_nevra (RpmOstreeContext *self, { g_autoptr(GVariant) header = NULL; - if (!rpmostree_pkgcache_find_pkg_header (self->pkgcache_repo, nevra, sha256, + if (!rpmostree_pkgcache_find_pkg_header (get_pkgcache_repo (self), nevra, sha256, &header, cancellable, error)) return FALSE; @@ -1630,14 +1630,15 @@ add_remaining_pkgcache_pkgs (RpmOstreeContext *self, GCancellable *cancellable, GError **error) { - g_assert (self->pkgcache_repo); + OstreeRepo *pkgcache_repo = get_pkgcache_repo (self); + g_assert (pkgcache_repo); g_assert (self->pkgcache_only); DnfSack *sack = dnf_context_get_sack (self->dnfctx); g_assert (sack); g_autoptr(GHashTable) refs = NULL; - if (!ostree_repo_list_refs_ext (self->pkgcache_repo, "rpmostree/pkg", &refs, + if (!ostree_repo_list_refs_ext (pkgcache_repo, "rpmostree/pkg", &refs, OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error)) return FALSE; @@ -1648,7 +1649,7 @@ add_remaining_pkgcache_pkgs (RpmOstreeContext *self, continue; g_autoptr(GVariant) header = NULL; - if (!get_header_variant (self->pkgcache_repo, ref, &header, cancellable, error)) + if (!get_header_variant (pkgcache_repo, ref, &header, cancellable, error)) return FALSE; if (!checkout_pkg_metadata (self, nevra, header, cancellable, error)) @@ -2125,6 +2126,10 @@ checkout_package_into_root (RpmOstreeContext *self, { OstreeRepo *pkgcache_repo = get_pkgcache_repo (self); + /* The below is currently TRUE only in the --ex-unified-core path. We probably want to + * migrate that over to always use a separate cache repo eventually, which would allow us + * to completely drop the pkgcache_repo/ostreerepo dichotomy in the core. See: + * https://github.com/projectatomic/rpm-ostree/pull/1055 */ if (pkgcache_repo != self->ostreerepo) { if (!rpmostree_pull_content_only (self->ostreerepo, pkgcache_repo, pkg_commit, diff --git a/src/libpriv/rpmostree-core.h b/src/libpriv/rpmostree-core.h index c874611830..976c349ff1 100644 --- a/src/libpriv/rpmostree-core.h +++ b/src/libpriv/rpmostree-core.h @@ -36,8 +36,9 @@ G_DECLARE_FINAL_TYPE (RpmOstreeContext, rpmostree_context, RPMOSTREE, CONTEXT, G #define RPMOSTREE_TYPE_TREESPEC (rpmostree_treespec_get_type ()) G_DECLARE_FINAL_TYPE (RpmOstreeTreespec, rpmostree_treespec, RPMOSTREE, TREESPEC, GObject) -RpmOstreeContext *rpmostree_context_new_system (GCancellable *cancellable, - GError **error); +RpmOstreeContext *rpmostree_context_new_system (OstreeRepo *repo, + GCancellable *cancellable, + GError **error); RpmOstreeContext *rpmostree_context_new_tree (int basedir_dfd, OstreeRepo *repo, diff --git a/src/libpriv/rpmostree-util.c b/src/libpriv/rpmostree-util.c index 122d5c5c3c..c5befbd52d 100644 --- a/src/libpriv/rpmostree-util.c +++ b/src/libpriv/rpmostree-util.c @@ -25,12 +25,15 @@ #include #include #include +#include #include "rpmostree-util.h" #include "rpmostree-origin.h" #include "rpmostree.h" #include "libglnx.h" +#define RPMOSTREE_OLD_PKGCACHE_DIR "extensions/rpmostree/pkgcache" + int rpmostree_ptrarray_sort_compare_strings (gconstpointer ap, gconstpointer bp) @@ -517,6 +520,79 @@ rpmostree_deployment_get_layered_info (OstreeRepo *repo, return TRUE; } +static gboolean +do_pkgcache_migration (OstreeRepo *repo, + OstreeRepo *pkgcache, + guint *out_n_migrated, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GHashTable) pkgcache_refs = NULL; + if (!ostree_repo_list_refs_ext (pkgcache, "rpmostree/pkg", &pkgcache_refs, + OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error)) + return FALSE; + + /* Always pull in any refs from the pkgcache into the system repo, regardless of whether + * the ref already exists there. This ensures that e.g. booting back into a tree with an + * older version of rpm-ostree and installing local pkgs will make use of those when it's + * time to relayer it again. */ + + if (g_hash_table_size (pkgcache_refs) == 0) + { + *out_n_migrated = 0; + return TRUE; /* Note early return */ + } + + g_autofree char **refs_strv = + (char **)g_hash_table_get_keys_as_array (pkgcache_refs, NULL); + + /* fd-relative pull API anyone? build the path manually at least to avoid one malloc */ + g_autofree char *pkgcache_uri = + g_strdup_printf ("file:///proc/self/fd/%d", ostree_repo_get_dfd (pkgcache)); + if (!ostree_repo_pull (repo, pkgcache_uri, refs_strv, OSTREE_REPO_PULL_FLAGS_NONE, NULL, + cancellable, error)) + return FALSE; + + *out_n_migrated = g_strv_length (refs_strv); + return TRUE; +} + +gboolean +rpmostree_migrate_pkgcache_repo (OstreeRepo *repo, + GCancellable *cancellable, + GError **error) +{ + int repo_dfd = ostree_repo_get_dfd (repo); + + struct stat stbuf; + if (!glnx_fstatat_allow_noent (repo_dfd, RPMOSTREE_OLD_PKGCACHE_DIR, &stbuf, 0, error)) + return FALSE; + + /* if pkgcache exists, we expect it to be valid; we don't want to nuke it just because of + * a transient error */ + if (errno == 0) + { + g_autoptr(OstreeRepo) pkgcache = ostree_repo_open_at (repo_dfd, + RPMOSTREE_OLD_PKGCACHE_DIR, + cancellable, error); + if (!pkgcache) + return FALSE; + + guint n_migrated; + if (!do_pkgcache_migration (repo, pkgcache, &n_migrated, cancellable, error)) + return FALSE; + + if (!glnx_shutil_rm_rf_at (repo_dfd, RPMOSTREE_OLD_PKGCACHE_DIR, cancellable, error)) + return FALSE; + + if (n_migrated > 0) + sd_journal_print (LOG_INFO, "migrated %u cached package%s to system repo", + n_migrated, _NS(n_migrated)); + } + + return TRUE; +} + gboolean rpmostree_decompose_sha256_nevra (const char **nevra, /* gets incremented */ char **out_sha256, diff --git a/src/libpriv/rpmostree-util.h b/src/libpriv/rpmostree-util.h index 1febd2f07c..e3aa120fb7 100644 --- a/src/libpriv/rpmostree-util.h +++ b/src/libpriv/rpmostree-util.h @@ -115,6 +115,11 @@ rpmostree_deployment_get_layered_info (OstreeRepo *repo, GVariant **out_replaced_base_pkgs, GError **error); +gboolean +rpmostree_migrate_pkgcache_repo (OstreeRepo *repo, + GCancellable *cancellable, + GError **error); + gboolean rpmostree_decompose_sha256_nevra (const char **nevra, char **sha256, diff --git a/tests/common/libvm.sh b/tests/common/libvm.sh index b3f5797dab..77075f26df 100644 --- a/tests/common/libvm.sh +++ b/tests/common/libvm.sh @@ -84,7 +84,7 @@ vm_cmdfile() { # Delete anything which we might change between runs vm_clean_caches() { - vm_cmd rm /ostree/repo/extensions/rpmostree/pkgcache/refs/heads/* -rf + vm_cmd rm /ostree/repo/refs/heads/rpmostree/pkg/* -rf } # run rpm-ostree in vm @@ -363,6 +363,8 @@ vm_get_journal_cursor() { } # Wait for a message logged after $cursor matching a regexp to appear +# $1 - cursor +# $2 - regex to wait for vm_wait_content_after_cursor() { from_cursor=$1; shift regex=$1; shift diff --git a/tests/vmcheck/test-layering-basic.sh b/tests/vmcheck/test-layering-basic.sh index f3bf7de9f9..9e91e39d8a 100755 --- a/tests/vmcheck/test-layering-basic.sh +++ b/tests/vmcheck/test-layering-basic.sh @@ -61,10 +61,10 @@ echo "ok failed to install in /opt and /usr/local" vm_build_rpm foo vm_rpmostree pkg-add foo-1.0 -vm_cmd ostree --repo=/sysroot/ostree/repo/extensions/rpmostree/pkgcache refs |grep /foo/> refs.txt +vm_cmd ostree refs |grep /foo/> refs.txt pkgref=$(head -1 refs.txt) # Verify we have a mapping from pkg-in-ostree → rpmmd-repo info -vm_cmd ostree --repo=/sysroot/ostree/repo/extensions/rpmostree/pkgcache show --print-metadata-key rpmostree.repo ${pkgref} >refdata.txt +vm_cmd ostree show --print-metadata-key rpmostree.repo ${pkgref} >refdata.txt assert_file_has_content refdata.txt 'id.*test-repo' assert_file_has_content refdata.txt 'timestamp' rm -f refs.txt refdata.txt @@ -166,3 +166,25 @@ vm_cmd cat /usr/lib/ostree-boot/EFI/efi/fedora/fonts/unicode.pf2 > unicode.txt assert_file_has_content unicode.txt unicode echo "ok failed installed in /boot" +# there should be a couple of pkgs already installed from the tests up above, +# though let's add our own to be self-contained +vm_build_rpm test-pkgcache-migrate-pkg1 +vm_build_rpm test-pkgcache-migrate-pkg2 +vm_rpmostree install test-pkgcache-migrate-pkg{1,2} + +# jury-rig a pkgcache of the olden days +OLD_PKGCACHE_DIR=/ostree/repo/extensions/rpmostree/pkgcache +vm_cmd systemctl stop rpm-ostreed +vm_cmd test ! -d ${OLD_PKGCACHE_DIR} +vm_cmd mkdir -p ${OLD_PKGCACHE_DIR} +vm_cmd ostree init --repo ${OLD_PKGCACHE_DIR} --mode=bare +vm_cmd ostree pull-local --repo ${OLD_PKGCACHE_DIR} /ostree/repo \ + rpmostree/pkg/test-pkgcache-migrate-pkg{1,2}/1.0-1.x86__64 +cursor=$(vm_get_journal_cursor) +vm_cmd systemctl start rpm-ostreed +vm_wait_content_after_cursor $cursor 'migrated 2 cached packages' +for ref in rpmostree/pkg/test-pkgcache-migrate-pkg{1,2}/1.0-1.x86__64; do + vm_cmd ostree show $ref +done +vm_cmd test ! -d ${OLD_PKGCACHE_DIR} +echo "ok migrate pkgcache"