daemon: Unify pkgcache with system repo

We originally needed the pkgcache to be a separate repo due to ostree's
overzealous pruning policies. The idea was to maintain multiple commits
in each pkg branch for different SELinux policies. In practice, there's
not much use in maintaining old copies and it's just easier to always
relabel on the fly. So then, the need for a separate repo completely
melts away.

This helps simplify the mental model a bit and allows us to avoid subtle
issues like coreos#1047. Note however that the core is still capable of
handling split repos for the `--ex-unified-core` compose use case. Once
that and the jigdo work are a bit more settled, we can have a clearer
picture of how to simplify the core further.

The tricky bit is migrating the cache. At daemon startup, we check if a
pkgcache repo exists and migrate its refs if so. This does mean that
rolling back to a tree with an older rpm-ostree and deploying from there
will require e.g. redownloading RPMs and reproviding any local RPMs to
layer. In such situations, we ensure that when eventually re-upgrading
from a newer rpm-ostree, we prioritize those newer refs.

src/daemon/rpmostree-sysroot-core.c

@@ -181,10 +181,6 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
   g_autoptr(GHashTable) referenced_pkgs = /* cache refs of packages we want to keep */
     g_hash_table_new_full (g_str_hash, g_str_equal, g_free, NULL);
 
-  g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-  if (!rpmostree_syscore_get_pkgcache_repo (repo, &pkgcache_repo, cancellable, error))
-    return FALSE;
-
   g_autoptr(GPtrArray) deployments = ostree_sysroot_get_deployments (sysroot);
   for (guint i = 0; i < deployments->len; i++)
     {
@@ -220,7 +216,7 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
       GLNX_HASH_TABLE_FOREACH (local_replace, const char*, nevra)
         {
           g_autofree char *cachebranch = NULL;
-          if (!rpmostree_find_cache_branch_by_nevra (pkgcache_repo, nevra, &cachebranch,
+          if (!rpmostree_find_cache_branch_by_nevra (repo, nevra, &cachebranch,
                                                      cancellable, error))
             return FALSE;
 
@@ -229,7 +225,7 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
     }
 
   g_autoptr(GHashTable) current_refs = NULL;
-  if (!ostree_repo_list_refs_ext (pkgcache_repo, "rpmostree/pkg", &current_refs,
+  if (!ostree_repo_list_refs_ext (repo, "rpmostree/pkg", &current_refs,
                                   OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error))
     return FALSE;
 
@@ -239,15 +235,17 @@ clean_pkgcache_orphans (OstreeSysroot            *sysroot,
       if (g_hash_table_contains (referenced_pkgs, ref))
         continue;
 
-      if (!ostree_repo_set_ref_immediate (pkgcache_repo, NULL, ref, NULL,
+      if (!ostree_repo_set_ref_immediate (repo, NULL, ref, NULL,
                                           cancellable, error))
         return FALSE;
       n_freed++;
     }
 
+  /* note that we're called right after an ostree_sysroot_cleanup(), so the stats reported
+   * accurately reflect pkgcache branches only */
   guint64 freed_space;
   gint n_objects_total, n_objects_pruned;
-  if (!ostree_repo_prune (pkgcache_repo, OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, 0,
+  if (!ostree_repo_prune (repo, OSTREE_REPO_PRUNE_FLAGS_REFS_ONLY, 0,
                           &n_objects_total, &n_objects_pruned, &freed_space,
                           cancellable, error))
     return FALSE;

src/daemon/rpmostree-sysroot-upgrader.c

@@ -612,15 +612,12 @@ finalize_replacement_overrides (RpmOstreeSysrootUpgrader *self,
     g_ptr_array_new_with_free_func (g_free);
 
   g_autoptr(GPtrArray) inactive_replacements = g_ptr_array_new ();
-  g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-  if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error))
-    return FALSE;
 
   GLNX_HASH_TABLE_FOREACH_KV (local_replacements, const char*, nevra, const char*, sha256)
     {
       /* use the pkgcache because there's no safe way to go from nevra --> pkgname */
       g_autofree char *pkgname = NULL;
-      if (!rpmostree_get_nevra_from_pkgcache (pkgcache_repo, nevra, &pkgname, NULL, NULL,
+      if (!rpmostree_get_nevra_from_pkgcache (self->repo, nevra, &pkgname, NULL, NULL,
                                               NULL, NULL, cancellable, error))
         return FALSE;
 
@@ -687,17 +684,13 @@ finalize_overlays (RpmOstreeSysrootUpgrader *self,
       if (!initialize_metatmpdir (self, error))
         return FALSE;
 
-      g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-      if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error))
-        return FALSE;
-
       GLNX_HASH_TABLE_FOREACH_KV (local_pkgs, const char*, nevra, const char*, sha256)
         {
           g_autoptr(GVariant) header = NULL;
           g_autofree char *path =
             g_strdup_printf ("%s/%s.rpm", self->metatmpdir.path, nevra);
 
-          if (!rpmostree_pkgcache_find_pkg_header (pkgcache_repo, nevra, sha256,
+          if (!rpmostree_pkgcache_find_pkg_header (self->repo, nevra, sha256,
                                                    &header, cancellable, error))
             return FALSE;
 
@@ -802,7 +795,7 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self,
                      GError                  **error)
 {
   g_assert (!self->ctx);
-  self->ctx = rpmostree_context_new_system (cancellable, error);
+  self->ctx = rpmostree_context_new_system (self->repo, cancellable, error);
   g_autofree char *tmprootfs_abspath = glnx_fdrel_abspath (self->tmprootfs_dfd, ".");
 
   if (!prepare_context_for_assembly (self, tmprootfs_abspath, cancellable, error))
@@ -821,12 +814,6 @@ prep_local_assembly (RpmOstreeSysrootUpgrader *self,
                                 cancellable, error))
     return FALSE;
 
-  g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-  if (!rpmostree_syscore_get_pkgcache_repo (self->repo, &pkgcache_repo, cancellable, error))
-    return FALSE;
-
-  rpmostree_context_set_repos (self->ctx, self->repo, pkgcache_repo);
-
   const gboolean have_packages = (self->overlay_packages->len > 0 ||
                                   g_hash_table_size (local_pkgs) > 0 ||
                                   self->override_remove_packages->len > 0 ||

src/daemon/rpmostreed-sysroot.c

@@ -727,6 +727,11 @@ rpmostreed_sysroot_populate (RpmostreedSysroot *self,
   if (!ostree_sysroot_get_repo (self->ot_sysroot, &self->repo, cancellable, error))
     return FALSE;
 
+  /* Migrate legacy pkgcache repo into system repo. After the first time, this boils down to
+   * one stat() call. */
+  if (!rpmostree_migrate_pkgcache_repo (self->repo, cancellable, error))
+    return FALSE;
+
   if (!sysroot_populate_deployments_unlocked (self, NULL, error))
     return FALSE;
 

src/daemon/rpmostreed-transaction-types.c

@@ -467,24 +467,17 @@ deploy_transaction_finalize (GObject *object)
 }
 
 static gboolean
-import_local_rpm (OstreeRepo    *parent,
+import_local_rpm (OstreeRepo    *repo,
                   int            fd,
                   char         **sha256_nevra,
                   GCancellable  *cancellable,
                   GError       **error)
 {
-  g_autoptr(OstreeRepo) pkgcache_repo = NULL;
-
-  /* It might seem risky to rely on the cache as the source of truth for local
-   * RPMs. However, the core will never re-import the same NEVRA if it's already
-   * present. To be safe, we do also record the SHA-256 of the RPM header in the
-   * origin. We don't record the checksum of the branch itself, because it may
-   * need relabeling and that's OK.
+  /* Note that we record the SHA-256 of the RPM header in the origin to make sure that e.g.
+   * if we somehow re-import the same NEVRA with different content, we error out. We don't
+   * record the checksum of the branch itself, because it may need relabeling and that's OK.
    * */
 
-  if (!rpmostree_syscore_get_pkgcache_repo (parent, &pkgcache_repo, cancellable, error))
-    return FALSE;
-
   /* let's just use the current sepolicy -- we'll just relabel it if the new
    * base turns out to have a different one */
   glnx_autofd int rootfs_dfd = -1;
@@ -498,8 +491,7 @@ import_local_rpm (OstreeRepo    *parent,
   if (unpacker == NULL)
     return FALSE;
 
-  if (!rpmostree_importer_run (unpacker, pkgcache_repo, policy,
-                               NULL, cancellable, error))
+  if (!rpmostree_importer_run (unpacker, repo, policy, NULL, cancellable, error))
     return FALSE;
 
   g_autofree char *nevra = rpmostree_importer_get_nevra (unpacker);
@@ -510,7 +502,7 @@ import_local_rpm (OstreeRepo    *parent,
 }
 
 static gboolean
-import_many_local_rpms (OstreeRepo    *parent,
+import_many_local_rpms (OstreeRepo    *repo,
                         GUnixFDList   *fdl,
                         GPtrArray    **out_pkgs,
                         GCancellable  *cancellable,
@@ -523,7 +515,7 @@ import_many_local_rpms (OstreeRepo    *parent,
   for (guint i = 0; i < nfds; i++)
     {
       g_autofree char *sha256_nevra = NULL;
-      if (!import_local_rpm (parent, fds[i], &sha256_nevra, cancellable, error))
+      if (!import_local_rpm (repo, fds[i], &sha256_nevra, cancellable, error))
         return FALSE;
 
       g_ptr_array_add (pkgs, g_steal_pointer (&sha256_nevra));
@@ -1348,7 +1340,8 @@ refresh_md_transaction_execute (RpmostreedTransaction *transaction,
   g_autofree char *origin_deployment_root =
     g_build_filename (sysroot_path, origin_deployment_dirpath, NULL);
 
-  g_autoptr(RpmOstreeContext) ctx = rpmostree_context_new_system (cancellable, error);
+  OstreeRepo *repo = ostree_sysroot_repo (sysroot);
+  g_autoptr(RpmOstreeContext) ctx = rpmostree_context_new_system (repo, cancellable, error);
 
   /* We could bypass rpmostree_context_setup() here and call dnf_context_setup() ourselves
    * since we're not actually going to perform any installation. Though it does provide us

src/libpriv/rpmostree-core.c

@@ -321,10 +321,14 @@ set_rpm_macro_define (const char *key, const char *value)
 }
 
 RpmOstreeContext *
-rpmostree_context_new_system (GCancellable *cancellable,
+rpmostree_context_new_system (OstreeRepo   *repo,
+                              GCancellable *cancellable,
                               GError      **error)
 {
+  g_return_val_if_fail (repo != NULL, FALSE);
+
   RpmOstreeContext *self = g_object_new (RPMOSTREE_TYPE_CONTEXT, NULL);
+  self->ostreerepo = g_object_ref (repo);
 
   /* We can always be control-c'd at any time; this is new API,
    * otherwise we keep calling _rpmostree_reset_rpm_sighandlers() in
@@ -366,8 +370,7 @@ rpmostree_context_new_tree (int               userroot_dfd,
                             GCancellable     *cancellable,
                             GError          **error)
 {
-  g_autoptr(RpmOstreeContext) ret =
-    rpmostree_context_new_system (cancellable, error);
+  g_autoptr(RpmOstreeContext) ret = rpmostree_context_new_system (repo, cancellable, error);
   if (!ret)
     return NULL;
 
@@ -387,9 +390,6 @@ rpmostree_context_new_tree (int               userroot_dfd,
     dnf_context_set_lock_dir (ret->dnfctx, lockdir);
   }
 
-  /* open user root repo if exists (container path) */
-  ret->ostreerepo = repo ? g_object_ref (repo) : NULL;
-
   return g_steal_pointer (&ret);
 }
 
@@ -927,7 +927,7 @@ checkout_pkg_metadata_by_nevra (RpmOstreeContext *self,
 {
   g_autoptr(GVariant) header = NULL;
 
-  if (!rpmostree_pkgcache_find_pkg_header (self->pkgcache_repo, nevra, sha256,
+  if (!rpmostree_pkgcache_find_pkg_header (get_pkgcache_repo (self), nevra, sha256,
                                            &header, cancellable, error))
     return FALSE;
 
@@ -1630,14 +1630,15 @@ add_remaining_pkgcache_pkgs (RpmOstreeContext *self,
                              GCancellable     *cancellable,
                              GError          **error)
 {
-  g_assert (self->pkgcache_repo);
+  OstreeRepo *pkgcache_repo = get_pkgcache_repo (self);
+  g_assert (pkgcache_repo);
   g_assert (self->pkgcache_only);
 
   DnfSack *sack = dnf_context_get_sack (self->dnfctx);
   g_assert (sack);
 
   g_autoptr(GHashTable) refs = NULL;
-  if (!ostree_repo_list_refs_ext (self->pkgcache_repo, "rpmostree/pkg", &refs,
+  if (!ostree_repo_list_refs_ext (pkgcache_repo, "rpmostree/pkg", &refs,
                                   OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error))
     return FALSE;
 
@@ -1648,7 +1649,7 @@ add_remaining_pkgcache_pkgs (RpmOstreeContext *self,
         continue;
 
       g_autoptr(GVariant) header = NULL;
-      if (!get_header_variant (self->pkgcache_repo, ref, &header, cancellable, error))
+      if (!get_header_variant (pkgcache_repo, ref, &header, cancellable, error))
         return FALSE;
 
       if (!checkout_pkg_metadata (self, nevra, header, cancellable, error))
@@ -2125,6 +2126,10 @@ checkout_package_into_root (RpmOstreeContext *self,
 {
   OstreeRepo *pkgcache_repo = get_pkgcache_repo (self);
 
+  /* The below is currently TRUE only in the --ex-unified-core path. We probably want to
+   * migrate that over to always use a separate cache repo eventually, which would allow us
+   * to completely drop the pkgcache_repo/ostreerepo dichotomy in the core. See:
+   * https://github.com/projectatomic/rpm-ostree/pull/1055 */
   if (pkgcache_repo != self->ostreerepo)
     {
       if (!rpmostree_pull_content_only (self->ostreerepo, pkgcache_repo, pkg_commit,

src/libpriv/rpmostree-core.h

@@ -36,8 +36,9 @@ G_DECLARE_FINAL_TYPE (RpmOstreeContext, rpmostree_context, RPMOSTREE, CONTEXT, G
 #define RPMOSTREE_TYPE_TREESPEC (rpmostree_treespec_get_type ())
 G_DECLARE_FINAL_TYPE (RpmOstreeTreespec, rpmostree_treespec, RPMOSTREE, TREESPEC, GObject)
 
-RpmOstreeContext *rpmostree_context_new_system (GCancellable *cancellable,
-                                                GError **error);
+RpmOstreeContext *rpmostree_context_new_system (OstreeRepo   *repo,
+                                                GCancellable *cancellable,
+                                                GError      **error);
 
 RpmOstreeContext *rpmostree_context_new_tree (int basedir_dfd,
                                               OstreeRepo  *repo,

src/libpriv/rpmostree-util.c

@@ -25,12 +25,15 @@
 #include <glib-unix.h>
 #include <json-glib/json-glib.h>
 #include <gio/gunixoutputstream.h>
+#include <systemd/sd-journal.h>
 
 #include "rpmostree-util.h"
 #include "rpmostree-origin.h"
 #include "rpmostree.h"
 #include "libglnx.h"
 
+#define RPMOSTREE_OLD_PKGCACHE_DIR "extensions/rpmostree/pkgcache"
+
 int
 rpmostree_ptrarray_sort_compare_strings (gconstpointer ap,
                                          gconstpointer bp)
@@ -517,6 +520,79 @@ rpmostree_deployment_get_layered_info (OstreeRepo        *repo,
   return TRUE;
 }
 
+static gboolean
+do_pkgcache_migration (OstreeRepo    *repo,
+                       OstreeRepo    *pkgcache,
+                       guint         *out_n_migrated,
+                       GCancellable  *cancellable,
+                       GError       **error)
+{
+  g_autoptr(GHashTable) pkgcache_refs = NULL;
+  if (!ostree_repo_list_refs_ext (pkgcache, "rpmostree/pkg", &pkgcache_refs,
+                                  OSTREE_REPO_LIST_REFS_EXT_NONE, cancellable, error))
+    return FALSE;
+
+  /* Always pull in any refs from the pkgcache into the system repo, regardless of whether
+   * the ref already exists there. This ensures that e.g. booting back into a tree with an
+   * older version of rpm-ostree and installing local pkgs will make use of those when it's
+   * time to relayer it again. */
+
+  if (g_hash_table_size (pkgcache_refs) == 0)
+    {
+      *out_n_migrated = 0;
+      return TRUE; /* Note early return */
+    }
+
+  g_autofree char **refs_strv =
+    (char **)g_hash_table_get_keys_as_array (pkgcache_refs, NULL);
+
+  /* fd-relative pull API anyone? build the path manually at least to avoid one malloc */
+  g_autofree char *pkgcache_uri =
+    g_strdup_printf ("file:///proc/self/fd/%d", ostree_repo_get_dfd (pkgcache));
+  if (!ostree_repo_pull (repo, pkgcache_uri, refs_strv, OSTREE_REPO_PULL_FLAGS_NONE, NULL,
+                         cancellable, error))
+    return FALSE;
+
+  *out_n_migrated = g_strv_length (refs_strv);
+  return TRUE;
+}
+
+gboolean
+rpmostree_migrate_pkgcache_repo (OstreeRepo   *repo,
+                                 GCancellable *cancellable,
+                                 GError      **error)
+{
+  int repo_dfd = ostree_repo_get_dfd (repo);
+
+  struct stat stbuf;
+  if (!glnx_fstatat_allow_noent (repo_dfd, RPMOSTREE_OLD_PKGCACHE_DIR, &stbuf, 0, error))
+    return FALSE;
+
+  /* if pkgcache exists, we expect it to be valid; we don't want to nuke it just because of
+   * a transient error */
+  if (errno == 0)
+    {
+      g_autoptr(OstreeRepo) pkgcache = ostree_repo_open_at (repo_dfd,
+                                                            RPMOSTREE_OLD_PKGCACHE_DIR,
+                                                            cancellable, error);
+      if (!pkgcache)
+        return FALSE;
+
+      guint n_migrated;
+      if (!do_pkgcache_migration (repo, pkgcache, &n_migrated, cancellable, error))
+        return FALSE;
+
+      if (!glnx_shutil_rm_rf_at (repo_dfd, RPMOSTREE_OLD_PKGCACHE_DIR, cancellable, error))
+        return FALSE;
+
+      if (n_migrated > 0)
+        sd_journal_print (LOG_INFO, "migrated %u cached package%s to system repo",
+                          n_migrated, _NS(n_migrated));
+    }
+
+  return TRUE;
+}
+
 gboolean
 rpmostree_decompose_sha256_nevra (const char **nevra, /* gets incremented */
                                   char       **out_sha256,

src/libpriv/rpmostree-util.h

@@ -115,6 +115,11 @@ rpmostree_deployment_get_layered_info (OstreeRepo        *repo,
                                        GVariant         **out_replaced_base_pkgs,
                                        GError           **error);
 
+gboolean
+rpmostree_migrate_pkgcache_repo (OstreeRepo   *repo,
+                                 GCancellable *cancellable,
+                                 GError      **error);
+
 gboolean
 rpmostree_decompose_sha256_nevra (const char **nevra,
                                   char       **sha256,