Rework treecompose kernel processing

Prep for changing `boot_location: new` to use `/usr/lib/ostree-boot`
and `/usr/lib/modules`.  Rework our kernel postprocessing
so that we unify the `boot_location` handling with initramfs generation.

Instead of doing the initramfs first in postprocessing, we do it nearly last,
after e.g. `etc` is renamed to `usr/etc`. This has some consequences, such as
the fact that `run_bwrap_mutably()` is now called in both situations. In
general, our handling of `etc` is inconsistent, although understandably so.

As part of this, I finally got around to implementing the bit from
systemd/systemd#4174 however suboptimal it is; need the
unified core so we can cleanly ignore the posttrans like we do others.  We
intentionally keep the file around in the generated tree so that installing a
kernel RPM per client doesn't try to do any of this either.

This all gets folded together so that the logic for handling the bootloader gets
simpler - in the Fedora case, we now know to find kernels in `/usr/lib/modules`
and can ignore `/boot`.

Closes: #959
Approved by: jlebon

docs/manual/treefile.md

@@ -32,7 +32,7 @@ It supports the following parameters:
     possible values:
     * "both": the default, kernel data goes in /boot and /usr/lib/ostree-boot
     * "legacy": Now an alias for "both"; historically meant just "boot"
-    * "new": kernel data goes in /usr/lib/ostree-boot
+    * "new": kernel data goes in /usr/lib/ostree-boot and /usr/lib/modules
 
  * `etc-group-members`: Array of strings, optional: Unix groups in this
    list will be stored in `/etc/group` instead of `/usr/lib/group`.  Use

src/app/rpmostree-compose-builtin-tree.c

@@ -477,6 +477,21 @@ install_packages_in_root (RpmOstreeTreeComposeContext  *self,
         return FALSE;
     }
 
+  /* Before we install packages, drop a file to suppress the kernel.rpm dracut run.
+   * <https://github.com/systemd/systemd/pull/4174> */
+  const char *kernel_installd_path = "usr/lib/kernel/install.d";
+  if (!glnx_shutil_mkdir_p_at (rootfs_dfd, kernel_installd_path, 0755, cancellable, error))
+    return FALSE;
+  const char skip_kernel_install_data[] = "#!/usr/bin/sh\nexit 77\n";
+  const char *kernel_skip_path = glnx_strjoina (kernel_installd_path, "/00-rpmostree-skip.install");
+  if (!glnx_file_replace_contents_with_perms_at (rootfs_dfd, kernel_skip_path,
+                                                 (guint8*)skip_kernel_install_data,
+                                                 strlen (skip_kernel_install_data),
+                                                 0755, 0, 0,
+                                                 GLNX_FILE_REPLACE_NODATASYNC,
+                                                 cancellable, error))
+    return FALSE;
+
   /* Now actually run through librpm to install the packages.  Note this bit
    * will be replaced in the future with a unified core:
    * https://github.com/projectatomic/rpm-ostree/issues/729

src/daemon/rpmostree-sysroot-upgrader.c

@@ -962,7 +962,7 @@ perform_local_assembly (RpmOstreeSysrootUpgrader *self,
         return FALSE;
 
       if (!rpmostree_finalize_kernel (self->tmprootfs_dfd, bootdir, kver, kernel_path,
-                                      &initramfs_tmpf,
+                                      &initramfs_tmpf, RPMOSTREE_FINALIZE_KERNEL_AUTO,
                                       cancellable, error))
         return FALSE;
 

src/libpriv/rpmostree-kernel.c

@@ -38,6 +38,12 @@
 #include "rpmostree-bwrap.h"
 #include "rpmostree-util.h"
 
+static const char usrlib_ostreeboot[] = "usr/lib/ostree-boot";
+
+/* Keep this in sync with ostree/src/libostree/ostree-sysroot-deploy.c:get_kernel_from_tree().
+ * Note they are of necessity slightly different since rpm-ostree needs
+ * to support grabbing wherever the Fedora kernel RPM dropped files as well.
+ */
 static gboolean
 find_kernel_and_initramfs_in_bootdir (int          rootfs_dfd,
                                       const char  *bootdir,
@@ -57,7 +63,7 @@ find_kernel_and_initramfs_in_bootdir (int          rootfs_dfd,
   if (dfd < 0)
     {
       if (errno == ENOENT)
-        return TRUE;
+        return TRUE; /* Note early return */
       else
         return glnx_throw_errno_prefix (error, "opendir(%s)", bootdir);
     }
@@ -80,25 +86,16 @@ find_kernel_and_initramfs_in_bootdir (int          rootfs_dfd,
       name = dent->d_name;
 
       /* Current Fedora 23 kernel.spec installs as just vmlinuz */
-      if (strcmp (name, "vmlinuz") == 0 || g_str_has_prefix (name, "vmlinuz-"))
+      if (g_str_equal (name, "vmlinuz") || g_str_has_prefix (name, "vmlinuz-"))
         {
           if (ret_kernel)
-            {
-              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "Multiple vmlinuz- in %s",
-                           bootdir);
-              return FALSE;
-            }
+            return glnx_throw (error, "Multiple vmlinuz- in %s", bootdir);
           ret_kernel = g_strconcat (bootdir, "/", name, NULL);
         }
-      else if (g_str_has_prefix (name, "initramfs-"))
+      else if (g_str_equal (name, "initramfs.img") || g_str_has_prefix (name, "initramfs-"))
         {
           if (ret_initramfs)
-            {
-              g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
-                           "Multiple initramfs- in %s", bootdir);
-              return FALSE;
-            }
+            return glnx_throw (error, "Multiple initramfs- in %s", bootdir);
           ret_initramfs = g_strconcat (bootdir, "/", name, NULL);
         }
     }
@@ -153,8 +150,8 @@ find_ensure_one_subdirectory (int            rootfs_dfd,
 /* Given a root filesystem, return a GVariant of format (sssms):
  *  - kver: uname -r equivalent
  *  - bootdir: Path to the boot directory
- *  - kernel_path: Relative path to kernel
- *  - initramfs_path: Relative path to initramfs (may be NULL if no initramfs)
+ *  - kernel_path: Relative (to rootfs) path to kernel
+ *  - initramfs_path: Relative (to rootfs) path to initramfs (may be NULL if no initramfs)
  */
 GVariant *
 rpmostree_find_kernel (int rootfs_dfd,
@@ -175,7 +172,7 @@ rpmostree_find_kernel (int rootfs_dfd,
   /* First, look for the kernel in the canonical ostree directory */
   g_autofree char* kernel_path = NULL;
   g_autofree char* initramfs_path = NULL;
-  g_autofree char *bootdir = g_strdup ("usr/lib/ostree-boot");
+  g_autofree char *bootdir = g_strdup (usrlib_ostreeboot);
   if (!find_kernel_and_initramfs_in_bootdir (rootfs_dfd, bootdir,
                                              &kernel_path, &initramfs_path,
                                              cancellable, error))
@@ -211,52 +208,146 @@ rpmostree_find_kernel (int rootfs_dfd,
   return g_variant_ref_sink (g_variant_new ("(sssms)", kver, bootdir, kernel_path, initramfs_path));
 }
 
-/* Given a kernel path and a temporary initramfs, compute their checksum and put
- * them in their final locations.
+/* Given a @rootfs_dfd and path to kernel/initramfs that live in
+ * usr/lib/modules/$kver, possibly update @bootdir to use them. @bootdir should
+ * be one of either /usr/lib/ostree-boot or /boot. If @only_if_found is set, we
+ * do the copy only if we find a kernel; this way we avoid e.g. touching /boot
+ * if it isn't being used.
+ */
+static gboolean
+copy_kernel_into (int rootfs_dfd,
+                  const char *kver,
+                  const char *boot_checksum_str,
+                  const char *kernel_modules_path,
+                  const char *initramfs_modules_path,
+                  gboolean only_if_found,
+                  const char *bootdir,
+                  GCancellable *cancellable,
+                  GError **error)
+{
+  g_autofree char *legacy_kernel_path = NULL;
+  g_autofree char* legacy_initramfs_path = NULL;
+  if (!find_kernel_and_initramfs_in_bootdir (rootfs_dfd, bootdir,
+                                             &legacy_kernel_path, &legacy_initramfs_path,
+                                             cancellable, error))
+    return FALSE;
+
+  /* No kernel found? Skip to the next if we're in "auto"
+   * mode i.e. only update if found.
+   */
+  if (!legacy_kernel_path && only_if_found)
+    return TRUE;
+
+  /* Update kernel */
+  if (legacy_kernel_path)
+    {
+      if (!glnx_unlinkat (rootfs_dfd, legacy_kernel_path, 0, error))
+        return FALSE;
+      g_free (legacy_kernel_path);
+    }
+  legacy_kernel_path = g_strconcat (bootdir, "/", "vmlinuz-", kver, "-", boot_checksum_str, NULL);
+  if (linkat (rootfs_dfd, kernel_modules_path, rootfs_dfd, legacy_kernel_path, 0) < 0)
+    return glnx_throw_errno_prefix (error, "linkat(%s)", legacy_kernel_path);
+
+  /* Update initramfs */
+  if (legacy_initramfs_path)
+    {
+      if (!glnx_unlinkat (rootfs_dfd, legacy_initramfs_path, 0, error))
+        return FALSE;
+      g_free (legacy_initramfs_path);
+    }
+  legacy_initramfs_path = g_strconcat (bootdir, "/", "initramfs-", kver, ".img-", boot_checksum_str, NULL);
+  if (linkat (rootfs_dfd, initramfs_modules_path, rootfs_dfd, legacy_initramfs_path, 0) < 0)
+    return glnx_throw_errno_prefix (error, "linkat(%s)", legacy_initramfs_path);
+
+  return TRUE;
+}
+
+/* Given a kernel path and a temporary initramfs, place them in their final
+ * location. We handle /usr/lib/modules as well as the /usr/lib/ostree-boot and
+ * /boot paths where we need to pre-compute their checksum.
  */
 gboolean
 rpmostree_finalize_kernel (int rootfs_dfd,
                            const char *bootdir,
                            const char *kver,
                            const char *kernel_path,
                            GLnxTmpfile *initramfs_tmpf,
+                           RpmOstreeFinalizeKernelDestination dest,
                            GCancellable *cancellable,
                            GError **error)
 {
-  g_autoptr(GChecksum) boot_checksum = NULL;
-  g_autofree char *kernel_final_path = NULL;
-  g_autofree char *initramfs_final_path = NULL;
-  const char *boot_checksum_str = NULL;
-
-  /* Now, calculate the combined sha256sum of the two. We checksum the initramfs
-   * from the tmpfile fd (via mmap()) to avoid writing it to disk in another
-   * temporary location.
+  const char slash_bootdir[] = "boot";
+  g_autofree char *modules_bootdir = g_strconcat ("usr/lib/modules/", kver, NULL);
+
+  /* Calculate the sha256sum of the kernel+initramfs (called the "boot
+   * checksum"). We checksum the initramfs from the tmpfile fd (via mmap()) to
+   * avoid writing it to disk in another temporary location.
    */
-  boot_checksum = g_checksum_new (G_CHECKSUM_SHA256);
+  g_autoptr(GChecksum) boot_checksum = g_checksum_new (G_CHECKSUM_SHA256);
   if (!_rpmostree_util_update_checksum_from_file (boot_checksum, rootfs_dfd, kernel_path,
                                                   cancellable, error))
     return FALSE;
-
   { g_autoptr(GMappedFile) mfile = g_mapped_file_new_from_fd (initramfs_tmpf->fd, FALSE, error);
     if (!mfile)
       return FALSE;
     g_checksum_update (boot_checksum, (guint8*)g_mapped_file_get_contents (mfile),
                        g_mapped_file_get_length (mfile));
   }
-  boot_checksum_str = g_checksum_get_string (boot_checksum);
+  const char *boot_checksum_str = g_checksum_get_string (boot_checksum);
 
-  kernel_final_path = g_strconcat (bootdir, "/", "vmlinuz-", kver, "-", boot_checksum_str, NULL);
-  initramfs_final_path = g_strconcat (bootdir, "/", "initramfs-", kver, ".img-", boot_checksum_str, NULL);
+  g_autofree char *kernel_modules_path = g_strconcat (modules_bootdir, "/vmlinuz", NULL);;
+  /* It's possible the bootdir is already the modules directory; in that case,
+   * we don't need to rename.
+   */
+  if (!g_str_equal (kernel_path, kernel_modules_path))
+    {
+      g_assert_cmpstr (bootdir, !=, modules_bootdir);
+      /* Ensure that the /usr/lib/modules kernel is the same as the source.
+       * Right now we don't support overriding the kernel, but to be
+       * conservative let's relink (unlink/link). We don't just rename() because
+       * for _AUTO mode we still want to find the kernel in the old path
+       * (probably /usr/lib/ostree-boot) and update as appropriate.
+       */
+      if (unlinkat (rootfs_dfd, kernel_modules_path, 0) < 0)
+        {
+          if (errno != ENOENT)
+            return glnx_throw_errno_prefix (error, "unlinkat(%s)", kernel_modules_path);
+        }
+      if (linkat (rootfs_dfd, kernel_path, rootfs_dfd, kernel_modules_path, 0) < 0)
+        return glnx_throw_errno_prefix (error, "linkat(%s)", kernel_modules_path);
+    }
 
-  /* Put the kernel in the final location */
-  if (!glnx_renameat (rootfs_dfd, kernel_path, rootfs_dfd, kernel_final_path, error))
-    return FALSE;
-  /* Link the initramfs directly to its final destination */
+  /* Replace the initramfs */
+  g_autofree char *initramfs_modules_path = g_strconcat (modules_bootdir, "/initramfs.img", NULL);
+  if (unlinkat (rootfs_dfd, initramfs_modules_path, 0) < 0)
+    {
+      if (errno != ENOENT)
+        return glnx_throw_errno_prefix (error, "unlinkat(%s)", initramfs_modules_path);
+    }
   if (!glnx_link_tmpfile_at (initramfs_tmpf, GLNX_LINK_TMPFILE_NOREPLACE,
-                             rootfs_dfd, initramfs_final_path,
+                             rootfs_dfd, initramfs_modules_path,
                              error))
     return FALSE;
 
+  /* Update /usr/lib/ostree-boot and /boot (if desired) */
+  const gboolean only_if_found = (dest == RPMOSTREE_FINALIZE_KERNEL_AUTO);
+  if (only_if_found || dest >= RPMOSTREE_FINALIZE_KERNEL_USRLIB_OSTREEBOOT)
+    {
+      if (!copy_kernel_into (rootfs_dfd, kver, boot_checksum_str,
+                             kernel_modules_path, initramfs_modules_path,
+                             only_if_found, usrlib_ostreeboot,
+                             cancellable, error))
+        return FALSE;
+    }
+  if (only_if_found || dest >= RPMOSTREE_FINALIZE_KERNEL_SLASH_BOOT)
+    {
+      if (!copy_kernel_into (rootfs_dfd, kver, boot_checksum_str,
+                             kernel_modules_path, initramfs_modules_path,
+                             only_if_found, slash_bootdir,
+                             cancellable, error))
+        return FALSE;
+    }
   return TRUE;
 }
 
@@ -341,12 +432,16 @@ rpmostree_run_dracut (int     rootfs_dfd,
                                       &tmpf, error))
     goto out;
 
+  /* If we're rebuilding, we use the *current* /etc so we pick up any modified
+   * config files.  Otherwise, we use the usr/etc defaults.
+   */
   if (rebuild_from_initramfs)
     bwrap = rpmostree_bwrap_new (rootfs_dfd, RPMOSTREE_BWRAP_IMMUTABLE, error,
                                  "--ro-bind", "/etc", "/etc",
                                  NULL);
   else
     bwrap = rpmostree_bwrap_new (rootfs_dfd, RPMOSTREE_BWRAP_IMMUTABLE, error,
+                                 "--ro-bind", "usr/etc", "/etc",
                                  NULL);
   if (!bwrap)
     return FALSE;

src/libpriv/rpmostree-kernel.h

@@ -22,6 +22,13 @@
 
 #include <ostree.h>
 
+typedef enum {
+  RPMOSTREE_FINALIZE_KERNEL_AUTO,
+  RPMOSTREE_FINALIZE_KERNEL_USRLIB_MODULES,
+  RPMOSTREE_FINALIZE_KERNEL_USRLIB_OSTREEBOOT,
+  RPMOSTREE_FINALIZE_KERNEL_SLASH_BOOT,
+} RpmOstreeFinalizeKernelDestination;
+
 GVariant *
 rpmostree_find_kernel (int rootfs_dfd,
                        GCancellable *cancellable,
@@ -33,6 +40,7 @@ rpmostree_finalize_kernel (int rootfs_dfd,
                            const char *kver,
                            const char *kernel_path,
                            GLnxTmpfile *initramfs_tmpf,
+                           RpmOstreeFinalizeKernelDestination dest,
                            GCancellable *cancellable,
                            GError **error);