libarchive: Add support for translating paths during commit

For rpm-ostree, I want to move RPM files in `/boot` to `/usr/lib/ostree-boot`.
This is currently impossible without forking the libarchive code.  Supporting
this is pretty straightforward; we already had pathname translation in
the libarchive code, we just need to expose it as an option.

On the command line side, I chose to wrap this as a regexp. That should be good
enough for a lot of use cases; sophisticated users should as always be making
use of the API. Note that this required some new `#ifdef LIBARCHIVE` bits to use
the new API. Following previous patterns here, we use the new API only if a
relevant option is enabled, ensuring unit test coverage of both paths.

For the test cases, I ended up changing the accounting to avoid having to
multiply the test count.

Closes: #1105
Approved by: jlebon

src/libostree/ostree-libarchive-private.h

@@ -38,6 +38,28 @@ typedef struct archive OtAutoArchiveWrite;
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtAutoArchiveWrite, archive_write_free)
 typedef struct archive OtAutoArchiveRead;
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtAutoArchiveRead, archive_read_free)
+
+static inline OtAutoArchiveRead *
+ot_open_archive_read (const char *path, GError **error)
+{
+  g_autoptr(OtAutoArchiveRead) a = archive_read_new ();
+
+#ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL
+  archive_read_support_filter_all (a);
+#else
+  archive_read_support_compression_all (a);
+#endif
+  archive_read_support_format_all (a);
+  if (archive_read_open_filename (a, path, 8192) != ARCHIVE_OK)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "%s", archive_error_string (a));
+      return NULL;
+    }
+
+  return g_steal_pointer (&a);
+}
+
 #endif
 
 G_END_DECLS

src/libostree/ostree-repo-libarchive.c

@@ -123,24 +123,30 @@ squash_trailing_slashes (char *path)
     *endp = '\0';
 }
 
-static GFileInfo *
-file_info_from_archive_entry (struct archive_entry *entry)
+/* Like archive_entry_stat(), but since some archives only store the permission
+ * mode bits in hardlink entries, so let's just make it into a regular file.
+ * Yes, this hack will work even if it's a hardlink to a symlink.
+ */
+static void
+read_archive_entry_stat (struct archive_entry *entry,
+                         struct stat          *stbuf)
 {
   const struct stat *st = archive_entry_stat (entry);
-  struct stat st_copy;
 
-  /* Some archives only store the permission mode bits in hardlink entries, so
-   * let's just make it into a regular file. Yes, this hack will work even if
-   * it's a hardlink to a symlink. */
+  *stbuf = *st;
   if (archive_entry_hardlink (entry))
-    {
-      st_copy = *st;
-      st_copy.st_mode |= S_IFREG;
-      st = &st_copy;
-    }
+    stbuf->st_mode |= S_IFREG;
+}
 
-  g_autoptr(GFileInfo) info = _ostree_stbuf_to_gfileinfo (st);
-  if (S_ISLNK (st->st_mode))
+/* Create a GFileInfo from archive_entry_stat() */
+static GFileInfo *
+file_info_from_archive_entry (struct archive_entry *entry)
+{
+  struct stat stbuf;
+  read_archive_entry_stat (entry, &stbuf);
+
+  g_autoptr(GFileInfo) info = _ostree_stbuf_to_gfileinfo (&stbuf);
+  if (S_ISLNK (stbuf.st_mode))
     g_file_info_set_attribute_byte_string (info, "standard::symlink-target",
                                            archive_entry_symlink (entry));
 
@@ -247,7 +253,18 @@ aic_get_final_path (OstreeRepoArchiveImportContext *ctx,
                     const char  *path,
                     GError     **error)
 {
-  if (ctx->opts->use_ostree_convention)
+  if (ctx->opts->translate_pathname)
+    {
+      struct stat stbuf;
+      path = path_relative (path, error);
+      read_archive_entry_stat (ctx->entry, &stbuf);
+      char *ret = ctx->opts->translate_pathname (ctx->repo, &stbuf, path,
+                                                 ctx->opts->translate_pathname_user_data);
+      if (ret)
+        return ret;
+      /* Fall through */
+    }
+  else if (ctx->opts->use_ostree_convention)
     return path_relative_ostree (path, error);
   return g_strdup (path_relative (path, error));
 }
@@ -258,7 +275,6 @@ aic_get_final_entry_pathname (OstreeRepoArchiveImportContext *ctx,
 {
   const char *pathname = archive_entry_pathname (ctx->entry);
   g_autofree char *final = aic_get_final_path (ctx, pathname, error);
-
   if (final == NULL)
     return NULL;
 
@@ -642,17 +658,17 @@ aic_import_entry (OstreeRepoArchiveImportContext *ctx,
                   GCancellable  *cancellable,
                   GError       **error)
 {
-  g_autoptr(GFileInfo) fi = NULL;
-  g_autoptr(OstreeMutableTree) parent = NULL;
   g_autofree char *path = aic_get_final_entry_pathname (ctx, error);
 
   if (path == NULL)
     return FALSE;
 
+  g_autoptr(GFileInfo) fi = NULL;
   if (aic_apply_modifier_filter (ctx, path, &fi)
         == OSTREE_REPO_COMMIT_FILTER_SKIP)
     return TRUE;
 
+  g_autoptr(OstreeMutableTree) parent = NULL;
   if (!aic_get_parent_dir (ctx, path, &parent, cancellable, error))
     return FALSE;
 
@@ -907,18 +923,9 @@ ostree_repo_write_archive_to_mtree (OstreeRepo                *self,
   g_autoptr(OtAutoArchiveRead) a = archive_read_new ();
   OstreeRepoImportArchiveOptions opts = { 0, };
 
-#ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL
-  archive_read_support_filter_all (a);
-#else
-  archive_read_support_compression_all (a);
-#endif
-  archive_read_support_format_all (a);
-  if (archive_read_open_filename (a, gs_file_get_path_cached (archive), 8192) != ARCHIVE_OK)
-    {
-      propagate_libarchive_error (error, a);
-      goto out;
-    }
-
+  a = ot_open_archive_read (gs_file_get_path_cached (archive), error);
+  if (!a)
+    goto out;
   opts.autocreate_parents = !!autocreate_parents;
 
   if (!ostree_repo_import_archive_to_mtree (self, &opts, a, mtree, modifier, cancellable, error))

src/libostree/ostree-repo.h

@@ -22,6 +22,8 @@
 
 #pragma once
 
+#include <sys/stat.h>
+
 #include "ostree-core.h"
 #include "ostree-types.h"
 #include "ostree-async-progress.h"
@@ -688,6 +690,31 @@ gboolean      ostree_repo_write_archive_to_mtree (OstreeRepo                   *
                                                   GCancellable                 *cancellable,
                                                   GError                      **error);
 
+/**
+ * OstreeRepoImportArchiveTranslatePathname:
+ * @repo: Repo
+ * @stbuf: Stat buffer
+ * @src_path: Path in the archive
+ * @user_data: User data
+ *
+ * Possibly change a pathname while importing an archive. If %NULL is returned,
+ * then @src_path will be used unchanged.  Otherwise, return a new pathname which
+ * will be freed via `g_free()`.
+ *
+ * This pathname translation will be performed *before* any processing from an
+ * active `OstreeRepoCommitModifier`. Will be invoked for all directory and file
+ * types, first with outer directories, then their sub-files and directories.
+ *
+ * Note that enabling pathname translation will always override the setting for
+ * `use_ostree_convention`.
+ *
+ * Since: 2017.11
+ */
+typedef char *(*OstreeRepoImportArchiveTranslatePathname) (OstreeRepo     *repo,
+                                                           const struct stat *stbuf,
+                                                           const char     *src_path,
+                                                           gpointer        user_data);
+
 /**
  * OstreeRepoImportArchiveOptions: (skip)
  *
@@ -703,7 +730,9 @@ typedef struct {
   guint reserved : 28;
 
   guint unused_uint[8];
-  gpointer unused_ptrs[8];
+  OstreeRepoImportArchiveTranslatePathname translate_pathname;
+  gpointer translate_pathname_user_data;
+  gpointer unused_ptrs[6];
 } OstreeRepoImportArchiveOptions;
 
 _OSTREE_PUBLIC

src/ostree/ot-builtin-commit.c

@@ -30,6 +30,7 @@
 #include "ot-tool-util.h"
 #include "parse-datetime.h"
 #include "ostree-repo-private.h"
+#include "ostree-libarchive-private.h"
 
 static char *opt_subject;
 static char *opt_body;
@@ -46,6 +47,7 @@ static char **opt_detached_metadata_strings;
 static gboolean opt_link_checkout_speedup;
 static gboolean opt_skip_if_unchanged;
 static gboolean opt_tar_autocreate_parents;
+static char *opt_tar_pathname_filter;
 static gboolean opt_no_xattrs;
 static char *opt_selinux_policy;
 static gboolean opt_canonical_permissions;
@@ -97,6 +99,7 @@ static GOptionEntry options[] = {
   { "selinux-policy", 0, 0, G_OPTION_ARG_FILENAME, &opt_selinux_policy, "Set SELinux labels based on policy in root filesystem PATH (may be /)", "PATH" },
   { "link-checkout-speedup", 0, 0, G_OPTION_ARG_NONE, &opt_link_checkout_speedup, "Optimize for commits of trees composed of hardlinks into the repository", NULL },
   { "tar-autocreate-parents", 0, 0, G_OPTION_ARG_NONE, &opt_tar_autocreate_parents, "When loading tar archives, automatically create parent directories as needed", NULL },
+  { "tar-pathname-filter", 0, 0, G_OPTION_ARG_STRING, &opt_tar_pathname_filter, "When loading tar archives, use REGEX,REPLACEMENT against path names", "REGEX,REPLACEMENT" },
   { "skip-if-unchanged", 0, 0, G_OPTION_ARG_NONE, &opt_skip_if_unchanged, "If the contents are unchanged from previous commit, do nothing", NULL },
   { "statoverride", 0, 0, G_OPTION_ARG_FILENAME, &opt_statoverride_file, "File containing list of modifications to make to permissions", "PATH" },
   { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" },
@@ -221,6 +224,28 @@ commit_filter (OstreeRepo         *self,
   return OSTREE_REPO_COMMIT_FILTER_ALLOW;
 }
 
+typedef struct {
+  GRegex *regex;
+  const char *replacement;
+} TranslatePathnameData;
+
+/* Implement --tar-pathname-filter */
+static char *
+handle_translate_pathname (OstreeRepo *repo,
+                           const struct stat *stbuf,
+                           const char *path,
+                           gpointer user_data)
+{
+  TranslatePathnameData *tpdata = user_data;
+  g_autoptr(GError) tmp_error = NULL;
+  char *ret =
+    g_regex_replace (tpdata->regex, path, -1, 0,
+                     tpdata->replacement, 0, &tmp_error);
+  g_assert_no_error (tmp_error);
+  g_assert (ret);
+  return ret;
+}
+
 static gboolean
 commit_editor (OstreeRepo     *repo,
                const char     *branch,
@@ -568,11 +593,50 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError
             }
           else if (strcmp (tree_type, "tar") == 0)
             {
-              object_to_commit = g_file_new_for_path (tree);
-              if (!ostree_repo_write_archive_to_mtree (repo, object_to_commit, mtree, modifier,
-                                                       opt_tar_autocreate_parents,
-                                                       cancellable, error))
-                goto out;
+              if (!opt_tar_pathname_filter)
+                {
+                  object_to_commit = g_file_new_for_path (tree);
+                  if (!ostree_repo_write_archive_to_mtree (repo, object_to_commit, mtree, modifier,
+                                                           opt_tar_autocreate_parents,
+                                                           cancellable, error))
+                    goto out;
+                }
+              else
+                {
+#ifdef HAVE_LIBARCHIVE
+                  const char *comma = strchr (opt_tar_pathname_filter, ',');
+                  if (!comma)
+                    {
+                      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                                           "Missing ',' in --tar-pathname-filter");
+                      goto out;
+                    }
+                  const char *replacement = comma + 1;
+                  g_autofree char *regexp_text = g_strndup (opt_tar_pathname_filter, comma - opt_tar_pathname_filter);
+                  /* Use new API if we have a pathname filter */
+                  OstreeRepoImportArchiveOptions opts = { 0, };
+                  opts.autocreate_parents = opt_tar_autocreate_parents;
+                  opts.translate_pathname = handle_translate_pathname;
+                  g_autoptr(GRegex) regexp = g_regex_new (regexp_text, 0, 0, error);
+                  TranslatePathnameData tpdata = { regexp, replacement };
+                  if (!regexp)
+                    {
+                      g_prefix_error (error, "--tar-pathname-filter: ");
+                      goto out;
+                    }
+                  opts.translate_pathname_user_data = &tpdata;
+                  g_autoptr(OtAutoArchiveRead) archive = ot_open_archive_read (tree, error);
+                  if (!archive)
+                    goto out;
+                  if (!ostree_repo_import_archive_to_mtree (repo, &opts, archive, mtree,
+                                                            modifier, cancellable, error))
+                    goto out;
+                }
+#else
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+                           "This version of ostree is not compiled with libarchive support");
+              return FALSE;
+#endif
             }
           else if (strcmp (tree_type, "ref") == 0)
             {