libarchive: Add support for translating paths during commit

For rpm-ostree, I want to move RPM files in `/boot` to `/usr/lib/ostree-boot`.
This is currently impossible without forking the libarchive code.  Supporting
this is pretty straightforward; we already had pathname translation in
the libarchive code, we just need to expose it as an option.

On the command line side, I chose to wrap this as a regexp. That should be good
enough for a lot of use cases; sophisticated users should as always be making
use of the API. Note that this required some new `#ifdef LIBARCHIVE` bits to use
the new API. Following previous patterns here, we use the new API only if a
relevant option is enabled, ensuring unit test coverage of both paths.

For the test cases, I ended up changing the accounting to avoid having to
multiply the test count.

src/libostree/ostree-core.c

@@ -1545,20 +1545,11 @@ GFileInfo *
 _ostree_stbuf_to_gfileinfo (const struct stat *stbuf)
 {
   GFileInfo *ret = g_file_info_new ();
-  GFileType ftype;
   const mode_t mode = stbuf->st_mode;
-  if (S_ISDIR (mode))
-    ftype = G_FILE_TYPE_DIRECTORY;
-  else if (S_ISREG (mode))
-    ftype = G_FILE_TYPE_REGULAR;
-  else if (S_ISLNK (mode))
-    ftype = G_FILE_TYPE_SYMBOLIC_LINK;
-  else if (S_ISBLK (mode) || S_ISCHR(mode) || S_ISFIFO(mode))
-    ftype = G_FILE_TYPE_SPECIAL;
-  else
-    ftype = G_FILE_TYPE_UNKNOWN;
+  GFileType ftype = ot_gfiletype_from_stbuf (stbuf);
   g_file_info_set_attribute_uint32 (ret, "standard::type", ftype);
-  g_file_info_set_attribute_boolean (ret, "standard::is-symlink", S_ISLNK (mode));
+  g_file_info_set_attribute_boolean (ret, "standard::is-symlink",
+                                     ftype == G_FILE_TYPE_SYMBOLIC_LINK);
   g_file_info_set_attribute_uint32 (ret, "unix::uid", stbuf->st_uid);
   g_file_info_set_attribute_uint32 (ret, "unix::gid", stbuf->st_gid);
   g_file_info_set_attribute_uint32 (ret, "unix::mode", mode);

src/libostree/ostree-libarchive-private.h

@@ -38,6 +38,28 @@ typedef struct archive OtAutoArchiveWrite;
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtAutoArchiveWrite, archive_write_free)
 typedef struct archive OtAutoArchiveRead;
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(OtAutoArchiveRead, archive_read_free)
+
+static inline OtAutoArchiveRead *
+ot_open_archive_read (const char *path, GError **error)
+{
+  g_autoptr(OtAutoArchiveRead) a = archive_read_new ();
+
+#ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL
+  archive_read_support_filter_all (a);
+#else
+  archive_read_support_compression_all (a);
+#endif
+  archive_read_support_format_all (a);
+  if (archive_read_open_filename (a, path, 8192) != ARCHIVE_OK)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "%s", archive_error_string (a));
+      return NULL;
+    }
+
+  return g_steal_pointer (&a);
+}
+
 #endif
 
 G_END_DECLS

src/libostree/ostree-repo-libarchive.c

@@ -123,24 +123,30 @@ squash_trailing_slashes (char *path)
     *endp = '\0';
 }
 
-static GFileInfo *
-file_info_from_archive_entry (struct archive_entry *entry)
+/* Like archive_entry_stat(), but since some archives only store the permission
+ * mode bits in hardlink entries, so let's just make it into a regular file.
+ * Yes, this hack will work even if it's a hardlink to a symlink.
+ */
+static void
+read_archive_entry_stat (struct archive_entry *entry,
+                         struct stat          *stbuf)
 {
   const struct stat *st = archive_entry_stat (entry);
-  struct stat st_copy;
 
-  /* Some archives only store the permission mode bits in hardlink entries, so
-   * let's just make it into a regular file. Yes, this hack will work even if
-   * it's a hardlink to a symlink. */
+  *stbuf = *st;
   if (archive_entry_hardlink (entry))
-    {
-      st_copy = *st;
-      st_copy.st_mode |= S_IFREG;
-      st = &st_copy;
-    }
+    stbuf->st_mode |= S_IFREG;
+}
 
-  g_autoptr(GFileInfo) info = _ostree_stbuf_to_gfileinfo (st);
-  if (S_ISLNK (st->st_mode))
+/* Create a GFileInfo from archive_entry_stat() */
+static GFileInfo *
+file_info_from_archive_entry (struct archive_entry *entry)
+{
+  struct stat stbuf;
+  read_archive_entry_stat (entry, &stbuf);
+
+  g_autoptr(GFileInfo) info = _ostree_stbuf_to_gfileinfo (&stbuf);
+  if (S_ISLNK (stbuf.st_mode))
     g_file_info_set_attribute_byte_string (info, "standard::symlink-target",
                                            archive_entry_symlink (entry));
 
@@ -247,6 +253,18 @@ aic_get_final_path (OstreeRepoArchiveImportContext *ctx,
                     const char  *path,
                     GError     **error)
 {
+  if (ctx->opts->translate_pathname)
+    {
+      struct stat stbuf;
+      path = path_relative (path, error);
+      read_archive_entry_stat (ctx->entry, &stbuf);
+      GFileType ftype = ot_gfiletype_from_stbuf (&stbuf);
+      char *ret = ctx->opts->translate_pathname (ctx->repo, ftype, path,
+                                                 ctx->opts->translate_pathname_user_data);
+      if (ret)
+        return ret;
+      /* Fall through */
+    }
   if (ctx->opts->use_ostree_convention)
     return path_relative_ostree (path, error);
   return g_strdup (path_relative (path, error));
@@ -258,7 +276,6 @@ aic_get_final_entry_pathname (OstreeRepoArchiveImportContext *ctx,
 {
   const char *pathname = archive_entry_pathname (ctx->entry);
   g_autofree char *final = aic_get_final_path (ctx, pathname, error);
-
   if (final == NULL)
     return NULL;
 
@@ -642,17 +659,17 @@ aic_import_entry (OstreeRepoArchiveImportContext *ctx,
                   GCancellable  *cancellable,
                   GError       **error)
 {
-  g_autoptr(GFileInfo) fi = NULL;
-  g_autoptr(OstreeMutableTree) parent = NULL;
   g_autofree char *path = aic_get_final_entry_pathname (ctx, error);
 
   if (path == NULL)
     return FALSE;
 
+  g_autoptr(GFileInfo) fi = NULL;
   if (aic_apply_modifier_filter (ctx, path, &fi)
         == OSTREE_REPO_COMMIT_FILTER_SKIP)
     return TRUE;
 
+  g_autoptr(OstreeMutableTree) parent = NULL;
   if (!aic_get_parent_dir (ctx, path, &parent, cancellable, error))
     return FALSE;
 
@@ -907,18 +924,9 @@ ostree_repo_write_archive_to_mtree (OstreeRepo                *self,
   g_autoptr(OtAutoArchiveRead) a = archive_read_new ();
   OstreeRepoImportArchiveOptions opts = { 0, };
 
-#ifdef HAVE_ARCHIVE_READ_SUPPORT_FILTER_ALL
-  archive_read_support_filter_all (a);
-#else
-  archive_read_support_compression_all (a);
-#endif
-  archive_read_support_format_all (a);
-  if (archive_read_open_filename (a, gs_file_get_path_cached (archive), 8192) != ARCHIVE_OK)
-    {
-      propagate_libarchive_error (error, a);
-      goto out;
-    }
-
+  a = ot_open_archive_read (gs_file_get_path_cached (archive), error);
+  if (!a)
+    goto out;
   opts.autocreate_parents = !!autocreate_parents;
 
   if (!ostree_repo_import_archive_to_mtree (self, &opts, a, mtree, modifier, cancellable, error))

src/libostree/ostree-repo.h

@@ -686,6 +686,28 @@ gboolean      ostree_repo_write_archive_to_mtree (OstreeRepo                   *
                                                   GCancellable                 *cancellable,
                                                   GError                      **error);
 
+/**
+ * OstreeRepoImportArchiveTranslatePathname:
+ * @repo: Repo
+ * @file_type: File type
+ * @src_path: Path in the archive
+ * @user_data: User data
+ *
+ * Possibly change a pathname while importing an archive. If %NULL is returned,
+ * then @src_path will be used unchanged.  Otherwise, return a new pathname which
+ * will be freed via `g_free()`.
+ *
+ * This pathname translation will be performed *before* any processing from an
+ * active `OstreeRepoCommitModifier`. Will be invoked for all directory and file
+ * types, first with outer directories, then their sub-files and directories.
+ *
+ * Since: 2017.11
+ */
+typedef char *(*OstreeRepoImportArchiveTranslatePathname) (OstreeRepo     *repo,
+                                                           GFileType       file_type,
+                                                           const char     *src_path,
+                                                           gpointer        user_data);
+
 /**
  * OstreeRepoImportArchiveOptions: (skip)
  *
@@ -701,7 +723,9 @@ typedef struct {
   guint reserved : 28;
 
   guint unused_uint[8];
-  gpointer unused_ptrs[8];
+  OstreeRepoImportArchiveTranslatePathname translate_pathname;
+  gpointer translate_pathname_user_data;
+  gpointer unused_ptrs[6];
 } OstreeRepoImportArchiveOptions;
 
 _OSTREE_PUBLIC

src/libotutil/ot-fs-utils.h

@@ -53,6 +53,23 @@ G_DEFINE_AUTO_CLEANUP_CLEAR_FUNC(OtCleanupUnlinkat, ot_cleanup_unlinkat)
 
 GFile * ot_fdrel_to_gfile (int dfd, const char *path);
 
+/* Given a `struct stat`, return its GFileType */
+static inline GFileType
+ot_gfiletype_from_stbuf (const struct stat *stbuf)
+{
+  const mode_t mode = stbuf->st_mode;
+  if (S_ISDIR (mode))
+    return G_FILE_TYPE_DIRECTORY;
+  else if (S_ISREG (mode))
+    return G_FILE_TYPE_REGULAR;
+  else if (S_ISLNK (mode))
+    return G_FILE_TYPE_SYMBOLIC_LINK;
+  else if (S_ISBLK (mode) || S_ISCHR(mode) || S_ISFIFO(mode))
+    return G_FILE_TYPE_SPECIAL;
+  else
+    return G_FILE_TYPE_UNKNOWN;
+}
+
 gboolean ot_readlinkat_gfile_info (int             dfd,
                                    const char     *path,
                                    GFileInfo      *target_info,

src/ostree/ot-builtin-commit.c

@@ -30,6 +30,7 @@
 #include "ot-tool-util.h"
 #include "parse-datetime.h"
 #include "ostree-repo-private.h"
+#include "ostree-libarchive-private.h"
 
 static char *opt_subject;
 static char *opt_body;
@@ -46,6 +47,7 @@ static char **opt_detached_metadata_strings;
 static gboolean opt_link_checkout_speedup;
 static gboolean opt_skip_if_unchanged;
 static gboolean opt_tar_autocreate_parents;
+static char *opt_tar_pathname_filter;
 static gboolean opt_no_xattrs;
 static gboolean opt_canonical_permissions;
 static char **opt_trees;
@@ -95,6 +97,7 @@ static GOptionEntry options[] = {
   { "no-xattrs", 0, 0, G_OPTION_ARG_NONE, &opt_no_xattrs, "Do not import extended attributes", NULL },
   { "link-checkout-speedup", 0, 0, G_OPTION_ARG_NONE, &opt_link_checkout_speedup, "Optimize for commits of trees composed of hardlinks into the repository", NULL },
   { "tar-autocreate-parents", 0, 0, G_OPTION_ARG_NONE, &opt_tar_autocreate_parents, "When loading tar archives, automatically create parent directories as needed", NULL },
+  { "tar-pathname-filter", 0, 0, G_OPTION_ARG_STRING, &opt_tar_pathname_filter, "When loading tar archives, use REGEX,REPLACEMENT against path names", "REGEX,REPLACEMENT" },
   { "skip-if-unchanged", 0, 0, G_OPTION_ARG_NONE, &opt_skip_if_unchanged, "If the contents are unchanged from previous commit, do nothing", NULL },
   { "statoverride", 0, 0, G_OPTION_ARG_FILENAME, &opt_statoverride_file, "File containing list of modifications to make to permissions", "PATH" },
   { "skip-list", 0, 0, G_OPTION_ARG_FILENAME, &opt_skiplist_file, "File containing list of files to skip", "PATH" },
@@ -219,6 +222,31 @@ commit_filter (OstreeRepo         *self,
   return OSTREE_REPO_COMMIT_FILTER_ALLOW;
 }
 
+typedef struct {
+  GRegex *regex;
+  const char *replacement;
+} TranslatePathnameData;
+
+/* Implement --tar-pathname-filter */
+static char *
+handle_translate_pathname (OstreeRepo *repo,
+                           GFileType ftype,
+                           const char *path,
+                           gpointer user_data)
+{
+  TranslatePathnameData *tpdata = user_data;
+  g_autoptr(GError) tmp_error = NULL;
+  char *ret =
+    g_regex_replace (tpdata->regex, path, -1, 0,
+                     tpdata->replacement, 0, &tmp_error);
+  if (!ret)
+    {
+      g_assert_no_error (tmp_error);
+      g_assert_not_reached ();
+    }
+  return ret;
+}
+
 static gboolean
 commit_editor (OstreeRepo     *repo,
                const char     *branch,
@@ -551,11 +579,50 @@ ostree_builtin_commit (int argc, char **argv, GCancellable *cancellable, GError
             }
           else if (strcmp (tree_type, "tar") == 0)
             {
-              object_to_commit = g_file_new_for_path (tree);
-              if (!ostree_repo_write_archive_to_mtree (repo, object_to_commit, mtree, modifier,
-                                                       opt_tar_autocreate_parents,
-                                                       cancellable, error))
-                goto out;
+              if (!opt_tar_pathname_filter)
+                {
+                  object_to_commit = g_file_new_for_path (tree);
+                  if (!ostree_repo_write_archive_to_mtree (repo, object_to_commit, mtree, modifier,
+                                                           opt_tar_autocreate_parents,
+                                                           cancellable, error))
+                    goto out;
+                }
+              else
+                {
+#ifdef HAVE_LIBARCHIVE
+                  const char *comma = strchr (opt_tar_pathname_filter, ',');
+                  if (!comma)
+                    {
+                      g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                                           "Missing ',' in --tar-pathname-filter");
+                      goto out;
+                    }
+                  const char *replacement = comma + 1;
+                  g_autofree char *regexp_text = g_strndup (opt_tar_pathname_filter, comma - opt_tar_pathname_filter);
+                  /* Use new API if we have a pathname filter */
+                  OstreeRepoImportArchiveOptions opts = { 0, };
+                  opts.autocreate_parents = opt_tar_autocreate_parents;
+                  opts.translate_pathname = handle_translate_pathname;
+                  g_autoptr(GRegex) regexp = g_regex_new (regexp_text, 0, 0, error);
+                  TranslatePathnameData tpdata = { regexp, replacement };
+                  if (!regexp)
+                    {
+                      g_prefix_error (error, "--tar-pathname-filter: ");
+                      goto out;
+                    }
+                  opts.translate_pathname_user_data = &tpdata;
+                  g_autoptr(OtAutoArchiveRead) archive = ot_open_archive_read (tree, error);
+                  if (!archive)
+                    goto out;
+                  if (!ostree_repo_import_archive_to_mtree (repo, &opts, archive, mtree,
+                                                            modifier, cancellable, error))
+                    goto out;
+                }
+#else
+              g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
+                           "This version of ostree is not compiled with libarchive support");
+              return FALSE;
+#endif
             }
           else if (strcmp (tree_type, "ref") == 0)
             {