From 213a84c854cd619362ad8e77d47acb579f2c1360 Mon Sep 17 00:00:00 2001 From: Rafael Fonseca Date: Wed, 23 Jan 2019 15:53:08 +0100 Subject: [PATCH] compose: Add --ex-lockfile and --ex-write-lockfile-to Fixes #1670 This patch introduces a new `compose tree --ex-write-lockfile-to=manifest.lock` argument and a new `compose tree --ex-lockfile=manifest.lock` to read it back for subsequent invocations. Signed-off-by: Rafael Fonseca --- src/app/rpmostree-compose-builtin-tree.c | 21 ++++++ src/app/rpmostree-composeutil.c | 93 ++++++++++++++++++++++++ src/app/rpmostree-composeutil.h | 8 ++ src/libpriv/rpmostree-core-private.h | 2 + src/libpriv/rpmostree-core.c | 18 ++++- src/libpriv/rpmostree-core.h | 2 + src/libpriv/rpmostree-rpm-util.c | 51 +++++++++++++ src/libpriv/rpmostree-rpm-util.h | 9 +++ tests/compose-tests/test-lockfile.sh | 42 +++++++++++ 9 files changed, 245 insertions(+), 1 deletion(-) create mode 100755 tests/compose-tests/test-lockfile.sh diff --git a/src/app/rpmostree-compose-builtin-tree.c b/src/app/rpmostree-compose-builtin-tree.c index fd27a0dedb..b13e79d180 100644 --- a/src/app/rpmostree-compose-builtin-tree.c +++ b/src/app/rpmostree-compose-builtin-tree.c @@ -70,6 +70,8 @@ static gboolean opt_print_only; static char *opt_write_commitid_to; static char *opt_write_composejson_to; static gboolean opt_no_parent; +static char *opt_write_lockfile; +static char *opt_read_lockfile; /* shared by both install & commit */ static GOptionEntry common_option_entries[] = { @@ -92,6 +94,8 @@ static GOptionEntry install_option_entries[] = { { "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if a new commit was created", "FILE" }, { "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "WORKDIR" }, { "workdir-tmpfs", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_workdir_tmpfs, "Use tmpfs for working state", NULL }, + { "ex-write-lockfile-to", 0, 0, G_OPTION_ARG_STRING, &opt_write_lockfile, "Write RPM versions information to FILE", "FILE" }, + { "ex-lockfile", 0, 0, G_OPTION_ARG_STRING, &opt_read_lockfile, "Read RPM version information from FILE", "FILE" }, { NULL } }; @@ -334,6 +338,10 @@ install_packages (RpmOstreeTreeComposeContext *self, rpmostree_print_transaction (dnfctx); + if (opt_write_lockfile && + !rpmostree_composeutil_write_lockfilejson (self->corectx, opt_write_lockfile, error)) + return FALSE; + /* FIXME - just do a depsolve here before we compute download requirements */ g_autofree char *ret_new_inputhash = NULL; if (!rpmostree_composeutil_checksum (dnf_context_get_goal (dnfctx), @@ -682,6 +690,19 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr, if (!self->corectx) return FALSE; + g_autoptr(GHashTable) vlockmap = NULL; + if (opt_read_lockfile) + { + vlockmap = rpmostree_composeutil_get_vlockmap (opt_read_lockfile, error); + if (!vlockmap) + return FALSE; + g_print ("Loaded lockfile: %s\n", opt_read_lockfile); + } + else + vlockmap = g_hash_table_new (g_str_hash, g_str_equal); + + rpmostree_context_set_vlockmap (self->corectx, vlockmap); + const char *arch = dnf_context_get_base_arch (rpmostree_context_get_dnf (self->corectx)); if (!parse_treefile_to_json (gs_file_get_path_cached (self->treefile_path), self->workdir_dfd, arch, diff --git a/src/app/rpmostree-composeutil.c b/src/app/rpmostree-composeutil.c index 85832d1398..66cd5a40a7 100644 --- a/src/app/rpmostree-composeutil.c +++ b/src/app/rpmostree-composeutil.c @@ -469,3 +469,96 @@ rpmostree_composeutil_write_composejson (OstreeRepo *repo, return TRUE; } + +/* Implements --write-lockfile-to. + * If `path` is NULL, this is a NO-OP. + */ +gboolean +rpmostree_composeutil_write_lockfilejson (RpmOstreeContext *ctx, + const char *path, + GError **error) +{ + if (!path) + return TRUE; + + g_autoptr(GPtrArray) pkgs = rpmostree_context_get_packages (ctx); + g_assert (pkgs); + + g_auto(GVariantBuilder) builder; + g_variant_builder_init (&builder, G_VARIANT_TYPE ("a{sv}")); + + g_autoptr(GVariant) pkglist_v = NULL; + if (!rpmostree_create_pkglist_variant (pkgs, &pkglist_v, NULL, error)) + return FALSE; + g_variant_builder_add (&builder, "{sv}", "packages", pkglist_v); + + g_autoptr(GVariant) lock_v = g_variant_builder_end (&builder); + g_assert (lock_v != NULL); + g_autoptr(JsonNode) lock_node = json_gvariant_serialize (lock_v); + g_assert (lock_node != NULL); + glnx_unref_object JsonGenerator *generator = json_generator_new (); + json_generator_set_root (generator, lock_node); + /* Let's make it somewhat introspectable by humans */ + json_generator_set_pretty (generator, TRUE); + + char *dnbuf = strdupa (path); + const char *dn = dirname (dnbuf); + g_auto(GLnxTmpfile) tmpf = { 0, }; + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, &tmpf, error)) + return FALSE; + g_autoptr(GOutputStream) out = g_unix_output_stream_new (tmpf.fd, FALSE); + /* See also similar code in status.c */ + if (json_generator_to_stream (generator, out, NULL, error) <= 0 || + (error != NULL && *error != NULL)) + return FALSE; + + /* World readable to match --write-commitid-to which uses mask */ + if (!glnx_fchmod (tmpf.fd, 0644, error)) + return FALSE; + + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, AT_FDCWD, path, error)) + return FALSE; + + return TRUE; +} + +/* compose tree accepts JSON package version lock via file; + * convert it to a hash table of a{sv}; suitable for further extension. + */ +GHashTable * +rpmostree_composeutil_get_vlockmap (const char *path, + GError **error) +{ + g_autoptr(JsonParser) parser = json_parser_new_immutable (); + if (!json_parser_load_from_file (parser, path, error)) + return glnx_null_throw (error, "Could not load lockfile %s", path); + + JsonNode *metarootval = json_parser_get_root (parser); + g_autoptr(GVariant) jsonmetav = json_gvariant_deserialize (metarootval, "a{sv}", error); + if (!jsonmetav) + return glnx_null_throw (error, "Could not parse %s", path); + + g_autoptr(GVariant) value = g_variant_lookup_value (jsonmetav, "packages", G_VARIANT_TYPE ("av")); + if (!value) + return glnx_null_throw (error, "Failed to find \"packages\" section in lockfile"); + + g_autoptr(GHashTable) nevra_to_chksum = + g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + + GVariantIter iter; + g_variant_iter_init (&iter, value); + GVariant *child; + while (g_variant_iter_loop (&iter, "v", &child)) + { + char *nevra = NULL, *repochksum = NULL; + g_autoptr(GVariant) nv = g_variant_get_child_value (child, 0); + g_autoptr(GVariant) nvv = g_variant_get_variant (nv); + nevra = g_variant_dup_string (nvv, NULL); + g_autoptr(GVariant) cv = g_variant_get_child_value (child, 1); + g_autoptr(GVariant) cvv = g_variant_get_variant (cv); + repochksum = g_variant_dup_string (cvv, NULL); + g_hash_table_insert (nevra_to_chksum, nevra, repochksum); + } + + return g_steal_pointer (&nevra_to_chksum); +} diff --git a/src/app/rpmostree-composeutil.h b/src/app/rpmostree-composeutil.h index 11a19328fb..e676fb41ce 100644 --- a/src/app/rpmostree-composeutil.h +++ b/src/app/rpmostree-composeutil.h @@ -68,5 +68,13 @@ rpmostree_composeutil_write_composejson (OstreeRepo *repo, GVariantBuilder *builder, GError **error); +gboolean +rpmostree_composeutil_write_lockfilejson (RpmOstreeContext *ctx, + const char *path, + GError **error); + +GHashTable * +rpmostree_composeutil_get_vlockmap (const char *path, + GError **error); G_END_DECLS diff --git a/src/libpriv/rpmostree-core-private.h b/src/libpriv/rpmostree-core-private.h index 9148728e9b..c0ce31bca2 100644 --- a/src/libpriv/rpmostree-core-private.h +++ b/src/libpriv/rpmostree-core-private.h @@ -72,6 +72,8 @@ struct _RpmOstreeContext { GHashTable *pkgs_to_remove; /* pkgname --> gv_nevra */ GHashTable *pkgs_to_replace; /* new gv_nevra --> old gv_nevra */ + GHashTable *vlockmap; /* nevra --> repochecksum */ + GLnxTmpDir tmpdir; gboolean kernel_changed; diff --git a/src/libpriv/rpmostree-core.c b/src/libpriv/rpmostree-core.c index 841b30f492..4e248da6a8 100644 --- a/src/libpriv/rpmostree-core.c +++ b/src/libpriv/rpmostree-core.c @@ -363,6 +363,8 @@ rpmostree_context_finalize (GObject *object) g_clear_pointer (&rctx->pkgs_to_remove, g_hash_table_unref); g_clear_pointer (&rctx->pkgs_to_replace, g_hash_table_unref); + g_clear_pointer (&rctx->vlockmap, g_hash_table_unref); + (void)glnx_tmpdir_delete (&rctx->tmpdir, NULL, NULL); (void)glnx_tmpdir_delete (&rctx->repo_tmpdir, NULL, NULL); @@ -1952,12 +1954,19 @@ rpmostree_context_prepare (RpmOstreeContext *self, /* And finally, handle repo packages to install */ g_autoptr(GPtrArray) missing_pkgs = NULL; + DnfSack *sack = dnf_context_get_sack (dnfctx); for (char **it = pkgnames; it && *it; it++) { const char *pkgname = *it; - g_autoptr(GError) local_error = NULL; + g_autoptr(DnfPackage) pkg = rpmostree_get_locked_package (sack, self->vlockmap, pkgname); + if (pkg) + { + hy_goal_install (goal, pkg); + continue; + } g_assert (!self->rojig_pure); + g_autoptr(GError) local_error = NULL; if (!dnf_context_install (dnfctx, pkgname, &local_error)) { /* Only keep going if it's ENOENT, so we coalesce into one msg at the end */ @@ -2064,6 +2073,13 @@ rpmostree_context_get_packages_to_import (RpmOstreeContext *self) return g_ptr_array_ref (self->pkgs_to_import); } +void +rpmostree_context_set_vlockmap (RpmOstreeContext *self, GHashTable *map) +{ + g_clear_pointer (&self->vlockmap, (GDestroyNotify)g_hash_table_unref); + self->vlockmap = g_hash_table_ref (map); +} + /* XXX: push this into libdnf */ static const char* convert_dnf_action_to_string (DnfStateAction action) diff --git a/src/libpriv/rpmostree-core.h b/src/libpriv/rpmostree-core.h index f86d59233e..38ee62b377 100644 --- a/src/libpriv/rpmostree-core.h +++ b/src/libpriv/rpmostree-core.h @@ -170,6 +170,8 @@ gboolean rpmostree_context_set_packages (RpmOstreeContext *self, GPtrArray *rpmostree_context_get_packages_to_import (RpmOstreeContext *self); +void rpmostree_context_set_vlockmap (RpmOstreeContext *self, GHashTable *map); + gboolean rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, GError **error); diff --git a/src/libpriv/rpmostree-rpm-util.c b/src/libpriv/rpmostree-rpm-util.c index 9509234884..1aa33cdd38 100644 --- a/src/libpriv/rpmostree-rpm-util.c +++ b/src/libpriv/rpmostree-rpm-util.c @@ -1257,6 +1257,57 @@ rpmostree_create_rpmdb_pkglist_variant (int dfd, return TRUE; } +gboolean +rpmostree_create_pkglist_variant (GPtrArray *pkglist, + GVariant **out_variant, + GCancellable *cancellable, + GError **error) +{ + /* we insert it sorted here so it can efficiently be searched on retrieval */ + g_ptr_array_sort (pkglist, (GCompareFunc)rpmostree_pkg_array_compare); + + GVariantBuilder pkglist_v_builder; + g_variant_builder_init (&pkglist_v_builder, G_VARIANT_TYPE ("a(ss)")); + for (guint i = 0; i < pkglist->len; i++) + { + DnfPackage *pkg = pkglist->pdata[i]; + g_autofree gchar *repodata_chksum = NULL; + + if (!rpmostree_get_repodata_chksum_repr (pkg, &repodata_chksum, error)) + return FALSE; + + g_variant_builder_add (&pkglist_v_builder, "(ss)", + dnf_package_get_nevra (pkg), + repodata_chksum); + } + + *out_variant = g_variant_ref_sink (g_variant_builder_end (&pkglist_v_builder)); + return TRUE; +} + +DnfPackage * +rpmostree_get_locked_package (DnfSack *sack, + GHashTable *lockmap, + const char *name) +{ + /* The manifest might specify not only package names (foo-1.x) but also + * something-that-foo-provides */ + g_autoptr(GPtrArray) alts = rpmostree_get_matching_packages (sack, name); + + for (gsize i = 0; i < alts->len; i++) + { + DnfPackage *pkg = alts->pdata[i]; + g_autofree gchar *repodata_chksum = NULL; + if (!rpmostree_get_repodata_chksum_repr (pkg, &repodata_chksum, NULL)) + continue; + + const char *chksum = g_hash_table_lookup (lockmap, dnf_package_get_nevra (pkg)); + if (chksum && !g_strcmp0 (chksum, repodata_chksum)) + return g_object_ref (pkg); + } + return NULL; +} + /* Simple wrapper around hy_split_nevra() that adds allow-none and GError convention */ gboolean rpmostree_decompose_nevra (const char *nevra, diff --git a/src/libpriv/rpmostree-rpm-util.h b/src/libpriv/rpmostree-rpm-util.h index 2c15f9fe2e..d01b6e33d3 100644 --- a/src/libpriv/rpmostree-rpm-util.h +++ b/src/libpriv/rpmostree-rpm-util.h @@ -210,6 +210,15 @@ rpmostree_create_rpmdb_pkglist_variant (int dfd, GCancellable *cancellable, GError **error); +gboolean +rpmostree_create_pkglist_variant (GPtrArray *pkglist, + GVariant **out_variant, + GCancellable *cancellable, + GError **error); + +DnfPackage * +rpmostree_get_locked_package (DnfSack *sack, GHashTable *lockmap, const char *name); + char * rpmostree_get_cache_branch_for_n_evr_a (const char *name, const char *evr, const char *arch); char *rpmostree_get_cache_branch_header (Header hdr); char *rpmostree_get_rojig_branch_header (Header hdr); diff --git a/tests/compose-tests/test-lockfile.sh b/tests/compose-tests/test-lockfile.sh new file mode 100755 index 0000000000..2c5b8f071e --- /dev/null +++ b/tests/compose-tests/test-lockfile.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -xeuo pipefail + +dn=$(cd $(dirname $0) && pwd) +. ${dn}/libcomposetest.sh + +prepare_compose_test "lockfile" +# Add a local rpm-md repo so we can mutate local test packages +pyappendjsonmember "repos" '["test-repo"]' +build_rpm test-pkg \ + files "/usr/bin/test-pkg" \ + install "mkdir -p %{buildroot}/usr/bin && echo localpkg data > %{buildroot}/usr/bin/test-pkg" +# The test suite writes to pwd, but we need repos in composedata +# Also we need to disable gpgcheck +echo gpgcheck=0 >> yumrepo.repo +ln yumrepo.repo composedata/test-repo.repo +pyappendjsonmember "packages" '["test-pkg"]' +pysetjsonmember "documentation" 'False' +mkdir cache +# Create lockfile +runcompose --ex-write-lockfile-to=$PWD/versions.lock --cachedir $(pwd)/cache +npkgs=$(rpm-ostree --repo=${repobuild} db list ${treeref} |grep -v '^ostree commit' | wc -l) +echo "npkgs=${npkgs}" +rpm-ostree --repo=${repobuild} db list ${treeref} test-pkg >test-pkg-list.txt +assert_file_has_content test-pkg-list.txt 'test-pkg-1.0-1.x86_64' +echo "ok compose" + +assert_has_file "versions.lock" +assert_file_has_content $PWD/versions.lock 'packages' +assert_file_has_content $PWD/versions.lock 'test-pkg-1.0-1.x86_64' +echo "lockfile created" +# Read lockfile back +build_rpm test-pkg \ + version 2.0 \ + files "/usr/bin/test-pkg" \ + install "mkdir -p %{buildroot}/usr/bin && echo localpkg data > %{buildroot}/usr/bin/test-pkg" +runcompose --ex-lockfile=$PWD/versions.lock --cachedir $(pwd)/cache +echo "ok compose with lockfile" + +rpm-ostree --repo=${repobuild} db list ${treeref} test-pkg >test-pkg-list.txt +assert_file_has_content test-pkg-list.txt 'test-pkg-1.0-1.x86_64' +echo "lockfile read"