diff --git a/rust/cbindgen.toml b/rust/cbindgen.toml index 643d6b5d3c..53ae680fd8 100644 --- a/rust/cbindgen.toml +++ b/rust/cbindgen.toml @@ -3,6 +3,7 @@ language = "C" header = "#pragma once\n#include \ntypedef GError RORGError;\ntypedef GHashTable RORGHashTable;" trailer = """ G_DEFINE_AUTOPTR_CLEANUP_FUNC(RORTreefile, ror_treefile_free) +G_DEFINE_AUTOPTR_CLEANUP_FUNC(RORLockfile, ror_lockfile_free) """ diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 758bd29f0d..f5677749d6 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -28,5 +28,7 @@ mod progress; pub use self::progress::*; mod treefile; pub use self::treefile::*; +mod lockfile; +pub use self::lockfile::*; mod utils; pub use self::utils::*; diff --git a/rust/src/lockfile.rs b/rust/src/lockfile.rs new file mode 100644 index 0000000000..b1b07a9863 --- /dev/null +++ b/rust/src/lockfile.rs @@ -0,0 +1,291 @@ +/* + * Copyright (C) 2018 Red Hat, Inc. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +use c_utf8::CUtf8Buf; +use serde_derive::{Deserialize, Serialize}; +use serde_json; +use serde_yaml; +use std::collections::HashMap; +use std::path::Path; +use std::{fs, io}; +use failure::Fallible; +use failure::ResultExt; + +pub struct Lockfile { + #[allow(dead_code)] // Not used in tests + parsed: LockfileConfig, + serialized: CUtf8Buf +} + +#[derive(PartialEq)] +enum InputFormat { + YAML, + JSON, +} + +#[derive(Serialize, Deserialize, Debug)] +struct LockfileConfig { + packages: HashMap> +} + +#[derive(Serialize, Deserialize, Debug)] +struct PermissiveLockfileConfig { + #[serde(flatten)] + config: LockfileConfig, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +struct StrictLockfileConfig { + #[serde(flatten)] + config: LockfileConfig, +} + +/// Parse a YAML/JSON lockfile definition. +fn lockfile_parse_stream( + fmt: InputFormat, + input: &mut R, +) -> Fallible { + let lockfile: LockfileConfig = match fmt { + InputFormat::YAML => { + let lf: StrictLockfileConfig = serde_yaml::from_reader(input).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("serde-yaml: {}", e.to_string()), + ) + })?; + lf.config + } + InputFormat::JSON => { + let lf: PermissiveLockfileConfig = serde_json::from_reader(input).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("serde-json: {}", e.to_string()), + ) + })?; + lf.config + } + }; + Ok(lockfile) +} + +/// Given a lockfile filename, parse it +fn lockfile_parse>( + filename: P, +) -> Fallible { + let filename = filename.as_ref(); + let mut f = io::BufReader::new(open_file(filename)?); + let basename = filename + .file_name() + .map(|s| s.to_string_lossy()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Expected a filename"))?; + let fmt = if basename.ends_with(".yaml") || basename.ends_with(".yml") { + InputFormat::YAML + } else { + InputFormat::JSON + }; + let lf = lockfile_parse_stream(fmt, &mut f).map_err(|e| { + io::Error::new( + io::ErrorKind::InvalidInput, + format!("Parsing {}: {}", filename.to_string_lossy(), e.to_string()), + ) + })?; + Ok(lf) +} + +/// Open file and provide context containing filename on failures. +fn open_file>(filename: P) -> Fallible { + return Ok(fs::File::open(filename.as_ref()).with_context( + |e| format!("Can't open file {:?}: {}", filename.as_ref().display(), e))?); +} + +impl Lockfile { + /// The main lockfile creation entrypoint. + fn new_boxed( + filename: &Path, + ) -> Fallible> { + let filename: &Path = filename.as_ref(); + let parsed = lockfile_parse(filename)?; + let serialized = Lockfile::serialize_json_string(&parsed)?; + Ok(Box::new(Lockfile { + parsed: parsed, + serialized: serialized + })) + } + + fn serialize_json_string(config: &LockfileConfig) -> Fallible { + let mut output = vec![]; + serde_json::to_writer_pretty(&mut output, config)?; + Ok(CUtf8Buf::from_string( + String::from_utf8(output).expect("utf-8 json"), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile; + use std::io::prelude::*; + + static VALID_PRELUDE: &str = r###" +packages: + fedora: + - - package1 + - repodata_chksum1 + - - package2 + - repodata_chksum2 + fedora-updates: + - - package3 + - repodata_chksum3 + - - package4 + - repodata_chksum4 +"###; + + #[test] + fn basic_valid() { + let mut input = io::BufReader::new(VALID_PRELUDE.as_bytes()); + let lockfile = + lockfile_parse_stream(InputFormat::YAML, &mut input).unwrap(); + assert!(lockfile.packages.len() == 2); + assert!(lockfile.packages.contains_key("fedora")); + assert!(lockfile.packages["fedora"].len() == 2); + assert!(lockfile.packages.contains_key("fedora-updates")); + assert!(lockfile.packages["fedora-updates"].len() == 2); + } + + fn test_invalid(data: &'static str) { + let mut buf = VALID_PRELUDE.to_string(); + buf.push_str(data); + let buf = buf.as_bytes(); + let mut input = io::BufReader::new(buf); + match lockfile_parse_stream(InputFormat::YAML, &mut input) { + Err(ref e) => { + match e.downcast_ref::() { + Some(ref ioe) if ioe.kind() == io::ErrorKind::InvalidInput => {}, + _ => panic!("Expected invalid lockfile, not {}", e.to_string()), + } + } + Ok(_) => panic!("Expected invalid lockfile"), + } + } + + #[test] + fn test_invalid_install_langs() { + test_invalid( + r###"install_langs: + - "klingon" + - "esperanto" +"###, + ); + } + + #[test] + fn test_invalid_arch() { + test_invalid( + r###"packages-hal9000: + - podbaydoor glowingredeye +"###, + ); + } + + #[test] + fn test_invalid_repo() { + test_invalid( + r###" - invalid: + - - "invalid_pkg" + "###, + ); + } + + struct LockfileTest { + lf: Box, + #[allow(dead_code)] + workdir: tempfile::TempDir, + } + + impl LockfileTest { + fn new<'a>(contents: &'a str) -> Fallible { + let workdir = tempfile::tempdir()?; + let lf_path = workdir.path().join("lockfile.yaml"); + { + let mut lf_stream = io::BufWriter::new(fs::File::create(&lf_path)?); + lf_stream.write_all(contents.as_bytes())?; + } + let lf = Lockfile::new_boxed(lf_path.as_path())?; + Ok(LockfileTest { lf, workdir }) + } + } + + #[test] + fn test_lockfile_new() { + let t = LockfileTest::new(VALID_PRELUDE).unwrap(); + let lf = &t.lf; + assert!(lf.parsed.packages.contains_key("fedora")); + assert!(lf.parsed.packages.contains_key("fedora-updates")); + } + + #[test] + fn test_open_file_nonexistent() { + let path = "/usr/share/empty/manifest.yaml"; + match lockfile_parse(path) { + Err(ref e) => assert!(e.to_string().starts_with( + format!("Can't open file {:?}:", path).as_str())), + Ok(_) => panic!("Expected nonexistent lockfile error for {}", path), + } + } +} + +mod ffi { + use super::*; + use glib_sys; + use libc; + + use crate::ffiutil::*; + + #[no_mangle] + pub extern "C" fn ror_lockfile_new( + filename: *const libc::c_char, + gerror: *mut *mut glib_sys::GError, + ) -> *mut Lockfile { + // Convert arguments + let filename = ffi_view_os_str(filename); + // Run code, map error if any, otherwise extract raw pointer, passing + // ownership back to C. + ptr_glib_error( + Lockfile::new_boxed(filename.as_ref()), + gerror, + ) + } + + #[no_mangle] + pub extern "C" fn ror_lockfile_get_json_string(lf: *mut Lockfile) -> *const libc::c_char { + ref_from_raw_ptr(lf).serialized.as_ptr() + } + + #[no_mangle] + pub extern "C" fn ror_lockfile_free(lf: *mut Lockfile) { + if lf.is_null() { + return; + } + unsafe { + Box::from_raw(lf); + } + } +} +pub use self::ffi::*; diff --git a/src/app/rpmostree-compose-builtin-tree.c b/src/app/rpmostree-compose-builtin-tree.c index fd27a0dedb..420859bad7 100644 --- a/src/app/rpmostree-compose-builtin-tree.c +++ b/src/app/rpmostree-compose-builtin-tree.c @@ -70,6 +70,8 @@ static gboolean opt_print_only; static char *opt_write_commitid_to; static char *opt_write_composejson_to; static gboolean opt_no_parent; +static char *opt_write_lockfile; +static char *opt_read_lockfile; /* shared by both install & commit */ static GOptionEntry common_option_entries[] = { @@ -92,6 +94,8 @@ static GOptionEntry install_option_entries[] = { { "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if a new commit was created", "FILE" }, { "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "WORKDIR" }, { "workdir-tmpfs", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_workdir_tmpfs, "Use tmpfs for working state", NULL }, + { "write-lockfile-to", 0, 0, G_OPTION_ARG_STRING, &opt_write_lockfile, "Write RPM versions information to FILE", "FILE" }, + { "lockfile", 0, 0, G_OPTION_ARG_STRING, &opt_read_lockfile, "Read RPM version information from FILE", "FILE" }, { NULL } }; @@ -332,6 +336,15 @@ install_packages (RpmOstreeTreeComposeContext *self, if (!rpmostree_context_prepare (self->corectx, cancellable, error)) return FALSE; + if (opt_write_lockfile) + { + g_autoptr(GPtrArray) pkgs = rpmostree_context_get_packages (self->corectx); + if (!rpmostree_composeutil_write_lockfilejson (pkgs, opt_write_lockfile, error)) + return FALSE; + + opt_dry_run = TRUE; /* Don't actually do a compose, just write pkgs */ + } + rpmostree_print_transaction (dnfctx); /* FIXME - just do a depsolve here before we compute download requirements */ @@ -484,6 +497,24 @@ parse_treefile_to_json (const char *treefile_path, return TRUE; } +static gboolean +parse_lockfile_to_json (const char *lockfile_path, + JsonParser **out_parser, + GError **error) +{ + g_autoptr(JsonParser) parser = json_parser_new (); + g_autoptr(RORLockfile) lockfile_rs = ror_lockfile_new (lockfile_path, error); + if (!lockfile_rs) + return glnx_prefix_error (error, "Failed to load YAML lockfile"); + + const char *serialized = ror_lockfile_get_json_string (lockfile_rs); + if (!json_parser_load_from_data (parser, serialized, -1, error)) + return FALSE; + + *out_parser = g_steal_pointer (&parser); + return TRUE; +} + static gboolean parse_metadata_keyvalue_strings (char **strings, GHashTable *metadata_hash, @@ -682,6 +713,18 @@ rpm_ostree_compose_context_new (const char *treefile_pathstr, if (!self->corectx) return FALSE; + g_autoptr(GHashTable) default_map = g_hash_table_new (g_str_hash, g_str_equal); + rpmostree_context_set_vlockmap (self->corectx, default_map); + if (opt_read_lockfile) + { + g_autoptr(JsonParser) parser = NULL; + if (!parse_lockfile_to_json (opt_read_lockfile, &parser, error)) + return FALSE; + g_autoptr(GHashTable) vlockmap = rpmostree_composeutil_get_vlockmap (parser, error); + if (vlockmap) + rpmostree_context_set_vlockmap (self->corectx, vlockmap); + } + const char *arch = dnf_context_get_base_arch (rpmostree_context_get_dnf (self->corectx)); if (!parse_treefile_to_json (gs_file_get_path_cached (self->treefile_path), self->workdir_dfd, arch, diff --git a/src/app/rpmostree-composeutil.c b/src/app/rpmostree-composeutil.c index e6dd971c36..f25c0faff2 100644 --- a/src/app/rpmostree-composeutil.c +++ b/src/app/rpmostree-composeutil.c @@ -468,3 +468,169 @@ rpmostree_composeutil_write_composejson (OstreeRepo *repo, return TRUE; } + +static GHashTable * +gather_packages_by_repo (GPtrArray *pkgs) +{ + g_autoptr(GHashTable) repo_to_packages = + g_hash_table_new_full (NULL, NULL, NULL, (GDestroyNotify)g_ptr_array_unref); + + for (guint i = 0; i < pkgs->len; i++) + { + DnfPackage *pkg = pkgs->pdata[i]; + DnfRepo *src = dnf_package_get_repo (pkg); + + GPtrArray *repo_src = g_hash_table_lookup (repo_to_packages, src); + if (!repo_src) + { + repo_src = g_ptr_array_new (); + g_hash_table_insert (repo_to_packages, src, repo_src); + } + g_ptr_array_add (repo_src, pkg); + } + + return g_steal_pointer (&repo_to_packages); +} + +/* Implements --write-lockfile, and also prints values. + * If `path` is NULL, we'll just print some data. + */ +gboolean +rpmostree_composeutil_write_lockfilejson (GPtrArray *pkgs, + const char *path, + GError **error) +{ + g_autoptr(GHashTable) repopkgs = gather_packages_by_repo (pkgs); + + g_print ("Gathering packages from %u repos\n", g_hash_table_size (repopkgs)); + + g_auto(GVariantBuilder) builder; + g_variant_builder_init (&builder, G_VARIANT_TYPE ("{sa{sa(ss)}}")); + g_variant_builder_add (&builder, "s", "packages"); + + g_autoptr(GList) repos = g_hash_table_get_keys (repopkgs); + + /* make sure the repos are sorted so the file is reproducible */ + repos = g_list_sort (repos, (GCompareFunc) g_strcmp0); + + /* "packages": */ + g_variant_builder_open (&builder, G_VARIANT_TYPE ("a{sa(ss)}")); + for (GList *list = repos; list != NULL; list = list->next) + { + DnfRepo *repo = list->data; + const gchar *id = dnf_repo_get_id (repo); + + /* "$repoid": */ + g_variant_builder_open (&builder, G_VARIANT_TYPE("{sa(ss)}")); + g_variant_builder_add (&builder, "s", id); + + GPtrArray *rpkgs = g_hash_table_lookup (repopkgs, repo); + g_print("Gathering %u packages on repo %s\n", rpkgs->len, id); + + /* Make sure the packages are sorted so the file is reproducible */ + g_ptr_array_sort (rpkgs, (GCompareFunc) rpmostree_pkg_array_compare); + + /* - [$pkg_nevra, $repodata_chksum] ... */ + g_variant_builder_open (&builder, G_VARIANT_TYPE ("a(ss)")); + for (guint i = 0; i < rpkgs->len; i++) + { + DnfPackage *pkg = rpkgs->pdata[i]; + g_autofree gchar *repodata_chksum = NULL; + + if (!rpmostree_get_repodata_chksum_repr (pkg, &repodata_chksum, error)) + return FALSE; + + const gchar *nevra = dnf_package_get_nevra (pkg); + g_variant_builder_add (&builder, "(ss)", nevra, repodata_chksum); + /* g_print(" - [%s, %s]\n", nevra, repodata_chksum); */ + } + g_variant_builder_close (&builder); /* - [pkg, chksum] ... */ + + g_variant_builder_close (&builder); /* $repoid */ + } + g_variant_builder_close (&builder); /* "packages" */ + + if (path) + { + g_autoptr(GVariant) lock_v = g_variant_builder_end (&builder); + g_assert (lock_v != NULL); + g_autoptr(JsonNode) lock_node = json_gvariant_serialize (lock_v); + g_assert (lock_node != NULL); + glnx_unref_object JsonGenerator *generator = json_generator_new (); + json_generator_set_root (generator, lock_node); + + char *dnbuf = strdupa (path); + const char *dn = dirname (dnbuf); + g_auto(GLnxTmpfile) tmpf = { 0, }; + if (!glnx_open_tmpfile_linkable_at (AT_FDCWD, dn, O_WRONLY | O_CLOEXEC, &tmpf, error)) + return FALSE; + g_autoptr(GOutputStream) out = g_unix_output_stream_new (tmpf.fd, FALSE); + /* See also similar code in status.c */ + if (json_generator_to_stream (generator, out, NULL, error) <= 0 || + (error != NULL && *error != NULL)) + return FALSE; + + /* World readable to match --write-commitid-to which uses mask */ + if (!glnx_fchmod (tmpf.fd, 0644, error)) + return FALSE; + + if (!glnx_link_tmpfile_at (&tmpf, GLNX_LINK_TMPFILE_REPLACE, AT_FDCWD, path, error)) + return FALSE; + + g_print("Saved lockfile to %s\n", path); + } + + return TRUE; +} + +/** FIXME **/ +/* compose tree accepts JSON package version lock via file; + * convert it to a hash table of a{sv}; suitable for further extension. + */ +GHashTable * +rpmostree_composeutil_get_vlockmap (JsonParser *parser, + GError **error) +{ + g_autoptr(GHashTable) name_to_nevra = + g_hash_table_new_full (g_str_hash, g_str_equal, g_free, g_free); + + /* lockfile = {sa{sa(ss)}} */ + JsonNode *metarootval = json_parser_get_root (parser); + g_autoptr(GVariant) jsonmetav = json_gvariant_deserialize (metarootval, "{sa{sa(ss)}}", error); + if (!jsonmetav) + return NULL; + + const char *pkgkey; + g_autoptr(GVariant) repometav = NULL; + g_variant_get (jsonmetav, "{&s@a{sa(ss)}}", &pkgkey, &repometav); + if (strcmp (pkgkey, "packages") != 0) + { + g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_FOUND, + "Failed to find \"packages\" section in lock file"); + return NULL; + } + + if (!repometav) + return NULL; + + char *repo; + GVariant *value; + GVariantIter viter; + g_variant_iter_init (&viter, repometav); + while (g_variant_iter_loop (&viter, "{&s@a(ss)}", &repo, &value)) + { + GVariantIter piter; + g_variant_iter_init (&piter, value); + char *nevra, *repochksum; + while (g_variant_iter_loop (&piter, "(s&s)", &nevra, &repochksum)) + { + char *name = NULL; + if (!rpmostree_decompose_nevra (nevra, &name, NULL, NULL, NULL, NULL, error)) + return NULL; + g_hash_table_insert (name_to_nevra, name, g_strdup (nevra)); + /* FIXME: how to return repochksum info? */ + } + } + + return g_steal_pointer (&name_to_nevra); +} diff --git a/src/app/rpmostree-composeutil.h b/src/app/rpmostree-composeutil.h index 11a19328fb..47edf13ce8 100644 --- a/src/app/rpmostree-composeutil.h +++ b/src/app/rpmostree-composeutil.h @@ -68,5 +68,13 @@ rpmostree_composeutil_write_composejson (OstreeRepo *repo, GVariantBuilder *builder, GError **error); +gboolean +rpmostree_composeutil_write_lockfilejson (GPtrArray *pkgs, + const char *path, + GError **error); + +GHashTable * +rpmostree_composeutil_get_vlockmap (JsonParser *parser, + GError **error); G_END_DECLS diff --git a/src/libpriv/rpmostree-core-private.h b/src/libpriv/rpmostree-core-private.h index 9148728e9b..dc5cbe8c5b 100644 --- a/src/libpriv/rpmostree-core-private.h +++ b/src/libpriv/rpmostree-core-private.h @@ -72,6 +72,8 @@ struct _RpmOstreeContext { GHashTable *pkgs_to_remove; /* pkgname --> gv_nevra */ GHashTable *pkgs_to_replace; /* new gv_nevra --> old gv_nevra */ + GHashTable *vlockmap; /* pkgname --> nevra */ + GLnxTmpDir tmpdir; gboolean kernel_changed; diff --git a/src/libpriv/rpmostree-core.c b/src/libpriv/rpmostree-core.c index 841b30f492..f8e6a1f8ef 100644 --- a/src/libpriv/rpmostree-core.c +++ b/src/libpriv/rpmostree-core.c @@ -363,6 +363,8 @@ rpmostree_context_finalize (GObject *object) g_clear_pointer (&rctx->pkgs_to_remove, g_hash_table_unref); g_clear_pointer (&rctx->pkgs_to_replace, g_hash_table_unref); + g_clear_pointer (&rctx->vlockmap, g_hash_table_unref); + (void)glnx_tmpdir_delete (&rctx->tmpdir, NULL, NULL); (void)glnx_tmpdir_delete (&rctx->repo_tmpdir, NULL, NULL); @@ -1954,7 +1956,7 @@ rpmostree_context_prepare (RpmOstreeContext *self, g_autoptr(GPtrArray) missing_pkgs = NULL; for (char **it = pkgnames; it && *it; it++) { - const char *pkgname = *it; + const char *pkgname = g_hash_table_lookup (self->vlockmap, *it) ?: *it; g_autoptr(GError) local_error = NULL; g_assert (!self->rojig_pure); @@ -2064,6 +2066,13 @@ rpmostree_context_get_packages_to_import (RpmOstreeContext *self) return g_ptr_array_ref (self->pkgs_to_import); } +void +rpmostree_context_set_vlockmap (RpmOstreeContext *self, GHashTable *map) +{ + g_clear_pointer (&self->vlockmap, (GDestroyNotify)g_hash_table_unref); + self->vlockmap = g_hash_table_ref (map); +} + /* XXX: push this into libdnf */ static const char* convert_dnf_action_to_string (DnfStateAction action) diff --git a/src/libpriv/rpmostree-core.h b/src/libpriv/rpmostree-core.h index f86d59233e..38ee62b377 100644 --- a/src/libpriv/rpmostree-core.h +++ b/src/libpriv/rpmostree-core.h @@ -170,6 +170,8 @@ gboolean rpmostree_context_set_packages (RpmOstreeContext *self, GPtrArray *rpmostree_context_get_packages_to_import (RpmOstreeContext *self); +void rpmostree_context_set_vlockmap (RpmOstreeContext *self, GHashTable *map); + gboolean rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, GError **error);