WIP: add lockfile for RPM versions

Signed-off-by: Rafael Fonseca <[email protected]>

rust/cbindgen.toml

@@ -3,6 +3,7 @@ language = "C"
 header = "#pragma once\n#include <gio/gio.h>\ntypedef GError RORGError;\ntypedef GHashTable RORGHashTable;"
 trailer = """
 G_DEFINE_AUTOPTR_CLEANUP_FUNC(RORTreefile, ror_treefile_free)
+G_DEFINE_AUTOPTR_CLEANUP_FUNC(RORLockfile, ror_lockfile_free)
 """
 
 

rust/src/lib.rs

@@ -28,5 +28,7 @@ mod progress;
 pub use self::progress::*;
 mod treefile;
 pub use self::treefile::*;
+mod lockfile;
+pub use self::lockfile::*;
 mod utils;
 pub use self::utils::*;

rust/src/lockfile.rs

@@ -0,0 +1,291 @@
+/*
+ * Copyright (C) 2018 Red Hat, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+use c_utf8::CUtf8Buf;
+use serde_derive::{Deserialize, Serialize};
+use serde_json;
+use serde_yaml;
+use std::collections::HashMap;
+use std::path::Path;
+use std::{fs, io};
+use failure::Fallible;
+use failure::ResultExt;
+
+pub struct Lockfile {
+    #[allow(dead_code)] // Not used in tests
+    parsed: LockfileConfig,
+    serialized: CUtf8Buf
+}
+
+#[derive(PartialEq)]
+enum InputFormat {
+    YAML,
+    JSON,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct LockfileConfig {
+    packages: HashMap<String, Vec<(String, String)>>
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+struct PermissiveLockfileConfig {
+    #[serde(flatten)]
+    config: LockfileConfig,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(deny_unknown_fields)]
+struct StrictLockfileConfig {
+    #[serde(flatten)]
+    config: LockfileConfig,
+}
+
+/// Parse a YAML/JSON lockfile definition.
+fn lockfile_parse_stream<R: io::Read>(
+    fmt: InputFormat,
+    input: &mut R,
+) -> Fallible<LockfileConfig> {
+    let lockfile: LockfileConfig = match fmt {
+        InputFormat::YAML => {
+            let lf: StrictLockfileConfig = serde_yaml::from_reader(input).map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::InvalidInput,
+                    format!("serde-yaml: {}", e.to_string()),
+                )
+            })?;
+            lf.config
+        }
+        InputFormat::JSON => {
+            let lf: PermissiveLockfileConfig = serde_json::from_reader(input).map_err(|e| {
+                io::Error::new(
+                    io::ErrorKind::InvalidInput,
+                    format!("serde-json: {}", e.to_string()),
+                )
+            })?;
+            lf.config
+        }
+    };
+    Ok(lockfile)
+}
+
+/// Given a lockfile filename, parse it
+fn lockfile_parse<P: AsRef<Path>>(
+    filename: P,
+) -> Fallible<LockfileConfig> {
+    let filename = filename.as_ref();
+    let mut f = io::BufReader::new(open_file(filename)?);
+    let basename = filename
+        .file_name()
+        .map(|s| s.to_string_lossy())
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Expected a filename"))?;
+    let fmt = if basename.ends_with(".yaml") || basename.ends_with(".yml") {
+        InputFormat::YAML
+    } else {
+        InputFormat::JSON
+    };
+    let lf = lockfile_parse_stream(fmt, &mut f).map_err(|e| {
+        io::Error::new(
+            io::ErrorKind::InvalidInput,
+            format!("Parsing {}: {}", filename.to_string_lossy(), e.to_string()),
+        )
+    })?;
+    Ok(lf)
+}
+
+/// Open file and provide context containing filename on failures.
+fn open_file<P: AsRef<Path>>(filename: P) -> Fallible<fs::File> {
+    return Ok(fs::File::open(filename.as_ref()).with_context(
+        |e| format!("Can't open file {:?}: {}", filename.as_ref().display(), e))?);
+}
+
+impl Lockfile {
+    /// The main lockfile creation entrypoint.
+    fn new_boxed(
+        filename: &Path,
+    ) -> Fallible<Box<Lockfile>> {
+        let filename: &Path = filename.as_ref();
+        let parsed = lockfile_parse(filename)?;
+        let serialized = Lockfile::serialize_json_string(&parsed)?;
+        Ok(Box::new(Lockfile {
+            parsed: parsed,
+            serialized: serialized
+        }))
+    }
+
+    fn serialize_json_string(config: &LockfileConfig) -> Fallible<CUtf8Buf> {
+        let mut output = vec![];
+        serde_json::to_writer_pretty(&mut output, config)?;
+        Ok(CUtf8Buf::from_string(
+            String::from_utf8(output).expect("utf-8 json"),
+        ))
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use tempfile;
+    use std::io::prelude::*;
+
+    static VALID_PRELUDE: &str = r###"
+packages:
+  fedora:
+  - - package1
+    - repodata_chksum1
+  - - package2
+    - repodata_chksum2
+  fedora-updates:
+  - - package3
+    - repodata_chksum3
+  - - package4
+    - repodata_chksum4
+"###;
+
+    #[test]
+    fn basic_valid() {
+        let mut input = io::BufReader::new(VALID_PRELUDE.as_bytes());
+        let lockfile =
+            lockfile_parse_stream(InputFormat::YAML, &mut input).unwrap();
+        assert!(lockfile.packages.len() == 2);
+        assert!(lockfile.packages.contains_key("fedora"));
+        assert!(lockfile.packages["fedora"].len() == 2);
+        assert!(lockfile.packages.contains_key("fedora-updates"));
+        assert!(lockfile.packages["fedora-updates"].len() == 2);
+    }
+
+    fn test_invalid(data: &'static str) {
+        let mut buf = VALID_PRELUDE.to_string();
+        buf.push_str(data);
+        let buf = buf.as_bytes();
+        let mut input = io::BufReader::new(buf);
+        match lockfile_parse_stream(InputFormat::YAML, &mut input) {
+            Err(ref e) => {
+                match e.downcast_ref::<io::Error>() {
+                    Some(ref ioe) if ioe.kind() == io::ErrorKind::InvalidInput => {},
+                    _ =>  panic!("Expected invalid lockfile, not {}", e.to_string()),
+                }
+            }
+            Ok(_) => panic!("Expected invalid lockfile"),
+        }
+    }
+
+    #[test]
+    fn test_invalid_install_langs() {
+        test_invalid(
+            r###"install_langs:
+  - "klingon"
+  - "esperanto"
+"###,
+        );
+    }
+
+    #[test]
+    fn test_invalid_arch() {
+        test_invalid(
+            r###"packages-hal9000:
+  - podbaydoor glowingredeye
+"###,
+        );
+    }
+
+    #[test]
+    fn test_invalid_repo() {
+        test_invalid(
+            r###" - invalid:
+             - - "invalid_pkg"
+             "###,
+        );
+    }
+
+    struct LockfileTest {
+        lf: Box<Lockfile>,
+        #[allow(dead_code)]
+        workdir: tempfile::TempDir,
+    }
+
+    impl LockfileTest {
+        fn new<'a>(contents: &'a str) -> Fallible<LockfileTest> {
+            let workdir = tempfile::tempdir()?;
+            let lf_path = workdir.path().join("lockfile.yaml");
+            {
+                let mut lf_stream = io::BufWriter::new(fs::File::create(&lf_path)?);
+                lf_stream.write_all(contents.as_bytes())?;
+            }
+            let lf = Lockfile::new_boxed(lf_path.as_path())?;
+            Ok(LockfileTest { lf, workdir })
+        }
+    }
+
+    #[test]
+    fn test_lockfile_new() {
+        let t = LockfileTest::new(VALID_PRELUDE).unwrap();
+        let lf = &t.lf;
+        assert!(lf.parsed.packages.contains_key("fedora"));
+        assert!(lf.parsed.packages.contains_key("fedora-updates"));
+    }
+
+    #[test]
+    fn test_open_file_nonexistent() {
+        let path = "/usr/share/empty/manifest.yaml";
+        match lockfile_parse(path) {
+            Err(ref e) => assert!(e.to_string().starts_with(
+                    format!("Can't open file {:?}:", path).as_str())),
+            Ok(_) => panic!("Expected nonexistent lockfile error for {}", path),
+        }
+    }
+}
+
+mod ffi {
+    use super::*;
+    use glib_sys;
+    use libc;
+
+    use crate::ffiutil::*;
+
+    #[no_mangle]
+    pub extern "C" fn ror_lockfile_new(
+        filename: *const libc::c_char,
+        gerror: *mut *mut glib_sys::GError,
+    ) -> *mut Lockfile {
+        // Convert arguments
+        let filename = ffi_view_os_str(filename);
+        // Run code, map error if any, otherwise extract raw pointer, passing
+        // ownership back to C.
+        ptr_glib_error(
+            Lockfile::new_boxed(filename.as_ref()),
+            gerror,
+        )
+    }
+
+    #[no_mangle]
+    pub extern "C" fn ror_lockfile_get_json_string(lf: *mut Lockfile) -> *const libc::c_char {
+        ref_from_raw_ptr(lf).serialized.as_ptr()
+    }
+
+    #[no_mangle]
+    pub extern "C" fn ror_lockfile_free(lf: *mut Lockfile) {
+        if lf.is_null() {
+            return;
+        }
+        unsafe {
+            Box::from_raw(lf);
+        }
+    }
+}
+pub use self::ffi::*;

src/app/rpmostree-compose-builtin-tree.c

@@ -70,6 +70,8 @@ static gboolean opt_print_only;
 static char *opt_write_commitid_to;
 static char *opt_write_composejson_to;
 static gboolean opt_no_parent;
+static char *opt_write_lockfile;
+static char *opt_read_lockfile;
 
 /* shared by both install & commit */
 static GOptionEntry common_option_entries[] = {
@@ -92,6 +94,8 @@ static GOptionEntry install_option_entries[] = {
   { "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if a new commit was created", "FILE" },
   { "workdir", 0, 0, G_OPTION_ARG_STRING, &opt_workdir, "Working directory", "WORKDIR" },
   { "workdir-tmpfs", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, &opt_workdir_tmpfs, "Use tmpfs for working state", NULL },
+  { "write-lockfile-to", 0, 0, G_OPTION_ARG_STRING, &opt_write_lockfile, "Write RPM versions information to FILE", "FILE" },
+  { "lockfile", 0, 0, G_OPTION_ARG_STRING, &opt_read_lockfile, "Read RPM version information from FILE", "FILE" },
   { NULL }
 };
 
@@ -332,6 +336,15 @@ install_packages (RpmOstreeTreeComposeContext  *self,
   if (!rpmostree_context_prepare (self->corectx, cancellable, error))
     return FALSE;
 
+  if (opt_write_lockfile)
+    {
+      g_autoptr(GPtrArray) pkgs = rpmostree_context_get_packages (self->corectx);
+      if (!rpmostree_composeutil_write_lockfilejson (pkgs, opt_write_lockfile, error))
+        return FALSE;
+
+      opt_dry_run = TRUE; /* Don't actually do a compose, just write pkgs */
+    }
+
   rpmostree_print_transaction (dnfctx);
 
   /* FIXME - just do a depsolve here before we compute download requirements */
@@ -484,6 +497,24 @@ parse_treefile_to_json (const char    *treefile_path,
   return TRUE;
 }
 
+static gboolean
+parse_lockfile_to_json (const char  *lockfile_path,
+                        JsonParser **out_parser,
+                        GError     **error)
+{
+  g_autoptr(JsonParser) parser = json_parser_new ();
+  g_autoptr(RORLockfile) lockfile_rs = ror_lockfile_new (lockfile_path, error);
+  if (!lockfile_rs)
+    return glnx_prefix_error (error, "Failed to load YAML lockfile");
+
+  const char *serialized = ror_lockfile_get_json_string (lockfile_rs);
+  if (!json_parser_load_from_data (parser, serialized, -1, error))
+    return FALSE;
+
+  *out_parser = g_steal_pointer (&parser);
+  return TRUE;
+}
+
 static gboolean
 parse_metadata_keyvalue_strings (char             **strings,
                                  GHashTable        *metadata_hash,
@@ -682,6 +713,18 @@ rpm_ostree_compose_context_new (const char    *treefile_pathstr,
   if (!self->corectx)
     return FALSE;
 
+  g_autoptr(GHashTable) default_map = g_hash_table_new (g_str_hash, g_str_equal);
+  rpmostree_context_set_vlockmap (self->corectx, default_map);
+  if (opt_read_lockfile)
+    {
+      g_autoptr(JsonParser) parser = NULL;
+      if (!parse_lockfile_to_json (opt_read_lockfile, &parser, error))
+        return FALSE;
+      g_autoptr(GHashTable) vlockmap = rpmostree_composeutil_get_vlockmap (parser, error);
+      if (vlockmap)
+        rpmostree_context_set_vlockmap (self->corectx, vlockmap);
+    }
+
   const char *arch = dnf_context_get_base_arch (rpmostree_context_get_dnf (self->corectx));
   if (!parse_treefile_to_json (gs_file_get_path_cached (self->treefile_path),
                                self->workdir_dfd, arch,