osmet: efficient packing/unpacking of CoreOS metal images

Introduce a new `osmet` command with two primary subcommands: `pack` and
`unpack`. Together, these allow offline bare metal installation of
CoreOS images while only slightly increasing the size of the initramfs.

The `pack` subcommand takes as input a block device of a CoreOS metal
image and the expected checksum to match. It mounts the root partition
from that device and generates a smaller version of the metal image
itself with the OSTree objects "punched out" (this is called the
"punched image" in the code). The command outputs this smaller version
as well as a lookup table of where the OSTree objects belonged into an
"osmet" binary file.

The `unpack` subcommand takes as input an osmet binary file and a path
to an OSTree repo and reconstrust the metal image, bit for bit. This
command is more for testing in practice. The following patch will teach
the `install` command to use the osmet path by default, which is how
users will interact with this.

Cargo.toml

@@ -24,13 +24,17 @@ path = "src/main.rs"
 lto = true
 
 [dependencies]
+bincode = "^1.2"
 byte-unit = "^3.0"
 clap = "^2.33"
 cpio = "^0.2"
 error-chain = { version = "^0.12", default-features = false }
 flate2 = "^1.0"
 hex = "^0.4"
+libc = "0.2.67"
 nix = "^0.17"
+openssl = "^0.10.28"
+pipe = "^0.2.0"
 progress-streams = "^1.1"
 regex = "^1.3"
 reqwest = { version = "^0.10", features = ["blocking"] }
@@ -39,6 +43,7 @@ serde_json = "^1.0"
 sha2 = "^0.8"
 tempfile = "^3.1"
 url = "^2.1"
+walkdir = "^2.3"
 xz2 = "^0.1"
 
 [dev-dependencies]

src/blockdev.rs

@@ -13,12 +13,14 @@
 // limitations under the License.
 
 use error_chain::bail;
+use nix::sys::stat::{major, minor};
 use nix::{errno::Errno, mount};
 use regex::Regex;
 use std::collections::HashMap;
 use std::convert::TryInto;
-use std::fs::{remove_dir, File, OpenOptions};
-use std::num::NonZeroU32;
+use std::fs::{metadata, read_to_string, remove_dir, File, OpenOptions};
+use std::num::{NonZeroU32, NonZeroU64};
+use std::os::linux::fs::MetadataExt;
 use std::os::raw::c_int;
 use std::os::unix::fs::FileTypeExt;
 use std::os::unix::io::AsRawFd;
@@ -147,6 +149,51 @@ impl Mount {
     pub fn mountpoint(&self) -> &Path {
         self.mountpoint.as_path()
     }
+
+    pub fn get_partition_offsets(&self) -> Result<(u64, u64)> {
+        let dev = metadata(&self.device)
+            .chain_err(|| format!("getting metadata for {}", &self.device))?
+            .st_rdev();
+        let maj: u64 = major(dev);
+        let min: u64 = minor(dev);
+
+        let start = read_sysfs_dev_block_value_u64(maj, min, "start")?;
+        let size = read_sysfs_dev_block_value_u64(maj, min, "size")?;
+
+        // We multiply by 512 here: the kernel values are always in 512 blocks, regardless of the
+        // actual sector size of the block device. We keep the values as bytes to make things
+        // easier.
+        let start_offset: u64 = start
+            .checked_mul(512)
+            .ok_or_else(|| "start offset mult overflow")?;
+        let end_offset: u64 = start_offset
+            .checked_add(
+                size.checked_mul(512)
+                    .ok_or_else(|| "end offset mult overflow")?,
+            )
+            .ok_or_else(|| "end offset add overflow")?;
+        Ok((start_offset, end_offset))
+    }
+}
+
+fn read_sysfs_dev_block_value_u64(maj: u64, min: u64, field: &str) -> Result<u64> {
+    let s = read_sysfs_dev_block_value(maj, min, field).chain_err(|| {
+        format!(
+            "reading partition {}:{} {} value from sysfs",
+            maj, min, field
+        )
+    })?;
+    Ok(s.parse().chain_err(|| {
+        format!(
+            "parsing partition {}:{} {} value \"{}\" as u64",
+            maj, min, field, &s
+        )
+    })?)
+}
+
+fn read_sysfs_dev_block_value(maj: u64, min: u64, field: &str) -> Result<String> {
+    let path = PathBuf::from(format!("/sys/dev/block/{}:{}/{}", maj, min, field));
+    Ok(read_to_string(&path)?.trim_end().into())
 }
 
 impl Drop for Mount {
@@ -202,6 +249,7 @@ pub fn reread_partition_table(file: &mut File) -> Result<()> {
     Ok(())
 }
 
+/// Get the sector size of the block device at a given path.
 pub fn get_sector_size_for_path(device: &Path) -> Result<NonZeroU32> {
     let dev = OpenOptions::new()
         .read(true)
@@ -235,13 +283,25 @@ pub fn get_sector_size(file: &File) -> Result<NonZeroU32> {
     }
 }
 
+/// Get the size of a block device.
+pub fn get_block_device_size(file: &File) -> Result<NonZeroU64> {
+    let fd = file.as_raw_fd();
+    let mut size: libc::size_t = 0;
+    match unsafe { ioctl::blkgetsize64(fd, &mut size) } {
+        // just cast using `as`: there is no platform we care about today where size_t > 64bits
+        Ok(_) => NonZeroU64::new(size as u64).ok_or_else(|| "found block size of zero".into()),
+        Err(e) => Err(Error::with_chain(e, "getting block size")),
+    }
+}
+
 // create unsafe ioctl wrappers
 #[allow(clippy::missing_safety_doc)]
 mod ioctl {
     use super::c_int;
-    use nix::{ioctl_none, ioctl_read_bad, request_code_none};
+    use nix::{ioctl_none, ioctl_read, ioctl_read_bad, request_code_none};
     ioctl_none!(blkrrpart, 0x12, 95);
     ioctl_read_bad!(blksszget, request_code_none!(0x12, 104), c_int);
+    ioctl_read!(blkgetsize64, 0x12, 114, libc::size_t);
 }
 
 pub fn udev_settle() -> Result<()> {