Add inscription compression (#1713)

Cargo.toml

@@ -23,6 +23,7 @@ bech32 = "0.9.1"
 bip39 = "2.0.0"
 bitcoin = { version = "0.30.1", features = ["rand"] }
 boilerplate = { version = "1.0.0", features = ["axum"] }
+brotli = "3.4.0"
 chrono = "0.4.19"
 ciborium = "0.2.1"
 clap = { version = "4.4.2", features = ["derive"] }
@@ -63,7 +64,7 @@ tower-http = { version = "0.4.0", features = ["compression-br", "compression-gzi
 [dev-dependencies]
 executable-path = "1.0.0"
 pretty_assertions = "1.2.1"
-reqwest = { version = "0.11.10", features = ["blocking", "json"] }
+reqwest = { version = "0.11.10", features = ["blocking", "brotli", "json"] }
 test-bitcoincore-rpc = { path = "test-bitcoincore-rpc" }
 unindent = "0.2.1"
 

src/envelope.rs

@@ -14,6 +14,7 @@ pub(crate) const POINTER_TAG: [u8; 1] = [2];
 pub(crate) const PARENT_TAG: [u8; 1] = [3];
 pub(crate) const METADATA_TAG: [u8; 1] = [5];
 pub(crate) const METAPROTOCOL_TAG: [u8; 1] = [7];
+pub(crate) const CONTENT_ENCODING_TAG: [u8; 1] = [9];
 
 type Result<T> = std::result::Result<T, script::Error>;
 type RawEnvelope = Envelope<Vec<Vec<u8>>>;
@@ -77,11 +78,12 @@ impl From<RawEnvelope> for ParsedEnvelope {
 
     let duplicate_field = fields.iter().any(|(_key, values)| values.len() > 1);
 
+    let content_encoding = remove_field(&mut fields, &CONTENT_ENCODING_TAG);
     let content_type = remove_field(&mut fields, &CONTENT_TYPE_TAG);
+    let metadata = remove_and_concatenate_field(&mut fields, &METADATA_TAG);
+    let metaprotocol = remove_field(&mut fields, &METAPROTOCOL_TAG);
     let parent = remove_field(&mut fields, &PARENT_TAG);
     let pointer = remove_field(&mut fields, &POINTER_TAG);
-    let metaprotocol = remove_field(&mut fields, &METAPROTOCOL_TAG);
-    let metadata = remove_and_concatenate_field(&mut fields, &METADATA_TAG);
 
     let unrecognized_even_field = fields
       .keys()
@@ -96,14 +98,15 @@ impl From<RawEnvelope> for ParsedEnvelope {
             .cloned()
             .collect()
         }),
+        content_encoding,
         content_type,
-        parent,
-        pointer,
-        unrecognized_even_field,
         duplicate_field,
         incomplete_field,
-        metaprotocol,
         metadata,
+        metaprotocol,
+        parent,
+        pointer,
+        unrecognized_even_field,
       },
       input: envelope.input,
       offset: envelope.offset,
@@ -394,13 +397,35 @@ mod tests {
   }
 
   #[test]
-  fn with_unknown_tag() {
+  fn with_content_encoding() {
     assert_eq!(
       parse(&[envelope(&[
         b"ord",
         &[1],
         b"text/plain;charset=utf-8",
         &[9],
+        b"br",
+        &[],
+        b"ord",
+      ])]),
+      vec![ParsedEnvelope {
+        payload: Inscription {
+          content_encoding: Some("br".as_bytes().to_vec()),
+          ..inscription("text/plain;charset=utf-8", "ord")
+        },
+        ..Default::default()
+      }]
+    );
+  }
+
+  #[test]
+  fn with_unknown_tag() {
+    assert_eq!(
+      parse(&[envelope(&[
+        b"ord",
+        &[1],
+        b"text/plain;charset=utf-8",
+        &[11],
         b"bar",
         &[],
         b"ord",
@@ -730,7 +755,7 @@ mod tests {
   #[test]
   fn unknown_odd_fields_are_ignored() {
     assert_eq!(
-      parse(&[envelope(&[b"ord", &[9], &[0]])]),
+      parse(&[envelope(&[b"ord", &[11], &[0]])]),
       vec![ParsedEnvelope {
         payload: Inscription::default(),
         ..Default::default()

src/inscription.rs

@@ -1,19 +1,23 @@
 use {
   super::*,
+  anyhow::ensure,
   bitcoin::{
     blockdata::{
       opcodes,
       script::{self, PushBytesBuf},
     },
     ScriptBuf,
   },
-  io::Cursor,
+  brotli::enc::{writer::CompressorWriter, BrotliEncoderParams},
+  http::header::HeaderValue,
+  io::{Cursor, Read, Write},
   std::str,
 };
 
 #[derive(Debug, PartialEq, Clone, Serialize, Deserialize, Eq, Default)]
 pub struct Inscription {
   pub body: Option<Vec<u8>>,
+  pub content_encoding: Option<Vec<u8>>,
   pub content_type: Option<Vec<u8>>,
   pub duplicate_field: bool,
   pub incomplete_field: bool,
@@ -41,23 +45,61 @@ impl Inscription {
     pointer: Option<u64>,
     metaprotocol: Option<String>,
     metadata: Option<Vec<u8>>,
+    compress: bool,
   ) -> Result<Self, Error> {
     let path = path.as_ref();
 
     let body = fs::read(path).with_context(|| format!("io error reading {}", path.display()))?;
 
+    let (content_type, compression_mode) = Media::content_type_for_path(path)?;
+
+    let (body, content_encoding) = if compress {
+      let mut compressed = Vec::new();
+
+      {
+        CompressorWriter::with_params(
+          &mut compressed,
+          body.len(),
+          &BrotliEncoderParams {
+            lgblock: 24,
+            lgwin: 24,
+            mode: compression_mode,
+            quality: 11,
+            size_hint: body.len(),
+            ..Default::default()
+          },
+        )
+        .write_all(&body)?;
+
+        let mut decompressor = brotli::Decompressor::new(compressed.as_slice(), compressed.len());
+
+        let mut decompressed = Vec::new();
+
+        decompressor.read_to_end(&mut decompressed)?;
+
+        ensure!(decompressed == body, "decompression roundtrip failed");
+      }
+
+      if compressed.len() < body.len() {
+        (compressed, Some("br".as_bytes().to_vec()))
+      } else {
+        (body, None)
+      }
+    } else {
+      (body, None)
+    };
+
     if let Some(limit) = chain.inscription_content_size_limit() {
       let len = body.len();
       if len > limit {
         bail!("content size of {len} bytes exceeds {limit} byte limit for {chain} inscriptions");
       }
     }
 
-    let content_type = Media::content_type_for_path(path)?;
-
     Ok(Self {
       body: Some(body),
       content_type: Some(content_type.into()),
+      content_encoding,
       metadata,
       metaprotocol: metaprotocol.map(|metaprotocol| metaprotocol.into_bytes()),
       parent: parent.map(|id| id.parent_value()),
@@ -91,6 +133,12 @@ impl Inscription {
         .push_slice(PushBytesBuf::try_from(content_type).unwrap());
     }
 
+    if let Some(content_encoding) = self.content_encoding.clone() {
+      builder = builder
+        .push_slice(envelope::CONTENT_ENCODING_TAG)
+        .push_slice(PushBytesBuf::try_from(content_encoding).unwrap());
+    }
+
     if let Some(protocol) = self.metaprotocol.clone() {
       builder = builder
         .push_slice(envelope::METAPROTOCOL_TAG)
@@ -177,6 +225,10 @@ impl Inscription {
     str::from_utf8(self.content_type.as_ref()?).ok()
   }
 
+  pub(crate) fn content_encoding(&self) -> Option<HeaderValue> {
+    HeaderValue::from_str(str::from_utf8(self.content_encoding.as_ref()?).unwrap_or_default()).ok()
+  }
+
   pub(crate) fn metadata(&self) -> Option<Value> {
     ciborium::from_reader(Cursor::new(self.metadata.as_ref()?)).ok()
   }
@@ -692,22 +744,46 @@ mod tests {
     write!(file, "foo").unwrap();
 
     let inscription =
-      Inscription::from_file(Chain::Mainnet, file.path(), None, None, None, None).unwrap();
+      Inscription::from_file(Chain::Mainnet, file.path(), None, None, None, None, false).unwrap();
 
     assert_eq!(inscription.pointer, None);
 
-    let inscription =
-      Inscription::from_file(Chain::Mainnet, file.path(), None, Some(0), None, None).unwrap();
+    let inscription = Inscription::from_file(
+      Chain::Mainnet,
+      file.path(),
+      None,
+      Some(0),
+      None,
+      None,
+      false,
+    )
+    .unwrap();
 
     assert_eq!(inscription.pointer, Some(Vec::new()));
 
-    let inscription =
-      Inscription::from_file(Chain::Mainnet, file.path(), None, Some(1), None, None).unwrap();
+    let inscription = Inscription::from_file(
+      Chain::Mainnet,
+      file.path(),
+      None,
+      Some(1),
+      None,
+      None,
+      false,
+    )
+    .unwrap();
 
     assert_eq!(inscription.pointer, Some(vec![1]));
 
-    let inscription =
-      Inscription::from_file(Chain::Mainnet, file.path(), None, Some(256), None, None).unwrap();
+    let inscription = Inscription::from_file(
+      Chain::Mainnet,
+      file.path(),
+      None,
+      Some(256),
+      None,
+      None,
+      false,
+    )
+    .unwrap();
 
     assert_eq!(inscription.pointer, Some(vec![0, 1]));
   }