msggen: Expose extratlvs of keysend

cln-grpc/proto/primitives.proto

@@ -72,4 +72,13 @@ message Routehint {
 }
 message RoutehintList {
 	repeated Routehint hints = 2;
-}
+}
+
+
+message TlvEntry {
+	uint64 type = 1;
+	bytes value = 2;
+}
+message TlvStream {
+	repeated TlvEntry entries = 1;
+}

cln-grpc/src/pb.rs

@@ -114,20 +114,40 @@ impl From<RouteHop> for cln_rpc::primitives::Routehop {
         }
     }
 }
+
 impl From<Routehint> for cln_rpc::primitives::Routehint {
     fn from(c: Routehint) -> Self {
         Self {
             hops: c.hops.into_iter().map(|h| h.into()).collect(),
         }
     }
 }
+
 impl From<RoutehintList> for cln_rpc::primitives::RoutehintList {
     fn from(c: RoutehintList) -> Self {
         Self {
             hints: c.hints.into_iter().map(|h| h.into()).collect(),
         }
     }
 }
+
+impl From<TlvStream> for cln_rpc::primitives::TlvStream {
+    fn from(s: TlvStream) -> Self {
+        Self {
+            entries: s.entries.into_iter().map(|e| e.into()).collect(),
+        }
+    }
+}
+
+impl From<TlvEntry> for cln_rpc::primitives::TlvEntry {
+    fn from(e: TlvEntry) -> Self {
+        Self {
+            typ: e.r#type,
+            value: e.value,
+        }
+    }
+}
+
 #[cfg(test)]
 mod test {
     use super::*;
@@ -347,6 +367,6 @@ mod test {
           ]
         });
         let u: cln_rpc::model::ListpeersResponse = serde_json::from_value(j).unwrap();
-        let g: ListpeersResponse = (&u).into();
+        let _g: ListpeersResponse = u.into();
     }
 }

cln-grpc/src/test.rs

@@ -278,6 +278,7 @@ fn test_keysend() {
                 }],
                 }],
             }),
+	    extratlvs: None,
         };
 
     let u: cln_rpc::model::KeysendRequest = g.into();

cln-rpc/src/lib.rs

@@ -146,7 +146,7 @@ mod test {
         let read_req = dbg!(read.next().await.unwrap().unwrap());
 
         assert_eq!(
-            json!({"id": "1", "method": "getinfo", "params": {}, "jsonrpc": "2.0"}),
+            json!({"id": 1, "method": "getinfo", "params": {}, "jsonrpc": "2.0"}),
             read_req
         );
     }

cln-rpc/src/primitives.rs

@@ -548,6 +548,19 @@ mod test {
         let serialized: String = serde_json::to_string(&od).unwrap();
         assert_eq!(a, serialized);
     }
+
+    #[test]
+    fn tlvstream() {
+	let stream = TlvStream {
+	    entries: vec![
+		TlvEntry { typ: 31337, value: vec![1,2,3,4,5]},
+			   TlvEntry { typ: 42, value: vec![]},
+	    ],
+	};
+
+	let res = serde_json::to_string(&stream).unwrap();
+        assert_eq!(res, "{\"31337\":\"0102030405\",\"42\":\"\"}");
+    }
 }
 
 #[derive(Clone, Debug, PartialEq)]
@@ -623,4 +636,50 @@ impl Display for RpcError {
     }
 }
 
-impl std::error::Error for RpcError {}
+impl std::error::Error for RpcError {}
+
+#[derive(Clone, Debug)]
+pub struct TlvEntry {
+    pub typ: u64,
+    pub value: Vec<u8>,
+}
+
+#[derive(Clone, Debug)]
+pub struct TlvStream {
+    pub entries: Vec<TlvEntry>,
+}
+
+impl<'de> Deserialize<'de> for TlvStream {
+    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        let map: std::collections::HashMap<u64, String> = Deserialize::deserialize(deserializer)?;
+
+        let entries = map
+            .iter()
+            .map(|(k, v)| TlvEntry {
+                typ: *k,
+                value: hex::decode(v).unwrap(),
+            })
+            .collect();
+
+        Ok(TlvStream { entries })
+    }
+}
+
+impl Serialize for TlvStream {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        use serde::ser::SerializeMap;
+
+        let mut map = serializer.serialize_map(Some(self.entries.len()))?;
+        for e in &self.entries {
+            map.serialize_key(&e.typ)?;
+            map.serialize_value(&hex::encode(&e.value))?;
+        }
+        map.end()
+    }
+}

common/json_param.c

@@ -883,7 +883,11 @@ struct command_result *param_extra_tlvs(struct command *cmd, const char *name,
 	temp = tal_arr(cmd, struct tlv_field, tok->size);
 	json_for_each_obj(i, curr, tok) {
 		f = &temp[i];
-		if (!json_to_u64(buffer, curr, &f->numtype)) {
+		/* Accept either bare ints as keys (not spec
+		 * compliant, but simpler), or ints in strings, which
+		 * are JSON spec compliant. */
+		if (!(json_str_to_u64(buffer, curr, &f->numtype) ||
+		      json_to_u64(buffer, curr, &f->numtype))) {
 			return command_fail(
 			    cmd, JSONRPC2_INVALID_PARAMS,
 			    "\"%s\" is not a valid numeric TLV type.",

common/json_parse_simple.c

@@ -88,6 +88,19 @@ bool json_to_u64(const char *buffer, const jsmntok_t *tok, u64 *num)
 	return true;
 }
 
+bool json_str_to_u64(const char *buffer, const jsmntok_t *tok, u64 *num)
+{
+	jsmntok_t temp;
+	if (tok->type != JSMN_STRING)
+		return false;
+
+	temp = *tok;
+	temp.start += 1;
+	temp.end -= 1;
+
+	return json_to_u64(buffer, &temp, num);
+}
+
 bool json_to_u32(const char *buffer, const jsmntok_t *tok, u32 *num)
 {
 	uint64_t u64;

common/json_parse_simple.h

@@ -35,6 +35,10 @@ char *json_strdup(const tal_t *ctx, const char *buffer, const jsmntok_t *tok);
 /* Extract number from this (may be a string, or a number literal) */
 bool json_to_u64(const char *buffer, const jsmntok_t *tok, u64 *num);
 
+/* Extract number from string. The number must be the entirety of the
+ * string between the '"' */
+bool json_str_to_u64(const char *buffer, const jsmntok_t *tok, u64 *num);
+
 /* Extract number from this (may be a string, or a number literal) */
 bool json_to_u32(const char *buffer, const jsmntok_t *tok, u32 *num);
 

contrib/msggen/msggen/gen/grpc.py

@@ -440,6 +440,7 @@ def generate_composite(self, prefix, field: CompositeField) -> None:
                     'hash': f'Sha256::from_slice(&c.{name}).unwrap()',
                     'hash?': f'c.{name}.map(|v| Sha256::from_slice(&v).unwrap())',
                     'txid': f'hex::encode(&c.{name})',
+                    'TlvStream?': f'c.{name}.map(|s| s.into())',
                 }.get(
                     typ,
                     f'c.{name}'  # default to just assignment

contrib/msggen/msggen/model.py

@@ -343,7 +343,21 @@ def __str__(self):
 DatastoreKeyField = ArrayField(itemtype=PrimitiveField("string", None, None), dims=1, path=None, description=None)
 InvoiceExposeprivatechannelsField = PrimitiveField("boolean", None, None)
 PayExclude = ArrayField(itemtype=PrimitiveField("string", None, None), dims=1, path=None, description=None)
-RoutehintListField = PrimitiveField("RoutehintList", None, None)
+RoutehintListField = PrimitiveField(
+    "RoutehintList",
+    None,
+    None
+)
+
+# TlvStreams are special, they don't have preset dict-keys, rather
+# they can specify `u64` keys pointing to hex payloads. So the schema
+# has to rely on additionalProperties to make it work.
+TlvStreamField = PrimitiveField(
+    "TlvStream",
+    None,
+    None
+)
+
 # Override fields with manually managed types, fieldpath -> field mapping
 overrides = {
     'Invoice.label': InvoiceLabelField,
@@ -355,6 +369,7 @@ def __str__(self):
     'Invoice.exposeprivatechannels': InvoiceExposeprivatechannelsField,
     'Pay.exclude': PayExclude,
     'KeySend.routehints': RoutehintListField,
+    'KeySend.extratlvs': TlvStreamField,
 }
 
 

tests/test_pay.py

@@ -3627,7 +3627,8 @@ def test_keysend_strip_tlvs(node_factory):
     ksinfo = """💕 ₿"'
 More info
 """
-    l1.rpc.keysend(l2.info['id'], amt, extratlvs={133773310: bytes(ksinfo, encoding='utf8').hex()})
+    # Since we're at it, use this to test string-keyed TLVs
+    l1.rpc.keysend(l2.info['id'], amt, extratlvs={"133773310": bytes(ksinfo, encoding='utf8').hex()})
     inv = only_one(l2.rpc.listinvoices()['invoices'])
     assert inv['description'] == 'keysend: ' + ksinfo
     l2.daemon.wait_for_log('Keysend payment uses illegal even field 133773310: stripping')