From 3336f1df94f99bd6f7d3eb58aadbeeda71cbb4bc Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 14:25:09 +0930 Subject: [PATCH 01/18] Makefile: use a library archive for CCAN The linker discards whole files in an archive if it doesn't need them, so saves a bit of space (and time). Also allows us to add more niche things to CCAN (e.g. runes support!) without bloating all the binaries. We also had many places which depended on $(CCAN_FILES), but that was already a dependent of $(ALL_PROGRAMS) and $(ALL_TEST_PROGRAMS). Before: ``` $ size lightningd/lightning*d text data bss dec hex filename 2247683 8696 39008 2295387 23065b lightningd/lightning_channeld 2086607 7432 38880 2132919 208bb7 lightningd/lightning_closingd 2227916 8056 39200 2275172 22b764 lightningd/lightning_connectd 3369236 119288 39240 3527764 35d454 lightningd/lightningd 2183551 8352 38880 2230783 2209ff lightningd/lightning_dualopend 2196389 8024 39136 2243549 223bdd lightningd/lightning_gossipd 2086216 7488 39264 2132968 208be8 lightningd/lightning_hsmd 2134396 8136 39424 2181956 214b44 lightningd/lightning_onchaind 2133391 8352 38880 2180623 21460f lightningd/lightning_openingd 1512168 2136 34384 1548688 17a190 lightningd/lightning_websocketd ``` After: ``` text data bss dec hex filename 2192065 8488 38912 2239465 222be9 lightningd/lightning_channeld 2030957 7224 38816 2076997 1fb145 lightningd/lightning_closingd 2179571 7968 39104 2226643 21f9d3 lightningd/lightning_connectd 3354296 119288 39208 3512792 3599d8 lightningd/lightningd 2127933 8144 38816 2174893 212fad lightningd/lightning_dualopend 2141699 7856 39072 2188627 216553 lightningd/lightning_gossipd 2024482 7288 5240 2037010 1f1512 lightningd/lightning_hsmd 2072074 7920 5400 2085394 1fd212 lightningd/lightning_onchaind 2077773 8144 38816 2124733 206bbd lightningd/lightning_openingd 1408958 1752 344 1411054 1587ee lightningd/lightning_websocketd ``` Signed-off-by: Rusty Russell --- .gitignore | 1 + Makefile | 12 ++++++++---- bitcoin/test/Makefile | 2 +- channeld/test/Makefile | 2 +- cli/Makefile | 2 +- cli/test/Makefile | 2 +- devtools/Makefile | 38 +++++++++++++++++++------------------- plugins/Makefile | 22 +++++++++++----------- plugins/test/Makefile | 2 +- tests/plugins/Makefile | 4 ++-- tools/Makefile | 8 ++++---- 11 files changed, 50 insertions(+), 45 deletions(-) diff --git a/.gitignore b/.gitignore index a242ad4cb214..cf5ccb4d3cd3 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ *.rej *.pyc *.tmp +libccan.a .cppcheck-suppress .mypy_cache TAGS diff --git a/Makefile b/Makefile index 85657e8fdf84..42c3a6a36c50 100644 --- a/Makefile +++ b/Makefile @@ -627,8 +627,12 @@ endif header_versions_gen.h: tools/headerversions @tools/headerversions $@ +# We make a static library, this way linker can discard unused parts. +libccan.a: $(CCAN_OBJS) + @$(call VERBOSE, "ar $@", $(AR) r $@ $(CCAN_OBJS)) + # All binaries require the external libs, ccan and system library versions. -$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): $(EXTERNAL_LIBS) $(CCAN_OBJS) +$(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): $(EXTERNAL_LIBS) libccan.a # Each test program depends on its own object. $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o @@ -638,7 +642,7 @@ $(ALL_TEST_PROGRAMS) $(ALL_FUZZ_TARGETS): %: %.o # uses some ccan modules internally). We want to rely on -lwallycore etc. # (as per EXTERNAL_LDLIBS) so we filter them out here. $(ALL_PROGRAMS) $(ALL_TEST_PROGRAMS): - @$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) -o $@) + @$(call VERBOSE, "ld $@", $(LINK.o) $(filter-out %.a,$^) $(LOADLIBES) $(EXTERNAL_LDLIBS) $(LDLIBS) libccan.a -o $@) # We special case the fuzzing target binaries, as they need to link against libfuzzer, # which brings its own main(). @@ -684,7 +688,7 @@ obsclean: $(RM) gen_*.h */gen_*.[ch] */*/gen_*.[ch] clean: obsclean - $(RM) $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS) + $(RM) libccan.a $(CCAN_OBJS) $(CDUMP_OBJS) $(ALL_OBJS) $(RM) $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) $(RM) $(ALL_PROGRAMS) $(RM) $(ALL_TEST_PROGRAMS) @@ -704,7 +708,7 @@ update-mocks: @echo Need DEVELOPER=1 and EXPERIMENTAL_FEATURES=1 to regenerate mocks >&2; exit 1 endif -$(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) $(CCAN_OBJS) ccan/ccan/cdump/tools/cdump-enumstr config.vars +$(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars update-mocks/%: % @MAKE=$(MAKE) tools/update-mocks.sh "$*" $(SUPPRESS_OUTPUT) diff --git a/bitcoin/test/Makefile b/bitcoin/test/Makefile index 47d8edc35b66..7b15ab9ae07f 100644 --- a/bitcoin/test/Makefile +++ b/bitcoin/test/Makefile @@ -4,7 +4,7 @@ BITCOIN_TEST_PROGRAMS := $(BITCOIN_TEST_OBJS:.o=) BITCOIN_TEST_COMMON_OBJS := common/utils.o common/setup.o common/autodata.o -$(BITCOIN_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_TEST_COMMON_OBJS) bitcoin/chainparams.o +$(BITCOIN_TEST_PROGRAMS): $(BITCOIN_TEST_COMMON_OBJS) bitcoin/chainparams.o $(BITCOIN_TEST_OBJS): $(CCAN_HEADERS) $(BITCOIN_HEADERS) $(BITCOIN_SRC) ALL_TEST_PROGRAMS += $(BITCOIN_TEST_PROGRAMS) diff --git a/channeld/test/Makefile b/channeld/test/Makefile index 77ea655abbf2..fb402a7e63e5 100644 --- a/channeld/test/Makefile +++ b/channeld/test/Makefile @@ -25,7 +25,7 @@ CHANNELD_TEST_COMMON_OBJS := \ common/type_to_string.o \ common/utils.o -$(CHANNELD_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(CHANNELD_TEST_COMMON_OBJS) +$(CHANNELD_TEST_PROGRAMS): $(BITCOIN_OBJS) $(WIRE_OBJS) $(CHANNELD_TEST_COMMON_OBJS) $(CHANNELD_TEST_OBJS): $(CHANNELD_HEADERS) $(CHANNELD_SRC) diff --git a/cli/Makefile b/cli/Makefile index fcbf84a6a4cd..5afdb39cae1e 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -15,6 +15,6 @@ LIGHTNING_CLI_COMMON_OBJS := \ $(LIGHTNING_CLI_OBJS): $(JSMN_HEADERS) $(COMMON_HEADERS) $(CCAN_HEADERS) -cli/lightning-cli: $(LIGHTNING_CLI_OBJS) $(LIGHTNING_CLI_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +cli/lightning-cli: $(LIGHTNING_CLI_OBJS) $(LIGHTNING_CLI_COMMON_OBJS) $(JSMN_OBJS) libccan.a include cli/test/Makefile diff --git a/cli/test/Makefile b/cli/test/Makefile index 3b5a800fefad..331018413d81 100644 --- a/cli/test/Makefile +++ b/cli/test/Makefile @@ -21,7 +21,7 @@ CLI_TEST_COMMON_OBJS := \ common/type_to_string.o \ common/permute_tx.o -$(CLI_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(CLI_TEST_COMMON_OBJS) +$(CLI_TEST_PROGRAMS): libccan.a $(BITCOIN_OBJS) $(WIRE_OBJS) $(CLI_TEST_COMMON_OBJS) $(CLI_TEST_OBJS): $(LIGHTNING_CLI_HEADERS) $(LIGHTNING_CLI_SRC) diff --git a/devtools/Makefile b/devtools/Makefile index 98fd57e6c2ef..2c32b024bb27 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -46,49 +46,49 @@ DEVTOOLS_COMMON_OBJS := \ wire/channel_type_wiregen.o \ wire/tlvstream.o -devtools/features: $(CCAN_OBJS) common/features.o common/utils.o wire/fromwire.o wire/towire.o devtools/features.o +devtools/features: common/features.o common/utils.o wire/fromwire.o wire/towire.o devtools/features.o -devtools/fp16: $(CCAN_OBJS) common/fp16.o common/utils.o common/setup.o common/autodata.o devtools/fp16.o +devtools/fp16: common/fp16.o common/utils.o common/setup.o common/autodata.o devtools/fp16.o -devtools/bolt11-cli: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o +devtools/bolt11-cli: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o devtools/encodeaddr: common/utils.o common/bech32.o devtools/encodeaddr.o -devtools/bolt12-cli: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/bolt12$(EXP)_wiregen.o wire/fromwire.o wire/towire.o common/bolt12.o common/bolt12_merkle.o devtools/bolt12-cli.o common/setup.o common/iso4217.o +devtools/bolt12-cli: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/bolt12$(EXP)_wiregen.o wire/fromwire.o wire/towire.o common/bolt12.o common/bolt12_merkle.o devtools/bolt12-cli.o common/setup.o common/iso4217.o -devtools/decodemsg: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_PRINT_OBJS) wire/fromwire.o wire/towire.o devtools/print_wire.o devtools/decodemsg.o +devtools/decodemsg: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) $(WIRE_PRINT_OBJS) wire/fromwire.o wire/towire.o devtools/print_wire.o devtools/decodemsg.o -devtools/dump-gossipstore: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/dump-gossipstore.o gossipd/gossip_store_wiregen.o +devtools/dump-gossipstore: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/dump-gossipstore.o gossipd/gossip_store_wiregen.o devtools/dump-gossipstore.o: gossipd/gossip_store_wiregen.h -devtools/create-gossipstore: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/create-gossipstore.o gossipd/gossip_store_wiregen.o +devtools/create-gossipstore: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/create-gossipstore.o gossipd/gossip_store_wiregen.o devtools/create-gossipstore.o: gossipd/gossip_store_wiregen.h devtools/onion.c: ccan/config.h -devtools/onion: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) common/onion.o common/onionreply.o wire/fromwire.o wire/towire.o devtools/onion.o common/sphinx.o +devtools/onion: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) common/onion.o common/onionreply.o wire/fromwire.o wire/towire.o devtools/onion.o common/sphinx.o -devtools/gossipwith: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/peer$(EXP)_wiregen.o devtools/gossipwith.o common/cryptomsg.o common/cryptomsg.o +devtools/gossipwith: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/peer$(EXP)_wiregen.o devtools/gossipwith.o common/cryptomsg.o common/cryptomsg.o $(DEVTOOLS_OBJS) $(DEVTOOLS_TOOL_OBJS): wire/wire.h -devtools/mkcommit: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) common/derive_basepoints.o common/channel_type.o common/keyset.o common/key_derive.o common/initial_commit_tx.o common/permute_tx.o wire/fromwire.o wire/towire.o devtools/mkcommit.o channeld/full_channel.o common/initial_channel.o common/htlc_state.o common/pseudorand.o common/htlc_tx.o channeld/commit_tx.o common/htlc_trim.o +devtools/mkcommit: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) common/derive_basepoints.o common/channel_type.o common/keyset.o common/key_derive.o common/initial_commit_tx.o common/permute_tx.o wire/fromwire.o wire/towire.o devtools/mkcommit.o channeld/full_channel.o common/initial_channel.o common/htlc_state.o common/pseudorand.o common/htlc_tx.o channeld/commit_tx.o common/htlc_trim.o -devtools/mkfunding: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/key_derive.o devtools/mkfunding.o +devtools/mkfunding: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/key_derive.o devtools/mkfunding.o -devtools/mkclose: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/mkclose.o +devtools/mkclose: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/mkclose.o -devtools/mkgossip: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/utxo.o common/permute_tx.o common/key_derive.o devtools/mkgossip.o +devtools/mkgossip: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o common/utxo.o common/permute_tx.o common/key_derive.o devtools/mkgossip.o -devtools/mkencoded: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/mkencoded.o +devtools/mkencoded: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/mkencoded.o -devtools/checkchannels: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) common/configdir.o wire/fromwire.o wire/towire.o devtools/checkchannels.o +devtools/checkchannels: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) common/configdir.o wire/fromwire.o wire/towire.o devtools/checkchannels.o -devtools/mkquery: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/mkquery.o +devtools/mkquery: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/mkquery.o -devtools/lightning-checkmessage: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/lightning-checkmessage.o +devtools/lightning-checkmessage: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/lightning-checkmessage.o -devtools/route: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o common/gossmap.o common/fp16.o common/random_select.o common/route.o common/dijkstra.o devtools/clean_topo.o devtools/route.o +devtools/route: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o common/gossmap.o common/fp16.o common/random_select.o common/route.o common/dijkstra.o devtools/clean_topo.o devtools/route.o -devtools/topology: $(DEVTOOLS_COMMON_OBJS) $(CCAN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o common/gossmap.o common/fp16.o common/random_select.o common/dijkstra.o common/route.o devtools/clean_topo.o devtools/topology.o +devtools/topology: $(DEVTOOLS_COMMON_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o wire/tlvstream.o common/gossmap.o common/fp16.o common/random_select.o common/dijkstra.o common/route.o devtools/clean_topo.o devtools/topology.o diff --git a/plugins/Makefile b/plugins/Makefile index 7b32bb82aea2..4ae876c746d3 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -161,30 +161,30 @@ PLUGIN_COMMON_OBJS := \ # Make all plugins depend on all plugin headers, for simplicity. $(PLUGIN_ALL_OBJS): $(PLUGIN_ALL_HEADER) -plugins/pay: bitcoin/chainparams.o $(PLUGIN_PAY_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) common/gossmap.o common/fp16.o common/route.o common/dijkstra.o common/bolt12.o common/bolt12_merkle.o wire/bolt12$(EXP)_wiregen.o bitcoin/block.o +plugins/pay: bitcoin/chainparams.o $(PLUGIN_PAY_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossmap.o common/fp16.o common/route.o common/dijkstra.o common/bolt12.o common/bolt12_merkle.o wire/bolt12$(EXP)_wiregen.o bitcoin/block.o -plugins/autoclean: bitcoin/chainparams.o $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/autoclean: bitcoin/chainparams.o $(PLUGIN_AUTOCLEAN_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) -plugins/chanbackup: bitcoin/chainparams.o $(PLUGIN_chanbackup_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/chanbackup: bitcoin/chainparams.o $(PLUGIN_chanbackup_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) # Topology wants to decode node_announcement, and peer_wiregen which # pulls in some of bitcoin/. -plugins/topology: common/route.o common/dijkstra.o common/gossmap.o common/fp16.o bitcoin/chainparams.o wire/peer$(EXP)_wiregen.o wire/channel_type_wiregen.o bitcoin/block.o bitcoin/preimage.o $(PLUGIN_TOPOLOGY_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/topology: common/route.o common/dijkstra.o common/gossmap.o common/fp16.o bitcoin/chainparams.o wire/peer$(EXP)_wiregen.o wire/channel_type_wiregen.o bitcoin/block.o bitcoin/preimage.o $(PLUGIN_TOPOLOGY_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) -plugins/txprepare: bitcoin/chainparams.o $(PLUGIN_TXPREPARE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/txprepare: bitcoin/chainparams.o $(PLUGIN_TXPREPARE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) -plugins/bcli: bitcoin/chainparams.o $(PLUGIN_BCLI_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/bcli: bitcoin/chainparams.o $(PLUGIN_BCLI_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) -plugins/keysend: bitcoin/chainparams.o wire/tlvstream.o wire/onion$(EXP)_wiregen.o $(PLUGIN_KEYSEND_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) common/gossmap.o common/fp16.o common/route.o common/dijkstra.o +plugins/keysend: bitcoin/chainparams.o wire/tlvstream.o wire/onion$(EXP)_wiregen.o $(PLUGIN_KEYSEND_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossmap.o common/fp16.o common/route.o common/dijkstra.o $(PLUGIN_KEYSEND_OBJS): $(PLUGIN_PAY_LIB_HEADER) -plugins/spenderp: bitcoin/block.o bitcoin/chainparams.o bitcoin/preimage.o bitcoin/psbt.o common/psbt_open.o wire/peer${EXP}_wiregen.o $(PLUGIN_SPENDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/spenderp: bitcoin/block.o bitcoin/chainparams.o bitcoin/preimage.o bitcoin/psbt.o common/psbt_open.o wire/peer${EXP}_wiregen.o $(PLUGIN_SPENDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) -plugins/offers: bitcoin/chainparams.o $(PLUGIN_OFFERS_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) common/bolt12.o common/bolt12_merkle.o common/bolt11_json.o common/iso4217.o $(WIRE_OBJS) bitcoin/block.o common/channel_id.o bitcoin/preimage.o $(JSMN_OBJS) $(CCAN_OBJS) +plugins/offers: bitcoin/chainparams.o $(PLUGIN_OFFERS_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) common/bolt12.o common/bolt12_merkle.o common/bolt11_json.o common/iso4217.o $(WIRE_OBJS) bitcoin/block.o common/channel_id.o bitcoin/preimage.o $(JSMN_OBJS) -plugins/fetchinvoice: bitcoin/chainparams.o $(PLUGIN_FETCHINVOICE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) common/bolt12.o common/bolt12_merkle.o common/iso4217.o $(WIRE_OBJS) bitcoin/block.o common/channel_id.o bitcoin/preimage.o $(JSMN_OBJS) $(CCAN_OBJS) common/gossmap.o common/fp16.o common/dijkstra.o common/route.o common/blindedpath.o common/hmac.o common/blinding.o +plugins/fetchinvoice: bitcoin/chainparams.o $(PLUGIN_FETCHINVOICE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) common/bolt12.o common/bolt12_merkle.o common/iso4217.o $(WIRE_OBJS) bitcoin/block.o common/channel_id.o bitcoin/preimage.o $(JSMN_OBJS) common/gossmap.o common/fp16.o common/dijkstra.o common/route.o common/blindedpath.o common/hmac.o common/blinding.o -plugins/funder: bitcoin/chainparams.o bitcoin/psbt.o common/psbt_open.o $(PLUGIN_FUNDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +plugins/funder: bitcoin/chainparams.o bitcoin/psbt.o common/psbt_open.o $(PLUGIN_FUNDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) # Generated from PLUGINS definition in plugins/Makefile ALL_C_HEADERS += plugins/list_of_builtin_plugins_gen.h diff --git a/plugins/test/Makefile b/plugins/test/Makefile index 465fc0331b5f..204a0f228d88 100644 --- a/plugins/test/Makefile +++ b/plugins/test/Makefile @@ -22,7 +22,7 @@ plugins/test/run-route-overlong: \ common/node_id.o \ common/route.o -$(PLUGIN_TEST_PROGRAMS): $(CCAN_OBJS) $(BITCOIN_OBJS) $(WIRE_OBJS) $(PLUGIN_TEST_COMMON_OBJS) +$(PLUGIN_TEST_PROGRAMS): $(BITCOIN_OBJS) $(WIRE_OBJS) $(PLUGIN_TEST_COMMON_OBJS) $(PLUGIN_TEST_OBJS): $(PLUGIN_FUNDER_HEADER) $(PLUGIN_FUNDER_SRC) diff --git a/tests/plugins/Makefile b/tests/plugins/Makefile index 3b2548221d23..386840be4e11 100644 --- a/tests/plugins/Makefile +++ b/tests/plugins/Makefile @@ -1,14 +1,14 @@ PLUGIN_TESTLIBPLUGIN_SRC := tests/plugins/test_libplugin.c PLUGIN_TESTLIBPLUGIN_OBJS := $(PLUGIN_TESTLIBPLUGIN_SRC:.c=.o) -tests/plugins/test_libplugin: bitcoin/chainparams.o $(PLUGIN_TESTLIBPLUGIN_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(CCAN_OBJS) +tests/plugins/test_libplugin: bitcoin/chainparams.o $(PLUGIN_TESTLIBPLUGIN_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) $(PLUGIN_TESTLIBPLUGIN_OBJS): $(PLUGIN_LIB_HEADER) PLUGIN_TESTSELFDISABLE_AFTER_GETMANIFEST_SRC := tests/plugins/test_selfdisable_after_getmanifest.c PLUGIN_TESTSELFDISABLE_AFTER_GETMANIFEST_OBJS := $(PLUGIN_TESTSELFDISABLE_AFTER_GETMANIFEST_SRC:.c=.o) -tests/plugins/test_selfdisable_after_getmanifest: bitcoin/chainparams.o $(PLUGIN_TESTSELFDISABLE_AFTER_GETMANIFEST_OBJS) common/autodata.o common/json_parse_simple.o common/setup.o common/utils.o $(JSMN_OBJS) $(CCAN_OBJS) +tests/plugins/test_selfdisable_after_getmanifest: bitcoin/chainparams.o $(PLUGIN_TESTSELFDISABLE_AFTER_GETMANIFEST_OBJS) common/autodata.o common/json_parse_simple.o common/setup.o common/utils.o $(JSMN_OBJS) # Make sure these depend on everything. ALL_TEST_PROGRAMS += tests/plugins/test_libplugin tests/plugins/test_selfdisable_after_getmanifest diff --git a/tools/Makefile b/tools/Makefile index 5dd478fcf4b2..6684b5bec48f 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -12,12 +12,12 @@ TOOLS_COMMON_OBJS = common/utils.o # We force make to relink this every time, to detect version changes. # Do it atomically, otherwise parallel builds can get upset! -tools/headerversions: $(FORCE) tools/headerversions.o $(CCAN_OBJS) - @trap "rm -f $@.tmp.$$$$" EXIT; $(LINK.o) tools/headerversions.o $(CCAN_OBJS) $(LOADLIBES) $(LDLIBS) -o $@.tmp.$$$$ && mv $@.tmp.$$$$ $@ +tools/headerversions: $(FORCE) tools/headerversions.o libccan.a + @trap "rm -f $@.tmp.$$$$" EXIT; $(LINK.o) tools/headerversions.o libccan.a $(LOADLIBES) $(LDLIBS) -o $@.tmp.$$$$ && mv $@.tmp.$$$$ $@ -tools/check-bolt: tools/check-bolt.o $(CCAN_OBJS) $(TOOLS_COMMON_OBJS) +tools/check-bolt: tools/check-bolt.o $(TOOLS_COMMON_OBJS) -tools/hsmtool: tools/hsmtool.o $(CCAN_OBJS) $(TOOLS_COMMON_OBJS) $(BITCOIN_OBJS) common/amount.o common/autodata.o common/bech32.o common/bigsize.o common/configdir.o common/derive_basepoints.o common/descriptor_checksum.o common/hsm_encryption.o common/node_id.o common/type_to_string.o common/version.o wire/fromwire.o wire/towire.o +tools/hsmtool: tools/hsmtool.o $(TOOLS_COMMON_OBJS) $(BITCOIN_OBJS) common/amount.o common/autodata.o common/bech32.o common/bigsize.o common/configdir.o common/derive_basepoints.o common/descriptor_checksum.o common/hsm_encryption.o common/node_id.o common/type_to_string.o common/version.o wire/fromwire.o wire/towire.o tools/lightning-hsmtool: tools/hsmtool cp $< $@ From 43ddaf8cb2eee54af3336e0b3c2b0473a537cdcf Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 14:25:11 +0930 Subject: [PATCH 02/18] ccan: upgrade to get ccan/runes. Signed-off-by: Rusty Russell --- Makefile | 11 + ccan/README | 2 +- ccan/ccan/base64/base64.h | 2 +- ccan/ccan/rune/LICENSE | 1 + ccan/ccan/rune/_info | 130 +++++ ccan/ccan/rune/coding.c | 422 +++++++++++++++ ccan/ccan/rune/internal.h | 8 + ccan/ccan/rune/rune.c | 491 ++++++++++++++++++ ccan/ccan/rune/rune.h | 379 ++++++++++++++ .../rune/test/run-alt-lexicographic-order.c | 33 ++ ccan/ccan/rune/test/run.c | 127 +++++ ccan/ccan/rune/test/test_vectors.csv | 151 ++++++ 12 files changed, 1755 insertions(+), 2 deletions(-) create mode 120000 ccan/ccan/rune/LICENSE create mode 100644 ccan/ccan/rune/_info create mode 100644 ccan/ccan/rune/coding.c create mode 100644 ccan/ccan/rune/internal.h create mode 100644 ccan/ccan/rune/rune.c create mode 100644 ccan/ccan/rune/rune.h create mode 100644 ccan/ccan/rune/test/run-alt-lexicographic-order.c create mode 100644 ccan/ccan/rune/test/run.c create mode 100644 ccan/ccan/rune/test/test_vectors.csv diff --git a/Makefile b/Makefile index 42c3a6a36c50..6c804b954884 100644 --- a/Makefile +++ b/Makefile @@ -92,6 +92,7 @@ FEATURES := CCAN_OBJS := \ ccan-asort.o \ + ccan-base64.o \ ccan-bitmap.o \ ccan-bitops.o \ ccan-breakpoint.o \ @@ -127,6 +128,8 @@ CCAN_OBJS := \ ccan-ptr_valid.o \ ccan-rbuf.o \ ccan-read_write_all.o \ + ccan-rune-coding.o \ + ccan-rune-rune.o \ ccan-str-base32.o \ ccan-str-hex.o \ ccan-str.o \ @@ -195,6 +198,8 @@ CCAN_HEADERS := \ $(CCANDIR)/ccan/ptrint/ptrint.h \ $(CCANDIR)/ccan/rbuf/rbuf.h \ $(CCANDIR)/ccan/read_write_all/read_write_all.h \ + $(CCANDIR)/ccan/rune/internal.h \ + $(CCANDIR)/ccan/rune/rune.h \ $(CCANDIR)/ccan/short_types/short_types.h \ $(CCANDIR)/ccan/str/base32/base32.h \ $(CCANDIR)/ccan/str/hex/hex.h \ @@ -840,6 +845,8 @@ endif ccan-breakpoint.o: $(CCANDIR)/ccan/breakpoint/breakpoint.c @$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<) +ccan-base64.o: $(CCANDIR)/ccan/base64/base64.c + @$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<) ccan-tal.o: $(CCANDIR)/ccan/tal/tal.c @$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<) ccan-tal-str.o: $(CCANDIR)/ccan/tal/str/str.c @@ -940,3 +947,7 @@ ccan-json_out.o: $(CCANDIR)/ccan/json_out/json_out.c @$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<) ccan-closefrom.o: $(CCANDIR)/ccan/closefrom/closefrom.c @$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<) +ccan-rune-rune.o: $(CCANDIR)/ccan/rune/rune.c + @$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<) +ccan-rune-coding.o: $(CCANDIR)/ccan/rune/coding.c + @$(call VERBOSE, "cc $<", $(CC) $(CFLAGS) -c -o $@ $<) diff --git a/ccan/README b/ccan/README index 25c2b543395e..acdc78064c9a 100644 --- a/ccan/README +++ b/ccan/README @@ -1,3 +1,3 @@ CCAN imported from http://ccodearchive.net. -CCAN version: init-2540-g8448fd28 +CCAN version: init-2541-g52b86922 diff --git a/ccan/ccan/base64/base64.h b/ccan/ccan/base64/base64.h index cef30d257673..a899af4a357f 100644 --- a/ccan/ccan/base64/base64.h +++ b/ccan/ccan/base64/base64.h @@ -116,7 +116,7 @@ ssize_t base64_decode_quartet_using_maps(const base64_maps_t *maps, * @note sets errno = EDOM if src contains invalid characters * @note sets errno = EINVAL if src is an invalid base64 tail */ -ssize_t base64_decode_tail_using_maps(const base64_maps_t *maps, char *dest, +ssize_t base64_decode_tail_using_maps(const base64_maps_t *maps, char dest[3], const char *src, size_t srclen); diff --git a/ccan/ccan/rune/LICENSE b/ccan/ccan/rune/LICENSE new file mode 120000 index 000000000000..2354d12945d3 --- /dev/null +++ b/ccan/ccan/rune/LICENSE @@ -0,0 +1 @@ +../../licenses/BSD-MIT \ No newline at end of file diff --git a/ccan/ccan/rune/_info b/ccan/ccan/rune/_info new file mode 100644 index 000000000000..2b2e2e8b98e8 --- /dev/null +++ b/ccan/ccan/rune/_info @@ -0,0 +1,130 @@ +#include "config.h" +#include +#include + +/** + * rune - Simple cookies you can extend (a-la Python runes class). + * + * This code is a form of cookies, but they are user-extensible, and + * contain a simple language to define what the cookie allows. + * + * A "rune" contains the hash of a secret (so the server can + * validate), such that you can add, but not subtract, conditions. + * This is a simplified form of Macaroons, See + * https://research.google/pubs/pub41892/ "Macaroons: Cookies with + * Contextual Caveats for Decentralized Authorization in the Cloud". + * It has one good idea, some extended ideas nobody implements, and + * lots and lots of words. + * + * License: BSD-MIT + * Author: Rusty Russell + * Example: + * // Given "generate secret 1" outputs kr7AW-eJ2Munhv5ftu4rHqAnhxUpPQM8aOyWOmqiytk9MQ== + * // Given "add kr7AW-eJ2Munhv5ftu4rHqAnhxUpPQM8aOyWOmqiytk9MQ== uid=rusty" outputs Xyt5S6FKUnA2ppGB62c6HTPGojt2S7k2n7Cf7Tjj6zM9MSZ1aWQ9cnVzdHk= + * // Given "test secret Xyt5S6FKUnA2ppGB62c6HTPGojt2S7k2n7Cf7Tjj6zM9MSZ1aWQ9cnVzdHk= rusty" outputs PASSED + * // Given "test secret Xyt5S6FKUnA2ppGB62c6HTPGojt2S7k2n7Cf7Tjj6zM9MSZ1aWQ9cnVzdHk= notrusty" outputs FAILED: uid is not equal to rusty + * // Given "add Xyt5S6FKUnA2ppGB62c6HTPGojt2S7k2n7Cf7Tjj6zM9MSZ1aWQ9cnVzdHk= t\<1655958616" outputs _YBFmeAedqlLigWHAmvyyGGHRrnI40BRQGh2hWdSZ9E9MSZ1aWQ9cnVzdHkmdDwxNjU1OTU4NjE2 + * // Given "test secret _YBFmeAedqlLigWHAmvyyGGHRrnI40BRQGh2hWdSZ9E9MSZ1aWQ9cnVzdHkmdDwxNjU1OTU4NjE2 rusty" outputs FAILED: t is greater or equal to 1655958616 + * #include + * #include + * #include + * #include + * #include + * + * // We support two values: current time (t), and user id (uid). + * static const char *check(const tal_t *ctx, + * const struct rune *rune, + * const struct rune_altern *alt, + * char *uid) + * { + * // t= means current time, in seconds, as integer + * if (streq(alt->fieldname, "t")) { + * struct timeval now; + * gettimeofday(&now, NULL); + * return rune_alt_single_int(ctx, alt, now.tv_sec); + * } + * if (streq(alt->fieldname, "uid")) { + * return rune_alt_single_str(ctx, alt, uid, strlen(uid)); + * } + * // Otherwise, field is missing + * return rune_alt_single_missing(ctx, alt); + * } + * + * int main(int argc, char *argv[]) + * { + * struct rune *master, *rune; + * + * if (argc < 3) + * goto usage; + * + * if (streq(argv[1], "generate")) { + * // Make master, derive a unique_id'd rune. + * if (argc != 3 && argc != 4) + * goto usage; + * master = rune_new(NULL, (u8 *)argv[2], strlen(argv[2]), NULL); + * rune = rune_derive_start(NULL, master, argv[3]); + * } else if (streq(argv[1], "add")) { + * // Add a restriction + * struct rune_restr *restr; + * if (argc != 4) + * goto usage; + * rune = rune_from_base64(NULL, argv[2]); + * if (!rune) + * errx(1, "Bad rune"); + * restr = rune_restr_from_string(NULL, argv[3], strlen(argv[3])); + * if (!restr) + * errx(1, "Bad restriction string"); + * rune_add_restr(rune, restr); + * } else if (streq(argv[1], "test")) { + * const char *err; + * if (argc != 5) + * goto usage; + * master = rune_new(NULL, (u8 *)argv[2], strlen(argv[2]), NULL); + * if (!master) + * errx(1, "Bad master rune"); + * rune = rune_from_base64(NULL, argv[3]); + * if (!rune) + * errx(1, "Bad rune"); + * err = rune_test(NULL, master, rune, check, argv[4]); + * if (err) + * printf("FAILED: %s\n", err); + * else + * printf("PASSED\n"); + * return 0; + * } else + * goto usage; + * + * printf("%s\n", rune_to_base64(NULL, rune)); + * return 0; + * + * usage: + * errx(1, "Usage: %s generate OR\n" + * "%s add OR\n" + * "%s test ", argv[0], argv[0], argv[0]); + * } + */ +int main(int argc, char *argv[]) +{ + /* Expect exactly one argument */ + if (argc != 2) + return 1; + + if (strcmp(argv[1], "depends") == 0) { + printf("ccan/base64\n"); + printf("ccan/crypto/sha256\n"); + printf("ccan/endian\n"); + printf("ccan/mem\n"); + printf("ccan/short_types\n"); + printf("ccan/str/hex\n"); + printf("ccan/tal/str\n"); + printf("ccan/tal\n"); + printf("ccan/typesafe_cb\n"); + return 0; + } + if (strcmp(argv[1], "testdepends") == 0) { + printf("ccan/tal/grab_file\n"); + return 0; + } + + return 1; +} diff --git a/ccan/ccan/rune/coding.c b/ccan/ccan/rune/coding.c new file mode 100644 index 000000000000..b9255b0b5b02 --- /dev/null +++ b/ccan/ccan/rune/coding.c @@ -0,0 +1,422 @@ +/* MIT (BSD) license - see LICENSE file for details */ +/* Routines to encode / decode a rune */ +#include +#include +#include +#include +#include +#include +#include + +/* From Python base64.urlsafe_b64encode: + * + * The alphabet uses '-' instead of '+' and '_' instead of '/'. + */ +static const base64_maps_t base64_maps_urlsafe = { + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", + + "\xff\xff\xff\xff\xff" /* 0 */ + "\xff\xff\xff\xff\xff" /* 5 */ + "\xff\xff\xff\xff\xff" /* 10 */ + "\xff\xff\xff\xff\xff" /* 15 */ + "\xff\xff\xff\xff\xff" /* 20 */ + "\xff\xff\xff\xff\xff" /* 25 */ + "\xff\xff\xff\xff\xff" /* 30 */ + "\xff\xff\xff\xff\xff" /* 35 */ + "\xff\xff\xff\xff\xff" /* 40 */ + "\x3e\xff\xff\x34\x35" /* 45 */ + "\x36\x37\x38\x39\x3a" /* 50 */ + "\x3b\x3c\x3d\xff\xff" /* 55 */ + "\xff\xff\xff\xff\xff" /* 60 */ + "\x00\x01\x02\x03\x04" /* 65 A */ + "\x05\x06\x07\x08\x09" /* 70 */ + "\x0a\x0b\x0c\x0d\x0e" /* 75 */ + "\x0f\x10\x11\x12\x13" /* 80 */ + "\x14\x15\x16\x17\x18" /* 85 */ + "\x19\xff\xff\xff\xff" /* 90 */ + "\x3f\xff\x1a\x1b\x1c" /* 95 */ + "\x1d\x1e\x1f\x20\x21" /* 100 */ + "\x22\x23\x24\x25\x26" /* 105 */ + "\x27\x28\x29\x2a\x2b" /* 110 */ + "\x2c\x2d\x2e\x2f\x30" /* 115 */ + "\x31\x32\x33\xff\xff" /* 120 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 125 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 135 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 145 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 155 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 165 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 175 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 185 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 195 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 205 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 215 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 225 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 235 */ + "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff" /* 245 */ +}; + +/* For encoding as a string */ +struct wbuf { + size_t off, len; + char *buf; +}; + +static void to_wbuf(const char *s, size_t len, void *vwbuf) +{ + struct wbuf *wbuf = vwbuf; + + while (wbuf->off + len > wbuf->len) + tal_resize(&wbuf->buf, wbuf->len *= 2); + memcpy(wbuf->buf + wbuf->off, s, len); + wbuf->off += len; +} + +/* For adding to sha256 */ +static void to_sha256(const char *s, size_t len, void *vshactx) +{ + struct sha256_ctx *shactx = vshactx; + sha256_update(shactx, s, len); +} + +static void rune_altern_encode(const struct rune_altern *altern, + void (*cb)(const char *s, size_t len, + void *arg), + void *arg) +{ + char cond = altern->condition; + const char *p; + + cb(altern->fieldname, strlen(altern->fieldname), arg); + cb(&cond, 1, arg); + + p = altern->value; + for (;;) { + char esc[2] = { '\\' }; + size_t len = strcspn(p, "\\|&"); + cb(p, len, arg); + if (!p[len]) + break; + esc[1] = p[len]; + cb(esc, 2, arg); + p++; + } +} + +static void rune_restr_encode(const struct rune_restr *restr, + void (*cb)(const char *s, size_t len, + void *arg), + void *arg) +{ + for (size_t i = 0; i < tal_count(restr->alterns); i++) { + if (i != 0) + cb("|", 1, arg); + rune_altern_encode(restr->alterns[i], cb, arg); + } +} + +void rune_sha256_add_restr(struct sha256_ctx *shactx, + struct rune_restr *restr) +{ + rune_restr_encode(restr, to_sha256, shactx); + rune_sha256_endmarker(shactx); +} + +const char *rune_is_derived(const struct rune *source, const struct rune *rune) +{ + if (!runestr_eq(source->version, rune->version)) + return "Version mismatch"; + + return rune_is_derived_anyversion(source, rune); +} + +const char *rune_is_derived_anyversion(const struct rune *source, + const struct rune *rune) +{ + struct sha256_ctx shactx; + size_t i; + + if (tal_count(rune->restrs) < tal_count(source->restrs)) + return "Fewer restrictions than master"; + + /* If we add the same restrictions to source rune, do we match? */ + shactx = source->shactx; + for (i = 0; i < tal_count(rune->restrs); i++) { + /* First restrictions must be identical */ + if (i < tal_count(source->restrs)) { + if (!rune_restr_eq(source->restrs[i], rune->restrs[i])) + return "Does not match master restrictions"; + } else + rune_sha256_add_restr(&shactx, rune->restrs[i]); + } + + if (memcmp(shactx.s, rune->shactx.s, sizeof(shactx.s)) != 0) + return "Not derived from master"; + return NULL; +} + +static bool peek_char(const char *data, size_t len, char *c) +{ + if (len == 0) + return false; + *c = *data; + return true; +} + +static void drop_char(const char **data, size_t *len) +{ + (*data)++; + (*len)--; +} + +static void pull_invalid(const char **data, size_t *len) +{ + *data = NULL; + *len = 0; +} + +static bool pull_char(const char **data, size_t *len, char *c) +{ + if (!peek_char(*data, *len, c)) { + pull_invalid(data, len); + return false; + } + drop_char(data, len); + return true; +} + +static bool is_valid_cond(enum rune_condition cond) +{ + switch (cond) { + case RUNE_COND_IF_MISSING: + case RUNE_COND_EQUAL: + case RUNE_COND_NOT_EQUAL: + case RUNE_COND_BEGINS: + case RUNE_COND_ENDS: + case RUNE_COND_CONTAINS: + case RUNE_COND_INT_LESS: + case RUNE_COND_INT_GREATER: + case RUNE_COND_LEXO_BEFORE: + case RUNE_COND_LEXO_AFTER: + case RUNE_COND_COMMENT: + return true; + } + return false; +} + +/* Sets *more on success: true if another altern follows */ +static struct rune_altern *rune_altern_decode(const tal_t *ctx, + const char **data, size_t *len, + bool *more) +{ + struct rune_altern *alt = tal(ctx, struct rune_altern); + const char *strstart = *data; + char *value; + size_t strlen = 0; + char c; + + /* Swallow field up to conditional */ + for (;;) { + if (!pull_char(data, len, &c)) + return tal_free(alt); + if (cispunct(c)) + break; + strlen++; + } + + alt->fieldname = tal_strndup(alt, strstart, strlen); + if (!is_valid_cond(c)) { + pull_invalid(data, len); + return tal_free(alt); + } + alt->condition = c; + + /* Assign worst case. */ + value = tal_arr(alt, char, *len + 1); + strlen = 0; + *more = false; + while (*len && pull_char(data, len, &c)) { + if (c == '|') { + *more = true; + break; + } + if (c == '&') + break; + + if (c == '\\' && !pull_char(data, len, &c)) + return tal_free(alt); + value[strlen++] = c; + } + value[strlen] = '\0'; + tal_resize(&value, strlen + 1); + alt->value = value; + return alt; +} + +static struct rune_restr *rune_restr_decode(const tal_t *ctx, + const char **data, size_t *len) +{ + struct rune_restr *restr = tal(ctx, struct rune_restr); + size_t num_alts = 0; + bool more; + + /* Must have at least one! */ + restr->alterns = tal_arr(restr, struct rune_altern *, 0); + do { + struct rune_altern *alt; + + alt = rune_altern_decode(restr, data, len, &more); + if (!alt) + return tal_free(restr); + tal_resize(&restr->alterns, num_alts+1); + restr->alterns[num_alts++] = alt; + } while (more); + return restr; +} + +static struct rune *from_string(const tal_t *ctx, + const char *str, + const u8 *hash32) +{ + size_t len = strlen(str); + struct rune *rune = tal(ctx, struct rune); + + /* Now count up how many bytes we should have hashed: secret uses + * first block. */ + rune->shactx.bytes = 64; + + rune->restrs = tal_arr(rune, struct rune_restr *, 0); + rune->unique_id = NULL; + rune->version = NULL; + + while (len) { + struct rune_restr *restr; + restr = rune_restr_decode(rune, &str, &len); + if (!restr) + return tal_free(rune); + if (!rune_add_restr(rune, restr)) + return tal_free(rune); + } + + /* Now we replace with canned hash state */ + memcpy(rune->shactx.s, hash32, 32); + for (size_t i = 0; i < 8; i++) + rune->shactx.s[i] = be32_to_cpu(rune->shactx.s[i]); + + return rune; +} + +struct rune_restr *rune_restr_from_string(const tal_t *ctx, + const char *str, + size_t len) +{ + struct rune_restr *restr; + + restr = rune_restr_decode(NULL, &str, &len); + /* Don't allow trailing chars */ + if (restr && len != 0) + restr = tal_free(restr); + return tal_steal(ctx, restr); +} + +static void to_string(struct wbuf *wbuf, const struct rune *rune, u8 *hash32) +{ + /* Copy hash in big-endian */ + for (size_t i = 0; i < 8; i++) { + be32 v = cpu_to_be32(rune->shactx.s[i]); + memcpy(hash32 + i*4, &v, sizeof(v)); + } + + for (size_t i = 0; i < tal_count(rune->restrs); i++) { + if (i != 0) + to_wbuf("&", 1, wbuf); + rune_restr_encode(rune->restrs[i], to_wbuf, wbuf); + } + to_wbuf("", 1, wbuf); +} + +struct rune *rune_from_base64n(const tal_t *ctx, const char *str, size_t len) +{ + size_t blen; + u8 *data; + struct rune *rune; + + data = tal_arr(NULL, u8, base64_decoded_length(len) + 1); + + blen = base64_decode_using_maps(&base64_maps_urlsafe, + (char *)data, tal_bytelen(data), + str, len); + if (blen == -1) + goto fail; + + if (blen < 32) + goto fail; + + data[blen] = '\0'; + /* Sanity check that it's a valid string! */ + if (strlen((char *)data + 32) != blen - 32) + goto fail; + + rune = from_string(ctx, (const char *)data + 32, data); + tal_free(data); + return rune; + +fail: + tal_free(data); + return NULL; +} + +struct rune *rune_from_base64(const tal_t *ctx, const char *str) +{ + return rune_from_base64n(ctx, str, strlen(str)); +} + +char *rune_to_base64(const tal_t *ctx, const struct rune *rune) +{ + u8 hash32[32]; + char *ret; + size_t ret_len; + struct wbuf wbuf; + + /* We're going to prepend hash */ + wbuf.off = sizeof(hash32); + wbuf.len = 64; + wbuf.buf = tal_arr(NULL, char, wbuf.len); + + to_string(&wbuf, rune, hash32); + /* Prepend hash */ + memcpy(wbuf.buf, hash32, sizeof(hash32)); + + ret = tal_arr(ctx, char, base64_encoded_length(wbuf.off) + 1); + ret_len = base64_encode_using_maps(&base64_maps_urlsafe, + ret, tal_bytelen(ret), + wbuf.buf, wbuf.off - 1); + ret[ret_len] = '\0'; + tal_free(wbuf.buf); + return ret; +} + +struct rune *rune_from_string(const tal_t *ctx, const char *str) +{ + u8 hash[32]; + if (!hex_decode(str, 64, hash, sizeof(hash))) + return NULL; + if (str[64] != ':') + return NULL; + return from_string(ctx, str + 65, hash); +} + +char *rune_to_string(const tal_t *ctx, const struct rune *rune) +{ + u8 hash32[32]; + struct wbuf wbuf; + + /* We're going to prepend hash (in hex), plus colon */ + wbuf.off = sizeof(hash32) * 2 + 1; + wbuf.len = 128; + wbuf.buf = tal_arr(ctx, char, wbuf.len); + + to_string(&wbuf, rune, hash32); + hex_encode(hash32, sizeof(hash32), wbuf.buf, sizeof(hash32) * 2 + 1); + wbuf.buf[sizeof(hash32) * 2] = ':'; + return wbuf.buf; +} diff --git a/ccan/ccan/rune/internal.h b/ccan/ccan/rune/internal.h new file mode 100644 index 000000000000..e4de06cc400d --- /dev/null +++ b/ccan/ccan/rune/internal.h @@ -0,0 +1,8 @@ +#ifndef CCAN_RUNE_INTERNAL_H +#define CCAN_RUNE_INTERNAL_H +/* MIT (BSD) license - see LICENSE file for details */ +void rune_sha256_endmarker(struct sha256_ctx *shactx); +void rune_sha256_add_restr(struct sha256_ctx *shactx, + struct rune_restr *restr); +bool runestr_eq(const char *a, const char *b); +#endif /* CCAN_RUNE_INTERNAL_H */ diff --git a/ccan/ccan/rune/rune.c b/ccan/ccan/rune/rune.c new file mode 100644 index 000000000000..84296c66b33d --- /dev/null +++ b/ccan/ccan/rune/rune.c @@ -0,0 +1,491 @@ +/* MIT (BSD) license - see LICENSE file for details */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Helper to produce an id field */ +static struct rune_restr *unique_id_restr(const tal_t *ctx, + const char *unique_id, + const char *version) +{ + const char *id; + struct rune_restr *restr; + + assert(!strchr(unique_id, '-')); + if (version) + id = tal_fmt(NULL, "%s-%s", unique_id, version); + else + id = tal_strdup(NULL, unique_id); + + restr = rune_restr_new(ctx); + /* We use the empty field for this, since it's always present. */ + rune_restr_add_altern(restr, + take(rune_altern_new(NULL, "", '=', take(id)))); + return restr; +} + +/* We pad between fields with something identical to the SHA end marker */ +void rune_sha256_endmarker(struct sha256_ctx *shactx) +{ + static const unsigned char pad[64] = {0x80}; + be64 sizedesc; + + sizedesc = cpu_to_be64((uint64_t)shactx->bytes << 3); + /* Add '1' bit to terminate, then all 0 bits, up to next block - 8. */ + sha256_update(shactx, pad, 1 + ((128 - 8 - (shactx->bytes % 64) - 1) % 64)); + /* Add number of bits of data (big endian) */ + sha256_update(shactx, &sizedesc, 8); +} + +struct rune *rune_new(const tal_t *ctx, const u8 *secret, size_t secret_len, + const char *version) +{ + struct rune *rune = tal(ctx, struct rune); + assert(secret_len + 1 + 8 <= 64); + + if (version) + rune->version = tal_strdup(rune, version); + else + rune->version = NULL; + rune->unique_id = NULL; + sha256_init(&rune->shactx); + sha256_update(&rune->shactx, secret, secret_len); + rune_sha256_endmarker(&rune->shactx); + rune->restrs = tal_arr(rune, struct rune_restr *, 0); + return rune; +} + +struct rune *rune_dup(const tal_t *ctx, const struct rune *rune TAKES) +{ + struct rune *dup; + + if (taken(rune)) + return tal_steal(ctx, (struct rune *)rune); + + dup = tal_dup(ctx, struct rune, rune); + dup->restrs = tal_arr(dup, struct rune_restr *, tal_count(rune->restrs)); + for (size_t i = 0; i < tal_count(rune->restrs); i++) { + dup->restrs[i] = rune_restr_dup(dup->restrs, + rune->restrs[i]); + } + return dup; +} + +struct rune *rune_derive_start(const tal_t *ctx, + const struct rune *master, + const char *unique_id TAKES) +{ + struct rune *rune = rune_dup(ctx, master); + + /* If they provide a unique_id, it goes first. */ + if (unique_id) { + if (taken(unique_id)) + rune->unique_id = tal_steal(rune, unique_id); + else + rune->unique_id = tal_strdup(rune, unique_id); + + rune_add_restr(rune, take(unique_id_restr(NULL, + rune->unique_id, + rune->version))); + } else { + assert(!rune->version); + } + return rune; +} + +struct rune_altern *rune_altern_new(const tal_t *ctx, + const char *fieldname TAKES, + enum rune_condition condition, + const char *value TAKES) +{ + struct rune_altern *altern = tal(ctx, struct rune_altern); + altern->condition = condition; + altern->fieldname = tal_strdup(altern, fieldname); + altern->value = tal_strdup(altern, value); + return altern; +} + +struct rune_altern *rune_altern_dup(const tal_t *ctx, + const struct rune_altern *altern TAKES) +{ + struct rune_altern *dup; + + if (taken(altern)) + return tal_steal(ctx, (struct rune_altern *)altern); + dup = tal(ctx, struct rune_altern); + dup->condition = altern->condition; + dup->fieldname = tal_strdup(dup, altern->fieldname); + dup->value = tal_strdup(dup, altern->value); + return dup; +} + +struct rune_restr *rune_restr_dup(const tal_t *ctx, + const struct rune_restr *restr TAKES) +{ + struct rune_restr *dup; + size_t num_altern; + + if (taken(restr)) + return tal_steal(ctx, (struct rune_restr *)restr); + + num_altern = tal_count(restr->alterns); + dup = tal(ctx, struct rune_restr); + dup->alterns = tal_arr(dup, struct rune_altern *, num_altern); + for (size_t i = 0; i < num_altern; i++) { + dup->alterns[i] = rune_altern_dup(dup->alterns, + restr->alterns[i]); + } + return dup; +} + +struct rune_restr *rune_restr_new(const tal_t *ctx) +{ + struct rune_restr *restr = tal(ctx, struct rune_restr); + restr->alterns = tal_arr(restr, struct rune_altern *, 0); + return restr; +} + +void rune_restr_add_altern(struct rune_restr *restr, + const struct rune_altern *alt TAKES) +{ + size_t num = tal_count(restr->alterns); + + tal_resize(&restr->alterns, num+1); + restr->alterns[num] = rune_altern_dup(restr->alterns, alt); +} + +static bool is_unique_id(const struct rune_altern *alt) +{ + return streq(alt->fieldname, ""); +} + +/* Return unique_id if valid, and sets *version */ +static const char *extract_unique_id(const tal_t *ctx, + const struct rune_altern *alt, + const char **version) +{ + size_t len; + /* Condition must be '='! */ + if (alt->condition != '=') + return NULL; + + len = strcspn(alt->value, "-"); + if (alt->value[len]) + *version = tal_strdup(ctx, alt->value + len + 1); + else + *version = NULL; + return tal_strndup(ctx, alt->value, len); +} + +bool rune_add_restr(struct rune *rune, + const struct rune_restr *restr TAKES) +{ + size_t num = tal_count(rune->restrs); + + /* An empty fieldname is additional correctness checks */ + for (size_t i = 0; i < tal_count(restr->alterns); i++) { + if (!is_unique_id(restr->alterns[i])) + continue; + + /* Must be the only alternative */ + if (tal_count(restr->alterns) != 1) + goto fail; + /* Must be the first restriction */ + if (num != 0) + goto fail; + + rune->unique_id = extract_unique_id(rune, + restr->alterns[i], + &rune->version); + if (!rune->unique_id) + goto fail; + } + + tal_resize(&rune->restrs, num+1); + rune->restrs[num] = rune_restr_dup(rune->restrs, restr); + + rune_sha256_add_restr(&rune->shactx, rune->restrs[num]); + return true; + +fail: + if (taken(restr)) + tal_free(restr); + return false; +} + +static const char *rune_restr_test(const tal_t *ctx, + const struct rune *rune, + const struct rune_restr *restr, + const char *(*check)(const tal_t *ctx, + const struct rune *rune, + const struct rune_altern *alt, + void *arg), + void *arg) +{ + size_t num = tal_count(restr->alterns); + const char **errs = tal_arr(NULL, const char *, num); + char *err; + + /* Only one alternative has to pass! */ + for (size_t i = 0; i < num; i++) { + errs[i] = check(errs, rune, restr->alterns[i], arg); + if (!errs[i]) { + tal_free(errs); + return NULL; + } + } + + err = tal_fmt(ctx, "%s", errs[0]); + for (size_t i = 1; i < num; i++) + tal_append_fmt(&err, " AND %s", errs[i]); + tal_free(errs); + return err; +} + +static const char *cond_test(const tal_t *ctx, + const struct rune_altern *alt, + const char *complaint, + bool cond) +{ + if (cond) + return NULL; + + return tal_fmt(ctx, "%s %s %s", alt->fieldname, complaint, alt->value); +} + +static const char *integer_compare_valid(const tal_t *ctx, + const s64 *fieldval_int, + const struct rune_altern *alt, + s64 *runeval_int) +{ + long l; + char *p; + + if (!fieldval_int) + return tal_fmt(ctx, "%s is not an integer field", + alt->fieldname); + + errno = 0; + l = strtol(alt->value, &p, 10); + if (p == alt->value + || *p + || ((l == LONG_MIN || l == LONG_MAX) && errno == ERANGE)) + return tal_fmt(ctx, "%s is not a valid integer", alt->value); + + *runeval_int = l; + return NULL; +} + +static int lexo_order(const char *fieldval_str, + size_t fieldval_strlen, + const char *alt) +{ + int ret = strncmp(fieldval_str, alt, fieldval_strlen); + + /* If alt is same but longer, fieldval is < */ + if (ret == 0 && strlen(alt) > fieldval_strlen) + ret = -1; + return ret; +} + +static const char *rune_alt_single(const tal_t *ctx, + const struct rune_altern *alt, + const char *fieldval_str, + size_t fieldval_strlen, + const s64 *fieldval_int) +{ + char strfield[STR_MAX_CHARS(s64) + 1]; + s64 runeval_int = 0 /* gcc v9.4.0 gets upset with uninitiaized var at -O3 */; + const char *err; + + /* Caller can't set both! */ + if (fieldval_int) { + assert(!fieldval_str); + sprintf(strfield, "%"PRIi64, *fieldval_int); + fieldval_str = strfield; + fieldval_strlen = strlen(strfield); + } + + switch (alt->condition) { + case RUNE_COND_IF_MISSING: + if (!fieldval_str) + return NULL; + return tal_fmt(ctx, "%s is present", alt->fieldname); + case RUNE_COND_EQUAL: + if (!fieldval_str) + return tal_fmt(ctx, "%s not present", alt->fieldname); + return cond_test(ctx, alt, "is not equal to", + memeqstr(fieldval_str, fieldval_strlen, alt->value)); + case RUNE_COND_NOT_EQUAL: + if (!fieldval_str) + return tal_fmt(ctx, "%s not present", alt->fieldname); + return cond_test(ctx, alt, "is equal to", + !memeqstr(fieldval_str, fieldval_strlen, alt->value)); + case RUNE_COND_BEGINS: + if (!fieldval_str) + return tal_fmt(ctx, "%s not present", alt->fieldname); + return cond_test(ctx, alt, "does not start with", + memstarts_str(fieldval_str, fieldval_strlen, alt->value)); + case RUNE_COND_ENDS: + if (!fieldval_str) + return tal_fmt(ctx, "%s not present", alt->fieldname); + return cond_test(ctx, alt, "does not end with", + memends_str(fieldval_str, fieldval_strlen, alt->value)); + case RUNE_COND_CONTAINS: + if (!fieldval_str) + return tal_fmt(ctx, "%s not present", alt->fieldname); + return cond_test(ctx, alt, "does not contain", + memmem(fieldval_str, fieldval_strlen, + alt->value, strlen(alt->value))); + case RUNE_COND_INT_LESS: + err = integer_compare_valid(ctx, fieldval_int, + alt, &runeval_int); + if (err) + return err; + return cond_test(ctx, alt, "is greater or equal to", + *fieldval_int < runeval_int); + case RUNE_COND_INT_GREATER: + err = integer_compare_valid(ctx, fieldval_int, + alt, &runeval_int); + if (err) + return err; + return cond_test(ctx, alt, "is less or equal to", + *fieldval_int > runeval_int); + case RUNE_COND_LEXO_BEFORE: + if (!fieldval_str) + return tal_fmt(ctx, "%s not present", alt->fieldname); + return cond_test(ctx, alt, "is equal to or ordered after", + lexo_order(fieldval_str, fieldval_strlen, alt->value) < 0); + case RUNE_COND_LEXO_AFTER: + if (!fieldval_str) + return tal_fmt(ctx, "%s not present", alt->fieldname); + return cond_test(ctx, alt, "is equal to or ordered before", + lexo_order(fieldval_str, fieldval_strlen, alt->value) > 0); + case RUNE_COND_COMMENT: + return NULL; + } + /* We should never create any other values! */ + abort(); +} + +const char *rune_alt_single_str(const tal_t *ctx, + const struct rune_altern *alt, + const char *fieldval_str, + size_t fieldval_strlen) +{ + return rune_alt_single(ctx, alt, fieldval_str, fieldval_strlen, NULL); +} + +const char *rune_alt_single_int(const tal_t *ctx, + const struct rune_altern *alt, + s64 fieldval_int) +{ + return rune_alt_single(ctx, alt, NULL, 0, &fieldval_int); +} + +const char *rune_alt_single_missing(const tal_t *ctx, + const struct rune_altern *alt) +{ + return rune_alt_single(ctx, alt, NULL, 0, NULL); +} + +const char *rune_meets_criteria_(const tal_t *ctx, + const struct rune *rune, + const char *(*check)(const tal_t *ctx, + const struct rune *rune, + const struct rune_altern *alt, + void *arg), + void *arg) +{ + for (size_t i = 0; i < tal_count(rune->restrs); i++) { + const char *err; + + /* Don't "check" unique id */ + if (i == 0 && is_unique_id(rune->restrs[i]->alterns[0])) + continue; + + err = rune_restr_test(ctx, rune, rune->restrs[i], check, arg); + if (err) + return err; + } + return NULL; +} + +const char *rune_test_(const tal_t *ctx, + const struct rune *master, + const struct rune *rune, + const char *(*check)(const tal_t *ctx, + const struct rune *rune, + const struct rune_altern *alt, + void *arg), + void *arg) +{ + const char *err; + + err = rune_is_derived(master, rune); + if (err) + return err; + return rune_meets_criteria_(ctx, rune, check, arg); +} + +bool rune_altern_eq(const struct rune_altern *alt1, + const struct rune_altern *alt2) +{ + return alt1->condition == alt2->condition + && streq(alt1->fieldname, alt2->fieldname) + && streq(alt1->value, alt2->value); +} + +bool rune_restr_eq(const struct rune_restr *rest1, + const struct rune_restr *rest2) +{ + if (tal_count(rest1->alterns) != tal_count(rest2->alterns)) + return false; + + for (size_t i = 0; i < tal_count(rest1->alterns); i++) + if (!rune_altern_eq(rest1->alterns[i], rest2->alterns[i])) + return false; + return true; +} + +/* Equal, as in both NULL, or both non-NULL and matching */ +bool runestr_eq(const char *a, const char *b) +{ + if (a) { + if (!b) + return false; + return streq(a, b); + } else + return b == NULL; +} + +bool rune_eq(const struct rune *rune1, const struct rune *rune2) +{ + if (!runestr_eq(rune1->unique_id, rune2->unique_id)) + return false; + if (!runestr_eq(rune1->version, rune2->version)) + return false; + + if (memcmp(rune1->shactx.s, rune2->shactx.s, sizeof(rune1->shactx.s))) + return false; + if (rune1->shactx.bytes != rune2->shactx.bytes) + return false; + if (memcmp(rune1->shactx.buf.u8, rune2->shactx.buf.u8, + rune1->shactx.bytes % 64)) + return false; + + if (tal_count(rune1->restrs) != tal_count(rune2->restrs)) + return false; + + for (size_t i = 0; i < tal_count(rune1->restrs); i++) + if (!rune_restr_eq(rune1->restrs[i], rune2->restrs[i])) + return false; + return true; +} diff --git a/ccan/ccan/rune/rune.h b/ccan/ccan/rune/rune.h new file mode 100644 index 000000000000..b67b78281eda --- /dev/null +++ b/ccan/ccan/rune/rune.h @@ -0,0 +1,379 @@ +/* MIT (BSD) license - see LICENSE file for details */ +#ifndef CCAN_RUNE_RUNE_H +#define CCAN_RUNE_RUNE_H +#include +#include +#include +#include + +/* A rune is a series of restrictions. */ +struct rune { + /* unique_id (if any) */ + const char *unique_id; + /* Version (if any) */ + const char *version; + + /* SHA-2 256 of restrictions so far. */ + struct sha256_ctx shactx; + /* Length given by tal_count() */ + struct rune_restr **restrs; +}; + +/* A restriction is one or more alternatives (altern) */ +struct rune_restr { + /* Length given by tal_count() */ + struct rune_altern **alterns; +}; + +enum rune_condition { + RUNE_COND_IF_MISSING = '!', + RUNE_COND_EQUAL = '=', + RUNE_COND_NOT_EQUAL = '/', + RUNE_COND_BEGINS = '^', + RUNE_COND_ENDS = '$', + RUNE_COND_CONTAINS = '~', + RUNE_COND_INT_LESS = '<', + RUNE_COND_INT_GREATER = '>', + RUNE_COND_LEXO_BEFORE = '{', + RUNE_COND_LEXO_AFTER = '}', + RUNE_COND_COMMENT = '#', +}; + +/* An alternative is a utf-8 fieldname, a condition, and a value */ +struct rune_altern { + enum rune_condition condition; + /* Strings. */ + const char *fieldname, *value; +}; + +/** + * rune_new - Create an unrestricted rune from this secret. + * @ctx: tal context, or NULL. Freeing @ctx will free the returned rune. + * @secret: secret bytes. + * @secret_len: number of @secret bytes (must be 55 bytes or less) + * @version: if non-NULL, sets a version for this rune. + * + * This allocates a new, unrestricted rune (sometimes called a master rune). + * + * Setting a version allows for different interpretations of a rune if + * things change in future, at cost of some space when it's used. + * + * Example: + * u8 secret[16]; + * struct rune *master; + * + * // A secret determined with a fair die roll! + * memset(secret, 5, sizeof(secret)); + * master = rune_new(NULL, secret, sizeof(secret), NULL); + * assert(master); + */ +struct rune *rune_new(const tal_t *ctx, const u8 *secret, size_t secret_len, + const char *version); + +/** + * rune_derive_start - Copy master rune, add a unique id. + * @ctx: context to allocate rune off + * @master: master rune. + * @unique_id: unique id; can be NULL, but that's not recommended. + * + * It's usually recommended to assign each rune a unique_id, so that + * specific runes can be blacklisted later (otherwise you need to disable + * all runes). This enlarges the rune string by '=' however. + * + * The rune version will be the same as the master: if that's non-zero, + * you *must* set unique_id. + * + * @unique_id cannot contain '-'. + * + * Example: + * struct rune *rune; + * // In reality, some global incrementing variable. + * const char *id = "1"; + * rune = rune_derive_start(NULL, master, id); + * assert(rune); + */ +struct rune *rune_derive_start(const tal_t *ctx, + const struct rune *master, + const char *unique_id); + +/** + * rune_dup - Copy a rune. + * @ctx: tal context, or NULL. + * @altern: the altern to copy. + * + * If @altern is take(), then simply returns it, otherwise copies. + */ +struct rune *rune_dup(const tal_t *ctx, const struct rune *rune TAKES); + +/** + * rune_altern_new - Create a new alternative. + * @ctx: tal context, or NULL. Freeing @ctx will free the returned altern. + * @fieldname: the UTF-8 field for the altern. You can only have + * alphanumerics, '.', '-' and '_' here. + * @condition: the condition, defined above. + * @value: the value for comparison; use "" if you don't care. Any UTF-8 value + * is allowed. + * + * An altern is the basis of rune restrictions (technically, a restriction + * is one or more alterns, but it's often just one). + * + * Example: + * struct rune_altern *a1, *a2; + * a1 = rune_altern_new(NULL, "val", RUNE_COND_EQUAL, "7"); + * a2 = rune_altern_new(NULL, "val2", '>', "-1"); + * assert(a1 && a2); + */ +struct rune_altern *rune_altern_new(const tal_t *ctx, + const char *fieldname TAKES, + enum rune_condition condition, + const char *value TAKES); + +/** + * rune_altern_dup - copy an alternative. + * @ctx: tal context, or NULL. + * @altern: the altern to copy. + * + * If @altern is take(), then simply returns it, otherwise copies. + */ +struct rune_altern *rune_altern_dup(const tal_t *ctx, + const struct rune_altern *altern TAKES); + +/** + * rune_restr_new - Create a new (empty) restriction. + * @ctx: tal context, or NULL. Freeing @ctx will free the returned restriction. + * + * Example: + * struct rune_restr *restr = rune_restr_new(NULL); + * assert(restr); + */ +struct rune_restr *rune_restr_new(const tal_t *ctx); + +/** + * rune_restr_dup - copy a restr. + * @ctx: tal context, or NULL. + * @restr: the restr to copy. + * + * If @resttr is take(), then simply returns it, otherwise copies. + */ +struct rune_restr *rune_restr_dup(const tal_t *ctx, + const struct rune_restr *restr TAKES); + +/** + * rune_restr_add_altern - add an altern to this restriction + * @restr: the restriction to add to + * @alt: the altern. + * + * If the alt is take(alt) then the alt will be owned by the restriction, + * otherwise it's copied. + * + * Example: + * rune_restr_add_altern(restr, take(a1)); + * rune_restr_add_altern(restr, take(a2)); + */ +void rune_restr_add_altern(struct rune_restr *restr, + const struct rune_altern *alt TAKES); + +/** + * rune_add_restr - add a restriction to this rune + * @rune: the rune to add to. + * @restr: the (non-empty) restriction. + * + * If the alt is take(alt) then the alt will be owned by the restr, + * otherwise it's copied (and all its children are copied!). + * + * This fails (and returns false) if restr tries to set unique_id/version + * and is not the first restriction, or has more than one alternative, + * or uses a non '=' condition. + * + * Example: + * rune_add_restr(rune, take(restr)); + */ +bool rune_add_restr(struct rune *rune, + const struct rune_restr *restr TAKES); + +/** + * rune_altern_eq - are two rune_altern equivalent? + * @alt1: the first + * @alt2: the second + */ +bool rune_altern_eq(const struct rune_altern *alt1, + const struct rune_altern *alt2); + +/** + * rune_restr_eq - are two rune_restr equivalent? + * @rest1: the first + * @rest2: the second + */ +bool rune_restr_eq(const struct rune_restr *rest1, + const struct rune_restr *rest2); + +/** + * rune_eq - are two runes equivalent? + * @rest1: the first + * @rest2: the second + */ +bool rune_eq(const struct rune *rune1, const struct rune *rune2); + +/** + * rune_alt_single_str - helper to implement check(). + * @ctx: context to allocate any error return from. + * @alt: alternative to test. + * @fieldval_str: field value as a string. + * @fieldval_strlen: length of @fieldval_str + */ +const char *rune_alt_single_str(const tal_t *ctx, + const struct rune_altern *alt, + const char *fieldval_str, + size_t fieldval_strlen); + +/** + * rune_alt_single_int - helper to implement check(). + * @ctx: context to allocate any error return from. + * @alt: alternative to test. + * @fieldval_int: field value as an integer. + */ +const char *rune_alt_single_int(const tal_t *ctx, + const struct rune_altern *alt, + s64 fieldval_int); + +/** + * rune_alt_single_missing - helper to implement check(). + * @ctx: context to allocate any error return from. + * @alt: alternative to test. + * + * Use this if alt->fieldname is unknown (it could still pass, if + * the test is that the fieldname is missing). + */ +const char *rune_alt_single_missing(const tal_t *ctx, + const struct rune_altern *alt); + + +/** + * rune_is_derived - is a rune derived from this other rune? + * @source: the base rune (usually the master rune) + * @rune: the rune to check. + * + * This is the first part of "is this rune valid?": does the cryptography + * check out, such that they validly made the rune from this source rune? + * + * It also checks that the versions match: if you want to allow more than + * one version, see rune_is_derived_anyversion. + */ +const char *rune_is_derived(const struct rune *source, const struct rune *rune); + +/** + * rune_is_derived_anyversion - is a rune derived from this other rune? + * @source: the base rune (usually the master rune) + * @rune: the rune to check. + * + * This does not check source->version against rune->version: if you issue + * different rune versions you will need to check that yourself. + */ +const char *rune_is_derived_anyversion(const struct rune *source, + const struct rune *rune); + +/** + * rune_meets_criteria - do we meet the criteria specified by the rune? + * @ctx: the tal context to allocate the returned error off. + * @rune: the rune to check. + * @check: the callback to check values + * @arg: data to hand to @check + * + * This is the second part of "is this rune valid?". + */ +const char *rune_meets_criteria_(const tal_t *ctx, + const struct rune *rune, + const char *(*check)(const tal_t *ctx, + const struct rune *rune, + const struct rune_altern *alt, + void *arg), + void *arg); + +/* Typesafe wrapper */ +#define rune_meets_criteria(ctx, rune, check, arg) \ + rune_meets_criteria_(typesafe_cb_preargs(const char *, void *, \ + (ctx), (rune), \ + (check), (arg), \ + const tal_t *, \ + const struct rune *, \ + const struct rune_altern *), \ + (arg)) + +/** + * rune_test - is a rune authorized? + * @ctx: the tal context to allocate @errstr off. + * @master: the master rune created from secret. + * @rune: the rune to check. + * @errstr: if non-NULL, descriptive string of failure. + * @get: the callback to get values + * @arg: data to hand to callback + * + * Simple call for rune_is_derived() and rune_meets_criteria(). If + * it's not OK, returns non-NULL. + */ +const char *rune_test_(const tal_t *ctx, + const struct rune *master, + const struct rune *rune, + const char *(*check)(const tal_t *ctx, + const struct rune *rune, + const struct rune_altern *alt, + void *arg), + void *arg); + +/* Typesafe wrapper */ +#define rune_test(ctx_, master_, rune_, check_, arg_) \ + rune_test_((ctx_), (master_), (rune_), \ + typesafe_cb_preargs(const char *, void *, \ + (check_), (arg_), \ + const tal_t *, \ + const struct rune *, \ + const struct rune_altern *), \ + (arg_)) + + +/** + * rune_from_base64 - convert base64 string to rune. + * @ctx: context to allocate rune off. + * @str: base64 string. + * + * Returns NULL if it's malformed. + */ +struct rune *rune_from_base64(const tal_t *ctx, const char *str); + +/** + * rune_from_base64n - convert base64 string to rune. + * @ctx: context to allocate rune off. + * @str: base64 string. + * @len: length of @str. + * + * Returns NULL if it's malformed. + */ +struct rune *rune_from_base64n(const tal_t *ctx, const char *str, size_t len); + +/** + * rune_to_base64 - convert run to base64 string. + * @ctx: context to allocate rune off. + * @rune: the rune. + * + * Only returns NULL if you've allowed tal allocations to return NULL. + */ +char *rune_to_base64(const tal_t *ctx, const struct rune *rune); + +/** + * This is a much more convenient working form. + */ +struct rune *rune_from_string(const tal_t *ctx, const char *str); +char *rune_to_string(const tal_t *ctx, const struct rune *rune); + +/** + * rune_restr_from_string - convenience routine to parse a single restriction. + * @ctx: context to allocate rune off. + * @str: the string of form "[|]*" + * @len: the length of @str. + * + * This is useful for writing simple tests and making simple runes. + */ +struct rune_restr *rune_restr_from_string(const tal_t *ctx, + const char *str, + size_t len); +#endif /* CCAN_RUNE_RUNE_H */ diff --git a/ccan/ccan/rune/test/run-alt-lexicographic-order.c b/ccan/ccan/rune/test/run-alt-lexicographic-order.c new file mode 100644 index 000000000000..a37ee581fb86 --- /dev/null +++ b/ccan/ccan/rune/test/run-alt-lexicographic-order.c @@ -0,0 +1,33 @@ +#include +#include +#include +#include + +int main(void) +{ + const char *str = "test string"; + plan_tests(strlen(str) * strlen(str)); + + for (size_t i = 0; str[i]; i++) { + char *stra = strdup(str); + stra[i] = '\0'; + for (size_t j = 0; str[j]; j++) { + char *strb = strdup(str); + strb[j] = '\0'; + int lexo, strc; + + lexo = lexo_order(str, i, strb); + strc = strcmp(stra, strb); + if (strc > 0) + ok1(lexo > 0); + else if (strc < 0) + ok1(lexo < 0); + else + ok1(lexo == 0); + free(strb); + } + free(stra); + } + /* This exits depending on whether all tests passed */ + return exit_status(); +} diff --git a/ccan/ccan/rune/test/run.c b/ccan/ccan/rune/test/run.c new file mode 100644 index 000000000000..86737b86be11 --- /dev/null +++ b/ccan/ccan/rune/test/run.c @@ -0,0 +1,127 @@ +#include +#include +#include +#include +#include + +static const char *check(const tal_t *ctx, + const struct rune *rune, + const struct rune_altern *alt, + char **parts) +{ + const char *val = NULL; + + for (size_t i = 1; parts[i]; i++) { + if (strstarts(parts[i], alt->fieldname) + && parts[i][strlen(alt->fieldname)] == '=') + val = parts[i] + strlen(alt->fieldname) + 1; + } + + /* If it's an integer, hand it like that */ + if (val) { + char *endp; + s64 v = strtol(val, &endp, 10); + if (*endp == '\0' && endp != val) + return rune_alt_single_int(ctx, alt, v); + return rune_alt_single_str(ctx, alt, val, strlen(val)); + } + return rune_alt_single_missing(ctx, alt); +} + +int main(void) +{ + char *vecs; + char **lines; + static const u8 secret_zero[16]; + struct rune *mr; + + /* Test vector rune uses all-zero secret */ + mr = rune_new(NULL, secret_zero, sizeof(secret_zero), NULL); + + /* Python runes library generates test vectors */ + vecs = grab_file(mr, "test/test_vectors.csv"); + assert(vecs); + lines = tal_strsplit(mr, take(vecs), "\n", STR_NO_EMPTY); + + plan_tests(343); + + for (size_t i = 0; lines[i]; i++) { + struct rune *rune1, *rune2; + char **parts; + + parts = tal_strsplit(lines, lines[i], ",", STR_EMPTY_OK); + if (streq(parts[0], "VALID")) { + diag("test %s %s", parts[0], parts[1]); + rune1 = rune_from_string(parts, parts[2]); + ok1(rune1); + rune2 = rune_from_base64(parts, parts[3]); + ok1(rune2); + ok1(rune_eq(rune1, rune2)); + ok1(streq(rune_to_string(parts, rune2), parts[2])); + ok1(streq(rune_to_base64(parts, rune1), parts[3])); + ok1(rune_is_derived_anyversion(mr, rune1) == NULL); + ok1(rune_is_derived_anyversion(mr, rune2) == NULL); + + if (parts[4]) { + if (parts[5]) + ok1(streq(rune1->version, parts[5])); + ok1(streq(rune1->unique_id, parts[4])); + } else { + ok1(!rune1->version); + ok1(!rune1->unique_id); + } + mr->version = NULL; + } else if (streq(parts[0], "DERIVE")) { + struct rune_restr *restr; + diag("test %s %s", parts[0], parts[1]); + rune1 = rune_from_base64(parts, parts[2]); + ok1(rune1); + rune2 = rune_from_base64(parts, parts[3]); + ok1(rune2); + ok1(rune_is_derived_anyversion(mr, rune1) == NULL); + ok1(rune_is_derived_anyversion(mr, rune2) == NULL); + ok1(rune_is_derived_anyversion(rune1, rune2) == NULL); + + restr = rune_restr_new(NULL); + for (size_t i = 4; parts[i]; i+=3) { + struct rune_altern *alt; + alt = rune_altern_new(NULL, + parts[i], + parts[i+1][0], + parts[i+2]); + rune_restr_add_altern(restr, take(alt)); + } + rune_add_restr(rune1, take(restr)); + ok1(rune_eq(rune1, rune2)); + } else if (streq(parts[0], "MALFORMED")) { + diag("test %s %s", parts[0], parts[1]); + rune1 = rune_from_string(parts, parts[2]); + ok1(!rune1); + rune2 = rune_from_base64(parts, parts[3]); + ok1(!rune2); + } else if (streq(parts[0], "BAD DERIVATION")) { + diag("test %s %s", parts[0], parts[1]); + rune1 = rune_from_string(parts, parts[2]); + ok1(rune1); + rune2 = rune_from_base64(parts, parts[3]); + ok1(rune2); + ok1(rune_eq(rune1, rune2)); + ok1(rune_is_derived(mr, rune1) != NULL); + ok1(rune_is_derived(mr, rune2) != NULL); + } else { + const char *err; + diag("test %s", parts[0]); + err = rune_test(parts, mr, rune1, check, parts); + if (streq(parts[0], "PASS")) { + ok1(!err); + } else { + assert(streq(parts[0], "FAIL")); + ok1(err); + } + } + } + + tal_free(mr); + /* This exits depending on whether all tests passed */ + return exit_status(); +} diff --git a/ccan/ccan/rune/test/test_vectors.csv b/ccan/ccan/rune/test/test_vectors.csv new file mode 100644 index 000000000000..a8411693c605 --- /dev/null +++ b/ccan/ccan/rune/test/test_vectors.csv @@ -0,0 +1,151 @@ +VALID,empty rune (secret = [0]*16),374708fff7719dd5979ec875d56cd2286f6d3cf7ec317a3b25632aab28ec37bb:,N0cI__dxndWXnsh11WzSKG9tPPfsMXo7JWMqqyjsN7s= +PASS +PASS,f1=1 +PASS,f1=var +PASS,f1=\|\&\\ +VALID,unique id 1,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:=1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL09MQ==,1 +VALID,unique id 2 version 1,4520773407c9658646326fdffe685ffbc3c8639a080dae4310b371830a205cf1:=2-1,RSB3NAfJZYZGMm_f_mhf-8PIY5oIDa5DELNxgwogXPE9Mi0x,2,1 +VALID,f1 is missing,64a926b7185d7cf98e10a07dfc4e83d2a826896ebdb112ac964566fa2d50b464:f1!,ZKkmtxhdfPmOEKB9_E6D0qgmiW69sRKslkVm-i1QtGRmMSE= +PASS +PASS,f2=f1 +FAIL,f1=1 +FAIL,f1=var +VALID,f1 equals v1,745c6e39cd41ee9f8388af8ad882bae4ee4e8f6b373f7682cc64d8574551fa5f:f1=v1,dFxuOc1B7p-DiK-K2IK65O5Oj2s3P3aCzGTYV0VR-l9mMT12MQ== +PASS,f1=v1 +FAIL,f1=v +FAIL,f1=v1a +FAIL +FAIL,f2=f1 +VALID,f1 not equal v1,c9236a6532bfa8e24bec9a66e96af3fb355f817770e79c5a81f6dd0b5ed20e47:f1/v1,ySNqZTK_qOJL7Jpm6Wrz-zVfgXdw55xagfbdC17SDkdmMS92MQ== +PASS,f1=v2 +PASS,f1=v +PASS,f1=v1a +FAIL +FAIL,f2=v1 +VALID,f1 ends with v1,71f2a1ec9631efc75b01db15fe1f025327ab467f8a83e6bfa7506da222adc5a2:f1$v1,cfKh7JYx78dbAdsV_h8CUyerRn-Kg-a_p1BtoiKtxaJmMSR2MQ== +PASS,f1=v1 +PASS,f1=2v1 +FAIL,f1=v1a +FAIL +VALID,f1 starts with v1,5b13dffbbd9f7b191b0557595d10b22c0acec0c567f8efeba1d7d047927d7bce:f1^v1,WxPf-72fexkbBVdZXRCyLArOwMVn-O_rodfQR5J9e85mMV52MQ== +PASS,f1=v1 +PASS,f1=v1a +FAIL,f1=2v1 +FAIL +VALID,f1 contains v1,ccbe593b72e0ab29446e46796ccd0c775ecd7a327fcc9ddc00fd3910cdacca00:f1~v1,zL5ZO3LgqylEbkZ5bM0Md17NejJ_zJ3cAP05EM2sygBmMX52MQ== +PASS,f1=v1 +PASS,f1=v1a +PASS,f1=2v1 +PASS,f1=2v12 +FAIL,f1=1v2 +FAIL +VALID,f1 less than v1,caff52cedb9241dc00aea7cefc2b89b0a7445b1a4e34c48a5a2b91d2fe76d31f:f1v1,ITV0jxlW2d-jxbCatq-da7BqQcW8-T0_gQXLJ4r1rFZmMT52MQ== +FAIL,f1=1 +FAIL,f1=2 +FAIL,f1=v1 +FAIL +VALID,f1 greater than 1,84e9991dd941bac97cc681eefec5dd7ac3668a4490ca6b0f19f0e79d2bb9c746:f1>1,hOmZHdlBusl8xoHu_sXdesNmikSQymsPGfDnnSu5x0ZmMT4x +PASS,f1=2 +PASS,f1=10000 +FAIL,f1=1 +FAIL,f1=-10000 +FAIL,f1=0 +FAIL,f1=v1 +FAIL +VALID,f1 sorts before 11,b9653ad0dcad7e5ed183f98cdd7e616acd07a98cc66a107a67626290bf000236:f1{11,uWU60Nytfl7Rg_mM3X5has0HqYzGahB6Z2JikL8AAjZmMXsxMQ== +PASS,f1=0 +PASS,f1=1 +PASS,f1= +PASS,f1=/ +FAIL,f1=11 +FAIL,f1=111 +FAIL,f1=v1 +FAIL,f1=: +FAIL +VALID,f1 sorts after 11,8c1f6c7c39badc5dea850192a0a4c6e9dd96bf33d410adc5a08fc375b22a1a52:f1}11,jB9sfDm63F3qhQGSoKTG6d2WvzPUEK3FoI_DdbIqGlJmMX0xMQ== +PASS,f1=111 +PASS,f1=v1 +PASS,f1=: +FAIL,f1=0 +FAIL,f1=1 +FAIL,f1= +FAIL,f1=/ +FAIL,f1=11 +FAIL +VALID,f1 comment 11,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1#11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMSMxMQ== +PASS,f1=111 +PASS,f1=v1 +PASS,f1=: +PASS,f1=0 +PASS,f1=1 +PASS,f1= +PASS,f1=/ +PASS,f1=11 +PASS +VALID,f1=1 or f2=3,85c3643dc102f0a0d6f20eeb8c294092151688fae41ef7c8ec7272ab23918376:f1=1|f2=3,hcNkPcEC8KDW8g7rjClAkhUWiPrkHvfI7HJyqyORg3ZmMT0xfGYyPTM= +PASS,f1=1 +PASS,f1=1,f2=2 +PASS,f2=3 +PASS,f1=var,f2=3 +PASS,f1=1,f2=3 +FAIL +FAIL,f1=2 +FAIL,f1=f1 +FAIL,f2=1 +FAIL,f2=f1 +DERIVE,unique_id 1 derivation,N0cI__dxndWXnsh11WzSKG9tPPfsMXo7JWMqqyjsN7s=,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL09MQ==,,=,1 +DERIVE,unique_id 2 version 1 derivation,N0cI__dxndWXnsh11WzSKG9tPPfsMXo7JWMqqyjsN7s=,RSB3NAfJZYZGMm_f_mhf-8PIY5oIDa5DELNxgwogXPE9Mi0x,,=,2-1 +DERIVE,f1=1 or f2=3,N0cI__dxndWXnsh11WzSKG9tPPfsMXo7JWMqqyjsN7s=,hcNkPcEC8KDW8g7rjClAkhUWiPrkHvfI7HJyqyORg3ZmMT0xfGYyPTM=,f1,=,1,f2,=,3 +DERIVE,AND f3 contains &|\,hcNkPcEC8KDW8g7rjClAkhUWiPrkHvfI7HJyqyORg3ZmMT0xfGYyPTM=,S253BW1Lragb1CpCSLXYGt9AdrE4iFMlXmnO0alV5vlmMT0xfGYyPTMmZjN-XCZcfFxc,f3,~,&|\ +PASS,f1=1,f3=&|\ +PASS,f2=3,f3=&|\x +FAIL +FAIL,f1=1 +FAIL,f2=3 +FAIL,f1=1,f2=3 +FAIL,f1=2,f3=&|\ +FAIL,f2=2,f3=&|\ +FAIL,f3=&|\ +MALFORMED,unique id must use = not !,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:!1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL0hMQ== +MALFORMED,unique id must use = not /,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:/1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL0vMQ== +MALFORMED,unique id must use = not ^,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:^1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL1eMQ== +MALFORMED,unique id must use = not $,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:$1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL0kMQ== +MALFORMED,unique id must use = not ~,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:~1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL1-MQ== +MALFORMED,unique id must use = not <,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:<1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL08MQ== +MALFORMED,unique id must use = not >,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:>1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL0-MQ== +MALFORMED,unique id must use = not },6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:}1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL19MQ== +MALFORMED,unique id must use = not {,6035731a2cbb022cbeb67645aa0f8a26653d8cc454e0e087d4d19d282b8da4bd:{1,YDVzGiy7Aiy-tnZFqg-KJmU9jMRU4OCH1NGdKCuNpL17MQ== +MALFORMED,unique id cannot be overridden,7a63a2966d38e6fed89256d4a6e983a6813bf084d4fc6c20b9cdaef24b23fa7e:=1-2&=3,emOilm045v7YklbUpumDpoE78ITU_Gwguc2u8ksj-n49MS0yJj0z +MALFORMED,version cannot be overridden,db823224f960976b3ee142ce8899fc7ea461b42617e7d16167b1886c5988c628:=1-2&=1-3,24IyJPlgl2s-4ULOiJn8fqRhtCYX59FhZ7GIbFmIxig9MS0yJj0xLTM= +MALFORMED,Bad condition ",76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1"11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMSIxMQ== +MALFORMED,Bad condition &,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1&11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMSYxMQ== +MALFORMED,Bad condition ',76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1'11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMScxMQ== +MALFORMED,Bad condition (,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1(11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMSgxMQ== +MALFORMED,Bad condition ),76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1)11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMSkxMQ== +MALFORMED,Bad condition *,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1*11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMSoxMQ== +MALFORMED,Bad condition +,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1+11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMSsxMQ== +MALFORMED,Bad condition -,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1-11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMS0xMQ== +MALFORMED,Bad condition .,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1.11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMS4xMQ== +MALFORMED,Bad condition :,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1:11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMToxMQ== +MALFORMED,Bad condition ;,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1;11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMTsxMQ== +MALFORMED,Bad condition ?,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1?11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMT8xMQ== +MALFORMED,Bad condition [,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1[11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMVsxMQ== +MALFORMED,Bad condition \,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1\11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMVwxMQ== +MALFORMED,Bad condition ],76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1]11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMV0xMQ== +MALFORMED,Bad condition _,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1_11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMV8xMQ== +MALFORMED,Bad condition `,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1`11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMWAxMQ== +MALFORMED,Bad condition |,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1|11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMXwxMQ== +BAD DERIVATION,Incremented sha,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0e:f1#11,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw5mMSMxMQ== +BAD DERIVATION,Unchanged sha,76bdd625de0e12058956e6c8a07cac58d7dc2253609a6bfb959f87cc094f3f0f:f1#11&a=1,dr3WJd4OEgWJVubIoHysWNfcIlNgmmv7lZ-HzAlPPw9mMSMxMSZhPTE= From d807af659245cf138299dba5ecc4944426229796 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 14:28:58 +0930 Subject: [PATCH 03/18] common/json_stream: make json_add_jsonstr take a length. This is useful when have have a jsmntok_t. Signed-off-by: Rusty Russell --- common/json_stream.c | 8 ++++---- common/json_stream.h | 7 ++++--- lightningd/jsonrpc.c | 8 +++++--- plugins/spender/multifundchannel.c | 9 ++++++--- plugins/spender/multiwithdraw.c | 2 +- plugins/spender/openchannel.c | 4 +++- 6 files changed, 23 insertions(+), 15 deletions(-) diff --git a/common/json_stream.c b/common/json_stream.c index f556033525bf..53b374e24260 100644 --- a/common/json_stream.c +++ b/common/json_stream.c @@ -199,13 +199,13 @@ static char *json_member_direct(struct json_stream *js, void json_add_jsonstr(struct json_stream *js, const char *fieldname, - const char *jsonstr) + const char *jsonstr, + size_t jsonstrlen) { char *p; - size_t len = strlen(jsonstr); - p = json_member_direct(js, fieldname, len); - memcpy(p, jsonstr, len); + p = json_member_direct(js, fieldname, jsonstrlen); + memcpy(p, jsonstr, jsonstrlen); } /* This is where we read the json_stream and write it to conn */ diff --git a/common/json_stream.h b/common/json_stream.h index 7743a29a83b1..316da33f4b42 100644 --- a/common/json_stream.h +++ b/common/json_stream.h @@ -157,12 +157,13 @@ void json_add_escaped_string(struct json_stream *result, * JSON-formatted. * @js: the json_stream. * @fieldname: fieldname (if in object), otherwise must be NULL. - * @jsonstr: the JSON entity, must be non-NULL, a null-terminated - * string that is already formatted in JSON. + * @jsonstr: the JSON entity + * @jsonstrlen: the length of @jsonstr */ void json_add_jsonstr(struct json_stream *js, const char *fieldname, - const char *jsonstr); + const char *jsonstr, + size_t jsonstrlen); /** * json_stream_output - start writing out a json_stream to this conn. diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index 317346ca1c75..357297f2f3a4 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -578,7 +578,7 @@ static struct json_stream *json_start(struct command *cmd) json_object_start(js, NULL); json_add_string(js, "jsonrpc", "2.0"); - json_add_jsonstr(js, "id", cmd->id); + json_add_jsonstr(js, "id", cmd->id, strlen(cmd->id)); return js; } @@ -742,13 +742,15 @@ static void rpc_command_hook_final(struct rpc_command_hook_payload *p STEALS) if (p->custom_result != NULL) { struct json_stream *s = json_start(p->cmd); - json_add_jsonstr(s, "result", p->custom_result); + json_add_jsonstr(s, "result", + p->custom_result, strlen(p->custom_result)); json_object_end(s); return was_pending(command_raw_complete(p->cmd, s)); } if (p->custom_error != NULL) { struct json_stream *s = json_start(p->cmd); - json_add_jsonstr(s, "error", p->custom_error); + json_add_jsonstr(s, "error", + p->custom_error, strlen(p->custom_error)); json_object_end(s); return was_pending(command_raw_complete(p->cmd, s)); } diff --git a/plugins/spender/multifundchannel.c b/plugins/spender/multifundchannel.c index b721d73c46d9..2c8a02ce577e 100644 --- a/plugins/spender/multifundchannel.c +++ b/plugins/spender/multifundchannel.c @@ -499,7 +499,8 @@ multifundchannel_finished(struct multifundchannel_command *mfc) mfc->removeds[i].error_message); if (mfc->removeds[i].error_data) json_add_jsonstr(out, "data", - mfc->removeds[i].error_data); + mfc->removeds[i].error_data, + strlen(mfc->removeds[i].error_data)); json_object_end(out); /* End error object */ json_object_end(out); } @@ -1382,7 +1383,8 @@ perform_fundpsbt(struct multifundchannel_command *mfc, u32 feerate) &after_fundpsbt, &mfc_forward_error, mfc); - json_add_jsonstr(req->js, "utxos", mfc->utxos_str); + json_add_jsonstr(req->js, "utxos", + mfc->utxos_str, strlen(mfc->utxos_str)); json_add_bool(req->js, "reservedok", false); } else { plugin_log(mfc->cmd->plugin, LOG_DBG, @@ -1814,7 +1816,8 @@ post_cleanup_redo_multifundchannel(struct multifundchannel_redo *redo) json_add_string(out, "method", failing_method); if (mfc->removeds[i].error_data) json_add_jsonstr(out, "data", - mfc->removeds[i].error_data); + mfc->removeds[i].error_data, + strlen(mfc->removeds[i].error_data)); /* Close 'data'. */ json_object_end(out); diff --git a/plugins/spender/multiwithdraw.c b/plugins/spender/multiwithdraw.c index 4bd7605f0bc2..9ab29bb9ef2c 100644 --- a/plugins/spender/multiwithdraw.c +++ b/plugins/spender/multiwithdraw.c @@ -356,7 +356,7 @@ static struct command_result *start_mw(struct multiwithdraw_command *mw) &mw_forward_error, mw); json_add_bool(req->js, "reservedok", false); - json_add_jsonstr(req->js, "utxos", mw->utxos); + json_add_jsonstr(req->js, "utxos", mw->utxos, strlen(mw->utxos)); } else { plugin_log(mw->cmd->plugin, LOG_DBG, "multiwithdraw %"PRIu64": fundpsbt.", diff --git a/plugins/spender/openchannel.c b/plugins/spender/openchannel.c index 7973bd0bedbb..0143c70704e2 100644 --- a/plugins/spender/openchannel.c +++ b/plugins/spender/openchannel.c @@ -346,7 +346,9 @@ openchannel_finished(struct multifundchannel_command *mfc) json_add_node_id(out, "id", &dest->id); json_add_string(out, "method", "openchannel_signed"); if (dest->error_data) - json_add_jsonstr(out, "data", dest->error_data); + json_add_jsonstr(out, "data", + dest->error_data, + strlen(dest->error_data)); json_object_end(out); return mfc_finished(mfc, out); From 7850d0f6ac44b5879556c0ee7ed1d979e6ec63e6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 14:30:17 +0930 Subject: [PATCH 04/18] libplugin: jsonrpc_request_whole_object_start() for more custom request handling. commando wants to see the whole reply object, and also not to assume params is an object. Signed-off-by: Rusty Russell --- plugins/libplugin.c | 26 +++++++++++++++++++------- plugins/libplugin.h | 14 ++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/plugins/libplugin.c b/plugins/libplugin.c index 74311b81c8b7..ee45e8874fcc 100644 --- a/plugins/libplugin.c +++ b/plugins/libplugin.c @@ -165,7 +165,8 @@ jsonrpc_request_start_(struct plugin *plugin, struct command *cmd, json_add_string(out->js, "jsonrpc", "2.0"); json_add_u64(out->js, "id", out->id); json_add_string(out->js, "method", method); - json_object_start(out->js, "params"); + if (out->errcb) + json_object_start(out->js, "params"); return out; } @@ -551,16 +552,26 @@ static void handle_rpc_reply(struct plugin *plugin, const jsmntok_t *toks) uintmap_del(&plugin->out_reqs, out->id); contenttok = json_get_member(plugin->rpc_buffer, toks, "error"); - if (contenttok) - res = out->errcb(out->cmd, plugin->rpc_buffer, - contenttok, out->arg); - else { + if (contenttok) { + if (out->errcb) + res = out->errcb(out->cmd, plugin->rpc_buffer, + contenttok, out->arg); + else + res = out->cb(out->cmd, plugin->rpc_buffer, + toks, out->arg); + } else { contenttok = json_get_member(plugin->rpc_buffer, toks, "result"); if (!contenttok) plugin_err(plugin, "Bad JSONRPC, no 'error' nor 'result': '%.*s'", json_tok_full_len(toks), json_tok_full(plugin->rpc_buffer, toks)); - res = out->cb(out->cmd, plugin->rpc_buffer, contenttok, out->arg); + /* errcb is NULL if it's a single whole-object callback */ + if (out->errcb) + res = out->cb(out->cmd, plugin->rpc_buffer, contenttok, + out->arg); + else + res = out->cb(out->cmd, plugin->rpc_buffer, toks, + out->arg); } assert(res == &pending || res == &complete); @@ -570,7 +581,8 @@ struct command_result * send_outreq(struct plugin *plugin, const struct out_req *req) { /* The "param" object. */ - json_object_end(req->js); + if (req->errcb) + json_object_end(req->js); json_object_end(req->js); json_stream_close(req->js, req->cmd); diff --git a/plugins/libplugin.h b/plugins/libplugin.h index feb6b4d8162e..6b90ef6e5793 100644 --- a/plugins/libplugin.h +++ b/plugins/libplugin.h @@ -117,6 +117,8 @@ struct out_req *jsonrpc_request_start_(struct plugin *plugin, void *arg), void *arg); +/* This variant has callbacks received whole obj, not "result" or + * "error" members. */ #define jsonrpc_request_start(plugin, cmd, method, cb, errcb, arg) \ jsonrpc_request_start_((plugin), (cmd), (method), \ typesafe_cb_preargs(struct command_result *, void *, \ @@ -132,6 +134,18 @@ struct out_req *jsonrpc_request_start_(struct plugin *plugin, (arg)) +/* This variant has callbacks received whole obj, not "result" or + * "error" members. It also doesn't start params{}. */ +#define jsonrpc_request_whole_object_start(plugin, cmd, method, cb, arg) \ + jsonrpc_request_start_((plugin), (cmd), (method), \ + typesafe_cb_preargs(struct command_result *, void *, \ + (cb), (arg), \ + struct command *command, \ + const char *buf, \ + const jsmntok_t *result), \ + NULL, \ + (arg)) + /* Helper to create a JSONRPC2 response stream with a "result" object. */ struct json_stream *jsonrpc_stream_success(struct command *cmd); From 097a5535fefd3ec41a56efb52445b791dc7cbf9a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 14:30:19 +0930 Subject: [PATCH 05/18] libplugin: datastore helpers. Plugins are supposed to store their data in the datastore, and commando does so: let's make it easier for them by providing convenience APIs. Signed-off-by: Rusty Russell --- plugins/libplugin.c | 123 +++++++++++++++++++++++++++++++++++++++----- plugins/libplugin.h | 58 +++++++++++++++++++++ 2 files changed, 169 insertions(+), 12 deletions(-) diff --git a/plugins/libplugin.c b/plugins/libplugin.c index ee45e8874fcc..6225289bc85c 100644 --- a/plugins/libplugin.c +++ b/plugins/libplugin.c @@ -490,20 +490,18 @@ static struct json_out *start_json_request(const tal_t *ctx, return jout; } -/* Synchronous routine to send command and extract fields from response */ -void rpc_scan(struct plugin *plugin, - const char *method, - const struct json_out *params TAKES, - const char *guide, - ...) +static const char *rpc_scan_core(const tal_t *ctx, + struct plugin *plugin, + const char *method, + const struct json_out *params TAKES, + const char *guide, + va_list ap) { bool error; - const char *err; const jsmntok_t *contents; int reqlen; const char *p; struct json_out *jout; - va_list ap; jout = start_json_request(tmpctx, 0, method, params); finish_and_send_json(plugin->rpc_conn->fd, jout); @@ -514,15 +512,116 @@ void rpc_scan(struct plugin *plugin, method, reqlen, membuf_elems(&plugin->rpc_conn->mb)); p = membuf_consume(&plugin->rpc_conn->mb, reqlen); + return json_scanv(ctx, p, contents, guide, ap); +} + +/* Synchronous routine to send command and extract fields from response */ +void rpc_scan(struct plugin *plugin, + const char *method, + const struct json_out *params TAKES, + const char *guide, + ...) +{ + const char *err; + va_list ap; va_start(ap, guide); - err = json_scanv(tmpctx, p, contents, guide, ap); + err = rpc_scan_core(tmpctx, plugin, method, params, guide, ap); va_end(ap); if (err) - plugin_err(plugin, "Could not parse %s in reply to %s: %s: '%.*s'", - guide, method, err, - reqlen, membuf_elems(&plugin->rpc_conn->mb)); + plugin_err(plugin, "Could not parse %s in reply to %s: %s", + guide, method, err); +} + +static void json_add_keypath(struct json_out *jout, const char *fieldname, const char *path) +{ + char **parts = tal_strsplit(tmpctx, path, "/", STR_EMPTY_OK); + + json_out_start(jout, fieldname, '['); + for (size_t i = 0; parts[i]; parts++) + json_out_addstr(jout, NULL, parts[i]); + json_out_end(jout, ']'); +} + +static bool rpc_scan_datastore(struct plugin *plugin, + const char *path, + const char *hex_or_string, + va_list ap) +{ + const char *guide; + struct json_out *params; + const char *err; + + params = json_out_new(NULL); + json_out_start(params, NULL, '{'); + json_add_keypath(params, "key", path); + json_out_end(params, '}'); + json_out_finished(params); + + guide = tal_fmt(tmpctx, "{datastore:[0:{%s:%%}]}", hex_or_string); + /* FIXME: Could be some other error, but that's probably a caller bug! */ + err = rpc_scan_core(tmpctx, plugin, "listdatastore", take(params), guide, ap); + if (!err) + return true; + plugin_log(plugin, LOG_DBG, "listdatastore error %s: %s", path, err); + return false; +} + +bool rpc_scan_datastore_str(struct plugin *plugin, + const char *path, + ...) +{ + bool ret; + va_list ap; + + va_start(ap, path); + ret = rpc_scan_datastore(plugin, path, "string", ap); + va_end(ap); + return ret; +} + +/* This variant scans the hex encoding, not the string */ +bool rpc_scan_datastore_hex(struct plugin *plugin, + const char *path, + ...) +{ + bool ret; + va_list ap; + + va_start(ap, path); + ret = rpc_scan_datastore(plugin, path, "hex", ap); + va_end(ap); + return ret; +} + +struct command_result *jsonrpc_set_datastore_(struct plugin *plugin, + struct command *cmd, + const char *path, + const void *value, + bool value_is_string, + const char *mode, + struct command_result *(*cb)(struct command *command, + const char *buf, + const jsmntok_t *result, + void *arg), + struct command_result *(*errcb)(struct command *command, + const char *buf, + const jsmntok_t *result, + void *arg), + void *arg) +{ + struct out_req *req; + + req = jsonrpc_request_start(plugin, cmd, "datastore", cb, errcb, arg); + + json_add_keypath(req->js->jout, "key", path); + if (value_is_string) + json_add_string(req->js, "string", value); + else + json_add_hex_talarr(req->js, "hex", value); + json_add_string(req->js, "mode", mode); + return send_outreq(plugin, req); } static void handle_rpc_reply(struct plugin *plugin, const jsmntok_t *toks) diff --git a/plugins/libplugin.h b/plugins/libplugin.h index 6b90ef6e5793..6342460cdc5e 100644 --- a/plugins/libplugin.h +++ b/plugins/libplugin.h @@ -160,6 +160,51 @@ struct json_stream *jsonrpc_stream_fail_data(struct command *cmd, int code, const char *err); +/* Helper to jsonrpc_request_start() and send_outreq() to update datastore. */ +struct command_result *jsonrpc_set_datastore_(struct plugin *plugin, + struct command *cmd, + const char *path, + const void *value, + bool value_is_string, + const char *mode, + struct command_result *(*cb)(struct command *command, + const char *buf, + const jsmntok_t *result, + void *arg), + struct command_result *(*errcb)(struct command *command, + const char *buf, + const jsmntok_t *result, + void *arg), + void *arg); + +#define jsonrpc_set_datastore_string(plugin, cmd, path, str, mode, cb, errcb, arg) \ + jsonrpc_set_datastore_((plugin), (cmd), (path), (str), true, (mode), \ + typesafe_cb_preargs(struct command_result *, void *, \ + (cb), (arg), \ + struct command *command, \ + const char *buf, \ + const jsmntok_t *result), \ + typesafe_cb_preargs(struct command_result *, void *, \ + (errcb), (arg), \ + struct command *command, \ + const char *buf, \ + const jsmntok_t *result), \ + (arg)) + +#define jsonrpc_set_datastore_binary(plugin, cmd, path, tal_ptr, mode, cb, errcb, arg) \ + jsonrpc_set_datastore_((plugin), (cmd), (path), (tal_ptr), false, (mode), \ + typesafe_cb_preargs(struct command_result *, void *, \ + (cb), (arg), \ + struct command *command, \ + const char *buf, \ + const jsmntok_t *result), \ + typesafe_cb_preargs(struct command_result *, void *, \ + (errcb), (arg), \ + struct command *command, \ + const char *buf, \ + const jsmntok_t *result), \ + (arg)) + /* This command is finished, here's the response (the content of the * "result" or "error" field) */ WARN_UNUSED_RESULT @@ -220,6 +265,19 @@ void rpc_scan(struct plugin *plugin, const char *guide, ...); +/* Helper to scan datastore: can only be used in init callback. * + Returns false if field does not exist. * path is /-separated. Final + arg is JSON_SCAN or JSON_SCAN_TAL. + */ +bool rpc_scan_datastore_str(struct plugin *plugin, + const char *path, + ...); +/* This variant scans the hex encoding, not the string */ +bool rpc_scan_datastore_hex(struct plugin *plugin, + const char *path, + ...); + + /* Send an async rpc request to lightningd. */ struct command_result *send_outreq(struct plugin *plugin, const struct out_req *req); From e579650d7527835559df026caafd0bdafdaa29fc Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:21 +0930 Subject: [PATCH 06/18] plugins/commando: basic commando plugin (no runes yet). Signed-off-by: Rusty Russell +#include +#include +#include +#include +#include +#include + +/* We (as your local commando command) detected an error. */ +#define COMMANDO_ERROR_LOCAL 0x4c4f +/* Remote (as executing your commando command) detected an error. */ +#define COMMANDO_ERROR_REMOTE 0x4c50 +/* Specifically: bad/missing rune */ +#define COMMANDO_ERROR_REMOTE_AUTH 0x4c51 + +enum commando_msgtype { + COMMANDO_MSG_CMD = 0x4c4f, + /* Replies are split across multiple CONTINUES, then TERM. */ + COMMANDO_MSG_REPLY_CONTINUES = 0x594b, + COMMANDO_MSG_REPLY_TERM = 0x594d, +}; + +struct commando { + struct command *cmd; + struct node_id peer; + u64 id; + + /* This is set to NULL if they seem to be spamming us! */ + u8 *contents; +}; + +static struct plugin *plugin; +static struct commando **outgoing_commands; + +/* NULL peer: don't care about peer. NULL id: don't care about id */ +static struct commando *find_commando(struct commando **arr, + const struct node_id *peer, + const u64 *id) +{ + for (size_t i = 0; i < tal_count(arr); i++) { + if (id && arr[i]->id != *id) + continue; + if (peer && !node_id_eq(&arr[i]->peer, peer)) + continue; + return arr[i]; + } + return NULL; +} + +static void destroy_commando(struct commando *commando, struct commando ***arr) +{ + for (size_t i = 0; i < tal_count(*arr); i++) { + if ((*arr)[i] == commando) { + tal_arr_remove(arr, i); + return; + } + } + abort(); +} + +/* Append to commando->contents: set to NULL if we've over max. */ +static void append_contents(struct commando *commando, const u8 *msg, size_t msglen, + size_t maxlen) +{ + size_t len = tal_count(commando->contents); + + if (!commando->contents) + return; + + if (len + msglen > maxlen) { + commando->contents = tal_free(commando->contents); + return; + } + + tal_resize(&commando->contents, len + msglen); + memcpy(commando->contents + len, msg, msglen); +} + +struct reply { + struct commando *incoming; + char *buf; + size_t off, len; +}; + +static struct command_result *send_response(struct command *command UNUSED, + const char *buf UNUSED, + const jsmntok_t *result UNUSED, + struct reply *reply) +{ + size_t msglen = reply->len - reply->off; + u8 *cmd_msg; + enum commando_msgtype msgtype; + struct out_req *req; + + /* Limit is 64k, but there's a little overhead */ + if (msglen > 65000) { + msglen = 65000; + msgtype = COMMANDO_MSG_REPLY_CONTINUES; + /* We need to make a copy first time before we call back, since + * plugin will reuse it! */ + if (reply->off == 0) + reply->buf = tal_dup_talarr(reply, char, reply->buf); + } else { + if (msglen == 0) { + tal_free(reply); + return command_done(); + } + msgtype = COMMANDO_MSG_REPLY_TERM; + } + + cmd_msg = tal_arr(NULL, u8, 0); + towire_u16(&cmd_msg, msgtype); + towire_u64(&cmd_msg, reply->incoming->id); + towire(&cmd_msg, reply->buf + reply->off, msglen); + reply->off += msglen; + + req = jsonrpc_request_start(plugin, NULL, "sendcustommsg", + send_response, send_response, + reply); + json_add_node_id(req->js, "node_id", &reply->incoming->peer); + json_add_hex_talarr(req->js, "msg", cmd_msg); + tal_free(cmd_msg); + send_outreq(plugin, req); + + return command_done(); +} + +static struct command_result *cmd_done(struct command *command, + const char *buf, + const jsmntok_t *obj, + struct commando *incoming) +{ + struct reply *reply = tal(plugin, struct reply); + reply->incoming = tal_steal(reply, incoming); + reply->buf = (char *)buf; + + /* result is contents of "error" or "response": we want top-leve + * object */ + reply->off = obj->start; + reply->len = obj->end; + + return send_response(command, buf, obj, reply); +} + +static void commando_error(struct commando *incoming, + int ecode, + const char *fmt, ...) + PRINTF_FMT(3,4); + +static void commando_error(struct commando *incoming, + int ecode, + const char *fmt, ...) +{ + struct reply *reply = tal(plugin, struct reply); + va_list ap; + + reply->incoming = tal_steal(reply, incoming); + reply->buf = tal_fmt(reply, "{\"error\":{\"code\":%i,\"message\":\"", ecode); + va_start(ap, fmt); + tal_append_vfmt(&reply->buf, fmt, ap); + va_end(ap); + tal_append_fmt(&reply->buf, "\"}}"); + reply->off = 0; + reply->len = tal_bytelen(reply->buf) - 1; + + send_response(NULL, NULL, NULL, reply); +} + +static const char *check_rune(struct commando *incoming, + const char *buf, + const jsmntok_t *method, + const jsmntok_t *params, + const jsmntok_t *rune) +{ + /* FIXME! */ + return NULL; +} + +static void try_command(struct node_id *peer, + u64 idnum, + const u8 *msg, size_t msglen) +{ + struct commando *incoming = tal(plugin, struct commando); + const jsmntok_t *toks, *method, *params, *rune; + const char *buf = (const char *)msg, *failmsg; + struct out_req *req; + + incoming->peer = *peer; + incoming->id = idnum; + + toks = json_parse_simple(incoming, buf, msglen); + if (!toks) { + commando_error(incoming, COMMANDO_ERROR_REMOTE, + "Invalid JSON"); + return; + } + + if (toks[0].type != JSMN_OBJECT) { + commando_error(incoming, COMMANDO_ERROR_REMOTE, + "Not a JSON object"); + return; + } + method = json_get_member(buf, toks, "method"); + if (!method) { + commando_error(incoming, COMMANDO_ERROR_REMOTE, + "No method"); + return; + } + params = json_get_member(buf, toks, "params"); + if (params && params->type != JSMN_OBJECT) { + commando_error(incoming, COMMANDO_ERROR_REMOTE, + "Params must be object"); + return; + } + rune = json_get_member(buf, toks, "rune"); + + failmsg = check_rune(incoming, buf, method, params, rune); + if (failmsg) { + commando_error(incoming, COMMANDO_ERROR_REMOTE_AUTH, + "Not authorized: %s", failmsg); + return; + } + + /* We handle success and failure the same */ + req = jsonrpc_request_whole_object_start(plugin, NULL, + json_strdup(tmpctx, buf, + method), + cmd_done, incoming); + if (params) { + size_t i; + const jsmntok_t *t; + + json_object_start(req->js, "params"); + /* FIXME: This is ugly! */ + json_for_each_obj(i, t, params) { + json_add_jsonstr(req->js, + json_strdup(tmpctx, buf, t), + json_tok_full(buf, t+1), + json_tok_full_len(t+1)); + } + json_object_end(req->js); + } else { + json_object_start(req->js, "params"); + json_object_end(req->js); + } + tal_free(toks); + send_outreq(plugin, req); +} + +static struct command_result *handle_reply(struct node_id *peer, + u64 idnum, + const u8 *msg, size_t msglen, + bool terminal) +{ + struct commando *ocmd; + struct json_stream *res; + const jsmntok_t *toks, *result, *err; + const char *replystr; + size_t i; + const jsmntok_t *t; + + ocmd = find_commando(outgoing_commands, peer, &idnum); + if (!ocmd) { + plugin_log(plugin, LOG_DBG, + "Ignoring unexpected %s reply from %s (id %"PRIu64")", + terminal ? "terminal" : "partial", + node_id_to_hexstr(tmpctx, peer), + idnum); + return NULL; + } + + /* FIXME: We buffer, but ideally we would stream! */ + /* listchannels is 71MB, so we need to allow some headroom! */ + append_contents(ocmd, msg, msglen, 500*1024*1024); + + if (!terminal) + return NULL; + + if (!ocmd->contents) + return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, "Reply was oversize"); + + replystr = (const char *)ocmd->contents; + toks = json_parse_simple(ocmd, replystr, tal_bytelen(ocmd->contents)); + if (!toks || toks[0].type != JSMN_OBJECT) + return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, "Reply was unparsable"); + + err = json_get_member(replystr, toks, "error"); + if (err) { + const jsmntok_t *code = json_get_member(replystr, err, "code"); + int ecode; + const jsmntok_t *message = json_get_member(replystr, err, "message"); + if (!code || !json_to_int(replystr, code, &ecode)) { + return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, + "Error '%.*s' had no valid code", + json_tok_full_len(err), + json_tok_full(replystr, err)); + } + if (!message) { + return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, + "Error had no message"); + } + /* FIXME: data! */ + return command_fail(ocmd->cmd, ecode, "%.*s", + message->end - message->start, + replystr + message->start); + } + + result = json_get_member(replystr, toks, "result"); + if (!result) + return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, "Reply had no result"); + + res = jsonrpc_stream_success(ocmd->cmd); + + /* FIXME: This is ugly! */ + json_for_each_obj(i, t, result) { + json_add_jsonstr(res, + json_strdup(tmpctx, replystr, t), + json_tok_full(replystr, t+1), + json_tok_full_len(t+1)); + } + + return command_finished(ocmd->cmd, res); +} + +static struct command_result *handle_custommsg(struct command *cmd, + const char *buf, + const jsmntok_t *params) +{ + struct node_id peer; + const u8 *msg; + size_t len; + enum commando_msgtype mtype; + u64 idnum; + + json_to_node_id(buf, json_get_member(buf, params, "peer_id"), &peer); + msg = json_tok_bin_from_hex(cmd, buf, + json_get_member(buf, params, "payload")); + + len = tal_bytelen(msg); + mtype = fromwire_u16(&msg, &len); + idnum = fromwire_u64(&msg, &len); + + if (msg) { + switch (mtype) { + case COMMANDO_MSG_CMD: + try_command(&peer, idnum, msg, len); + break; + case COMMANDO_MSG_REPLY_CONTINUES: + case COMMANDO_MSG_REPLY_TERM: + handle_reply(&peer, idnum, msg, len, + mtype == COMMANDO_MSG_REPLY_TERM); + break; + } + } + + return command_hook_success(cmd); +} + +static const struct plugin_hook hooks[] = { + { + "custommsg", + handle_custommsg + }, +}; + +static struct command_result *send_success(struct command *command, + const char *buf, + const jsmntok_t *result, + struct commando *incoming) +{ + return command_still_pending(command); +} + + +static struct command_result *json_commando(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct node_id *peer; + const char *method, *cparams; + const char *rune; + struct commando *ocmd; + struct out_req *req; + u8 *cmd_msg; + char *json; + + if (!param(cmd, buffer, params, + p_req("peer_id", param_node_id, &peer), + p_req("method", param_string, &method), + p_opt("params", param_string, &cparams), + p_opt("rune", param_string, &rune), + NULL)) + return command_param_failed(); + + ocmd = tal(cmd, struct commando); + ocmd->cmd = cmd; + ocmd->peer = *peer; + ocmd->contents = tal_arr(ocmd, u8, 0); + do { + ocmd->id = pseudorand_u64(); + } while (find_commando(outgoing_commands, NULL, &ocmd->id)); + tal_arr_expand(&outgoing_commands, ocmd); + tal_add_destructor2(ocmd, destroy_commando, &outgoing_commands); + + json = tal_fmt(tmpctx, + "{\"method\":\"%s\",\"params\":%s", method, + cparams ? cparams : "{}"); + if (rune) + tal_append_fmt(&json, ",\"rune\":\"%s\"", rune); + tal_append_fmt(&json, "}"); + + cmd_msg = tal_arr(NULL, u8, 0); + towire_u16(&cmd_msg, COMMANDO_MSG_CMD); + towire_u64(&cmd_msg, ocmd->id); + towire(&cmd_msg, json, strlen(json)); + req = jsonrpc_request_start(plugin, NULL, "sendcustommsg", + send_success, forward_error, ocmd); + json_add_node_id(req->js, "node_id", &ocmd->peer); + json_add_hex_talarr(req->js, "msg", cmd_msg); + tal_free(cmd_msg); + + /* Keep memleak code happy! */ + tal_free(peer); + tal_free(method); + tal_free(cparams); + + return send_outreq(plugin, req); +} + +#if DEVELOPER +static void memleak_mark_globals(struct plugin *p, struct htable *memtable) +{ + memleak_remove_region(memtable, outgoing_commands, tal_bytelen(outgoing_commands)); +} +#endif + +static const char *init(struct plugin *p, + const char *buf UNUSED, const jsmntok_t *config UNUSED) +{ + outgoing_commands = tal_arr(p, struct commando *, 0); + plugin = p; +#if DEVELOPER + plugin_set_memleak_handler(p, memleak_mark_globals); +#endif + return NULL; +} + +static const struct plugin_command commands[] = { { + "commando", + "utility", + "Send a commando message to a direct peer, wait for response", + "Sends {peer_id} {method} with optional {params} and {rune}", + json_commando, + } +}; + +int main(int argc, char *argv[]) +{ + setup_locale(); + plugin_main(argv, init, PLUGIN_STATIC, true, NULL, + commands, ARRAY_SIZE(commands), + NULL, 0, + hooks, ARRAY_SIZE(hooks), + NULL, 0, + NULL); +} diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 214dadf09197..74490cf93078 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2544,3 +2544,46 @@ def test_plugin_shutdown(node_factory): l1.daemon.wait_for_logs(['test_libplugin: shutdown called', 'misc_notifications.py: via lightningd shutdown, datastore failed', 'test_libplugin: failed to self-terminate in time, killing.']) + + +def test_commando(node_factory): + l1, l2 = node_factory.line_graph(2, fundchannel=False) + + # This works + res = l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'listpeers'}) + assert len(res['peers']) == 1 + assert res['peers'][0]['id'] == l2.info['id'] + + res = l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'listpeers', + 'params': {'id': l2.info['id']}}) + assert len(res['peers']) == 1 + assert res['peers'][0]['id'] == l2.info['id'] + + with pytest.raises(RpcError, match='missing required parameter'): + l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'withdraw'}) + + with pytest.raises(RpcError, match='unknown parameter: foobar'): + l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'invoice', + 'params': {'foobar': 1}}) + + ret = l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'ping', + 'params': {'id': l2.info['id']}}) + assert 'totlen' in ret + + # Now, reply will go over a multiple messages! + ret = l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'getlog', + 'params': {'level': 'io'}}) + + assert len(json.dumps(ret)) > 65535 From 7e73e4d1d4318b38883956a444dbfda686c43b60 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 07/18] commando: support commands larger than 64k. This is needed for invoice, which can be asked to commit to giant descriptions (though that's antisocial!). Signed-off-by: Rusty Russell --- plugins/commando.c | 129 ++++++++++++++++++++++++++++++++++--------- tests/test_plugin.py | 11 ++++ 2 files changed, 115 insertions(+), 25 deletions(-) diff --git a/plugins/commando.c b/plugins/commando.c index 9c6070b986d3..3103c4d86aeb 100644 --- a/plugins/commando.c +++ b/plugins/commando.c @@ -15,7 +15,9 @@ #define COMMANDO_ERROR_REMOTE_AUTH 0x4c51 enum commando_msgtype { - COMMANDO_MSG_CMD = 0x4c4f, + /* Requests are split across multiple CONTINUES, then TERM. */ + COMMANDO_MSG_CMD_CONTINUES = 0x4c4d, + COMMANDO_MSG_CMD_TERM = 0x4c4f, /* Replies are split across multiple CONTINUES, then TERM. */ COMMANDO_MSG_REPLY_CONTINUES = 0x594b, COMMANDO_MSG_REPLY_TERM = 0x594d, @@ -32,6 +34,7 @@ struct commando { static struct plugin *plugin; static struct commando **outgoing_commands; +static struct commando **incoming_commands; /* NULL peer: don't care about peer. NULL id: don't care about id */ static struct commando *find_commando(struct commando **arr, @@ -83,9 +86,10 @@ struct reply { size_t off, len; }; +/* Calls itself repeatedly: first time, result is NULL */ static struct command_result *send_response(struct command *command UNUSED, const char *buf UNUSED, - const jsmntok_t *result UNUSED, + const jsmntok_t *result, struct reply *reply) { size_t msglen = reply->len - reply->off; @@ -99,7 +103,7 @@ static struct command_result *send_response(struct command *command UNUSED, msgtype = COMMANDO_MSG_REPLY_CONTINUES; /* We need to make a copy first time before we call back, since * plugin will reuse it! */ - if (reply->off == 0) + if (!result) reply->buf = tal_dup_talarr(reply, char, reply->buf); } else { if (msglen == 0) { @@ -140,7 +144,7 @@ static struct command_result *cmd_done(struct command *command, reply->off = obj->start; reply->len = obj->end; - return send_response(command, buf, obj, reply); + return send_response(command, NULL, NULL, reply); } static void commando_error(struct commando *incoming, @@ -248,6 +252,43 @@ static void try_command(struct node_id *peer, send_outreq(plugin, req); } +static void handle_incmd(struct node_id *peer, + u64 idnum, + const u8 *msg, size_t msglen, + bool terminal) +{ + struct commando *incmd; + + incmd = find_commando(incoming_commands, peer, NULL); + /* Don't let them buffer multiple commands: discard old. */ + if (incmd && incmd->id != idnum) + incmd = tal_free(incmd); + + if (!incmd) { + incmd = tal(plugin, struct commando); + incmd->id = idnum; + incmd->cmd = NULL; + incmd->peer = *peer; + incmd->contents = tal_arr(incmd, u8, 0); + tal_arr_expand(&incoming_commands, incmd); + tal_add_destructor2(incmd, destroy_commando, &incoming_commands); + } + + /* 1MB should be enough for anybody! */ + append_contents(incmd, msg, msglen, 1024*1024); + + if (!terminal) + return; + + if (!incmd->contents) { + plugin_log(plugin, LOG_UNUSUAL, "%s: ignoring oversize request", + node_id_to_hexstr(tmpctx, peer)); + return; + } + + try_command(peer, idnum, incmd->contents, tal_bytelen(incmd->contents)); +} + static struct command_result *handle_reply(struct node_id *peer, u64 idnum, const u8 *msg, size_t msglen, @@ -283,7 +324,9 @@ static struct command_result *handle_reply(struct node_id *peer, replystr = (const char *)ocmd->contents; toks = json_parse_simple(ocmd, replystr, tal_bytelen(ocmd->contents)); if (!toks || toks[0].type != JSMN_OBJECT) - return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, "Reply was unparsable"); + return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, + "Reply was unparsable: '%.*s'", + (int)tal_bytelen(ocmd->contents), replystr); err = json_get_member(replystr, toks, "error"); if (err) { @@ -343,8 +386,10 @@ static struct command_result *handle_custommsg(struct command *cmd, if (msg) { switch (mtype) { - case COMMANDO_MSG_CMD: - try_command(&peer, idnum, msg, len); + case COMMANDO_MSG_CMD_CONTINUES: + case COMMANDO_MSG_CMD_TERM: + handle_incmd(&peer, idnum, msg, len, + mtype == COMMANDO_MSG_CMD_TERM); break; case COMMANDO_MSG_REPLY_CONTINUES: case COMMANDO_MSG_REPLY_TERM: @@ -364,14 +409,31 @@ static const struct plugin_hook hooks[] = { }, }; -static struct command_result *send_success(struct command *command, - const char *buf, - const jsmntok_t *result, - struct commando *incoming) +struct outgoing { + struct node_id peer; + size_t msg_off; + u8 **msgs; +}; + +static struct command_result *send_more_cmd(struct command *cmd, + const char *buf UNUSED, + const jsmntok_t *result UNUSED, + struct outgoing *outgoing) { - return command_still_pending(command); -} + struct out_req *req; + + if (outgoing->msg_off == tal_count(outgoing->msgs)) { + tal_free(outgoing); + return command_still_pending(cmd); + } + + req = jsonrpc_request_start(plugin, cmd, "sendcustommsg", + send_more_cmd, forward_error, outgoing); + json_add_node_id(req->js, "node_id", &outgoing->peer); + json_add_hex_talarr(req->js, "msg", outgoing->msgs[outgoing->msg_off++]); + return send_outreq(plugin, req); +} static struct command_result *json_commando(struct command *cmd, const char *buffer, @@ -381,9 +443,9 @@ static struct command_result *json_commando(struct command *cmd, const char *method, *cparams; const char *rune; struct commando *ocmd; - struct out_req *req; - u8 *cmd_msg; + struct outgoing *outgoing; char *json; + size_t jsonlen; if (!param(cmd, buffer, params, p_req("peer_id", param_node_id, &peer), @@ -410,28 +472,44 @@ static struct command_result *json_commando(struct command *cmd, tal_append_fmt(&json, ",\"rune\":\"%s\"", rune); tal_append_fmt(&json, "}"); - cmd_msg = tal_arr(NULL, u8, 0); - towire_u16(&cmd_msg, COMMANDO_MSG_CMD); - towire_u64(&cmd_msg, ocmd->id); - towire(&cmd_msg, json, strlen(json)); - req = jsonrpc_request_start(plugin, NULL, "sendcustommsg", - send_success, forward_error, ocmd); - json_add_node_id(req->js, "node_id", &ocmd->peer); - json_add_hex_talarr(req->js, "msg", cmd_msg); - tal_free(cmd_msg); + /* This is not a leak, but we don't keep a pointer. */ + outgoing = notleak(tal(cmd, struct outgoing)); + outgoing->peer = *peer; + outgoing->msg_off = 0; + /* 65000 per message gives sufficient headroom. */ + jsonlen = tal_bytelen(json)-1; + outgoing->msgs = notleak(tal_arr(cmd, u8 *, (jsonlen + 64999) / 65000)); + for (size_t i = 0; i < tal_count(outgoing->msgs); i++) { + u8 *cmd_msg = tal_arr(outgoing, u8, 0); + bool terminal = (i == tal_count(outgoing->msgs) - 1); + size_t off = i * 65000, len; + + if (terminal) + len = jsonlen - off; + else + len = 65000; + + towire_u16(&cmd_msg, + terminal ? COMMANDO_MSG_CMD_TERM + : COMMANDO_MSG_CMD_CONTINUES); + towire_u64(&cmd_msg, ocmd->id); + towire(&cmd_msg, json + off, len); + outgoing->msgs[i] = cmd_msg; + } /* Keep memleak code happy! */ tal_free(peer); tal_free(method); tal_free(cparams); - return send_outreq(plugin, req); + return send_more_cmd(cmd, NULL, NULL, outgoing); } #if DEVELOPER static void memleak_mark_globals(struct plugin *p, struct htable *memtable) { memleak_remove_region(memtable, outgoing_commands, tal_bytelen(outgoing_commands)); + memleak_remove_region(memtable, incoming_commands, tal_bytelen(incoming_commands)); } #endif @@ -439,6 +517,7 @@ static const char *init(struct plugin *p, const char *buf UNUSED, const jsmntok_t *config UNUSED) { outgoing_commands = tal_arr(p, struct commando *, 0); + incoming_commands = tal_arr(p, struct commando *, 0); plugin = p; #if DEVELOPER plugin_set_memleak_handler(p, memleak_mark_globals); diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 74490cf93078..77683b24e8d5 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2587,3 +2587,14 @@ def test_commando(node_factory): 'params': {'level': 'io'}}) assert len(json.dumps(ret)) > 65535 + + # Command will go over multiple messages. + ret = l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'invoice', + 'params': {'amount_msat': 'any', + 'label': 'label', + 'description': 'A' * 200000, + 'deschashonly': True}}) + + assert 'bolt11' in ret From 048093ee0e84aaaea5fd80116e2974229e112da7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 08/18] commando: correctly reflect error data field. Some JSON error include "data", and we should reflect that. Signed-off-by: Rusty Russell --- plugins/commando.c | 20 +++++++++++++++----- tests/test_plugin.py | 12 ++++++++++++ 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/plugins/commando.c b/plugins/commando.c index 3103c4d86aeb..4eed51d56116 100644 --- a/plugins/commando.c +++ b/plugins/commando.c @@ -1,5 +1,6 @@ #include "config.h" #include +#include #include #include #include @@ -331,8 +332,10 @@ static struct command_result *handle_reply(struct node_id *peer, err = json_get_member(replystr, toks, "error"); if (err) { const jsmntok_t *code = json_get_member(replystr, err, "code"); - int ecode; const jsmntok_t *message = json_get_member(replystr, err, "message"); + const jsmntok_t *datatok = json_get_member(replystr, err, "data"); + struct json_out *data; + int ecode; if (!code || !json_to_int(replystr, code, &ecode)) { return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, "Error '%.*s' had no valid code", @@ -343,10 +346,17 @@ static struct command_result *handle_reply(struct node_id *peer, return command_fail(ocmd->cmd, COMMANDO_ERROR_LOCAL, "Error had no message"); } - /* FIXME: data! */ - return command_fail(ocmd->cmd, ecode, "%.*s", - message->end - message->start, - replystr + message->start); + if (datatok) { + data = json_out_new(ocmd->cmd); + memcpy(json_out_direct(data, json_tok_full_len(datatok)), + json_tok_full(replystr, datatok), + json_tok_full_len(datatok)); + } else + data = NULL; + + return command_done_err(ocmd->cmd, ecode, + json_strdup(tmpctx, replystr, message), + data); } result = json_get_member(replystr, toks, "result"); diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 77683b24e8d5..f0e656e04301 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2598,3 +2598,15 @@ def test_commando(node_factory): 'deschashonly': True}}) assert 'bolt11' in ret + + # This will fail, will include data. + with pytest.raises(RpcError, match='No connection to first peer found') as exc_info: + l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'sendpay', + 'params': {'route': [{'amount_msat': 1000, + 'id': l1.info['id'], + 'delay': 12, + 'channel': '1x2x3'}], + 'payment_hash': '00' * 32}}) + assert exc_info.value.error['data']['erring_index'] == 0 From 13a53ad1943533105b68511bc7a3b9bc8710b0e9 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 09/18] commando: runes infrastructure. We support the old commando.py plugin, which stores a random secret, as well as a more modern approach which uses makesecret. Signed-off-by: Rusty Russell --- plugins/commando.c | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/plugins/commando.c b/plugins/commando.c index 4eed51d56116..1521b3baf459 100644 --- a/plugins/commando.c +++ b/plugins/commando.c @@ -1,6 +1,7 @@ #include "config.h" #include #include +#include #include #include #include @@ -36,6 +37,8 @@ struct commando { static struct plugin *plugin; static struct commando **outgoing_commands; static struct commando **incoming_commands; +static u64 *rune_counter; +static struct rune *master_rune; /* NULL peer: don't care about peer. NULL id: don't care about id */ static struct commando *find_commando(struct commando **arr, @@ -260,6 +263,8 @@ static void handle_incmd(struct node_id *peer, { struct commando *incmd; + /* FIXME: don't do *anything* unless they've set up a rune. */ + incmd = find_commando(incoming_commands, peer, NULL); /* Don't let them buffer multiple commands: discard old. */ if (incmd && incmd->id != idnum) @@ -520,18 +525,50 @@ static void memleak_mark_globals(struct plugin *p, struct htable *memtable) { memleak_remove_region(memtable, outgoing_commands, tal_bytelen(outgoing_commands)); memleak_remove_region(memtable, incoming_commands, tal_bytelen(incoming_commands)); + if (rune_counter) + memleak_remove_region(memtable, rune_counter, sizeof(*rune_counter)); } #endif static const char *init(struct plugin *p, const char *buf UNUSED, const jsmntok_t *config UNUSED) { + struct secret rune_secret; + outgoing_commands = tal_arr(p, struct commando *, 0); incoming_commands = tal_arr(p, struct commando *, 0); plugin = p; #if DEVELOPER plugin_set_memleak_handler(p, memleak_mark_globals); #endif + + rune_counter = tal(p, u64); + if (!rpc_scan_datastore_str(plugin, "commando/rune_counter", + JSON_SCAN(json_to_u64, rune_counter))) + rune_counter = tal_free(rune_counter); + + /* Old python commando used to store secret */ + if (!rpc_scan_datastore_hex(plugin, "commando/secret", + JSON_SCAN(json_to_secret, &rune_secret))) { + rpc_scan(plugin, "makesecret", + /* $ i commando + * 99 0x63 0143 0b1100011 'c' + * 111 0x6F 0157 0b1101111 'o' + * 109 0x6D 0155 0b1101101 'm' + * 109 0x6D 0155 0b1101101 'm' + * 97 0x61 0141 0b1100001 'a' + * 110 0x6E 0156 0b1101110 'n' + * 100 0x64 0144 0b1100100 'd' + * 111 0x6F 0157 0b1101111 'o' + */ + take(json_out_obj(NULL, "hex", "636F6D6D616E646F")), + "{secret:%}", + JSON_SCAN(json_to_secret, &rune_secret)); + } + + master_rune = rune_new(plugin, rune_secret.data, ARRAY_SIZE(rune_secret.data), + NULL); + return NULL; } From c6a74712c5391748a46716c0514abe14fda9a401 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 10/18] devtools/rune: simple decode tool. Signed-off-by: Rusty Russell --- devtools/Makefile | 4 ++- devtools/rune.c | 81 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 devtools/rune.c diff --git a/devtools/Makefile b/devtools/Makefile index 2c32b024bb27..a18548c666fc 100644 --- a/devtools/Makefile +++ b/devtools/Makefile @@ -1,4 +1,4 @@ -DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/mkquery devtools/lightning-checkmessage devtools/topology devtools/route devtools/bolt12-cli devtools/encodeaddr devtools/features devtools/fp16 +DEVTOOLS := devtools/bolt11-cli devtools/decodemsg devtools/onion devtools/dump-gossipstore devtools/gossipwith devtools/create-gossipstore devtools/mkcommit devtools/mkfunding devtools/mkclose devtools/mkgossip devtools/mkencoded devtools/mkquery devtools/lightning-checkmessage devtools/topology devtools/route devtools/bolt12-cli devtools/encodeaddr devtools/features devtools/fp16 devtools/rune ifeq ($(HAVE_SQLITE3),1) DEVTOOLS += devtools/checkchannels endif @@ -48,6 +48,8 @@ DEVTOOLS_COMMON_OBJS := \ devtools/features: common/features.o common/utils.o wire/fromwire.o wire/towire.o devtools/features.o +devtools/rune: common/utils.o common/autodata.o common/setup.o common/version.o wire/fromwire.o wire/towire.o devtools/rune.o + devtools/fp16: common/fp16.o common/utils.o common/setup.o common/autodata.o devtools/fp16.o devtools/bolt11-cli: $(DEVTOOLS_COMMON_OBJS) $(JSMN_OBJS) $(BITCOIN_OBJS) wire/fromwire.o wire/towire.o devtools/bolt11-cli.o diff --git a/devtools/rune.c b/devtools/rune.c new file mode 100644 index 000000000000..37322e06c475 --- /dev/null +++ b/devtools/rune.c @@ -0,0 +1,81 @@ +/* Decodes a rune. */ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include + +int main(int argc, char *argv[]) +{ + struct rune *rune; + common_setup(argv[0]); + + opt_register_noarg("--help|-h", opt_usage_and_exit, + "", "Show this message"); + opt_register_version(); + + opt_early_parse(argc, argv, opt_log_stderr_exit); + opt_parse(&argc, argv, opt_log_stderr_exit); + if (argc != 2) + opt_usage_exit_fail("needs rune"); + + rune = rune_from_base64(NULL, argv[1]); + if (!rune) + opt_usage_exit_fail("invalid rune"); + + printf("string encoding: %s\n", rune_to_string(rune, rune)); + for (size_t i = 0; i < tal_count(rune->restrs); i++) { + const struct rune_restr *restr = rune->restrs[i]; + const char *sep = "- "; + for (size_t j = 0; j < tal_count(restr->alterns); j++) { + const struct rune_altern *alt = restr->alterns[j]; + if (streq(alt->fieldname, "")) { + printf("Unique id is %s", alt->value); + } else { + printf("%s", sep); + switch (alt->condition) { + case RUNE_COND_IF_MISSING: + printf("%s is missing", alt->fieldname); + break; + case RUNE_COND_EQUAL: + printf("%s equal to %s", alt->fieldname, alt->value); + break; + case RUNE_COND_NOT_EQUAL: + printf("%s unequal to %s", alt->fieldname, alt->value); + break; + case RUNE_COND_BEGINS: + printf("%s starts with %s", alt->fieldname, alt->value); + break; + case RUNE_COND_ENDS: + printf("%s ends with %s", alt->fieldname, alt->value); + break; + case RUNE_COND_CONTAINS: + printf("%s contains %s", alt->fieldname, alt->value); + break; + case RUNE_COND_INT_LESS: + printf("%s < %s", alt->fieldname, alt->value); + break; + case RUNE_COND_INT_GREATER: + printf("%s > %s", alt->fieldname, alt->value); + break; + case RUNE_COND_LEXO_BEFORE: + printf("%s sorts before %s", alt->fieldname, alt->value); + break; + case RUNE_COND_LEXO_AFTER: + printf("%s sorts after %s", alt->fieldname, alt->value); + break; + case RUNE_COND_COMMENT: + printf("comment: %s%s", alt->fieldname, alt->value); + break; + } + sep = " OR "; + } + } + printf("\n"); + } + common_shutdown(); +} From 2fd81d9238fb971fec2dbf882e7c7fb3941fb2ee Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 11/18] commando: add commando-rune command. Can both mint new runes, and add one or more restrictions to existing ones. Signed-off-by: Rusty Russell --- plugins/commando.c | 149 ++++++++++++++++++++++++++++++++++++++++++- tests/test_plugin.py | 35 +++++++++- 2 files changed, 182 insertions(+), 2 deletions(-) diff --git a/plugins/commando.c b/plugins/commando.c index 1521b3baf459..c3e3aca2bf2a 100644 --- a/plugins/commando.c +++ b/plugins/commando.c @@ -520,11 +520,152 @@ static struct command_result *json_commando(struct command *cmd, return send_more_cmd(cmd, NULL, NULL, outgoing); } +static struct command_result *param_rune(struct command *cmd, const char *name, + const char * buffer, const jsmntok_t *tok, + struct rune **rune) +{ + *rune = rune_from_base64n(cmd, buffer + tok->start, tok->end - tok->start); + if (!*rune) + return command_fail_badparam(cmd, name, buffer, tok, + "should be base64 string"); + + return NULL; +} + +static struct rune_restr **readonly_restrictions(const tal_t *ctx) +{ + struct rune_restr **restrs = tal_arr(ctx, struct rune_restr *, 2); + + /* Any list*, get*, or summary: + * method^list|method^get|method=summary + */ + restrs[0] = rune_restr_new(restrs); + rune_restr_add_altern(restrs[0], + take(rune_altern_new(NULL, + "method", + RUNE_COND_BEGINS, + "list"))); + rune_restr_add_altern(restrs[0], + take(rune_altern_new(NULL, + "method", + RUNE_COND_BEGINS, + "get"))); + rune_restr_add_altern(restrs[0], + take(rune_altern_new(NULL, + "method", + RUNE_COND_EQUAL, + "summary"))); + /* But not listdatastore! + * method/listdatastore + */ + restrs[1] = rune_restr_new(restrs); + rune_restr_add_altern(restrs[1], + take(rune_altern_new(NULL, + "method", + RUNE_COND_NOT_EQUAL, + "listdatastore"))); + + return restrs; +} + +static struct command_result *param_restrictions(struct command *cmd, + const char *name, + const char *buffer, + const jsmntok_t *tok, + struct rune_restr ***restrs) +{ + if (json_tok_streq(buffer, tok, "readonly")) + *restrs = readonly_restrictions(cmd); + else if (tok->type == JSMN_ARRAY) { + size_t i; + const jsmntok_t *t; + + *restrs = tal_arr(cmd, struct rune_restr *, tok->size); + json_for_each_arr(i, t, tok) { + (*restrs)[i] = rune_restr_from_string(*restrs, + buffer + t->start, + t->end - t->start); + if (!(*restrs)[i]) + return command_fail_badparam(cmd, name, buffer, t, + "not a valid restriction"); + } + } else { + *restrs = tal_arr(cmd, struct rune_restr *, 1); + (*restrs)[0] = rune_restr_from_string(*restrs, + buffer + tok->start, + tok->end - tok->start); + if (!(*restrs)[0]) + return command_fail_badparam(cmd, name, buffer, tok, + "not a valid restriction"); + } + return NULL; +} + +static struct command_result *reply_with_rune(struct command *cmd, + const char *buf UNUSED, + const jsmntok_t *result UNUSED, + struct rune *rune) +{ + struct json_stream *js = jsonrpc_stream_success(cmd); + + json_add_string(js, "rune", rune_to_base64(tmpctx, rune)); + json_add_string(js, "unique_id", rune->unique_id); + return command_finished(cmd, js); +} + +static struct command_result *json_commando_rune(struct command *cmd, + const char *buffer, + const jsmntok_t *params) +{ + struct rune *rune; + struct rune_restr **restrs; + struct out_req *req; + + if (!param(cmd, buffer, params, + p_opt("rune", param_rune, &rune), + p_opt("restrictions", param_restrictions, &restrs), + NULL)) + return command_param_failed(); + + if (rune) { + for (size_t i = 0; i < tal_count(restrs); i++) + rune_add_restr(rune, restrs[i]); + return reply_with_rune(cmd, NULL, NULL, rune); + } + + rune = rune_derive_start(cmd, master_rune, + tal_fmt(tmpctx, "%"PRIu64, + rune_counter ? *rune_counter : 0)); + for (size_t i = 0; i < tal_count(restrs); i++) + rune_add_restr(rune, restrs[i]); + + /* Now update datastore, before returning rune */ + req = jsonrpc_request_start(plugin, cmd, "datastore", + reply_with_rune, forward_error, rune); + json_array_start(req->js, "key"); + json_add_string(req->js, NULL, "commando"); + json_add_string(req->js, NULL, "rune_counter"); + json_array_end(req->js); + if (rune_counter) { + (*rune_counter)++; + json_add_string(req->js, "mode", "must-replace"); + } else { + /* This used to say "🌩🤯🧨🔫!" but our log filters are too strict :( */ + plugin_log(plugin, LOG_INFORM, "Commando powers enabled: BOOM!"); + rune_counter = tal(plugin, u64); + *rune_counter = 1; + json_add_string(req->js, "mode", "must-create"); + } + json_add_u64(req->js, "string", *rune_counter); + return send_outreq(plugin, req); +} + #if DEVELOPER static void memleak_mark_globals(struct plugin *p, struct htable *memtable) { memleak_remove_region(memtable, outgoing_commands, tal_bytelen(outgoing_commands)); memleak_remove_region(memtable, incoming_commands, tal_bytelen(incoming_commands)); + memleak_remove_region(memtable, master_rune, sizeof(*master_rune)); if (rune_counter) memleak_remove_region(memtable, rune_counter, sizeof(*rune_counter)); } @@ -578,7 +719,13 @@ static const struct plugin_command commands[] = { { "Send a commando message to a direct peer, wait for response", "Sends {peer_id} {method} with optional {params} and {rune}", json_commando, - } + }, { + "commando-rune", + "utility", + "Create or restrict a rune", + "Takes an optional {rune} with optional {restrictions} and returns {rune}", + json_commando_rune, + }, }; int main(int argc, char *argv[]) diff --git a/tests/test_plugin.py b/tests/test_plugin.py index f0e656e04301..b706baaaffa1 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2546,7 +2546,7 @@ def test_plugin_shutdown(node_factory): 'test_libplugin: failed to self-terminate in time, killing.']) -def test_commando(node_factory): +def test_commando(node_factory, executor): l1, l2 = node_factory.line_graph(2, fundchannel=False) # This works @@ -2610,3 +2610,36 @@ def test_commando(node_factory): 'channel': '1x2x3'}], 'payment_hash': '00' * 32}}) assert exc_info.value.error['data']['erring_index'] == 0 + + +def test_commando_rune(node_factory): + l1, l2 = node_factory.line_graph(2, fundchannel=False) + + # l1's commando secret is 1241faef85297127c2ac9bde95421b2c51e5218498ae4901dc670c974af4284b. + # I put that into a test node's commando.py to generate these runes (modified readonly to match ours): + # $ l1-cli commando-rune + # "rune": "zKc2W88jopslgUBl0UE77aEe5PNCLn5WwqSusU_Ov3A9MA==" + # $ l1-cli commando-rune restrictions=readonly + # "rune": "1PJnoR9a7u4Bhglj2s7rVOWqRQnswIwUoZrDVMKcLTY9MSZtZXRob2RebGlzdHxtZXRob2ReZ2V0fG1ldGhvZD1zdW1tYXJ5Jm1ldGhvZC9saXN0ZGF0YXN0b3Jl" + # $ l1-cli commando-rune restrictions='time>1656675211' + # "rune": "RnlWC4lwBULFaObo6ZP8jfqYRyTbfWPqcMT3qW-Wmso9MiZ0aW1lPjE2NTY2NzUyMTE=" + # $ l1-cli commando-rune restrictions='["id^022d223620a359a47ff7","method=listpeers"]' + # "rune": "lXFWzb51HjWxKV5TmfdiBgd74w0moeyChj3zbLoxmws9MyZpZF4wMjJkMjIzNjIwYTM1OWE0N2ZmNyZtZXRob2Q9bGlzdHBlZXJz" + # $ l1-cli commando-rune lXFWzb51HjWxKV5TmfdiBgd74w0moeyChj3zbLoxmws9MyZpZF4wMjJkMjIzNjIwYTM1OWE0N2ZmNyZtZXRob2Q9bGlzdHBlZXJz 'pnamelevel!|pnamelevel/io' + # "rune": "Dw2tzGCoUojAyT0JUw7fkYJYqExpEpaDRNTkyvWKoJY9MyZpZF4wMjJkMjIzNjIwYTM1OWE0N2ZmNyZtZXRob2Q9bGlzdHBlZXJzJnBuYW1lbGV2ZWwhfHBuYW1lbGV2ZWwvaW8=" + + rune1 = l1.rpc.commando_rune() + assert rune1['rune'] == 'zKc2W88jopslgUBl0UE77aEe5PNCLn5WwqSusU_Ov3A9MA==' + assert rune1['unique_id'] == '0' + rune2 = l1.rpc.commando_rune(restrictions="readonly") + assert rune2['rune'] == '1PJnoR9a7u4Bhglj2s7rVOWqRQnswIwUoZrDVMKcLTY9MSZtZXRob2RebGlzdHxtZXRob2ReZ2V0fG1ldGhvZD1zdW1tYXJ5Jm1ldGhvZC9saXN0ZGF0YXN0b3Jl' + assert rune2['unique_id'] == '1' + rune3 = l1.rpc.commando_rune(restrictions="time>1656675211") + assert rune3['rune'] == 'RnlWC4lwBULFaObo6ZP8jfqYRyTbfWPqcMT3qW-Wmso9MiZ0aW1lPjE2NTY2NzUyMTE=' + assert rune3['unique_id'] == '2' + rune4 = l1.rpc.commando_rune(restrictions=["id^022d223620a359a47ff7", "method=listpeers"]) + assert rune4['rune'] == 'lXFWzb51HjWxKV5TmfdiBgd74w0moeyChj3zbLoxmws9MyZpZF4wMjJkMjIzNjIwYTM1OWE0N2ZmNyZtZXRob2Q9bGlzdHBlZXJz' + assert rune4['unique_id'] == '3' + rune5 = l1.rpc.commando_rune(rune4['rune'], "pnamelevel!|pnamelevel/io") + assert rune5['rune'] == 'Dw2tzGCoUojAyT0JUw7fkYJYqExpEpaDRNTkyvWKoJY9MyZpZF4wMjJkMjIzNjIwYTM1OWE0N2ZmNyZtZXRob2Q9bGlzdHBlZXJzJnBuYW1lbGV2ZWwhfHBuYW1lbGV2ZWwvaW8=' + assert rune5['unique_id'] == '3' From df8242b268a69f0e3dc79713d08583aa0f0ea511 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 12/18] commando: don't look at messages *at all* unless they've created a rune. This means we can leave commando on by default, without an explicit config flag. Signed-off-by: Rusty Russell --- plugins/commando.c | 3 ++- tests/test_plugin.py | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/plugins/commando.c b/plugins/commando.c index c3e3aca2bf2a..fd9c86a0d089 100644 --- a/plugins/commando.c +++ b/plugins/commando.c @@ -263,7 +263,8 @@ static void handle_incmd(struct node_id *peer, { struct commando *incmd; - /* FIXME: don't do *anything* unless they've set up a rune. */ + if (!rune_counter) + return; incmd = find_commando(incoming_commands, peer, NULL); /* Don't let them buffer multiple commands: discard old. */ diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b706baaaffa1..471d3bc5ae4e 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -13,6 +13,7 @@ ) import ast +import concurrent.futures import json import os import pytest @@ -2549,6 +2550,15 @@ def test_plugin_shutdown(node_factory): def test_commando(node_factory, executor): l1, l2 = node_factory.line_graph(2, fundchannel=False) + # Nothing works until we've issued a rune. + fut = executor.submit(l2.rpc.call, method='commando', + payload={'peer_id': l1.info['id'], + 'method': 'listpeers'}) + with pytest.raises(concurrent.futures.TimeoutError): + fut.result(10) + + l1.rpc.commando_rune() + # This works res = l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], From 3b2767a9d372cb54098eb39192561f9060c556b6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 13/18] commando: require runes for operation. Signed-off-by: Rusty Russell --- plugins/commando.c | 94 +++++++++++++++++++++++++++++++++++++++++--- tests/test_plugin.py | 70 ++++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 7 deletions(-) diff --git a/plugins/commando.c b/plugins/commando.c index fd9c86a0d089..1fcff8d8e88c 100644 --- a/plugins/commando.c +++ b/plugins/commando.c @@ -1,8 +1,10 @@ #include "config.h" #include +#include #include #include #include +#include #include #include #include @@ -175,14 +177,96 @@ static void commando_error(struct commando *incoming, send_response(NULL, NULL, NULL, reply); } -static const char *check_rune(struct commando *incoming, +struct cond_info { + const struct node_id *peer; + const char *buf; + const jsmntok_t *method; + const jsmntok_t *params; + STRMAP(const jsmntok_t *) cached_params; +}; + +static const char *check_condition(const tal_t *ctx, + const struct rune *rune, + const struct rune_altern *alt, + struct cond_info *cinfo) +{ + const jsmntok_t *ptok; + + if (streq(alt->fieldname, "time")) { + return rune_alt_single_int(ctx, alt, time_now().ts.tv_sec); + } else if (streq(alt->fieldname, "id")) { + const char *id = node_id_to_hexstr(tmpctx, cinfo->peer); + return rune_alt_single_str(ctx, alt, id, strlen(id)); + } else if (streq(alt->fieldname, "method")) { + return rune_alt_single_str(ctx, alt, + cinfo->buf + cinfo->method->start, + cinfo->method->end - cinfo->method->start); + } + + /* Rest are params looksup: generate this once! */ + if (cinfo->params) { + /* Note: we require that params be an obj! */ + const jsmntok_t *t; + size_t i; + + json_for_each_obj(i, t, cinfo->params) { + char *pmemname = tal_fmt(tmpctx, + "pname%.*s", + t->end - t->start, + cinfo->buf + t->start); + size_t off = strlen("pname"); + /* Remove punctuation! */ + for (size_t n = off; pmemname[n]; n++) { + if (cispunct(pmemname[n])) + continue; + pmemname[off++] = pmemname[n]; + } + pmemname[off++] = '\0'; + strmap_add(&cinfo->cached_params, pmemname, t+1); + } + cinfo->params = NULL; + } + + ptok = strmap_get(&cinfo->cached_params, alt->fieldname); + if (!ptok) + return rune_alt_single_missing(ctx, alt); + + return rune_alt_single_str(ctx, alt, + cinfo->buf + ptok->start, + ptok->end - ptok->start); +} + +static const char *check_rune(const tal_t *ctx, + struct commando *incoming, + const struct node_id *peer, const char *buf, const jsmntok_t *method, const jsmntok_t *params, - const jsmntok_t *rune) + const jsmntok_t *runetok) { - /* FIXME! */ - return NULL; + struct rune *rune; + struct cond_info cinfo; + const char *err; + + if (!runetok) + return "Missing rune"; + + rune = rune_from_base64n(tmpctx, buf + runetok->start, + runetok->end - runetok->start); + if (!rune) + return "Invalid rune"; + + cinfo.peer = peer; + cinfo.buf = buf; + cinfo.method = method; + cinfo.params = params; + strmap_init(&cinfo.cached_params); + err = rune_test(ctx, master_rune, rune, check_condition, &cinfo); + /* Just in case they manage to make us speak non-JSON, escape! */ + if (err) + err = json_escape(ctx, take(err))->s; + strmap_clear(&cinfo.cached_params); + return err; } static void try_command(struct node_id *peer, @@ -223,7 +307,7 @@ static void try_command(struct node_id *peer, } rune = json_get_member(buf, toks, "rune"); - failmsg = check_rune(incoming, buf, method, params, rune); + failmsg = check_rune(tmpctx, incoming, peer, buf, method, params, rune); if (failmsg) { commando_error(incoming, COMMANDO_ERROR_REMOTE_AUTH, "Not authorized: %s", failmsg); diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 471d3bc5ae4e..1d39f13de4dc 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2557,17 +2557,18 @@ def test_commando(node_factory, executor): with pytest.raises(concurrent.futures.TimeoutError): fut.result(10) - l1.rpc.commando_rune() - + rune = l1.rpc.commando_rune()['rune'] # This works res = l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], + 'rune': rune, 'method': 'listpeers'}) assert len(res['peers']) == 1 assert res['peers'][0]['id'] == l2.info['id'] res = l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], + 'rune': rune, 'method': 'listpeers', 'params': {'id': l2.info['id']}}) assert len(res['peers']) == 1 @@ -2576,16 +2577,19 @@ def test_commando(node_factory, executor): with pytest.raises(RpcError, match='missing required parameter'): l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], + 'rune': rune, 'method': 'withdraw'}) with pytest.raises(RpcError, match='unknown parameter: foobar'): l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], 'method': 'invoice', + 'rune': rune, 'params': {'foobar': 1}}) ret = l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], + 'rune': rune, 'method': 'ping', 'params': {'id': l2.info['id']}}) assert 'totlen' in ret @@ -2593,6 +2597,7 @@ def test_commando(node_factory, executor): # Now, reply will go over a multiple messages! ret = l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], + 'rune': rune, 'method': 'getlog', 'params': {'level': 'io'}}) @@ -2601,6 +2606,7 @@ def test_commando(node_factory, executor): # Command will go over multiple messages. ret = l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], + 'rune': rune, 'method': 'invoice', 'params': {'amount_msat': 'any', 'label': 'label', @@ -2613,6 +2619,7 @@ def test_commando(node_factory, executor): with pytest.raises(RpcError, match='No connection to first peer found') as exc_info: l2.rpc.call(method='commando', payload={'peer_id': l1.info['id'], + 'rune': rune, 'method': 'sendpay', 'params': {'route': [{'amount_msat': 1000, 'id': l1.info['id'], @@ -2653,3 +2660,62 @@ def test_commando_rune(node_factory): rune5 = l1.rpc.commando_rune(rune4['rune'], "pnamelevel!|pnamelevel/io") assert rune5['rune'] == 'Dw2tzGCoUojAyT0JUw7fkYJYqExpEpaDRNTkyvWKoJY9MyZpZF4wMjJkMjIzNjIwYTM1OWE0N2ZmNyZtZXRob2Q9bGlzdHBlZXJzJnBuYW1lbGV2ZWwhfHBuYW1lbGV2ZWwvaW8=' assert rune5['unique_id'] == '3' + + # Replace rune3 with a more useful timestamp! + expiry = int(time.time()) + 15 + rune3 = l1.rpc.commando_rune(restrictions="time<{}".format(expiry)) + successes = ((rune1, "listpeers", {}), + (rune2, "listpeers", {}), + (rune2, "getinfo", {}), + (rune2, "getinfo", {}), + (rune3, "getinfo", {}), + (rune4, "listpeers", {}), + (rune5, "listpeers", {'id': l2.info['id']}), + (rune5, "listpeers", {'id': l2.info['id'], 'level': 'broken'})) + failures = ((rune2, "withdraw", {}), + (rune2, "plugin", {'subcommand': 'list'}), + (rune3, "getinfo", {}), + (rune4, "listnodes", {}), + (rune5, "listpeers", {'id': l2.info['id'], 'level': 'io'})) + + for rune, cmd, params in successes: + l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'rune': rune['rune'], + 'method': cmd, + 'params': params}) + + while time.time() < expiry: + time.sleep(1) + + for rune, cmd, params in failures: + print("{} {}".format(cmd, params)) + with pytest.raises(RpcError, match='Not authorized:') as exc_info: + l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'rune': rune['rune'], + 'method': cmd, + 'params': params}) + assert exc_info.value.error['code'] == 0x4c51 + + # rune5 can only be used by l2: + l3 = node_factory.get_node() + l3.connect(l1) + with pytest.raises(RpcError, match='Not authorized:') as exc_info: + l3.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'rune': rune5['rune'], + 'method': "listpeers", + 'params': {}}) + assert exc_info.value.error['code'] == 0x4c51 + + # Remote doesn't allow array parameters. + l2.rpc.check_request_schemas = False + with pytest.raises(RpcError, match='Params must be object') as exc_info: + l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'rune': rune5['rune'], + 'method': "listpeers", + 'params': [l2.info['id'], 'io']}) + assert exc_info.value.error['code'] == 0x4c50 + l2.rpc.check_request_schemas = True From cbe36422bdc48de6de06059750020ab5ab840bec Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 14/18] doc: document commando and commando-rune. Signed-off-by: Rusty Russell --- doc/Makefile | 2 + doc/index.rst | 2 + doc/lightning-commando-rune.7.md | 101 +++++++++++++++++++++++++ doc/lightning-commando.7.md | 52 +++++++++++++ doc/schemas/commando-rune.request.json | 27 +++++++ doc/schemas/commando-rune.schema.json | 19 +++++ doc/schemas/commando.request.json | 27 +++++++ 7 files changed, 230 insertions(+) create mode 100644 doc/lightning-commando-rune.7.md create mode 100644 doc/lightning-commando.7.md create mode 100644 doc/schemas/commando-rune.request.json create mode 100644 doc/schemas/commando-rune.schema.json create mode 100644 doc/schemas/commando.request.json diff --git a/doc/Makefile b/doc/Makefile index 3f5a39453a58..04be559e9cd8 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -13,6 +13,8 @@ MANPAGES := doc/lightning-cli.1 \ doc/lightning-checkmessage.7 \ doc/lightning-close.7 \ doc/lightning-connect.7 \ + doc/lightning-commando.7 \ + doc/lightning-commando-rune.7 \ doc/lightning-createonion.7 \ doc/lightning-createinvoice.7 \ doc/lightning-datastore.7 \ diff --git a/doc/index.rst b/doc/index.rst index 87e6601af46e..b60ced919aca 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -35,6 +35,8 @@ Core Lightning Documentation lightning-checkmessage lightning-cli lightning-close + lightning-commando + lightning-commando-rune lightning-connect lightning-createinvoice lightning-createonion diff --git a/doc/lightning-commando-rune.7.md b/doc/lightning-commando-rune.7.md new file mode 100644 index 000000000000..23a8e9c6e684 --- /dev/null +++ b/doc/lightning-commando-rune.7.md @@ -0,0 +1,101 @@ +lightning-commando-rune -- Command to Authorize Remote Peer Access +=================================================================== + +SYNOPSIS +-------- + +**commando-rune** [*rune*] [*restrictions*] + +DESCRIPTION +----------- + +The **commando-rune** RPC command creates a base64 string called a +*rune* which can be used to access commands on this node. Each *rune* +contains a unique id (a number starting at 0), and can have +restrictions inside it. Nobody can remove restrictions from a rune: if +you try, the rune will be rejected. There is no limit on how many +runes you can issue: the node doesn't store them, but simply decodes +and checks them as they are received. + +If *rune* is supplied, the restrictions are simple appended to that +*rune* (it doesn't need to be a rune belonging to this node). If no +*rune* is supplied, a new one is constructed, with a new unique id. + +*restrictions* can be the string "readonly" (creates a rune which +allows most *get* and *list* commands, and the *summary* command), or +an array of restrictions, or a single resriction. + +Each restriction is a set of one or more alternatives, such as "method +is listpeers", or "method is listpeers OR time is before 2023". +Alternatives use a simple language to examine the command which is +being run: + +* time: the current UNIX time, e.g. "time<1656759180". +* id: the node_id of the peer, e.g. "id=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605". +* method: the command being run, e.g. "method=withdraw". +* pnameX: the parameter named X. e.g. "pnamedestination=1RustyRX2oai4EYYDpQGWvEL62BBGqN9T". + +RESTRICTION FORMAT +------------------ + +Restrictions are one or more altneratives, separated by `|`. Each +alternative is *name* *operator* *value*. The valid names are shown +above. If a value contains `|`, `&` or `\\`, it must be preceeded by +a `\\`. + +* `=`: passes if equal ie. identical. e.g. `method=withdraw` +* `/`: not equals, e.g. `method/withdraw` +* `^`: starts with, e.g. `id^024b9a1fa8e006f1e3937f` +* `$`: ends with, e.g. `id$381df1cc449605`. +* `~`: contains, e.g. `id~006f1e3937f65f66c40`. +* `<`: is a decimal integer, and is less than. e.g. `time<1656759180` +* `>`: is a decimal integer, and is greater than. e.g. `time>1656759180` +* `{`: preceeds in alphabetical order (or matches but is shorter), e.g. `id{02ff`. +* `}`: follows in alphabetical order (or matches but is longer), e.g. `id}02ff`. +* `#`: a comment, ignored, e.g. `dumb example#`. +* `!`: only passes if the *name* does *not* exist. e.g. `pnamedestination!`. + Every other operator except `#` fails if *name* does not exist! + +For example, the "readonly" restriction is actually two restrictions: + +1. `method^list|method^get|method=summary`: You may call list, get or summary. +2. `method/listdatastore`: But not listdatastore: that contains sensitive stuff! + +SHARING RUNES +------------- + +Because anyone can add a restriction to a rune, you can always turn a +normal rune into a read-only rune, or restrict access for 30 minutes +from the time you give it to someone. Adding restrictions before +sharing runes is best practice. + +RETURN VALUE +------------ + +[comment]: # (GENERATE-FROM-SCHEMA-START) +On success, an object is returned, containing: +- **rune** (string): the resulting rune +- **unique_id** (string): the id of this rune: this is set at creation and cannot be changed (even as restrictions are added) + +[comment]: # (GENERATE-FROM-SCHEMA-END) + +AUTHOR +------ + +Rusty Russell <> wrote the original Python +commando.py plugin, the in-tree commando plugin, and this manual page. + +Christian Decker came up with the name "commando", which almost +excuses his previous adoption of the name "Eltoo". + +SEE ALSO +-------- + +lightning-commando(7) + +RESOURCES +--------- + +Main web site: + +[comment]: # ( SHA256STAMP:598337212d2e8a6833698e931f838d8cb424c353af4d7adf6891803ff0ee604b) diff --git a/doc/lightning-commando.7.md b/doc/lightning-commando.7.md new file mode 100644 index 000000000000..871083f12083 --- /dev/null +++ b/doc/lightning-commando.7.md @@ -0,0 +1,52 @@ +lightning-commando -- Command to Send a Command to a Remote Peer +================================================================ + +SYNOPSIS +-------- + +**commando** *peer_id* *method* [*params*] [*rune*] + +DESCRIPTION +----------- + +The **commando** RPC command is a homage to bad 80s movies. It also +sends a directly-connected *peer_id* a custom message, containing a +request to run *method* (with an optional dictionary of *params*); +generally the peer will only allow you to run a command if it has +provided you with a *rune* which allows it. + +RETURN VALUE +------------ + +On success, the return depends on the *method* invoked. + +On failure, one of the following error codes may be returned: + +- -32600: Usually means peer is not connected +- 19535: the local commando plugin discovered an error. +- 19536: the remote commando plugin discovered an error. +- 19537: the remote commando plugin said we weren't authorized. + +It can also fail if the peer does not respond, in which case it will simply +hang awaiting a response. + +AUTHOR +------ + +Rusty Russell <> wrote the original Python +commando.py plugin, the in-tree commando plugin, and this manual page. + +Christian Decker came up with the name "commando", which almost +excuses his previous adoption of the name "Eltoo". + +SEE ALSO +-------- + +lightning-commando-rune(7) + +RESOURCES +--------- + +Main web site: + +[comment]: # ( SHA256STAMP:6f4406cae30cab813b3bf4e1242af914276716a057e558474e29340665ee8c2f) diff --git a/doc/schemas/commando-rune.request.json b/doc/schemas/commando-rune.request.json new file mode 100644 index 000000000000..a93ae80adc79 --- /dev/null +++ b/doc/schemas/commando-rune.request.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [], + "properties": { + "rune": { + "type": "string", + "description": "optional rune to add to" + }, + "restrictions": { + "oneOf": [ + { + "type": "array", + "description": "array of restrictions to add to rune", + "items": { + "type": "string" + } + }, + { + "type": "string", + "description": "single restrictions to add to rune, or readonly." + } + ] + } + } +} diff --git a/doc/schemas/commando-rune.schema.json b/doc/schemas/commando-rune.schema.json new file mode 100644 index 000000000000..c0519e51cb40 --- /dev/null +++ b/doc/schemas/commando-rune.schema.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "rune", + "unique_id" + ], + "properties": { + "rune": { + "type": "string", + "description": "the resulting rune" + }, + "unique_id": { + "type": "string", + "description": "the id of this rune: this is set at creation and cannot be changed (even as restrictions are added)" + } + } +} diff --git a/doc/schemas/commando.request.json b/doc/schemas/commando.request.json new file mode 100644 index 000000000000..354f83e86683 --- /dev/null +++ b/doc/schemas/commando.request.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": false, + "required": [ + "peer_id", + "method" + ], + "properties": { + "peer_id": { + "type": "pubkey", + "description": "peer to command" + }, + "method": { + "type": "string", + "description": "method to invoke on peer" + }, + "params": { + "type": "object", + "description": "parameters for method" + }, + "rune": { + "type": "string", + "description": "rune to authorize the command" + } + } +} From a06bcfb621208e34fcc8ac423f9da841c7348bfd Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 15/18] commando: add support for parameters by array, parameter count. Awkward to filter, but they're really practical for many commands. Signed-off-by: Rusty Russell --- doc/lightning-commando-rune.7.md | 61 ++++++++++++++++++++++++++- doc/schemas/commando.request.json | 12 +++++- plugins/commando.c | 69 ++++++++++++++++++++----------- tests/test_plugin.py | 28 +++++++------ 4 files changed, 128 insertions(+), 42 deletions(-) diff --git a/doc/lightning-commando-rune.7.md b/doc/lightning-commando-rune.7.md index 23a8e9c6e684..ca31e2e96171 100644 --- a/doc/lightning-commando-rune.7.md +++ b/doc/lightning-commando-rune.7.md @@ -33,7 +33,9 @@ being run: * time: the current UNIX time, e.g. "time<1656759180". * id: the node_id of the peer, e.g. "id=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605". * method: the command being run, e.g. "method=withdraw". +* pnum: the number of parameters. e.g. "pnum<2". * pnameX: the parameter named X. e.g. "pnamedestination=1RustyRX2oai4EYYDpQGWvEL62BBGqN9T". +* parrN: the N'th parameter. e.g. "parr0=1RustyRX2oai4EYYDpQGWvEL62BBGqN9T". RESTRICTION FORMAT ------------------ @@ -56,10 +58,65 @@ a `\\`. * `!`: only passes if the *name* does *not* exist. e.g. `pnamedestination!`. Every other operator except `#` fails if *name* does not exist! -For example, the "readonly" restriction is actually two restrictions: +EXAMPLES +-------- + +This creates a fresh rune which can do anything: + + $ lightning-cli commando-rune + { + "rune": "KUhZzNlECC7pYsz3QVbF1TqjIUYi3oyESTI7n60hLMs9MA==", + "unique_id": "0" + } + +We can add restrictions to that rune, like so: + + $ lightning-cli commando-rune rune=KUhZzNlECC7pYsz3QVbF1TqjIUYi3oyESTI7n60hLMs9MA== restrictions=readonly + { + "rune": "NbL7KkXcPQsVseJ9TdJNjJK2KsPjnt_q4cE_wvc873I9MCZtZXRob2RebGlzdHxtZXRob2ReZ2V0fG1ldGhvZD1zdW1tYXJ5Jm1ldGhvZC9saXN0ZGF0YXN0b3Jl", + "unique_id": "0" + } + +The "readonly" restriction is a short-cut for two restrictions: 1. `method^list|method^get|method=summary`: You may call list, get or summary. -2. `method/listdatastore`: But not listdatastore: that contains sensitive stuff! +2. `method/listdatastore`: But not listdatastore: that contains sensitive stuff! + +We can do the same manually, like so: + + $ lightning-cli commando-rune rune=KUhZzNlECC7pYsz3QVbF1TqjIUYi3oyESTI7n60hLMs9MA== restrictions='["method^list|method^get|method=summary","method/listdatastore"]' + { + "rune": "NbL7KkXcPQsVseJ9TdJNjJK2KsPjnt_q4cE_wvc873I9MCZtZXRob2RebGlzdHxtZXRob2ReZ2V0fG1ldGhvZD1zdW1tYXJ5Jm1ldGhvZC9saXN0ZGF0YXN0b3Jl", + "unique_id": "0" + } + +Let's create a rune which lets a specific peer +(024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605) +run "listpeers" on themselves: + + $ lightning-cli commando-rune restrictions='["id=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605","method=listpeers","pnum=1","pnameid=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605|parr0=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605"]' + { + "rune": "FE8GHiGVvxcFqCQcClVRRiNE_XEeLYQzyG2jmqto4jM9MiZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDV8cGFycjA9MDI0YjlhMWZhOGUwMDZmMWUzOTM3ZjY1ZjY2YzQwOGU2ZGE4ZTFjYTcyOGVhNDMyMjJhNzM4MWRmMWNjNDQ5NjA1", + "unique_id": "2" + } + +This allows `listpeers` with 1 argument (`pnum=1`), which is either by name (`pnameid`), or position (`parr0`). We could shorten this in several ways: either allowing only positional or named parameters, or by testing the start of the parameters only. Here's an example which only checks the first 9 bytes of the `listpeers` parameter: + + $ lightning-cli commando-rune restrictions='["id=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605","method=listpeers","pnum=1","pnameid^024b9a1fa8e006f1e393|parr0^024b9a1fa8e006f1e393"]' + { + "rune": "fTQnfL05coEbiBO8SS0cvQwCcPLxE9c02pZCC6HRVEY9MyZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZF4wMjRiOWExZmE4ZTAwNmYxZTM5M3xwYXJyMF4wMjRiOWExZmE4ZTAwNmYxZTM5Mw==", + "unique_id": "3" + } + +Before we give this to our peer, let's add another restriction: that +it only be usable for 24 hours from now. `date +%s` can give us the +current time in seconds: + + $ lightning-cli commando-rune rune=fTQnfL05coEbiBO8SS0cvQwCcPLxE9c02pZCC6HRVEY9MyZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZF4wMjRiOWExZmE4ZTAwNmYxZTM5M3xwYXJyMF4wMjRiOWExZmE4ZTAwNmYxZTM5Mw== restrictions="t<$(($(date +%s) + 24*60*60))" + { + "rune": "Sh-jGdfO9UGByLvah2AHgc_VwgoNujckPNkxTx54ugg9MyZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZF4wMjRiOWExZmE4ZTAwNmYxZTM5M3xwYXJyMF4wMjRiOWExZmE4ZTAwNmYxZTM5MyZ0PDE2NTY4OTc5MjU=", + "unique_id": "3" + } SHARING RUNES ------------- diff --git a/doc/schemas/commando.request.json b/doc/schemas/commando.request.json index 354f83e86683..52c52773f6e7 100644 --- a/doc/schemas/commando.request.json +++ b/doc/schemas/commando.request.json @@ -16,8 +16,16 @@ "description": "method to invoke on peer" }, "params": { - "type": "object", - "description": "parameters for method" + "oneOf": [ + { + "type": "array", + "description": "array of positional parameters" + }, + { + "type": "object", + "description": "parameters for method" + } + ] }, "rune": { "type": "string", diff --git a/plugins/commando.c b/plugins/commando.c index 1fcff8d8e88c..b7e44f82f3dd 100644 --- a/plugins/commando.c +++ b/plugins/commando.c @@ -201,30 +201,37 @@ static const char *check_condition(const tal_t *ctx, return rune_alt_single_str(ctx, alt, cinfo->buf + cinfo->method->start, cinfo->method->end - cinfo->method->start); + } else if (streq(alt->fieldname, "pnum")) { + return rune_alt_single_int(ctx, alt, cinfo->params->size); } /* Rest are params looksup: generate this once! */ - if (cinfo->params) { - /* Note: we require that params be an obj! */ + if (strmap_empty(&cinfo->cached_params)) { const jsmntok_t *t; size_t i; - json_for_each_obj(i, t, cinfo->params) { - char *pmemname = tal_fmt(tmpctx, - "pname%.*s", - t->end - t->start, - cinfo->buf + t->start); - size_t off = strlen("pname"); - /* Remove punctuation! */ - for (size_t n = off; pmemname[n]; n++) { - if (cispunct(pmemname[n])) - continue; - pmemname[off++] = pmemname[n]; + if (cinfo->params->type == JSMN_OBJECT) { + json_for_each_obj(i, t, cinfo->params) { + char *pmemname = tal_fmt(tmpctx, + "pname%.*s", + t->end - t->start, + cinfo->buf + t->start); + size_t off = strlen("pname"); + /* Remove punctuation! */ + for (size_t n = off; pmemname[n]; n++) { + if (cispunct(pmemname[n])) + continue; + pmemname[off++] = pmemname[n]; + } + pmemname[off++] = '\0'; + strmap_add(&cinfo->cached_params, pmemname, t+1); + } + } else if (cinfo->params->type == JSMN_ARRAY) { + json_for_each_arr(i, t, cinfo->params) { + char *pmemname = tal_fmt(tmpctx, "parr%zu", i); + strmap_add(&cinfo->cached_params, pmemname, t); } - pmemname[off++] = '\0'; - strmap_add(&cinfo->cached_params, pmemname, t+1); } - cinfo->params = NULL; } ptok = strmap_get(&cinfo->cached_params, alt->fieldname); @@ -300,9 +307,9 @@ static void try_command(struct node_id *peer, return; } params = json_get_member(buf, toks, "params"); - if (params && params->type != JSMN_OBJECT) { + if (!params || (params->type != JSMN_OBJECT && params->type != JSMN_ARRAY)) { commando_error(incoming, COMMANDO_ERROR_REMOTE, - "Params must be object"); + "Params must be object or array"); return; } rune = json_get_member(buf, toks, "rune"); @@ -323,15 +330,27 @@ static void try_command(struct node_id *peer, size_t i; const jsmntok_t *t; - json_object_start(req->js, "params"); /* FIXME: This is ugly! */ - json_for_each_obj(i, t, params) { - json_add_jsonstr(req->js, - json_strdup(tmpctx, buf, t), - json_tok_full(buf, t+1), - json_tok_full_len(t+1)); + if (params->type == JSMN_OBJECT) { + json_object_start(req->js, "params"); + json_for_each_obj(i, t, params) { + json_add_jsonstr(req->js, + json_strdup(tmpctx, buf, t), + json_tok_full(buf, t+1), + json_tok_full_len(t+1)); + } + json_object_end(req->js); + } else { + assert(params->type == JSMN_ARRAY); + json_array_start(req->js, "params"); + json_for_each_arr(i, t, params) { + json_add_jsonstr(req->js, + NULL, + json_tok_full(buf, t), + json_tok_full_len(t)); + } + json_array_end(req->js); } - json_object_end(req->js); } else { json_object_start(req->js, "params"); json_object_end(req->js); diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 1d39f13de4dc..b7ddd03502cd 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2660,6 +2660,12 @@ def test_commando_rune(node_factory): rune5 = l1.rpc.commando_rune(rune4['rune'], "pnamelevel!|pnamelevel/io") assert rune5['rune'] == 'Dw2tzGCoUojAyT0JUw7fkYJYqExpEpaDRNTkyvWKoJY9MyZpZF4wMjJkMjIzNjIwYTM1OWE0N2ZmNyZtZXRob2Q9bGlzdHBlZXJzJnBuYW1lbGV2ZWwhfHBuYW1lbGV2ZWwvaW8=' assert rune5['unique_id'] == '3' + rune6 = l1.rpc.commando_rune(rune5['rune'], "parr1!|parr1/io") + assert rune6['rune'] == '2Wh6F4R51D3esZzp-7WWG51OhzhfcYKaaI8qiIonaHE9MyZpZF4wMjJkMjIzNjIwYTM1OWE0N2ZmNyZtZXRob2Q9bGlzdHBlZXJzJnBuYW1lbGV2ZWwhfHBuYW1lbGV2ZWwvaW8mcGFycjEhfHBhcnIxL2lv' + assert rune6['unique_id'] == '3' + rune7 = l1.rpc.commando_rune(restrictions="pnum=0") + assert rune7['rune'] == 'QJonN6ySDFw-P5VnilZxlOGRs_tST1ejtd-bAYuZfjk9NCZwbnVtPTA=' + assert rune7['unique_id'] == '4' # Replace rune3 with a more useful timestamp! expiry = int(time.time()) + 15 @@ -2671,12 +2677,19 @@ def test_commando_rune(node_factory): (rune3, "getinfo", {}), (rune4, "listpeers", {}), (rune5, "listpeers", {'id': l2.info['id']}), - (rune5, "listpeers", {'id': l2.info['id'], 'level': 'broken'})) + (rune5, "listpeers", {'id': l2.info['id'], 'level': 'broken'}), + (rune6, "listpeers", [l2.info['id'], 'broken']), + (rune6, "listpeers", [l2.info['id']]), + (rune7, "listpeers", []), + (rune7, "getinfo", {})) failures = ((rune2, "withdraw", {}), (rune2, "plugin", {'subcommand': 'list'}), (rune3, "getinfo", {}), (rune4, "listnodes", {}), - (rune5, "listpeers", {'id': l2.info['id'], 'level': 'io'})) + (rune5, "listpeers", {'id': l2.info['id'], 'level': 'io'}), + (rune6, "listpeers", [l2.info['id'], 'io']), + (rune7, "listpeers", [l2.info['id']]), + (rune7, "listpeers", {'id': l2.info['id']})) for rune, cmd, params in successes: l2.rpc.call(method='commando', @@ -2708,14 +2721,3 @@ def test_commando_rune(node_factory): 'method': "listpeers", 'params': {}}) assert exc_info.value.error['code'] == 0x4c51 - - # Remote doesn't allow array parameters. - l2.rpc.check_request_schemas = False - with pytest.raises(RpcError, match='Params must be object') as exc_info: - l2.rpc.call(method='commando', - payload={'peer_id': l1.info['id'], - 'rune': rune5['rune'], - 'method': "listpeers", - 'params': [l2.info['id'], 'io']}) - assert exc_info.value.error['code'] == 0x4c50 - l2.rpc.check_request_schemas = True From ee6b174dbba4b26d118d8a375898d7e55aac3ae7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 16/18] commando: add rate for maximum successful rune use per minute. I'm assuming that nobody wants a rate slower than 1 per minute; we can introduce 'drate' if we want a per-day kind of limit. Signed-off-by: Rusty Russell --- doc/lightning-commando-rune.7.md | 19 ++++++-- plugins/commando.c | 84 ++++++++++++++++++++++++++++++++ tests/test_plugin.py | 25 +++++++++- 3 files changed, 121 insertions(+), 7 deletions(-) diff --git a/doc/lightning-commando-rune.7.md b/doc/lightning-commando-rune.7.md index ca31e2e96171..97bc4ec9afed 100644 --- a/doc/lightning-commando-rune.7.md +++ b/doc/lightning-commando-rune.7.md @@ -33,6 +33,7 @@ being run: * time: the current UNIX time, e.g. "time<1656759180". * id: the node_id of the peer, e.g. "id=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605". * method: the command being run, e.g. "method=withdraw". +* rate: the rate limit, per minute, e.g. "rate=60". * pnum: the number of parameters. e.g. "pnum<2". * pnameX: the parameter named X. e.g. "pnamedestination=1RustyRX2oai4EYYDpQGWvEL62BBGqN9T". * parrN: the N'th parameter. e.g. "parr0=1RustyRX2oai4EYYDpQGWvEL62BBGqN9T". @@ -108,13 +109,14 @@ This allows `listpeers` with 1 argument (`pnum=1`), which is either by name (`pn "unique_id": "3" } -Before we give this to our peer, let's add another restriction: that -it only be usable for 24 hours from now. `date +%s` can give us the -current time in seconds: +Before we give this to our peer, let's add two more restrictions: that +it only be usable for 24 hours from now (`time<`), and that it can only +be used twice a minute (`rate=2`). `date +%s` can give us the current +time in seconds: - $ lightning-cli commando-rune rune=fTQnfL05coEbiBO8SS0cvQwCcPLxE9c02pZCC6HRVEY9MyZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZF4wMjRiOWExZmE4ZTAwNmYxZTM5M3xwYXJyMF4wMjRiOWExZmE4ZTAwNmYxZTM5Mw== restrictions="t<$(($(date +%s) + 24*60*60))" + $ lightning-cli commando-rune rune=fTQnfL05coEbiBO8SS0cvQwCcPLxE9c02pZCC6HRVEY9MyZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZF4wMjRiOWExZmE4ZTAwNmYxZTM5M3xwYXJyMF4wMjRiOWExZmE4ZTAwNmYxZTM5Mw== restrictions='["time<'$(($(date +%s) + 24*60*60))'","rate=2"]' { - "rune": "Sh-jGdfO9UGByLvah2AHgc_VwgoNujckPNkxTx54ugg9MyZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZF4wMjRiOWExZmE4ZTAwNmYxZTM5M3xwYXJyMF4wMjRiOWExZmE4ZTAwNmYxZTM5MyZ0PDE2NTY4OTc5MjU=", + "rune": "tU-RLjMiDpY2U0o3W1oFowar36RFGpWloPbW9-RuZdo9MyZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZF4wMjRiOWExZmE4ZTAwNmYxZTM5M3xwYXJyMF4wMjRiOWExZmE4ZTAwNmYxZTM5MyZ0aW1lPDE2NTY5MjA1MzgmcmF0ZT0y", "unique_id": "3" } @@ -126,6 +128,13 @@ normal rune into a read-only rune, or restrict access for 30 minutes from the time you give it to someone. Adding restrictions before sharing runes is best practice. +If a rune has a ratelimit, any derived rune will have the same id, and +thus will compete for that ratelimit. You might want to consider +adding a tighter ratelimit to a rune before sharing it, so you will +keep the remainder. For example, if you rune has a limit of 60 times +per minute, adding a limit of 5 times per minute and handing that rune +out means you can still use your original rune 55 times per minute. + RETURN VALUE ------------ diff --git a/plugins/commando.c b/plugins/commando.c index b7e44f82f3dd..2f39079a2665 100644 --- a/plugins/commando.c +++ b/plugins/commando.c @@ -1,5 +1,7 @@ #include "config.h" #include +#include +#include #include #include #include @@ -42,6 +44,45 @@ static struct commando **incoming_commands; static u64 *rune_counter; static struct rune *master_rune; +struct usage { + /* If you really issue more than 2^32 runes, they'll share ratelimit buckets */ + u32 id; + u32 counter; +}; + +static u64 usage_id(const struct usage *u) +{ + return u->id; +} + +static size_t id_hash(u64 id) +{ + return siphash24(siphash_seed(), &id, sizeof(id)); +} + +static bool usage_eq_id(const struct usage *u, u64 id) +{ + return u->id == id; +} +HTABLE_DEFINE_TYPE(struct usage, usage_id, id_hash, usage_eq_id, usage_table); +static struct usage_table usage_table; + +/* Every minute we forget entries. */ +static void flush_usage_table(void *unused) +{ + struct usage *u; + struct usage_table_iter it; + + for (u = usage_table_first(&usage_table, &it); + u; + u = usage_table_next(&usage_table, &it)) { + usage_table_delval(&usage_table, &it); + tal_free(u); + } + + notleak(plugin_timer(plugin, time_from_sec(60), flush_usage_table, NULL)); +} + /* NULL peer: don't care about peer. NULL id: don't care about id */ static struct commando *find_commando(struct commando **arr, const struct node_id *peer, @@ -183,8 +224,40 @@ struct cond_info { const jsmntok_t *method; const jsmntok_t *params; STRMAP(const jsmntok_t *) cached_params; + struct usage *usage; }; +static const char *rate_limit_check(const tal_t *ctx, + const struct rune *rune, + const struct rune_altern *alt, + struct cond_info *cinfo) +{ + unsigned long r; + char *endp; + if (alt->condition != '=') + return "rate operator must be ="; + + r = strtoul(alt->value, &endp, 10); + if (endp == alt->value || *endp || r == 0 || r >= UINT32_MAX) + return "malformed rate"; + + /* We cache this: we only add usage counter if whole rune succeeds! */ + if (!cinfo->usage) { + cinfo->usage = usage_table_get(&usage_table, atol(rune->unique_id)); + if (!cinfo->usage) { + cinfo->usage = tal(plugin, struct usage); + cinfo->usage->id = atol(rune->unique_id); + cinfo->usage->counter = 0; + usage_table_add(&usage_table, cinfo->usage); + } + } + + /* >= becuase if we allow this, counter will increment */ + if (cinfo->usage->counter >= r) + return tal_fmt(ctx, "Rate of %lu per minute exceeded", r); + return NULL; +} + static const char *check_condition(const tal_t *ctx, const struct rune *rune, const struct rune_altern *alt, @@ -203,6 +276,8 @@ static const char *check_condition(const tal_t *ctx, cinfo->method->end - cinfo->method->start); } else if (streq(alt->fieldname, "pnum")) { return rune_alt_single_int(ctx, alt, cinfo->params->size); + } else if (streq(alt->fieldname, "rate")) { + return rate_limit_check(ctx, rune, alt, cinfo); } /* Rest are params looksup: generate this once! */ @@ -267,12 +342,17 @@ static const char *check_rune(const tal_t *ctx, cinfo.buf = buf; cinfo.method = method; cinfo.params = params; + cinfo.usage = NULL; strmap_init(&cinfo.cached_params); err = rune_test(ctx, master_rune, rune, check_condition, &cinfo); /* Just in case they manage to make us speak non-JSON, escape! */ if (err) err = json_escape(ctx, take(err))->s; strmap_clear(&cinfo.cached_params); + + /* If it succeeded, *now* we increment any associated usage counter. */ + if (!err && cinfo.usage) + cinfo.usage->counter++; return err; } @@ -770,6 +850,7 @@ static void memleak_mark_globals(struct plugin *p, struct htable *memtable) memleak_remove_region(memtable, outgoing_commands, tal_bytelen(outgoing_commands)); memleak_remove_region(memtable, incoming_commands, tal_bytelen(incoming_commands)); memleak_remove_region(memtable, master_rune, sizeof(*master_rune)); + memleak_remove_htable(memtable, &usage_table.raw); if (rune_counter) memleak_remove_region(memtable, rune_counter, sizeof(*rune_counter)); } @@ -782,6 +863,7 @@ static const char *init(struct plugin *p, outgoing_commands = tal_arr(p, struct commando *, 0); incoming_commands = tal_arr(p, struct commando *, 0); + usage_table_init(&usage_table); plugin = p; #if DEVELOPER plugin_set_memleak_handler(p, memleak_mark_globals); @@ -814,6 +896,8 @@ static const char *init(struct plugin *p, master_rune = rune_new(plugin, rune_secret.data, ARRAY_SIZE(rune_secret.data), NULL); + /* Start flush timer. */ + flush_usage_table(NULL); return NULL; } diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b7ddd03502cd..9061ac7154e7 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2666,10 +2666,19 @@ def test_commando_rune(node_factory): rune7 = l1.rpc.commando_rune(restrictions="pnum=0") assert rune7['rune'] == 'QJonN6ySDFw-P5VnilZxlOGRs_tST1ejtd-bAYuZfjk9NCZwbnVtPTA=' assert rune7['unique_id'] == '4' + rune8 = l1.rpc.commando_rune(rune7['rune'], "rate=3") + assert rune8['rune'] == 'kSYFx6ON9hr_ExcQLwVkm1ABnvc1TcMFBwLrAVee0EA9NCZwbnVtPTAmcmF0ZT0z' + assert rune8['unique_id'] == '4' + rune9 = l1.rpc.commando_rune(rune8['rune'], "rate=1") + assert rune9['rune'] == 'O8Zr-ULTBKO3_pKYz0QKE9xYl1vQ4Xx9PtlHuist9Rk9NCZwbnVtPTAmcmF0ZT0zJnJhdGU9MQ==' + assert rune9['unique_id'] == '4' # Replace rune3 with a more useful timestamp! expiry = int(time.time()) + 15 rune3 = l1.rpc.commando_rune(restrictions="time<{}".format(expiry)) + ratelimit_successes = ((rune9, "getinfo", {}), + (rune8, "getinfo", {}), + (rune8, "getinfo", {})) successes = ((rune1, "listpeers", {}), (rune2, "listpeers", {}), (rune2, "getinfo", {}), @@ -2681,7 +2690,7 @@ def test_commando_rune(node_factory): (rune6, "listpeers", [l2.info['id'], 'broken']), (rune6, "listpeers", [l2.info['id']]), (rune7, "listpeers", []), - (rune7, "getinfo", {})) + (rune7, "getinfo", {})) + ratelimit_successes failures = ((rune2, "withdraw", {}), (rune2, "plugin", {'subcommand': 'list'}), (rune3, "getinfo", {}), @@ -2689,7 +2698,9 @@ def test_commando_rune(node_factory): (rune5, "listpeers", {'id': l2.info['id'], 'level': 'io'}), (rune6, "listpeers", [l2.info['id'], 'io']), (rune7, "listpeers", [l2.info['id']]), - (rune7, "listpeers", {'id': l2.info['id']})) + (rune7, "listpeers", {'id': l2.info['id']}), + (rune9, "getinfo", {}), + (rune8, "getinfo", {})) for rune, cmd, params in successes: l2.rpc.call(method='commando', @@ -2721,3 +2732,13 @@ def test_commando_rune(node_factory): 'method': "listpeers", 'params': {}}) assert exc_info.value.error['code'] == 0x4c51 + + # Now wait for ratelimit expiry, ratelimits should reset. + time.sleep(61) + + for rune, cmd, params in ratelimit_successes: + l2.rpc.call(method='commando', + payload={'peer_id': l1.info['id'], + 'rune': rune['rune'], + 'method': cmd, + 'params': params}) From 96ab5048cd12eec4797ea223a5292def38128d69 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 16 Jul 2022 22:48:27 +0930 Subject: [PATCH 17/18] decode: support decoding runes. This is a bit weird since it lives in the offers plugin, but it works well. This should make runes much more approachable for people! Signed-off-by: Rusty Russell --- doc/lightning-commando-rune.7.md | 52 +++++++++- doc/lightning-decode.7.md | 28 ++++-- doc/schemas/decode.schema.json | 69 ++++++++++++- plugins/offers.c | 161 +++++++++++++++++++++++++++++++ tests/test_plugin.py | 53 ++++++++++ 5 files changed, 354 insertions(+), 9 deletions(-) diff --git a/doc/lightning-commando-rune.7.md b/doc/lightning-commando-rune.7.md index 97bc4ec9afed..c2285b65054e 100644 --- a/doc/lightning-commando-rune.7.md +++ b/doc/lightning-commando-rune.7.md @@ -120,6 +120,56 @@ time in seconds: "unique_id": "3" } +You can also use lightning-decode(7) to examine runes you have been given: + + $ .lightning-cli decode tU-RLjMiDpY2U0o3W1oFowar36RFGpWloPbW9-RuZdo9MyZpZD0wMjRiOWExZmE4ZTAwNmYxZTM5MzdmNjVmNjZjNDA4ZTZkYThlMWNhNzI4ZWE0MzIyMmE3MzgxZGYxY2M0NDk2MDUmbWV0aG9kPWxpc3RwZWVycyZwbnVtPTEmcG5hbWVpZF4wMjRiOWExZmE4ZTAwNmYxZTM5M3xwYXJyMF4wMjRiOWExZmE4ZTAwNmYxZTM5MyZ0aW1lPDE2NTY5MjA1MzgmcmF0ZT0y + { + "type": "rune", + "unique_id": "3", + "string": "b54f912e33220e9636534a375b5a05a306abdfa4451a95a5a0f6d6f7e46e65da:=3&id=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605&method=listpeers&pnum=1&pnameid^024b9a1fa8e006f1e393|parr0^024b9a1fa8e006f1e393&time<1656920538&rate=2", + "restrictions": [ + { + "alternatives": [ + "id=024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605" + ], + "summary": "id (of commanding peer) equal to '024b9a1fa8e006f1e3937f65f66c408e6da8e1ca728ea43222a7381df1cc449605'" + }, + { + "alternatives": [ + "method=listpeers" + ], + "summary": "method (of command) equal to 'listpeers'" + }, + { + "alternatives": [ + "pnum=1" + ], + "summary": "pnum (number of command parameters) equal to 1" + }, + { + "alternatives": [ + "pnameid^024b9a1fa8e006f1e393", + "parr0^024b9a1fa8e006f1e393" + ], + "summary": "pnameid (object parameter 'id') starts with '024b9a1fa8e006f1e393' OR parr0 (array parameter #0) starts with '024b9a1fa8e006f1e393'" + }, + { + "alternatives": [ + "time<1656920538" + ], + "summary": "time (in seconds since 1970) less than 1656920538 (approximately 19 hours 18 minutes from now)" + }, + { + "alternatives": [ + "rate=2" + ], + "summary": "rate (max per minute) equal to 2" + } + ], + "valid": true + } + + SHARING RUNES ------------- @@ -157,7 +207,7 @@ excuses his previous adoption of the name "Eltoo". SEE ALSO -------- -lightning-commando(7) +lightning-commando(7), lightning-decode(7) RESOURCES --------- diff --git a/doc/lightning-decode.7.md b/doc/lightning-decode.7.md index 7c916562b30f..01b3a3c6ea4e 100644 --- a/doc/lightning-decode.7.md +++ b/doc/lightning-decode.7.md @@ -9,17 +9,21 @@ SYNOPSIS DESCRIPTION ----------- -The **decode** RPC command checks and parses a *bolt11* or *bolt12* -string (optionally prefixed by `lightning:` or `LIGHTNING:`) as -specified by the BOLT 11 and BOLT 12 specifications. It may decode -other formats in future. +The **decode** RPC command checks and parses: + +- a *bolt11* or *bolt12* string (optionally prefixed by `lightning:` + or `LIGHTNING:`) as specified by the BOLT 11 and BOLT 12 + specifications. +- a *rune* as created by lightning-commando-rune(7). + +It may decode other formats in future. RETURN VALUE ------------ [comment]: # (GENERATE-FROM-SCHEMA-START) On success, an object is returned, containing: -- **type** (string): what kind of object it decoded to (one of "bolt12 offer", "bolt12 invoice", "bolt12 invoice_request", "bolt11 invoice") +- **type** (string): what kind of object it decoded to (one of "bolt12 offer", "bolt12 invoice", "bolt12 invoice_request", "bolt11 invoice", "rune") - **valid** (boolean): if this is false, you *MUST* not use the result except for diagnostics! If **type** is "bolt12 offer", and **valid** is *true*: @@ -159,6 +163,16 @@ If **type** is "bolt11 invoice", and **valid** is *true*: - **tag** (string): The bech32 letter which identifies this field (always 1 characters) - **data** (string): The bech32 data for this field +If **type** is "rune": + - **string** (string): the string encoding of the rune + - **restrictions** (array of objects): restrictions built into the rune: all must pass: + - **alternatives** (array of strings): each way restriction can be met: any can pass: + - the alternative of form fieldname condition fieldname + - **summary** (string): human-readable summary of this restriction + - **unique_id** (string, optional): unique id (always a numeric id on runes we create) + - **version** (string, optional): rune version, not currently set on runes we create + - **valid** (boolean, optional) (always *true*) + [comment]: # (GENERATE-FROM-SCHEMA-END) AUTHOR @@ -169,7 +183,7 @@ Rusty Russell <> is mainly responsible. SEE ALSO -------- -lightning-pay(7), lightning-offer(7), lightning-offerout(7), lightning-fetchinvoice(7), lightning-sendinvoice(7) +lightning-pay(7), lightning-offer(7), lightning-offerout(7), lightning-fetchinvoice(7), lightning-sendinvoice(7), lightning-commando-rune(7) [BOLT \#11](https://github.com/lightningnetwork/lightning-rfc/blob/master/11-payment-encoding.md). @@ -181,4 +195,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:bc3778965137591623ce08ff51adf411bc42e6d1a4200692961b69962da39be7) +[comment]: # ( SHA256STAMP:d1e1f044c2e67ec169728dbc551903c97f9a9daa1f42e9d2f1686fc692d25be8) diff --git a/doc/schemas/decode.schema.json b/doc/schemas/decode.schema.json index 2c84e820bd54..1ff631d28882 100644 --- a/doc/schemas/decode.schema.json +++ b/doc/schemas/decode.schema.json @@ -12,7 +12,8 @@ "bolt12 offer", "bolt12 invoice", "bolt12 invoice_request", - "bolt11 invoice" + "bolt11 invoice", + "rune" ], "description": "what kind of object it decoded to" }, @@ -909,6 +910,72 @@ } } } + }, + { + "if": { + "properties": { + "type": { + "type": "string", + "enum": [ + "rune" + ] + } + } + }, + "then": { + "required": [ + "string", + "restrictions" + ], + "additionalProperties": false, + "properties": { + "unique_id": { + "type": "string", + "description": "unique id (always a numeric id on runes we create)" + }, + "version": { + "type": "string", + "description": "rune version, not currently set on runes we create" + }, + "valid": { + "type": "boolean", + "enum": [ + true + ] + }, + "type": {}, + "string": { + "type": "string", + "description": "the string encoding of the rune" + }, + "restrictions": { + "type": "array", + "description": "restrictions built into the rune: all must pass", + "items": { + "type": "object", + "required": [ + "alternatives", + "summary" + ], + "additionalProperties": false, + "properties": { + "alternatives": { + "type": "array", + "description": "each way restriction can be met: any can pass", + "items": { + "type": "string", + "description": "the alternative of form fieldname condition fieldname" + } + }, + "summary": { + "type": "string", + "description": "human-readable summary of this restriction" + } + } + } + } + } + } } ] } diff --git a/plugins/offers.c b/plugins/offers.c index 5ca4ea4d7760..72442234c112 100644 --- a/plugins/offers.c +++ b/plugins/offers.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -204,6 +205,7 @@ struct decodable { struct tlv_offer *offer; struct tlv_invoice *invoice; struct tlv_invoice_request *invreq; + struct rune *rune; }; static struct command_result *param_decodable(struct command *cmd, @@ -271,6 +273,13 @@ static struct command_result *param_decodable(struct command *cmd, return NULL; } + decodable->rune = rune_from_base64n(decodable, buffer + tok.start, + tok.end - tok.start); + if (decodable->rune) { + decodable->type = "rune"; + return NULL; + } + /* Return failure message from most likely parsing candidate */ return command_fail_badparam(cmd, name, buffer, &tok, likely_fail); } @@ -808,6 +817,156 @@ static void json_add_invoice_request(struct json_stream *js, json_add_bool(js, "valid", valid); } +static void json_add_rune(struct command *cmd, struct json_stream *js, const struct rune *rune) +{ + if (rune->unique_id) + json_add_string(js, "unique_id", rune->unique_id); + if (rune->version) + json_add_string(js, "version", rune->version); + json_add_string(js, "string", take(rune_to_string(NULL, rune))); + + json_array_start(js, "restrictions"); + for (size_t i = rune->unique_id ? 1 : 0; i < tal_count(rune->restrs); i++) { + const struct rune_restr *restr = rune->restrs[i]; + char *summary = tal_strdup(tmpctx, ""); + const char *sep = ""; + + json_object_start(js, NULL); + json_array_start(js, "alternatives"); + for (size_t j = 0; j < tal_count(restr->alterns); j++) { + const struct rune_altern *alt = restr->alterns[j]; + const char *annotation, *value; + bool int_val = false, time_val = false; + + if (streq(alt->fieldname, "time")) { + annotation = "in seconds since 1970"; + time_val = true; + } else if (streq(alt->fieldname, "id")) + annotation = "of commanding peer"; + else if (streq(alt->fieldname, "method")) + annotation = "of command"; + else if (streq(alt->fieldname, "pnum")) { + annotation = "number of command parameters"; + int_val = true; + } else if (streq(alt->fieldname, "rate")) { + annotation = "max per minute"; + int_val = true; + } else if (strstarts(alt->fieldname, "parr")) { + annotation = tal_fmt(tmpctx, "array parameter #%s", alt->fieldname+4); + } else if (strstarts(alt->fieldname, "pname")) + annotation = tal_fmt(tmpctx, "object parameter '%s'", alt->fieldname+5); + else + annotation = "unknown condition?"; + + tal_append_fmt(&summary, "%s", sep); + + /* Where it's ambiguous, quote if it's not treated as an int */ + if (int_val) + value = alt->value; + else if (time_val) { + u64 t = atol(alt->value); + + if (t) { + u64 diff, now = time_now().ts.tv_sec; + /* Need a non-const during construction */ + char *v; + + if (now > t) + diff = now - t; + else + diff = t - now; + if (diff < 60) + v = tal_fmt(tmpctx, "%"PRIu64" seconds", diff); + else if (diff < 60 * 60) + v = tal_fmt(tmpctx, "%"PRIu64" minutes %"PRIu64" seconds", + diff / 60, diff % 60); + else { + v = tal_strdup(tmpctx, "approximately "); + /* diff is in minutes */ + diff /= 60; + if (diff < 48 * 60) + tal_append_fmt(&v, "%"PRIu64" hours %"PRIu64" minutes", + diff / 60, diff % 60); + else { + /* hours */ + diff /= 60; + if (diff < 60 * 24) + tal_append_fmt(&v, "%"PRIu64" days %"PRIu64" hours", + diff / 24, diff % 24); + else { + /* days */ + diff /= 24; + if (diff < 365 * 2) + tal_append_fmt(&v, "%"PRIu64" months %"PRIu64" days", + diff / 30, diff % 30); + else { + /* months */ + diff /= 30; + tal_append_fmt(&v, "%"PRIu64" years %"PRIu64" months", + diff / 12, diff % 12); + } + } + } + } + if (now > t) + tal_append_fmt(&v, " ago"); + else + tal_append_fmt(&v, " from now"); + value = tal_fmt(tmpctx, "%s (%s)", alt->value, v); + } else + value = alt->value; + } else + value = tal_fmt(tmpctx, "'%s'", alt->value); + + switch (alt->condition) { + case RUNE_COND_IF_MISSING: + tal_append_fmt(&summary, "%s (%s) is missing", alt->fieldname, annotation); + break; + case RUNE_COND_EQUAL: + tal_append_fmt(&summary, "%s (%s) equal to %s", alt->fieldname, annotation, value); + break; + case RUNE_COND_NOT_EQUAL: + tal_append_fmt(&summary, "%s (%s) unequal to %s", alt->fieldname, annotation, value); + break; + case RUNE_COND_BEGINS: + tal_append_fmt(&summary, "%s (%s) starts with '%s'", alt->fieldname, annotation, alt->value); + break; + case RUNE_COND_ENDS: + tal_append_fmt(&summary, "%s (%s) ends with '%s'", alt->fieldname, annotation, alt->value); + break; + case RUNE_COND_CONTAINS: + tal_append_fmt(&summary, "%s (%s) contains '%s'", alt->fieldname, annotation, alt->value); + break; + case RUNE_COND_INT_LESS: + tal_append_fmt(&summary, "%s (%s) less than %s", alt->fieldname, annotation, + time_val ? value : alt->value); + break; + case RUNE_COND_INT_GREATER: + tal_append_fmt(&summary, "%s (%s) greater than %s", alt->fieldname, annotation, + time_val ? value : alt->value); + break; + case RUNE_COND_LEXO_BEFORE: + tal_append_fmt(&summary, "%s (%s) sorts before '%s'", alt->fieldname, annotation, alt->value); + break; + case RUNE_COND_LEXO_AFTER: + tal_append_fmt(&summary, "%s (%s) sorts after '%s'", alt->fieldname, annotation, alt->value); + break; + case RUNE_COND_COMMENT: + tal_append_fmt(&summary, "[comment: %s%s]", alt->fieldname, alt->value); + break; + } + sep = " OR "; + json_add_str_fmt(js, NULL, "%s%c%s", alt->fieldname, alt->condition, alt->value); + } + json_array_end(js); + json_add_string(js, "summary", summary); + json_object_end(js); + } + json_array_end(js); + /* FIXME: do some sanity checks? */ + json_add_bool(js, "valid", true); +} + static struct command_result *json_decode(struct command *cmd, const char *buffer, const jsmntok_t *params) @@ -833,6 +992,8 @@ static struct command_result *json_decode(struct command *cmd, json_add_bolt11(response, decodable->b11); json_add_bool(response, "valid", true); } + if (decodable->rune) + json_add_rune(cmd, response, decodable->rune); return command_finished(cmd, response); } diff --git a/tests/test_plugin.py b/tests/test_plugin.py index 9061ac7154e7..3d7a76ca048f 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -2673,6 +2673,59 @@ def test_commando_rune(node_factory): assert rune9['rune'] == 'O8Zr-ULTBKO3_pKYz0QKE9xYl1vQ4Xx9PtlHuist9Rk9NCZwbnVtPTAmcmF0ZT0zJnJhdGU9MQ==' assert rune9['unique_id'] == '4' + runedecodes = ((rune1, []), + (rune2, [{'alternatives': ['method^list', 'method^get', 'method=summary'], + 'summary': "method (of command) starts with 'list' OR method (of command) starts with 'get' OR method (of command) equal to 'summary'"}, + {'alternatives': ['method/listdatastore'], + 'summary': "method (of command) unequal to 'listdatastore'"}]), + (rune4, [{'alternatives': ['id^022d223620a359a47ff7'], + 'summary': "id (of commanding peer) starts with '022d223620a359a47ff7'"}, + {'alternatives': ['method=listpeers'], + 'summary': "method (of command) equal to 'listpeers'"}]), + (rune5, [{'alternatives': ['id^022d223620a359a47ff7'], + 'summary': "id (of commanding peer) starts with '022d223620a359a47ff7'"}, + {'alternatives': ['method=listpeers'], + 'summary': "method (of command) equal to 'listpeers'"}, + {'alternatives': ['pnamelevel!', 'pnamelevel/io'], + 'summary': "pnamelevel (object parameter 'level') is missing OR pnamelevel (object parameter 'level') unequal to 'io'"}]), + (rune6, [{'alternatives': ['id^022d223620a359a47ff7'], + 'summary': "id (of commanding peer) starts with '022d223620a359a47ff7'"}, + {'alternatives': ['method=listpeers'], + 'summary': "method (of command) equal to 'listpeers'"}, + {'alternatives': ['pnamelevel!', 'pnamelevel/io'], + 'summary': "pnamelevel (object parameter 'level') is missing OR pnamelevel (object parameter 'level') unequal to 'io'"}, + {'alternatives': ['parr1!', 'parr1/io'], + 'summary': "parr1 (array parameter #1) is missing OR parr1 (array parameter #1) unequal to 'io'"}]), + (rune7, [{'alternatives': ['pnum=0'], + 'summary': "pnum (number of command parameters) equal to 0"}]), + (rune8, [{'alternatives': ['pnum=0'], + 'summary': "pnum (number of command parameters) equal to 0"}, + {'alternatives': ['rate=3'], + 'summary': "rate (max per minute) equal to 3"}]), + (rune9, [{'alternatives': ['pnum=0'], + 'summary': "pnum (number of command parameters) equal to 0"}, + {'alternatives': ['rate=3'], + 'summary': "rate (max per minute) equal to 3"}, + {'alternatives': ['rate=1'], + 'summary': "rate (max per minute) equal to 1"}])) + for decode in runedecodes: + rune = decode[0] + restrictions = decode[1] + decoded = l1.rpc.decode(rune['rune']) + assert decoded['type'] == 'rune' + assert decoded['unique_id'] == rune['unique_id'] + assert decoded['valid'] is True + assert decoded['restrictions'] == restrictions + + # Time handling is a bit special, since we annotate the timestamp with how far away it is. + decoded = l1.rpc.decode(rune3['rune']) + assert decoded['type'] == 'rune' + assert decoded['unique_id'] == rune3['unique_id'] + assert decoded['valid'] is True + assert len(decoded['restrictions']) == 1 + assert decoded['restrictions'][0]['alternatives'] == ['time>1656675211'] + assert decoded['restrictions'][0]['summary'].startswith("time (in seconds since 1970) greater than 1656675211 (") + # Replace rune3 with a more useful timestamp! expiry = int(time.time()) + 15 rune3 = l1.rpc.commando_rune(restrictions="time<{}".format(expiry)) From 2e13b72f55080be07ea68de77976eb990a043f5d Mon Sep 17 00:00:00 2001 From: zero fee routing <90521529+zerofeerouting@users.noreply.github.com> Date: Sat, 16 Jul 2022 22:48:28 +0930 Subject: [PATCH 18/18] fix typo in commando documentation --- doc/lightning-commando-rune.7.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/lightning-commando-rune.7.md b/doc/lightning-commando-rune.7.md index c2285b65054e..699cab88dd2f 100644 --- a/doc/lightning-commando-rune.7.md +++ b/doc/lightning-commando-rune.7.md @@ -41,7 +41,7 @@ being run: RESTRICTION FORMAT ------------------ -Restrictions are one or more altneratives, separated by `|`. Each +Restrictions are one or more alternatives, separated by `|`. Each alternative is *name* *operator* *value*. The valid names are shown above. If a value contains `|`, `&` or `\\`, it must be preceeded by a `\\`.