From c3bed51b2d626449384af955fd9bba70f87cf496 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 00:02:16 +1030 Subject: [PATCH 0001/1428] test_lightningd.py: make HSM seeds constant for tests. Makes it easier to compare before/after failures. Ideally, we should run under Travis both with this option and with the seed based on the entire tmp path (which is still reproducible with determination, but not fixed every run like this is). Signed-off-by: Rusty Russell --- lightningd/lightningd.c | 1 + lightningd/lightningd.h | 3 +++ lightningd/options.c | 28 ++++++++++++++++++++++++++++ tests/utils.py | 3 +++ 4 files changed, 35 insertions(+) diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 2381557e4dff..2921fe996b30 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -75,6 +75,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx, htlc_in_map_init(&ld->htlcs_in); htlc_out_map_init(&ld->htlcs_out); ld->dev_disconnect_fd = -1; + ld->dev_hsm_seed = NULL; ld->log_book = log_book; ld->log = new_log(log_book, log_book, "lightningd(%u):", (int)getpid()); diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index eff6cda6a189..e0c6165d5199 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -113,6 +113,9 @@ struct lightningd { /* If we want to debug a subdaemon. */ const char *dev_debug_subdaemon; + /* If we want to set a specific non-random HSM seed. */ + const u8 *dev_hsm_seed; + /* If we have a --dev-disconnect file */ int dev_disconnect_fd; diff --git a/lightningd/options.c b/lightningd/options.c index 816236c007bd..b147ed037db1 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -2,12 +2,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -20,6 +22,8 @@ #include #include #include +#include +#include #include #include #include @@ -253,6 +257,15 @@ static void config_register_opts(struct lightningd *ld) "regtest, or litecoin)"); } +static char *opt_set_hsm_seed(const char *arg, struct lightningd *ld) +{ + ld->dev_hsm_seed = tal_hexdata(ld, arg, strlen(arg)); + if (ld->dev_hsm_seed) + return NULL; + + return tal_fmt(NULL, "bad hex string '%s'", arg); +} + static void dev_register_opts(struct lightningd *ld) { opt_register_noarg("--dev-no-broadcast", opt_set_bool, @@ -266,6 +279,8 @@ static void dev_register_opts(struct lightningd *ld) "Time between gossip broadcasts in milliseconds (default: 30000)"); opt_register_arg("--dev-disconnect=", opt_subd_dev_disconnect, NULL, ld, "File containing disconnection points"); + opt_register_arg("--dev-hsm-seed=", opt_set_hsm_seed, + NULL, ld, "Hex-encoded seed for HSM"); } static const struct config testnet_config = { @@ -546,5 +561,18 @@ bool handle_opts(struct lightningd *ld, int argc, char *argv[]) errx(1, "no arguments accepted"); check_config(ld); + + if (ld->dev_hsm_seed) { + int fd; + unlink("hsm_secret"); + fd = open("hsm_secret", O_CREAT|O_WRONLY, 0400); + if (fd < 0 || + !write_all(fd, ld->dev_hsm_seed, tal_len(ld->dev_hsm_seed)) + || fsync(fd) != 0) + fatal("dev-hsm-seed: Could not write file: %s", + strerror(errno)); + close(fd); + } + return newdir; } diff --git a/tests/utils.py b/tests/utils.py index 395b865722ca..891a77ac6374 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -225,6 +225,8 @@ def __init__(self, lightning_dir, bitcoin_dir, port=9735): TailableProc.__init__(self, lightning_dir) self.lightning_dir = lightning_dir self.port = port + # Last 32-bytes of final part of dir -> seed. + seed = (bytes(re.search('([^/]+)/*$', lightning_dir).group(1), encoding='utf-8') + bytes(32))[:32] self.cmd_line = [ 'lightningd/lightningd', '--bitcoin-datadir={}'.format(bitcoin_dir), @@ -232,6 +234,7 @@ def __init__(self, lightning_dir, bitcoin_dir, port=9735): '--port={}'.format(port), '--network=regtest', '--dev-broadcast-interval=1000', + '--dev-hsm-seed={}'.format(seed.hex()) ] self.cmd_line += ["--{}={}".format(k, v) for k, v in LIGHTNINGD_CONFIG.items()] From ffaa15c7dab91e22e279e63a6061570245fc7b2b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 00:02:46 +1030 Subject: [PATCH 0002/1428] hsm: remove unique_id. It was only for error messages, so replace it with pubkey. Signed-off-by: Rusty Russell --- hsmd/Makefile | 1 + hsmd/hsm.c | 28 ++++++++++++++++------------ hsmd/hsm_wire.csv | 7 +++---- lightningd/gossip_control.c | 2 +- lightningd/peer_control.c | 2 +- 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/hsmd/Makefile b/hsmd/Makefile index 4ac793650a56..bb2c1ff21e1c 100644 --- a/hsmd/Makefile +++ b/hsmd/Makefile @@ -34,6 +34,7 @@ HSMD_COMMON_OBJS := \ common/msg_queue.o \ common/permute_tx.o \ common/status.o \ + common/type_to_string.o \ common/utils.o \ common/utxo.o \ common/version.o \ diff --git a/hsmd/hsm.c b/hsmd/hsm.c index 3f72c7ef0849..50e5e51ee892 100644 --- a/hsmd/hsm.c +++ b/hsmd/hsm.c @@ -18,6 +18,7 @@ #include #include #include +#include #include #include #include @@ -47,7 +48,7 @@ struct client { struct daemon_conn dc; struct daemon_conn *master; - u64 id; + struct pubkey id; struct io_plan *(*handle)(struct io_conn *, struct daemon_conn *); }; @@ -74,13 +75,13 @@ static void node_key(struct privkey *node_privkey, struct pubkey *node_id) } static struct client *new_client(struct daemon_conn *master, - u64 id, + const struct pubkey *id, struct io_plan *(*handle)(struct io_conn *, struct daemon_conn *), int fd) { struct client *c = tal(master, struct client); - c->id = id; + c->id = *id; c->handle = handle; c->master = master; daemon_conn_init(c, &c->dc, fd, handle, NULL); @@ -102,7 +103,7 @@ static struct io_plan *handle_ecdh(struct io_conn *conn, struct daemon_conn *dc) if (!fromwire_hsm_ecdh_req(dc->msg_in, NULL, &point)) { daemon_conn_send(c->master, take(towire_hsmstatus_client_bad_request(c, - c->id, + &c->id, dc->msg_in))); return io_close(conn); } @@ -110,10 +111,11 @@ static struct io_plan *handle_ecdh(struct io_conn *conn, struct daemon_conn *dc) node_key(&privkey, NULL); if (secp256k1_ecdh(secp256k1_ctx, ss.data, &point.pubkey, privkey.secret.data) != 1) { - status_trace("secp256k1_ecdh fail for client %"PRIu64, c->id); + status_trace("secp256k1_ecdh fail for client %s", + type_to_string(trc, struct pubkey, &c->id)); daemon_conn_send(c->master, take(towire_hsmstatus_client_bad_request(c, - c->id, + &c->id, dc->msg_in))); return io_close(conn); } @@ -235,7 +237,7 @@ static struct io_plan *handle_channeld(struct io_conn *conn, daemon_conn_send(c->master, take(towire_hsmstatus_client_bad_request(c, - c->id, + &c->id, dc->msg_in))); return io_close(conn); } @@ -415,16 +417,18 @@ static void init_hsm(struct daemon_conn *master, const u8 *msg) static void pass_hsmfd_ecdh(struct daemon_conn *master, const u8 *msg) { int fds[2]; - u64 id; + struct pubkey id; - if (!fromwire_hsmctl_hsmfd_ecdh(msg, NULL, &id)) + if (!fromwire_hsmctl_hsmfd_ecdh(msg, NULL)) master_badmsg(WIRE_HSMCTL_HSMFD_ECDH, msg); if (socketpair(AF_UNIX, SOCK_STREAM, 0, fds) != 0) status_failed(STATUS_FAIL_INTERNAL_ERROR, "creating fds: %s", strerror(errno)); - new_client(master, id, handle_ecdh, fds[0]); + /* This is gossipd, so we use our own id */ + node_key(NULL, &id); + new_client(master, &id, handle_ecdh, fds[0]); daemon_conn_send(master, take(towire_hsmctl_hsmfd_ecdh_fd_reply(master))); daemon_conn_send_fd(master, fds[1]); @@ -434,7 +438,7 @@ static void pass_hsmfd_ecdh(struct daemon_conn *master, const u8 *msg) static void pass_hsmfd_channeld(struct daemon_conn *master, const u8 *msg) { int fds[2]; - u64 id; + struct pubkey id; if (!fromwire_hsmctl_hsmfd_channeld(msg, NULL, &id)) master_badmsg(WIRE_HSMCTL_HSMFD_CHANNELD, msg); @@ -443,7 +447,7 @@ static void pass_hsmfd_channeld(struct daemon_conn *master, const u8 *msg) status_failed(STATUS_FAIL_INTERNAL_ERROR, "creating fds: %s", strerror(errno)); - new_client(master, id, handle_channeld, fds[0]); + new_client(master, &id, handle_channeld, fds[0]); daemon_conn_send(master, take(towire_hsmctl_hsmfd_channeld_reply(master))); daemon_conn_send_fd(master, fds[1]); diff --git a/hsmd/hsm_wire.csv b/hsmd/hsm_wire.csv index 1f86f06098a8..3e79a3f96aa5 100644 --- a/hsmd/hsm_wire.csv +++ b/hsmd/hsm_wire.csv @@ -1,6 +1,6 @@ # Clients should not give a bad request but not the HSM's decision to crash. hsmstatus_client_bad_request,1000 -hsmstatus_client_bad_request,,unique_id,u64 +hsmstatus_client_bad_request,,id,struct pubkey hsmstatus_client_bad_request,,len,u16 hsmstatus_client_bad_request,,msg,len*u8 @@ -14,9 +14,8 @@ hsmctl_init_reply,,node_id,struct pubkey hsmctl_init_reply,,peer_seed,struct secret hsmctl_init_reply,,bip32,struct ext_key -# ECDH returns an fd. +# ECDH returns an fd (for gossipd to do handshake) hsmctl_hsmfd_ecdh,3 -hsmctl_hsmfd_ecdh,,unique_id,u64 # No contents, just an fd. hsmctl_hsmfd_ecdh_fd_reply,103 @@ -39,7 +38,7 @@ hsmctl_sign_funding_reply,,sig,num_sigs*secp256k1_ecdsa_signature # Request a client socket for a `channeld`, allows signing announcements hsmctl_hsmfd_channeld,5 -hsmctl_hsmfd_channeld,,unique_id,u64 +hsmctl_hsmfd_channeld,,id,struct pubkey # Empty reply, just an fd hsmctl_hsmfd_channeld_reply,105 diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index d770e1e17ce9..d7df9d35d4ac 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -97,7 +97,7 @@ void gossip_init(struct lightningd *ld) u8 *msg; int hsmfd; - msg = towire_hsmctl_hsmfd_ecdh(tmpctx, 0); + msg = towire_hsmctl_hsmfd_ecdh(tmpctx); if (!wire_sync_write(ld->hsm_fd, msg)) fatal("Could not write to HSM: %s", strerror(errno)); diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 1240d40d5a6f..97c1cb56f27c 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -2021,7 +2021,7 @@ static bool peer_start_channeld(struct peer *peer, } else assert(peer->our_msatoshi); - msg = towire_hsmctl_hsmfd_channeld(tmpctx, peer->unique_id); + msg = towire_hsmctl_hsmfd_channeld(tmpctx, &peer->id); if (!wire_sync_write(peer->ld->hsm_fd, take(msg))) fatal("Could not write to HSM: %s", strerror(errno)); From 48cedef756b60970262f2530f066881005a78808 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 00:02:46 +1030 Subject: [PATCH 0003/1428] peer_control: remove unique_id field. It's now completely useless. Signed-off-by: Rusty Russell --- lightningd/peer_control.c | 1 - lightningd/peer_control.h | 3 --- wallet/db.c | 1 - wallet/wallet.c | 7 ++----- wallet/wallet_tests.c | 1 - 5 files changed, 2 insertions(+), 11 deletions(-) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 97c1cb56f27c..f3df54078196 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -890,7 +890,6 @@ static void json_getpeers(struct command *cmd, json_array_start(response, "peers"); list_for_each(&cmd->ld->peers, p, list) { json_object_start(response, NULL); - json_add_u64(response, "unique_id", p->unique_id); json_add_string(response, "state", peer_state_name(p->state)); json_add_string(response, "netaddr", netaddr_name(response, &p->netaddr)); diff --git a/lightningd/peer_control.h b/lightningd/peer_control.h index f2dc672584de..f89d4a83dbf1 100644 --- a/lightningd/peer_control.h +++ b/lightningd/peer_control.h @@ -23,9 +23,6 @@ struct peer { /* Database ID of the peer */ u64 dbid; - /* Unique ID of connection (works even if we have multiple to same id) */ - u64 unique_id; - /* ID of peer */ struct pubkey id; diff --git a/wallet/db.c b/wallet/db.c index efb850704403..f7c5f5ffa2b1 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -38,7 +38,6 @@ char *dbmigrations[] = { PRIMARY KEY (shachain_id, pos));", "CREATE TABLE channels (" " id INTEGER," /* chan->id */ - " unique_id INTEGER," " peer_id INTEGER REFERENCES peers(id) ON DELETE CASCADE," " short_channel_id BLOB," " channel_config_local INTEGER," diff --git a/wallet/wallet.c b/wallet/wallet.c index b05fdb176c27..247c3195af8c 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -445,7 +445,6 @@ static bool wallet_stmt2channel(struct wallet *w, sqlite3_stmt *stmt, chan->peer = talz(chan, struct peer); } chan->id = sqlite3_column_int64(stmt, col++); - chan->peer->unique_id = sqlite3_column_int64(stmt, col++); chan->peer->dbid = sqlite3_column_int64(stmt, col++); wallet_peer_load(w, chan->peer->dbid, chan->peer); @@ -549,7 +548,7 @@ static bool wallet_stmt2channel(struct wallet *w, sqlite3_stmt *stmt, col += 2; } - assert(col == 33); + assert(col == 32); chan->peer->channel = chan; @@ -559,7 +558,7 @@ static bool wallet_stmt2channel(struct wallet *w, sqlite3_stmt *stmt, /* List of fields to retrieve from the channels DB table, in the order * that wallet_stmt2channel understands and will parse correctly */ const char *channel_fields = - "id, unique_id, peer_id, short_channel_id, channel_config_local, " + "id, peer_id, short_channel_id, channel_config_local, " "channel_config_remote, state, funder, channel_flags, " "minimum_depth, " "next_index_local, next_index_remote, " @@ -723,7 +722,6 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ /* Now do the real update */ ok &= db_exec(__func__, w->db, "UPDATE channels SET" - " unique_id=%"PRIu64"," " shachain_remote_id=%"PRIu64"," " short_channel_id=%s," " state=%d," @@ -744,7 +742,6 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ " channel_config_local=%"PRIu64"," " last_tx=%s, last_sig=%s" " WHERE id=%"PRIu64, - p->unique_id, p->their_shachain.id, p->scid?tal_fmt(tmpctx,"'%s'", short_channel_id_to_str(tmpctx, p->scid)):"null", p->state, diff --git a/wallet/wallet_tests.c b/wallet/wallet_tests.c index a6c05086dc69..709a0d073fc9 100644 --- a/wallet/wallet_tests.c +++ b/wallet/wallet_tests.c @@ -212,7 +212,6 @@ static bool test_channel_crud(const tal_t *ctx) memset(&p.id, 'A', sizeof(p.id)); c1.peer = &p; p.id = pk; - p.unique_id = 42; p.our_msatoshi = NULL; p.last_tx = NULL; memset(&ci.their_config, 0, sizeof(struct channel_config)); From 79962b3588f29463b71bd652be12d473117235a1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 00:02:46 +1030 Subject: [PATCH 0004/1428] lightningd: return transaction from fundchannel RPC. Lets tests figure out the short channel name, for example. Signed-off-by: Rusty Russell --- lightningd/peer_control.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index f3df54078196..d290847508a9 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -1454,7 +1454,9 @@ static void opening_got_hsm_funding_sig(struct funding_channel *fc, { secp256k1_ecdsa_signature *sigs; struct bitcoin_tx *tx = fc->funding_tx; + u8 *linear; u64 change_satoshi; + struct json_result *response = new_json_result(fc->cmd); size_t i; if (!fromwire_hsmctl_sign_funding_reply(fc, resp, NULL, &sigs)) @@ -1495,7 +1497,12 @@ static void opening_got_hsm_funding_sig(struct funding_channel *fc, /* We could defer until after funding locked, but makes testing * harder. */ tal_del_destructor(fc, fail_fundchannel_command); - command_success(fc->cmd, null_response(fc->cmd)); + + json_object_start(response, NULL); + linear = linearize_tx(response, tx); + json_add_hex(response, "tx", linear, tal_len(linear)); + json_object_end(response); + command_success(fc->cmd, response); /* Start normal channel daemon. */ peer_start_channeld(fc->peer, cs, peer_fd, gossip_fd, NULL, false); @@ -2581,7 +2588,7 @@ static const struct json_command fund_channel_command = { "fundchannel", json_fund_channel, "Fund channel with {id} using {satoshi} satoshis", - "Returns once channel established" + "Returns {tx} once channel established" }; AUTODATA(json_command, &fund_channel_command); From c2dd0cb295a4c52699f03b96a615b0e488164962 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 00:02:46 +1030 Subject: [PATCH 0005/1428] test_lightningd.py: return short channel id from fund_channel helper. Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 6bc418a6b5e3..57f74df6fecc 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -216,6 +216,7 @@ def connect(self): l2.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') return l1,l2 + # Returns the short channel-id: :: def fund_channel(self, l1, l2, amount): addr = l1.rpc.newaddr()['address'] @@ -223,15 +224,22 @@ def fund_channel(self, l1, l2, amount): tx = l1.bitcoin.rpc.getrawtransaction(txid) l1.rpc.addfunds(tx) - l1.rpc.fundchannel(l2.info['id'], amount) + # Generate a block, so we know next tx will be first in block. + l1.bitcoin.rpc.generate(1) + + tx = l1.rpc.fundchannel(l2.info['id'], amount)['tx'] # Technically, this is async to fundchannel. l1.daemon.wait_for_log('sendrawtx exit 0') - l1.bitcoin.rpc.generate(1) - l1.daemon.wait_for_log('-> CHANNELD_NORMAL') l2.daemon.wait_for_log('-> CHANNELD_NORMAL') + # Hacky way to find our output. + for out in bitcoind.rpc.decoderawtransaction(tx)['vout']: + if out['scriptPubKey']['type'] == 'witness_v0_scripthash' and out['value'] == Decimal(amount) / 10**8: + return "{}:1:{}".format(bitcoind.rpc.getblockcount(), out['n']) + raise ValueError("Can't find {} payment in {}".format(amount, tx)) + def pay(self, lsrc, ldst, amt, label=None, async=False): if not label: label = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(20)) From 7e022b522cb9c23c9f2a4d43831d29302c0988d7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 00:03:58 +1030 Subject: [PATCH 0006/1428] gossipd: don't try to handle padding inside fromwire_ipaddr. It makes it impossible to embed an ipaddr in another structure, since we always try to skip over any zeroes, which may swallow a following field. Do the skip specially for the case where we're parsing routing messages: we never use padding for our own internal messages anyway. Signed-off-by: Rusty Russell --- gossipd/routing.c | 4 ++++ gossipd/test/run-find_route-specific.c | 3 +++ gossipd/test/run-find_route.c | 3 +++ lightningd/test/run-find_my_path.c | 6 ------ wire/fromwire.c | 4 ---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/gossipd/routing.c b/gossipd/routing.c index fd583a020b6e..e51ef4992ec9 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -637,6 +637,10 @@ static struct ipaddr *read_addresses(const tal_t *ctx, const u8 *ser) while (cursor && cursor < ser + max) { struct ipaddr ipaddr; + /* Skip any padding */ + while (max && cursor[0] == ADDR_TYPE_PADDING) + fromwire_u8(&cursor, &max); + /* BOLT #7: * * The receiving node SHOULD ignore the first `address diff --git a/gossipd/test/run-find_route-specific.c b/gossipd/test/run-find_route-specific.c index 7af3930cffd9..247376db6d19 100644 --- a/gossipd/test/run-find_route-specific.c +++ b/gossipd/test/run-find_route-specific.c @@ -31,6 +31,9 @@ bool fromwire_ipaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct ip /* Generated stub for fromwire_node_announcement */ bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, size_t *plen UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct pubkey *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED) { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); } +/* Generated stub for fromwire_u8 */ +u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); } /* Generated stub for queue_broadcast */ void queue_broadcast(struct broadcast_state *bstate UNNEEDED, const int type UNNEEDED, diff --git a/gossipd/test/run-find_route.c b/gossipd/test/run-find_route.c index b187fb4124c8..b286e78a9be6 100644 --- a/gossipd/test/run-find_route.c +++ b/gossipd/test/run-find_route.c @@ -24,6 +24,9 @@ bool fromwire_ipaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct ip /* Generated stub for fromwire_node_announcement */ bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, size_t *plen UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct pubkey *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED) { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); } +/* Generated stub for fromwire_u8 */ +u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) +{ fprintf(stderr, "fromwire_u8 called!\n"); abort(); } /* Generated stub for queue_broadcast */ void queue_broadcast(struct broadcast_state *bstate UNNEEDED, const int type UNNEEDED, diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 425a6de97090..472031e80b48 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -40,12 +40,6 @@ struct log_book *new_log_book(const tal_t *ctx UNNEEDED, /* Generated stub for new_topology */ struct chain_topology *new_topology(const tal_t *ctx UNNEEDED, struct log *log UNNEEDED) { fprintf(stderr, "new_topology called!\n"); abort(); } -/* Generated stub for opt_subd_debug */ -char *opt_subd_debug(const char *optarg UNNEEDED, struct lightningd *ld UNNEEDED) -{ fprintf(stderr, "opt_subd_debug called!\n"); abort(); } -/* Generated stub for opt_subd_dev_disconnect */ -char *opt_subd_dev_disconnect(const char *optarg UNNEEDED, struct lightningd *ld UNNEEDED) -{ fprintf(stderr, "opt_subd_dev_disconnect called!\n"); abort(); } /* Generated stub for populate_peer */ void populate_peer(struct lightningd *ld UNNEEDED, struct peer *peer UNNEEDED) { fprintf(stderr, "populate_peer called!\n"); abort(); } diff --git a/wire/fromwire.c b/wire/fromwire.c index 975b56c21754..f9e118c93c85 100644 --- a/wire/fromwire.c +++ b/wire/fromwire.c @@ -198,10 +198,6 @@ bool fromwire_ipaddr(const u8 **cursor, size_t *max, struct ipaddr *addr) fromwire(cursor, max, addr->addr, addr->addrlen); addr->port = fromwire_u16(cursor, max); - /* Skip any post-padding */ - while (*max && (*cursor)[0] == ADDR_TYPE_PADDING) - fromwire_u8(cursor, max); - return *cursor != NULL; } From 7f389439563fa5aba260cc690ebb2908dab25644 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 00:04:44 +1030 Subject: [PATCH 0007/1428] options: show the default network setting in --help. Signed-off-by: Rusty Russell --- lightningd/options.c | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/lightningd/options.c b/lightningd/options.c index b147ed037db1..a2bb07cfb8f9 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -179,14 +179,11 @@ static char *opt_set_network(const char *arg, struct lightningd *ld) return NULL; } -/* FIXME: Uncomment once legacy daemon has been removed */ -/* static void opt_show_network(char buf[OPT_SHOW_LEN], const struct lightningd *ld) { - snprintf(buf, OPT_SHOW_LEN, "%s", ld->chainparams->network_name); + snprintf(buf, OPT_SHOW_LEN, "%s", get_chainparams(ld)->network_name); } -*/ static void config_register_opts(struct lightningd *ld) { @@ -251,10 +248,10 @@ static void config_register_opts(struct lightningd *ld) &ld->config.ipaddr, "Set the IP address (v4 or v6) to announce to the network for incoming connections"); - /* FIXME: Register opt_show_network with the option */ - opt_register_early_arg("--network", opt_set_network, NULL, ld, - "Select the network parameters (bitcoin, testnet, " - "regtest, or litecoin)"); + opt_register_early_arg("--network", opt_set_network, opt_show_network, + ld, + "Select the network parameters (bitcoin, testnet," + " regtest, or litecoin)"); } static char *opt_set_hsm_seed(const char *arg, struct lightningd *ld) From 66a0c55322800925aead845e42a19ea996f31ccd Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 07:02:10 +1030 Subject: [PATCH 0008/1428] test_lightning.py: fix float insanity with values. When is 0.01 != 0.01? When there are floats involved! Jenkins hit an error once, I have no idea why. This works around the following intermittant error: ERROR: test_closing_negotiation_reconnect (__main__.LightningDTests) ---------------------------------------------------------------------- Traceback (most recent call last): File "tests/test_lightningd.py", line 1601, in test_closing_negotiation_reconnect self.fund_channel(l1, l2, 10**6) File "tests/test_lightningd.py", line 241, in fund_channel raise ValueError("Can't find {} payment in {}".format(amount, tx)) ValueError: Can't find 1000000 payment in 02000000000101b0b27be92916faee59f197c263e1ae7b44c0e59acdf2c385c55b5b04670ac026010000001716001401fad90abcd66697e2592164722de4a95ebee165ffffffff02651d0f0000000000160014c2ccab171c2a5be9dab52ec41b825863024c546640420f00000000002200205b8cd3b914cf67cdd8fa6273c930353dd36476734fbd962102c2df53b90880cd02473044022017bd19a0ee85532f67a280c71ed02d1321f08975334bd281527478022265225702202ec448bf9c0890a31a26f0ef4f03d298c8ec16b277faff09a70ddd335df44b6e012103d745445c9362665f22e0d96e9e766f273f3260dea39c8a76bfa05dd2684ddccf00000000 For testing, that tx decodes to: { "txid": "0165e92be762b352b665b76b9872d5189e1b2a8faf4918ab3cca7cd5d4b6a5fa", "hash": "8af9a36c79ee5243468c5cbed1c80f10238fba405f0ad957a0c2cfc46fb632f5", "version": 2, "size": 257, "vsize": 176, "locktime": 0, "vin": [ { "txid": "26c00a67045b5bc585c3f2cd9ae5c0447baee163c297f159eefa1629e97bb2b0", "vout": 1, "scriptSig": { "asm": "001401fad90abcd66697e2592164722de4a95ebee165", "hex": "16001401fad90abcd66697e2592164722de4a95ebee165" }, "txinwitness": [ "3044022017bd19a0ee85532f67a280c71ed02d1321f08975334bd281527478022265225702202ec448bf9c0890a31a26f0ef4f03d298c8ec16b277faff09a70ddd335df44b6e01", "03d745445c9362665f22e0d96e9e766f273f3260dea39c8a76bfa05dd2684ddccf" ], "sequence": 4294967295 } ], "vout": [ { "value": 0.00990565, "n": 0, "scriptPubKey": { "asm": "0 c2ccab171c2a5be9dab52ec41b825863024c5466", "hex": "0014c2ccab171c2a5be9dab52ec41b825863024c5466", "type": "witness_v0_keyhash" } }, { "value": 0.01000000, "n": 1, "scriptPubKey": { "asm": "0 5b8cd3b914cf67cdd8fa6273c930353dd36476734fbd962102c2df53b90880cd", "hex": "00205b8cd3b914cf67cdd8fa6273c930353dd36476734fbd962102c2df53b90880cd", "type": "witness_v0_scripthash" } } ] } Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 57f74df6fecc..ec13486d8cb6 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -236,8 +236,10 @@ def fund_channel(self, l1, l2, amount): # Hacky way to find our output. for out in bitcoind.rpc.decoderawtransaction(tx)['vout']: - if out['scriptPubKey']['type'] == 'witness_v0_scripthash' and out['value'] == Decimal(amount) / 10**8: - return "{}:1:{}".format(bitcoind.rpc.getblockcount(), out['n']) + # Sometimes a float? Sometimes a decimal? WTF Python?! + if out['scriptPubKey']['type'] == 'witness_v0_scripthash': + if out['value'] == Decimal(amount) / 10**8 or out['value'] * 10**8 == amount: + return "{}:1:{}".format(bitcoind.rpc.getblockcount(), out['n']) raise ValueError("Can't find {} payment in {}".format(amount, tx)) def pay(self, lsrc, ldst, amt, label=None, async=False): From ebdecebb1a89f7dcd8daa53c57ec58af32f7c40d Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:41:38 +1030 Subject: [PATCH 0009/1428] channeld: send channel_announce and initial update to master, not gossipd. There is a race we see sometimes under valgrind on Travis which shows gossipd receiving the node_announce from master before it reads the channel_announce from channeld, and thus fails. The simplest solution is to send the channel_announce and channel_update to master as well, so it can ensure it sends them to gossipd in order Signed-off-by: Rusty Russell --- channeld/channel.c | 42 +++++++++++++++++++-------------------- channeld/channel_wire.csv | 8 ++++++-- lightningd/peer_control.c | 18 +++++++++++++---- 3 files changed, 40 insertions(+), 28 deletions(-) diff --git a/channeld/channel.c b/channeld/channel.c index 463aeeddbad7..11c01eb9dd99 100644 --- a/channeld/channel.c +++ b/channeld/channel.c @@ -231,9 +231,10 @@ static void send_announcement_signatures(struct peer *peer) tal_free(tmpctx); } -static void send_channel_update(struct peer *peer, bool disabled) +static u8 *create_channel_update(const tal_t *ctx, + struct peer *peer, bool disabled) { - tal_t *tmpctx = tal_tmpctx(peer); + tal_t *tmpctx = tal_tmpctx(ctx); u32 timestamp = time_now().ts.tv_sec; u16 flags; u8 *cupdate, *msg; @@ -257,14 +258,12 @@ static void send_channel_update(struct peer *peer, bool disabled) strerror(errno)); msg = wire_sync_read(tmpctx, HSM_FD); - if (!msg || !fromwire_hsm_cupdate_sig_reply(tmpctx, msg, NULL, &cupdate)) + if (!msg || !fromwire_hsm_cupdate_sig_reply(ctx, msg, NULL, &cupdate)) status_failed(STATUS_FAIL_HSM_IO, "Reading cupdate_sig_req: %s", strerror(errno)); - - daemon_conn_send(&peer->gossip_client, cupdate); - msg_enqueue(&peer->peer_out, cupdate); tal_free(tmpctx); + return cupdate; } /* Tentatively create a channel_announcement, possibly with invalid @@ -297,15 +296,6 @@ static u8 *create_channel_announcement(const tal_t *ctx, struct peer *peer) return cannounce; } -static void send_channel_announcement(struct peer *peer) -{ - u8 *msg = create_channel_announcement(peer, peer); - /* Makes a copy */ - msg_enqueue(&peer->peer_out, msg); - /* Takes ownership */ - daemon_conn_send(&peer->gossip_client, take(msg)); -} - static struct io_plan *peer_out(struct io_conn *conn, struct peer *peer) { const u8 *out = msg_dequeue(&peer->peer_out); @@ -362,11 +352,16 @@ static struct io_plan *handle_peer_funding_locked(struct io_conn *conn, static void announce_channel(struct peer *peer) { - send_channel_announcement(peer); - send_channel_update(peer, false); - /* Tell the master that we just announced the channel, - * so it may announce the node */ - wire_sync_write(MASTER_FD, take(towire_channel_announced(peer))); + u8 *cannounce, *cupdate; + + cannounce = create_channel_announcement(peer, peer); + cupdate = create_channel_update(cannounce, peer, false); + + /* Tell the master that we to announce channel (it does node) */ + wire_sync_write(MASTER_FD, take(towire_channel_announce(peer, + cannounce, + cupdate))); + tal_free(cannounce); } static struct io_plan *handle_peer_announcement_signatures(struct io_conn *conn, @@ -1406,7 +1401,10 @@ static void peer_conn_broken(struct io_conn *conn, struct peer *peer) { /* If we have signatures, send an update to say we're disabled. */ if (peer->have_sigs[LOCAL] && peer->have_sigs[REMOTE]) { - send_channel_update(peer, true); + u8 *cupdate = create_channel_update(conn, peer, true); + + daemon_conn_send(&peer->gossip_client, cupdate); + msg_enqueue(&peer->peer_out, take(cupdate)); /* Make sure gossipd actually gets this message before dying */ daemon_conn_sync_flush(&peer->gossip_client); @@ -1954,7 +1952,7 @@ static void req_in(struct peer *peer, const u8 *msg) case WIRE_CHANNEL_INIT: case WIRE_CHANNEL_OFFER_HTLC_REPLY: case WIRE_CHANNEL_PING_REPLY: - case WIRE_CHANNEL_ANNOUNCED: + case WIRE_CHANNEL_ANNOUNCE: case WIRE_CHANNEL_SENDING_COMMITSIG: case WIRE_CHANNEL_GOT_COMMITSIG: case WIRE_CHANNEL_GOT_REVOKE: diff --git a/channeld/channel_wire.csv b/channeld/channel_wire.csv index 5ad595fde061..8c761e852c9f 100644 --- a/channeld/channel_wire.csv +++ b/channeld/channel_wire.csv @@ -102,8 +102,12 @@ channel_ping,,len,u16 channel_ping_reply,1111 channel_ping_reply,,totlen,u16 -# Channeld tells the master that the channel has been announced -channel_announced,1012 +# Channeld tells the master to announce the channel (with first update) +channel_announce,1012 +channel_announce,,announce_len,u16 +channel_announce,,announce,announce_len*u8 +channel_announce,,update_len,u16 +channel_announce,,update,update_len*u8 # When we receive funding_locked. channel_got_funding_locked,1019 diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index d290847508a9..0284fa4a7867 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -1546,15 +1546,16 @@ static u8 *create_node_announcement(const tal_t *ctx, struct lightningd *ld, * an update, so we can now start sending a node_announcement. The * first step is to build the provisional announcement and ask the HSM * to sign it. */ -static void peer_channel_announced(struct peer *peer, const u8 *msg) +static void peer_channel_announce(struct peer *peer, const u8 *msg) { struct lightningd *ld = peer->ld; tal_t *tmpctx = tal_tmpctx(peer); secp256k1_ecdsa_signature sig; + u8 *cannounce, *cupdate; u8 *announcement, *wrappedmsg; u32 timestamp = time_now().ts.tv_sec; - if (!fromwire_channel_announced(msg, NULL)) { + if (!fromwire_channel_announce(msg, msg, NULL, &cannounce, &cupdate)) { peer_internal_error(peer, "bad fromwire_channel_announced %s", tal_hex(peer, msg)); return; @@ -1574,6 +1575,15 @@ static void peer_channel_announced(struct peer *peer, const u8 *msg) * from the HSM, create the real announcement and forward it to * gossipd so it can take care of forwarding it. */ announcement = create_node_announcement(tmpctx, ld, &sig, timestamp); + + /* We have to send channel_announce before channel_update and + * node_announcement */ + wrappedmsg = towire_gossip_forwarded_msg(tmpctx, cannounce); + subd_send_msg(ld->gossip, take(wrappedmsg)); + + wrappedmsg = towire_gossip_forwarded_msg(tmpctx, cupdate); + subd_send_msg(ld->gossip, take(wrappedmsg)); + wrappedmsg = towire_gossip_forwarded_msg(tmpctx, announcement); subd_send_msg(ld->gossip, take(wrappedmsg)); tal_free(tmpctx); @@ -1956,8 +1966,8 @@ static unsigned channel_msg(struct subd *sd, const u8 *msg, const int *fds) case WIRE_CHANNEL_GOT_REVOKE: peer_got_revoke(sd->peer, msg); break; - case WIRE_CHANNEL_ANNOUNCED: - peer_channel_announced(sd->peer, msg); + case WIRE_CHANNEL_ANNOUNCE: + peer_channel_announce(sd->peer, msg); break; case WIRE_CHANNEL_GOT_FUNDING_LOCKED: peer_got_funding_locked(sd->peer, msg); From b6a2b8c58be9730261f1cfca143185b056559ebd Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 15:35:28 +1030 Subject: [PATCH 0010/1428] Add --rgb and --alias options. And derive random ones from nodeid if they don't choose. Signed-off-by: Rusty Russell --- lightningd/lightningd.c | 10 +++- lightningd/lightningd.h | 4 ++ lightningd/options.c | 75 ++++++++++++++++++++++++++++++ lightningd/options.h | 3 ++ lightningd/peer_control.c | 6 +-- lightningd/test/run-find_my_path.c | 3 ++ tests/utils.py | 5 ++ 7 files changed, 100 insertions(+), 6 deletions(-) diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 2921fe996b30..97a7cd62de9a 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -78,7 +78,8 @@ static struct lightningd *new_lightningd(const tal_t *ctx, ld->dev_hsm_seed = NULL; ld->log_book = log_book; ld->log = new_log(log_book, log_book, "lightningd(%u):", (int)getpid()); - + ld->alias = NULL; + ld->rgb = NULL; list_head_init(&ld->pay_commands); list_head_init(&ld->connects); ld->portnum = DEFAULT_PORT; @@ -251,6 +252,9 @@ int main(int argc, char *argv[]) /* Set up HSM. */ hsm_init(ld, newdir); + /* Now we know our ID, we can set our color/alias if not already. */ + setup_color_and_alias(ld); + /* Initialize block topology. */ setup_topology(ld->topology, &ld->timers, @@ -288,7 +292,9 @@ int main(int argc, char *argv[]) setup_jsonrpc(ld, ld->rpc_filename); /* Mark ourselves live. */ - log_info(ld->log, "Hello world from %s!", version()); + log_info(ld->log, "Hello world from %s aka %s #%s (version %s)!", + type_to_string(ltmp, struct pubkey, &ld->id), + ld->alias, tal_hex(ltmp, ld->rgb), version()); #if 0 /* Load peers from database. */ diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index e0c6165d5199..9303e982b8de 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -87,6 +87,10 @@ struct lightningd { /* This is us. */ struct pubkey id; + /* My name is... my favorite color is... */ + char *alias; /* At least 32 bytes (zero-filled) */ + u8 *rgb; /* tal_len() == 3. */ + /* Any pending timers. */ struct timers timers; diff --git a/lightningd/options.c b/lightningd/options.c index a2bb07cfb8f9..9f2e39696f05 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -185,6 +186,36 @@ static void opt_show_network(char buf[OPT_SHOW_LEN], snprintf(buf, OPT_SHOW_LEN, "%s", get_chainparams(ld)->network_name); } +static char *opt_set_rgb(const char *arg, struct lightningd *ld) +{ + ld->rgb = tal_free(ld->rgb); + /* BOLT #7: + * + * the first byte of `rgb` is the red value, the second byte is the + * green value and the last byte is the blue value */ + ld->rgb = tal_hexdata(ld, arg, strlen(arg)); + if (!ld->rgb || tal_len(ld->rgb) != 3) + return tal_fmt(NULL, "rgb '%s' is not six hex digits", arg); + return NULL; +} + +static char *opt_set_alias(const char *arg, struct lightningd *ld) +{ + ld->alias = tal_free(ld->alias); + /* BOLT #7: + * + * * [`32`:`alias`] + *... + * It MUST set `alias` to a valid UTF-8 string, with any `alias` bytes + * following equal to zero. + */ + if (strlen(arg) > 32) + return tal_fmt(NULL, "Alias '%s' is over 32 characters", arg); + ld->alias = tal_arrz(ld, char, 33); + strncpy(ld->alias, arg, 32); + return NULL; +} + static void config_register_opts(struct lightningd *ld) { opt_register_arg("--locktime-blocks", opt_set_u32, opt_show_u32, @@ -520,6 +551,10 @@ void register_opts(struct lightningd *ld) opt_register_arg("--bitcoin-datadir", opt_set_charp, NULL, &ld->topology->bitcoind->datadir, "-datadir arg for bitcoin-cli"); + opt_register_arg("--rgb", opt_set_rgb, NULL, ld, + "RRGGBB hex color for node"); + opt_register_arg("--alias", opt_set_alias, NULL, ld, + "Up to 32-byte alias for node"); opt_register_logging(ld->log); opt_register_version(); @@ -528,6 +563,46 @@ void register_opts(struct lightningd *ld) dev_register_opts(ld); } +/* Names stolen from https://github.com/ternus/nsaproductgenerator/blob/master/nsa.js */ +static const char *codename_adjective[] += { "LOUD", "RED", "BLUE", "GREEN", "YELLOW", "IRATE", "ANGRY", "PEEVED", + "HAPPY", "SLIMY", "SLEEPY", "JUNIOR", "SLICKER", "UNITED", "SOMBER", + "BIZARRE", "ODD", "WEIRD", "WRONG", "LATENT", "CHILLY", "STRANGE", "LOUD", + "SILENT", "HOPPING", "ORANGE", "VIOLET", "VIOLENT", "LIGHTNING" }; + +static const char *codename_noun[] += { "WHISPER", "FELONY", "MOON", "SUCKER", "PENGUIN", "WAFFLE", "MAESTRO", + "NIGHT", "TRINITY", "DEITY", "MONKEY", "ARK", "SQUIRREL", "IRON", "BOUNCE", + "FARM", "CHEF", "TROUGH", "NET", "TRAWL", "GLEE", "WATER", "SPORK", "PLOW", + "FEED", "SOUFFLE", "ROUTE", "BAGEL", "MONTANA", "ANALYST", "AUTO", "WATCH", + "PHOTO", "YARD", "SOURCE", "MONKEY", "SEAGULL", "TOLL", "SPAWN", "GOPHER", + "CHIPMUNK", "SET", "CALENDAR", "ARTIST", "CHASER", "SCAN", "TOTE", "BEAM", + "ENTOURAGE", "GENESIS", "WALK", "SPATULA", "RAGE", "FIRE", "MASTER" }; + +void setup_color_and_alias(struct lightningd *ld) +{ + u8 der[PUBKEY_DER_LEN]; + pubkey_to_der(der, &ld->id); + + if (!ld->rgb) + /* You can't get much red by default */ + ld->rgb = tal_dup_arr(ld, u8, der, 3, 0); + + if (!ld->alias) { + u64 adjective, noun; + + memcpy(&adjective, der+3, sizeof(adjective)); + memcpy(&noun, der+3+sizeof(adjective), sizeof(noun)); + noun %= ARRAY_SIZE(codename_noun); + adjective %= ARRAY_SIZE(codename_adjective); + ld->alias = tal_arrz(ld, char, 33); + assert(strlen(codename_adjective[adjective]) + + strlen(codename_noun[noun]) < 33); + strcpy(ld->alias, codename_adjective[adjective]); + strcat(ld->alias, codename_noun[noun]); + } +} + bool handle_opts(struct lightningd *ld, int argc, char *argv[]) { bool newdir = false; diff --git a/lightningd/options.h b/lightningd/options.h index f25254d8a820..dab0e38188ca 100644 --- a/lightningd/options.h +++ b/lightningd/options.h @@ -14,4 +14,7 @@ void register_opts(struct lightningd *ld); bool handle_opts(struct lightningd *ld, int argc, char *argv[]); bool parse_ipaddr(const char *arg, struct ipaddr *addr); + +/* Derive default color and alias from the pubkey. */ +void setup_color_and_alias(struct lightningd *ld); #endif /* LIGHTNING_LIGHTNINGD_OPTIONS_H */ diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 0284fa4a7867..45fa94eb43fd 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -1523,8 +1523,6 @@ static u8 *create_node_announcement(const tal_t *ctx, struct lightningd *ld, secp256k1_ecdsa_signature *sig, u32 timestamp) { - u8 rgb[3] = {0x77, 0x88, 0x99}; - u8 alias[32]; u8 *features = NULL; u8 *addresses = tal_arr(ctx, u8, 0); u8 *announcement; @@ -1535,10 +1533,10 @@ static u8 *create_node_announcement(const tal_t *ctx, struct lightningd *ld, if (ld->config.ipaddr.type != ADDR_TYPE_PADDING) { towire_ipaddr(&addresses, &ld->config.ipaddr); } - memset(alias, 0, sizeof(alias)); announcement = towire_node_announcement(ctx, sig, features, timestamp, - &ld->id, rgb, alias, addresses); + &ld->id, ld->rgb, (u8 *)ld->alias, + addresses); return announcement; } diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 472031e80b48..364091dbb194 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -46,6 +46,9 @@ void populate_peer(struct lightningd *ld UNNEEDED, struct peer *peer UNNEEDED) /* Generated stub for register_opts */ void register_opts(struct lightningd *ld UNNEEDED) { fprintf(stderr, "register_opts called!\n"); abort(); } +/* Generated stub for setup_color_and_alias */ +void setup_color_and_alias(struct lightningd *ld UNNEEDED) +{ fprintf(stderr, "setup_color_and_alias called!\n"); abort(); } /* Generated stub for setup_jsonrpc */ void setup_jsonrpc(struct lightningd *ld UNNEEDED, const char *rpc_filename UNNEEDED) { fprintf(stderr, "setup_jsonrpc called!\n"); abort(); } diff --git a/tests/utils.py b/tests/utils.py index 891a77ac6374..bd4ec1914a52 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -219,6 +219,11 @@ def start(self): logging.info("BitcoinD started") +# lightning-1 => 0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518 aka JUNIORBEAM #0266e4 +# lightning-2 => 022d223620a359a47ff7f7ac447c85c46c923da53389221a0054c11c1e3ca31d59 aka SILENTARTIST #022d22 +# lightning-3 => 035d2b1192dfba134e10e540875d366ebc8bc353d5aa766b80c090b39c3a5d885d aka HOPPINGFIRE #035d2b +# lightning-4 => 0382ce59ebf18be7d84677c2e35f23294b9992ceca95491fcf8a56c6cb2d9de199 aka JUNIORFELONY #0382ce +# lightning-5 => 032cf15d1ad9c4a08d26eab1918f732d8ef8fdc6abb9640bf3db174372c491304e aka SOMBERFIRE #032cf1 class LightningD(TailableProc): def __init__(self, lightning_dir, bitcoin_dir, port=9735): From 7d62de86324faece9420dd76ad7671658970ae96 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 25 Oct 2017 19:47:08 +1030 Subject: [PATCH 0011/1428] lightningd: fix typo in fatal error. Signed-off-by: Rusty Russell --- lightningd/hsm_control.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightningd/hsm_control.c b/lightningd/hsm_control.c index 88b585d13e9c..99b50eab94c6 100644 --- a/lightningd/hsm_control.c +++ b/lightningd/hsm_control.c @@ -19,7 +19,7 @@ u8 *hsm_sync_read(const tal_t *ctx, struct lightningd *ld) for (;;) { u8 *msg = wire_sync_read(ctx, ld->hsm_fd); if (!msg) - fatal("Could not write from HSM: %s", strerror(errno)); + fatal("Could not read from HSM: %s", strerror(errno)); if (fromwire_peektype(msg) != STATUS_TRACE) return msg; From 9e869e641a2e997a15b51ae9f3a29821217d34ac Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 25 Oct 2017 19:47:14 +1030 Subject: [PATCH 0012/1428] take: turn labels on. Gives us meaningful errors when there's a take() leak. Signed-off-by: Rusty Russell --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 287080500615..c1987dd7a33f 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,7 @@ ALL_PROGRAMS = CWARNFLAGS := -Werror -Wall -Wundef -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes -Wold-style-definition CDEBUGFLAGS := -std=gnu11 -g -fstack-protector -CFLAGS = $(CWARNFLAGS) $(CDEBUGFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . $(FEATURES) $(COVFLAGS) -DSHACHAIN_BITS=48 +CFLAGS = $(CWARNFLAGS) $(CDEBUGFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . $(FEATURES) $(COVFLAGS) -DSHACHAIN_BITS=48 -DCCAN_TAKE_DEBUG=1 LDLIBS = -lgmp -lsqlite3 $(COVFLAGS) From a2dc71b0a127826246610949a6a3a471153d89ec Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 25 Oct 2017 19:47:20 +1030 Subject: [PATCH 0013/1428] lightningd: close a take() leak. test_routing_gossip (__main__.LightningDTests) ... lightningd: Outstanding taken pointers: lightningd/peer_control.c:2352:towire_errorfmt(ld, ((void *)0), "Can't resolve your address") This caused by the other end closing due to the next bug. Signed-off-by: Rusty Russell --- lightningd/peer_control.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 45fa94eb43fd..2540d55eeec2 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -2348,8 +2348,7 @@ static void peer_accept_channel(struct lightningd *ld, /* FIXME: Only happens due to netaddr fail. */ if (!peer) { - errmsg = take(towire_errorfmt(ld, NULL, - "Can't resolve your address")); + errmsg = towire_errorfmt(ld, NULL, "Can't resolve your address"); goto peer_to_gossipd; } @@ -2411,6 +2410,7 @@ static void peer_accept_channel(struct lightningd *ld, subd_send_msg(ld->gossip, take(msg)); subd_send_fd(ld->gossip, peer_fd); close(gossip_fd); + tal_free(errmsg); return; } From a8f033f6ae9980793243c2d43275cff99754fc13 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 25 Oct 2017 19:47:24 +1030 Subject: [PATCH 0014/1428] ccan: update to get new ccan/io changes. Signed-off-by: Rusty Russell --- ccan/README | 2 +- ccan/ccan/io/backend.h | 5 +- ccan/ccan/io/io.c | 23 +++- ccan/ccan/io/io.h | 28 +++++ ccan/ccan/io/poll.c | 6 +- ccan/ccan/io/test/run-43-io_plan_in_started.c | 75 +++++++++++++ .../ccan/io/test/run-44-io_plan_out_started.c | 100 ++++++++++++++++++ 7 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 ccan/ccan/io/test/run-43-io_plan_in_started.c create mode 100644 ccan/ccan/io/test/run-44-io_plan_out_started.c diff --git a/ccan/README b/ccan/README index fb5efa185f1c..139fec08647c 100644 --- a/ccan/README +++ b/ccan/README @@ -1,3 +1,3 @@ CCAN imported from http://ccodearchive.net. -CCAN version: init-2397-g261b1086 +CCAN version: init-2398-g7082f7d0 diff --git a/ccan/ccan/io/backend.h b/ccan/ccan/io/backend.h index c8ceb4e86439..2ee5a830e123 100644 --- a/ccan/ccan/io/backend.h +++ b/ccan/ccan/io/backend.h @@ -25,8 +25,9 @@ struct io_listener { enum io_plan_status { /* As before calling next function. */ IO_UNSET, - /* Normal. */ - IO_POLLING, + /* Normal, but haven't started yet. */ + IO_POLLING_NOTSTARTED, + IO_POLLING_STARTED, /* Waiting for io_wake */ IO_WAITING, /* Always do this. */ diff --git a/ccan/ccan/io/io.c b/ccan/ccan/io/io.c index 99f0f7c90fd8..c0dd9b838e84 100644 --- a/ccan/ccan/io/io.c +++ b/ccan/ccan/io/io.c @@ -131,7 +131,7 @@ struct io_plan_arg *io_plan_arg(struct io_conn *conn, enum io_direction dir) { assert(conn->plan[dir].status == IO_UNSET); - conn->plan[dir].status = IO_POLLING; + conn->plan[dir].status = IO_POLLING_NOTSTARTED; return &conn->plan[dir].arg; } @@ -368,7 +368,8 @@ static bool do_plan(struct io_conn *conn, struct io_plan *plan, bool idle_on_epipe) { /* We shouldn't have polled for this event if this wasn't true! */ - assert(plan->status == IO_POLLING); + assert(plan->status == IO_POLLING_NOTSTARTED + || plan->status == IO_POLLING_STARTED); switch (plan->io(conn->fd.fd, &plan->arg)) { case -1: @@ -380,6 +381,7 @@ static bool do_plan(struct io_conn *conn, struct io_plan *plan, io_close(conn); return false; case 0: + plan->status = IO_POLLING_STARTED; return true; case 1: return next_plan(conn, plan); @@ -399,7 +401,8 @@ void io_ready(struct io_conn *conn, int pollflags) /* If we're writing to a closed pipe, we need to wait for * read to fail if we're duplex: we want to drain it! */ do_plan(conn, &conn->plan[IO_OUT], - conn->plan[IO_IN].status == IO_POLLING); + conn->plan[IO_IN].status == IO_POLLING_NOTSTARTED + || conn->plan[IO_IN].status == IO_POLLING_STARTED); } void io_do_always(struct io_conn *conn) @@ -509,13 +512,24 @@ struct io_plan *io_set_plan(struct io_conn *conn, enum io_direction dir, return plan; } +bool io_plan_in_started(const struct io_conn *conn) +{ + return conn->plan[IO_IN].status == IO_POLLING_STARTED; +} + +bool io_plan_out_started(const struct io_conn *conn) +{ + return conn->plan[IO_OUT].status == IO_POLLING_STARTED; +} + bool io_flush_sync(struct io_conn *conn) { struct io_plan *plan = &conn->plan[IO_OUT]; bool ok; /* Not writing? Nothing to do. */ - if (plan->status != IO_POLLING) + if (plan->status != IO_POLLING_STARTED + && plan->status != IO_POLLING_NOTSTARTED) return true; /* Synchronous please. */ @@ -528,6 +542,7 @@ bool io_flush_sync(struct io_conn *conn) break; /* Incomplete, try again. */ case 0: + plan->status = IO_POLLING_STARTED; goto again; case 1: ok = true; diff --git a/ccan/ccan/io/io.h b/ccan/ccan/io/io.h index 1e4e80e7db77..c9ab228c7a04 100644 --- a/ccan/ccan/io/io.h +++ b/ccan/ccan/io/io.h @@ -674,6 +674,34 @@ void *io_loop(struct timers *timers, struct timer **expired); */ int io_conn_fd(const struct io_conn *conn); +/** + * io_plan_in_started - is this conn doing input I/O now? + * @conn: the conn. + * + * This returns true if input I/O has been performed on the conn but + * @next hasn't been called yet. For example, io_read() may have done + * a partial read. + * + * This can be useful if we want to terminate a connection only after + * reading a whole packet: if this returns true, we would wait until + * @next is called. + */ +bool io_plan_in_started(const struct io_conn *conn); + +/** + * io_plan_out_started - is this conn doing output I/O now? + * @conn: the conn. + * + * This returns true if output I/O has been performed on the conn but + * @next hasn't been called yet. For example, io_write() may have done + * a partial write. + * + * This can be useful if we want to terminate a connection only after + * writing a whole packet: if this returns true, we would wait until + * @next is called. + */ +bool io_plan_out_started(const struct io_conn *conn); + /** * io_flush_sync - (synchronously) complete any outstanding output. * @conn: the connection. diff --git a/ccan/ccan/io/poll.c b/ccan/ccan/io/poll.c index a4e83ed761e7..3354abe01a2b 100644 --- a/ccan/ccan/io/poll.c +++ b/ccan/ccan/io/poll.c @@ -130,9 +130,11 @@ void backend_new_plan(struct io_conn *conn) num_waiting--; pfd->events = 0; - if (conn->plan[IO_IN].status == IO_POLLING) + if (conn->plan[IO_IN].status == IO_POLLING_NOTSTARTED + || conn->plan[IO_IN].status == IO_POLLING_STARTED) pfd->events |= POLLIN; - if (conn->plan[IO_OUT].status == IO_POLLING) + if (conn->plan[IO_OUT].status == IO_POLLING_NOTSTARTED + || conn->plan[IO_OUT].status == IO_POLLING_STARTED) pfd->events |= POLLOUT; if (pfd->events) { diff --git a/ccan/ccan/io/test/run-43-io_plan_in_started.c b/ccan/ccan/io/test/run-43-io_plan_in_started.c new file mode 100644 index 000000000000..f63f87790f33 --- /dev/null +++ b/ccan/ccan/io/test/run-43-io_plan_in_started.c @@ -0,0 +1,75 @@ +#include +/* Include the C files directly. */ +#include +#include +#include + +static struct io_conn *in_conn; + +static struct io_plan *in_conn_done(struct io_conn *conn, void *unused) +{ + ok1(!io_plan_in_started(conn)); + return io_close(conn); +} + +static struct io_plan *init_in_conn(struct io_conn *conn, char *buf) +{ + ok1(!io_plan_in_started(conn)); + return io_read(conn, buf, 2, in_conn_done, NULL); +} + +static int do_nothing(int fd, struct io_plan_arg *arg) +{ + return 1; +} + +static struct io_plan *dummy_write(struct io_conn *conn, + struct io_plan *(*next) + (struct io_conn *, void *), + void *next_arg) +{ + io_plan_arg(conn, IO_OUT); + return io_set_plan(conn, IO_OUT, do_nothing, next, next_arg); +} + +static struct io_plan *out_post_write(struct io_conn *conn, void *unused) +{ + /* It might not have started yet: try again. */ + if (!io_plan_in_started(in_conn)) + return dummy_write(conn, out_post_write, NULL); + ok1(io_plan_in_started(in_conn)); + + /* Final write, then close */ + return io_write(conn, "2", 1, io_close_cb, NULL); +} + +static struct io_plan *init_out_conn(struct io_conn *conn, void *unused) +{ + ok1(!io_plan_in_started(in_conn)); + return io_write(conn, "1", 1, out_post_write, NULL); +} + +int main(void) +{ + int fds[2]; + const tal_t *ctx = tal(NULL, char); + char *buf = tal_arr(ctx, char, 3); + + /* This is how many tests you plan to run */ + plan_tests(5); + + if (pipe(fds) != 0) + abort(); + + buf[2] = '\0'; + + in_conn = io_new_conn(ctx, fds[0], init_in_conn, buf); + io_new_conn(ctx, fds[1], init_out_conn, NULL); + + io_loop(NULL, NULL); + ok1(strcmp(buf, "12") == 0); + tal_free(ctx); + + /* This exits depending on whether all tests passed */ + return exit_status(); +} diff --git a/ccan/ccan/io/test/run-44-io_plan_out_started.c b/ccan/ccan/io/test/run-44-io_plan_out_started.c new file mode 100644 index 000000000000..71a63dd5c9a2 --- /dev/null +++ b/ccan/ccan/io/test/run-44-io_plan_out_started.c @@ -0,0 +1,100 @@ +#include +/* Include the C files directly. */ +#include +#include +#include + +static struct io_conn *out_conn; + +/* Write one byte at a time. */ +static int do_slow_write(int fd, struct io_plan_arg *arg) +{ + ssize_t ret = write(fd, arg->u1.cp, 1); + if (ret < 0) + return -1; + + arg->u1.cp += ret; + arg->u2.s -= ret; + return arg->u2.s == 0; +} + +static struct io_plan *io_slow_write(struct io_conn *conn, + const void *data, size_t len, + struct io_plan *(*next)(struct io_conn *, + void *), + void *next_arg) +{ + struct io_plan_arg *arg = io_plan_arg(conn, IO_OUT); + + arg->u1.const_vp = data; + arg->u2.s = len; + + return io_set_plan(conn, IO_OUT, do_slow_write, next, next_arg); +} + +static struct io_plan *out_conn_done(struct io_conn *conn, void *unused) +{ + ok1(!io_plan_out_started(conn)); + return io_close(conn); +} + +static struct io_plan *init_out_conn(struct io_conn *conn, void *unused) +{ + ok1(!io_plan_out_started(conn)); + return io_slow_write(conn, "12", 2, out_conn_done, NULL); +} + +static int do_nothing(int fd, struct io_plan_arg *arg) +{ + return 1; +} + +static struct io_plan *dummy_read(struct io_conn *conn, + struct io_plan *(*next) + (struct io_conn *, void *), + void *next_arg) +{ + io_plan_arg(conn, IO_IN); + return io_set_plan(conn, IO_IN, do_nothing, next, next_arg); +} + +static struct io_plan *in_post_read(struct io_conn *conn, void *buf) +{ + /* It might not have started yet: try again. */ + if (!io_plan_out_started(out_conn)) + return dummy_read(conn, in_post_read, NULL); + ok1(io_plan_out_started(out_conn)); + + /* Final read, then close */ + return io_read(conn, (char *)buf+1, 1, io_close_cb, NULL); +} + +static struct io_plan *init_in_conn(struct io_conn *conn, char *buf) +{ + return io_read(conn, buf, 1, in_post_read, buf); +} + +int main(void) +{ + int fds[2]; + const tal_t *ctx = tal(NULL, char); + char *buf = tal_arr(ctx, char, 3); + + /* This is how many tests you plan to run */ + plan_tests(4); + + if (pipe(fds) != 0) + abort(); + + buf[2] = '\0'; + + io_new_conn(ctx, fds[0], init_in_conn, buf); + out_conn = io_new_conn(ctx, fds[1], init_out_conn, NULL); + + io_loop(NULL, NULL); + ok1(strcmp(buf, "12") == 0); + tal_free(ctx); + + /* This exits depending on whether all tests passed */ + return exit_status(); +} From 4e141859613532cf345c846c099f24030a1ab338 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 25 Oct 2017 19:47:48 +1030 Subject: [PATCH 0015/1428] cryptomsg: add helpers to determine if we're partway through msg read/write. For message read, we do it as header then body, so we can have io_plan_in_started(conn) false, but we're between header and body. Signed-off-by: Rusty Russell --- common/cryptomsg.c | 20 ++++++++++++++++++++ common/cryptomsg.h | 9 +++++++++ 2 files changed, 29 insertions(+) diff --git a/common/cryptomsg.c b/common/cryptomsg.c index f3f18e717fc7..c29254e4e884 100644 --- a/common/cryptomsg.c +++ b/common/cryptomsg.c @@ -132,6 +132,8 @@ static struct io_plan *peer_decrypt_body(struct io_conn *conn, struct io_plan *plan; u8 *in, *decrypted; + pcs->reading_body = false; + decrypted = cryptomsg_decrypt_body(pcs->in, &pcs->cs, pcs->in); if (!decrypted) return io_close(conn); @@ -198,6 +200,8 @@ static struct io_plan *peer_decrypt_header(struct io_conn *conn, tal_free(pcs->in); + pcs->reading_body = true; + /* BOLT #8: * * * Read _exactly_ `l+16` bytes from the network buffer, let @@ -224,6 +228,7 @@ struct io_plan *peer_read_message(struct io_conn *conn, * * * Read _exactly_ `18-bytes` from the network buffer. */ + pcs->reading_body = false; pcs->in = tal_arr(conn, u8, 18); pcs->next_in = next; return io_read(conn, pcs->in, 18, peer_decrypt_header, pcs); @@ -366,8 +371,23 @@ struct io_plan *peer_write_message(struct io_conn *conn, return io_write(conn, pcs->out, tal_count(pcs->out), post, pcs); } +/* We write in one op, so it's all or nothing. */ +bool peer_out_started(const struct io_conn *conn, + const struct peer_crypto_state *cs) +{ + return io_plan_out_started(conn); +} + +/* We read in two parts, so we might have started body. */ +bool peer_in_started(const struct io_conn *conn, + const struct peer_crypto_state *cs) +{ + return io_plan_in_started(conn) || cs->reading_body; +} + void init_peer_crypto_state(struct peer *peer, struct peer_crypto_state *pcs) { pcs->peer = peer; pcs->out = pcs->in = NULL; + pcs->reading_body = false; } diff --git a/common/cryptomsg.h b/common/cryptomsg.h index 7d2de5d306d6..d9f1d2ca9faf 100644 --- a/common/cryptomsg.h +++ b/common/cryptomsg.h @@ -14,6 +14,9 @@ struct peer_crypto_state { /* Peer who owns us: peer->crypto_state == this */ struct peer *peer; + /* Where we are up to in reading (we do in two parts). */ + bool reading_body; + /* Output and input buffers. */ u8 *out, *in; struct io_plan *(*next_in)(struct io_conn *, struct peer *, u8 *); @@ -30,6 +33,12 @@ struct io_plan *peer_read_message(struct io_conn *conn, struct peer *, u8 *msg)); +/* Have we already started writing/reading a message? */ +bool peer_out_started(const struct io_conn *conn, + const struct peer_crypto_state *cs); +bool peer_in_started(const struct io_conn *conn, + const struct peer_crypto_state *cs); + /* Sends message: frees if taken(msg). */ struct io_plan *peer_write_message(struct io_conn *conn, struct peer_crypto_state *cs, From 954a3990fa0399522eb75de7adc58a40de6ebdd8 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 25 Oct 2017 19:48:05 +1030 Subject: [PATCH 0016/1428] gossipd: don't send a peer to master with half-written or half-read packet. In this case, it was a gossip message half-sent, when we asked the peer to be released. Fix the problem in general by making send_peer_with_fds() wait until after the next packet. test_routing_gossip/lightning-4/log: b'lightning_openingd(8738): TRACE: First per_commit_point = 02e2ff759ed70c71f154695eade1983664a72546ebc552861f844bff5ea5b933bf' b'lightning_openingd(8738): TRACE: Failed hdr decrypt with rn=11' b'lightning_openingd(8738): STATUS_FAIL_PEER_IO: Reading accept_channel: Success' test_routing_gossip/lightning-5/log: b'lightning_gossipd(8461): UPDATE WIRE_GOSSIP_PEER_NONGOSSIP' b'lightning_gossipd(8461): UPDATE WIRE_GOSSIP_PEER_NONGOSSIP' b'lightningd(8308): Failed to get netaddr for outgoing: Transport endpoint is not connected' The problem occurs here on release, but could be on any place where we hand a peer over when using ccan/io. Note the other case (channel.c). Signed-off-by: Rusty Russell --- channeld/channel.c | 5 +++ gossipd/gossip.c | 106 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 86 insertions(+), 25 deletions(-) diff --git a/channeld/channel.c b/channeld/channel.c index 11c01eb9dd99..b55e086158c1 100644 --- a/channeld/channel.c +++ b/channeld/channel.c @@ -2107,6 +2107,11 @@ static void gossip_gone(struct io_conn *unused, struct daemon_conn *dc) "Gossip connection closed"); } +/* FIXME: This doesn't cover partly read packets! We could be halfway + * through receiving a gossip msg, for example. We'll simply reconnect + * in this case, but the real fix is to wean off ccan/io here, as it doesn't + * buy us anything: a poll for read on gossipfd, masterfd and peerfd then acting + * synchronous would be a simpler model. */ static void send_shutdown_complete(struct peer *peer) { const u8 *msg; diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 6d5065afd038..9a26ae8513a4 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -126,6 +126,9 @@ struct peer { /* If we die, should we reach again? */ bool reach_again; + + /* Waiting to send_peer_with_fds to master? */ + const u8 *send_to_master; }; struct addrhint { @@ -140,7 +143,7 @@ struct addrhint { /* FIXME: Reorder */ static struct io_plan *peer_start_gossip(struct io_conn *conn, struct peer *peer); -static void send_peer_with_fds(struct peer *peer, const u8 *msg); +static bool send_peer_with_fds(struct peer *peer, const u8 *msg); static void wake_pkt_out(struct peer *peer); static void try_reach_peer(struct daemon *daemon, const struct pubkey *id); @@ -191,6 +194,7 @@ static struct peer *new_peer(const tal_t *ctx, peer->daemon = daemon; peer->local = true; peer->reach_again = false; + peer->send_to_master = NULL; peer->num_pings_outstanding = 0; peer->broadcast_index = 0; msg_queue_init(&peer->peer_out, peer); @@ -295,9 +299,11 @@ static struct io_plan *peer_init_received(struct io_conn *conn, * gossipfd closed (forget_peer) or reconnect. */ peer_finalized(peer); + /* We will not have anything queued, since we're not duplex. */ msg = towire_gossip_peer_connected(peer, &peer->id, &peer->pcs.cs, peer->gfeatures, peer->lfeatures); - send_peer_with_fds(peer, msg); + if (!send_peer_with_fds(peer, msg)) + return io_close(conn); /* Start the gossip flowing. */ /* FIXME: This is a bit wasteful in the common case where master @@ -398,10 +404,49 @@ static void handle_pong(struct peer *peer, const u8 *pong) tal_len(pong)))); } +/* If master asks us to release peer, we attach this destructor in case it + * dies while we're waiting for it to finish IO */ +static void fail_release(struct peer *peer) +{ + u8 *msg = towire_gossipctl_release_peer_replyfail(peer); + daemon_conn_send(&peer->daemon->master, take(msg)); +} + +static struct io_plan *wait_until_ready_for_master(struct io_conn *conn, + struct peer *peer) +{ + /* One of these is always true, since we've just finished read/write */ + if (!peer_in_started(conn, &peer->pcs) + && !peer_out_started(conn, &peer->pcs)) { + if (send_peer_with_fds(peer, take(peer->send_to_master))) { + /* In case we set this earlier. */ + tal_del_destructor(peer, fail_release); + peer->send_to_master = NULL; + return io_close_taken_fd(conn); + } else + return io_close(conn); + } + + /* Don't do any more I/O. */ + return io_wait(conn, peer, wait_until_ready_for_master, peer); +} + +static struct io_plan *peer_msgin(struct io_conn *conn, + struct peer *peer, u8 *msg); + +/* Wrapper around peer_read_message: don't read another if we want to + * pass up to master */ +static struct io_plan *peer_next_in(struct io_conn *conn, struct peer *peer) +{ + if (peer->send_to_master) + return wait_until_ready_for_master(conn, peer); + + return peer_read_message(conn, &peer->pcs, peer_msgin); +} + static struct io_plan *peer_msgin(struct io_conn *conn, struct peer *peer, u8 *msg) { - u8 *s; enum wire_type t = fromwire_peektype(msg); switch (t) { @@ -415,15 +460,15 @@ static struct io_plan *peer_msgin(struct io_conn *conn, case WIRE_NODE_ANNOUNCEMENT: case WIRE_CHANNEL_UPDATE: handle_gossip_msg(peer->daemon->rstate, msg); - return peer_read_message(conn, &peer->pcs, peer_msgin); + return peer_next_in(conn, peer); case WIRE_PING: handle_ping(peer, msg); - return peer_read_message(conn, &peer->pcs, peer_msgin); + return peer_next_in(conn, peer); case WIRE_PONG: handle_pong(peer, msg); - return peer_read_message(conn, &peer->pcs, peer_msgin); + return peer_next_in(conn, peer); case WIRE_OPEN_CHANNEL: case WIRE_CHANNEL_REESTABLISH: @@ -443,13 +488,15 @@ static struct io_plan *peer_msgin(struct io_conn *conn, case WIRE_REVOKE_AND_ACK: case WIRE_INIT: /* Not our place to handle this, so we punt */ - s = towire_gossip_peer_nongossip(msg, &peer->id, - &peer->pcs.cs, - peer->gfeatures, - peer->lfeatures, - msg); - send_peer_with_fds(peer, take(s)); - return io_close_taken_fd(conn); + peer->send_to_master + = towire_gossip_peer_nongossip(peer, &peer->id, + &peer->pcs.cs, + peer->gfeatures, + peer->lfeatures, + msg); + + /* This will wait. */ + return peer_next_in(conn, peer); } /* BOLT #1: @@ -463,7 +510,7 @@ static struct io_plan *peer_msgin(struct io_conn *conn, } else peer_error(peer, "Unknown packet %u", t); - return peer_read_message(conn, &peer->pcs, peer_msgin); + return peer_next_in(conn, peer); } /* Wake up the outgoing direction of the connection and write any @@ -495,6 +542,10 @@ static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer) peer_pkt_out); } + /* Do we want to send this peer to the master daemon? */ + if (peer->send_to_master) + return wait_until_ready_for_master(conn, peer); + /* If we're supposed to be sending gossip, do so now. */ if (peer->gossip_sync) { struct queued_message *next; @@ -518,7 +569,7 @@ static struct io_plan *peer_start_gossip(struct io_conn *conn, struct peer *peer { wake_pkt_out(peer); return io_duplex(conn, - peer_read_message(conn, &peer->pcs, peer_msgin), + peer_next_in(conn, peer), peer_pkt_out(conn, peer)); } @@ -554,7 +605,7 @@ static void forget_peer(struct io_conn *conn, struct daemon_conn *dc) /* When a peer is to be owned by another daemon, we create a socket * pair to send/receive gossip from it */ -static void send_peer_with_fds(struct peer *peer, const u8 *msg) +static bool send_peer_with_fds(struct peer *peer, const u8 *msg) { int fds[2]; @@ -564,7 +615,7 @@ static void send_peer_with_fds(struct peer *peer, const u8 *msg) /* FIXME: Send error to peer? */ /* Peer will be freed when caller closes conn. */ - return; + return false; } /* Now we talk to socket to get to peer's owner daemon. */ @@ -583,6 +634,7 @@ static void send_peer_with_fds(struct peer *peer, const u8 *msg) /* Don't get confused: we can't use this any more. */ peer->fd = -1; + return true; } /** @@ -678,7 +730,7 @@ static struct io_plan *handle_peer(struct io_conn *conn, struct daemon *daemon, static struct io_plan *release_peer(struct io_conn *conn, struct daemon *daemon, const u8 *msg) - { +{ struct pubkey id; struct peer *peer; @@ -686,20 +738,24 @@ static struct io_plan *release_peer(struct io_conn *conn, struct daemon *daemon, master_badmsg(WIRE_GOSSIPCTL_RELEASE_PEER, msg); peer = find_peer(daemon, &id); - if (!peer || !peer->local) { - status_trace("release_peer: peer %s not %s", - type_to_string(trc, struct pubkey, &id), - peer ? "local" : "found"); + if (!peer || !peer->local || peer->send_to_master) { /* This can happen with dying peers, or reconnect */ + status_trace("release_peer: peer %s %s", + type_to_string(trc, struct pubkey, &id), + !peer ? "not found" + : !peer->send_to_master ? "already releasing" + : "not local"); msg = towire_gossipctl_release_peer_replyfail(msg); daemon_conn_send(&daemon->master, take(msg)); } else { - msg = towire_gossipctl_release_peer_reply(msg, + msg = towire_gossipctl_release_peer_reply(peer, &peer->pcs.cs, peer->gfeatures, peer->lfeatures); - send_peer_with_fds(peer, take(msg)); - io_close_taken_fd(peer->conn); + peer->send_to_master = msg; + + /* Wake output, in case it's idle. */ + msg_wake(&peer->peer_out); } return daemon_conn_read_next(conn, &daemon->master); } From 8d9818ff9ccd235fd9129e34e2c3ca23f43d42b3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:41:21 +1030 Subject: [PATCH 0017/1428] gossipd: receive global/local features the right way around Fixes: #323 Signed-off-by: Rusty Russell --- gossipd/gossip.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 9a26ae8513a4..c93622365a31 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -1009,8 +1009,8 @@ static struct io_plan *gossip_init(struct daemon_conn *master, if (!fromwire_gossipctl_init(daemon, msg, NULL, &daemon->broadcast_interval, &chain_hash, &daemon->id, &port, - &daemon->localfeatures, - &daemon->globalfeatures)) { + &daemon->globalfeatures, + &daemon->localfeatures)) { master_badmsg(WIRE_GOSSIPCTL_INIT, msg); } daemon->rstate = new_routing_state(daemon, &chain_hash); From 3c6eec87e38e20e1f1a362ee8642ea3ea7694f3f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 24 Oct 2017 12:36:14 +1030 Subject: [PATCH 0018/1428] Add DEVELOPER flag, set by default. This is a bit messier than I'd like, but we want to clearly remove all dev code (not just have it uncalled), so we remove fields and functions altogether rather than stub them out. This means we put #ifdefs in callers in some places, but at least it's explicit. We still run tests, but only a subset, and we run with NO_VALGRIND under Travis to avoid increasing test times too much. See-also: #176 Signed-off-by: Rusty Russell --- .travis.yml | 11 +- Makefile | 11 +- channeld/channel.c | 6 + common/crypto_sync.c | 8 +- common/cryptomsg.c | 7 + common/debug.c | 4 + common/dev_disconnect.c | 2 + common/dev_disconnect.h | 2 + lightningd/chaintopology.c | 18 +- lightningd/chaintopology.h | 5 +- lightningd/jsonrpc.c | 2 + lightningd/lightningd.c | 10 +- lightningd/lightningd.h | 26 +-- lightningd/options.c | 6 + lightningd/peer_control.c | 259 +++++++++++++++-------------- lightningd/subd.c | 39 ++++- lightningd/subd.h | 2 + lightningd/test/run-cryptomsg.c | 2 + lightningd/test/run-find_my_path.c | 10 ++ tests/test_lightningd.py | 30 +++- tests/utils.py | 9 +- 21 files changed, 301 insertions(+), 168 deletions(-) diff --git a/.travis.yml b/.travis.yml index 44e87fec6a1f..721cbc62bf54 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,13 +3,14 @@ dist: trusty sudo: true env: - - NO_VALGRIND=1 ARCH=32 - - NO_VALGRIND=1 ARCH=64 - - NO_VALGRIND=0 ARCH=64 + - NO_VALGRIND=1 ARCH=32 DEVELOPER=1 + - NO_VALGRIND=1 ARCH=64 DEVELOPER=1 + - NO_VALGRIND=0 ARCH=64 DEVELOPER=1 + - NO_VALGRIND=0 ARCH=64 DEVELOPER=0 # Trusty (aka 14.04) is way way too old, so run in docker... script: - docker pull cdecker/lightning-ci:${ARCH}bit > /dev/null - - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make -j3 - - docker run --rm=true -e NO_VALGRIND=${NO_VALGRIND:-0} -e TEST_DEBUG=${TEST_DEBUG:-0} -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check + - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make -j3 DEVELOPER=${DEVELOPER} + - docker run --rm=true -e NO_VALGRIND=${NO_VALGRIND:-0} -e TEST_DEBUG=${TEST_DEBUG:-0} -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check DEVELOPER=${DEVELOPER} - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check-source diff --git a/Makefile b/Makefile index c1987dd7a33f..de8a917762c5 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,15 @@ VALGRIND=valgrind -q --error-exitcode=7 VALGRIND_TEST_ARGS = --track-origins=yes --leak-check=full --show-reachable=yes --errors-for-leak-kinds=all endif +# By default, we are in DEVELOPER mode, use DEVELOPER= on cmdline to override. +DEVELOPER := 1 + +ifeq ($(DEVELOPER),1) +DEV_CFLAGS=-DDEVELOPER=1 +else +DEV_CFLAGS=-DDEVELOPER=0 +endif + ifeq ($(COVERAGE),1) COVFLAGS = --coverage endif @@ -132,7 +141,7 @@ ALL_PROGRAMS = CWARNFLAGS := -Werror -Wall -Wundef -Wmissing-prototypes -Wmissing-declarations -Wstrict-prototypes -Wold-style-definition CDEBUGFLAGS := -std=gnu11 -g -fstack-protector -CFLAGS = $(CWARNFLAGS) $(CDEBUGFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . $(FEATURES) $(COVFLAGS) -DSHACHAIN_BITS=48 -DCCAN_TAKE_DEBUG=1 +CFLAGS = $(CWARNFLAGS) $(CDEBUGFLAGS) -I $(CCANDIR) $(EXTERNAL_INCLUDE_FLAGS) -I . $(FEATURES) $(COVFLAGS) $(DEV_CFLAGS) -DSHACHAIN_BITS=48 -DCCAN_TAKE_DEBUG=1 LDLIBS = -lgmp -lsqlite3 $(COVFLAGS) diff --git a/channeld/channel.c b/channeld/channel.c index b55e086158c1..648b6570c192 100644 --- a/channeld/channel.c +++ b/channeld/channel.c @@ -615,12 +615,14 @@ static void send_commit(struct peer *peer) u8 *msg; const struct htlc **changed_htlcs; +#if DEVELOPER /* Hack to suppress all commit sends if dev_disconnect says to */ if (dev_suppress_commit) { peer->commit_timer = NULL; tal_free(tmpctx); return; } +#endif /* FIXME: Document this requirement in BOLT 2! */ /* We can't send two commits in a row. */ @@ -1910,6 +1912,7 @@ static void handle_shutdown_cmd(struct peer *peer, const u8 *inmsg) start_commit_timer(peer); } +#if DEVELOPER static void handle_dev_reenable_commit(struct peer *peer) { dev_suppress_commit = false; @@ -1918,6 +1921,7 @@ static void handle_dev_reenable_commit(struct peer *peer) wire_sync_write(MASTER_FD, take(towire_channel_dev_reenable_commit_reply(peer))); } +#endif static void req_in(struct peer *peer, const u8 *msg) { @@ -1946,8 +1950,10 @@ static void req_in(struct peer *peer, const u8 *msg) handle_shutdown_cmd(peer, msg); goto out; case WIRE_CHANNEL_DEV_REENABLE_COMMIT: +#if DEVELOPER handle_dev_reenable_commit(peer); goto out; +#endif /* DEVELOPER */ case WIRE_CHANNEL_NORMAL_OPERATION: case WIRE_CHANNEL_INIT: case WIRE_CHANNEL_OFFER_HTLC_REPLY: diff --git a/common/crypto_sync.c b/common/crypto_sync.c index e3b53f54a420..69e4e9744ee1 100644 --- a/common/crypto_sync.c +++ b/common/crypto_sync.c @@ -11,11 +11,14 @@ bool sync_crypto_write(struct crypto_state *cs, int fd, const void *msg TAKES) { +#if DEVELOPER + bool post_sabotage = false; int type = fromwire_peektype(msg); +#endif u8 *enc = cryptomsg_encrypt_msg(NULL, cs, msg); bool ret; - bool post_sabotage = false; +#if DEVELOPER switch (dev_disconnect(type)) { case DEV_DISCONNECT_BEFORE: dev_sabotage_fd(fd); @@ -31,11 +34,14 @@ bool sync_crypto_write(struct crypto_state *cs, int fd, const void *msg TAKES) case DEV_DISCONNECT_NORMAL: break; } +#endif ret = write_all(fd, enc, tal_len(enc)); tal_free(enc); +#if DEVELOPER if (post_sabotage) dev_sabotage_fd(fd); +#endif return ret; } diff --git a/common/cryptomsg.c b/common/cryptomsg.c index c29254e4e884..976e380ba39e 100644 --- a/common/cryptomsg.c +++ b/common/cryptomsg.c @@ -326,6 +326,7 @@ u8 *cryptomsg_encrypt_msg(const tal_t *ctx, return out; } +#if DEVELOPER static struct io_plan *peer_write_postclose(struct io_conn *conn, struct peer_crypto_state *pcs) { @@ -333,6 +334,7 @@ static struct io_plan *peer_write_postclose(struct io_conn *conn, dev_sabotage_fd(io_conn_fd(conn)); return pcs->next_out(conn, pcs->peer); } +#endif struct io_plan *peer_write_message(struct io_conn *conn, struct peer_crypto_state *pcs, @@ -341,7 +343,10 @@ struct io_plan *peer_write_message(struct io_conn *conn, struct peer *)) { struct io_plan *(*post)(struct io_conn *, struct peer_crypto_state *); +#if DEVELOPER int type = fromwire_peektype(msg); +#endif + assert(!pcs->out); pcs->out = cryptomsg_encrypt_msg(conn, &pcs->cs, msg); @@ -349,6 +354,7 @@ struct io_plan *peer_write_message(struct io_conn *conn, post = peer_write_done; +#if DEVELOPER switch (dev_disconnect(type)) { case DEV_DISCONNECT_BEFORE: dev_sabotage_fd(io_conn_fd(conn)); @@ -364,6 +370,7 @@ struct io_plan *peer_write_message(struct io_conn *conn, case DEV_DISCONNECT_NORMAL: break; } +#endif /* DEVELOPER */ /* BOLT #8: * * Send `lc || c` over the network buffer. diff --git a/common/debug.c b/common/debug.c index dc6e70ecf5d4..99a2e2ba15ed 100644 --- a/common/debug.c +++ b/common/debug.c @@ -50,13 +50,16 @@ static void crashlog_activate(void) void subdaemon_debug(int argc, char *argv[]) { +#if DEVELOPER int i; bool printed = false; +#endif err_set_progname(argv[0]); backtrace_state = backtrace_create_state(argv[0], 0, NULL, NULL); crashlog_activate(); +#if DEVELOPER for (i = 1; i < argc; i++) { if (strstarts(argv[i], "--dev-disconnect=")) { dev_disconnect_init(atoi(argv[i] @@ -73,4 +76,5 @@ void subdaemon_debug(int argc, char *argv[]) printed = true; } } +#endif } diff --git a/common/dev_disconnect.c b/common/dev_disconnect.c index ae0e1596163b..0b278a13d7a9 100644 --- a/common/dev_disconnect.c +++ b/common/dev_disconnect.c @@ -11,6 +11,7 @@ #include #include +#if DEVELOPER /* We move the fd IFF we do a disconnect. */ static int dev_disconnect_fd = -1; static char dev_disconnect_line[200]; @@ -132,3 +133,4 @@ void dev_blackhole_fd(int fd) dup2(fds[1], fd); close(fds[1]); } +#endif diff --git a/common/dev_disconnect.h b/common/dev_disconnect.h index 026c672f0de1..8760f35ee305 100644 --- a/common/dev_disconnect.h +++ b/common/dev_disconnect.h @@ -3,6 +3,7 @@ #include "config.h" #include +#if DEVELOPER enum dev_disconnect { /* Do nothing. */ DEV_DISCONNECT_NORMAL = '=', @@ -30,5 +31,6 @@ void dev_disconnect_init(int fd); /* Hack for channeld to do DEV_DISCONNECT_SUPPRESS_COMMIT. */ extern bool dev_suppress_commit; +#endif /* DEVELOPER */ #endif /* LIGHTNING_COMMON_DEV_DISCONNECT_H */ diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index f4c7fe4b5b67..fa5a0b273be1 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -188,8 +188,10 @@ static void rebroadcast_txs(struct chain_topology *topo, struct command *cmd) struct txs_to_broadcast *txs; struct outgoing_tx *otx; +#if DEVELOPER if (topo->dev_no_broadcast) return; +#endif /* DEVELOPER */ txs = tal(topo, struct txs_to_broadcast); txs->cmd = cmd; @@ -263,11 +265,13 @@ void broadcast_tx(struct chain_topology *topo, log_add(topo->log, " (tx %s)", type_to_string(ltmp, struct sha256_double, &otx->txid)); - if (topo->dev_no_broadcast) +#if DEVELOPER + if (topo->dev_no_broadcast) { broadcast_done(topo->bitcoind, 0, "dev_no_broadcast", otx); - else - bitcoind_sendrawtx(topo->bitcoind, otx->hextx, - broadcast_done, otx); + return; + } +#endif + bitcoind_sendrawtx(topo->bitcoind, otx->hextx, broadcast_done, otx); } static void free_blocks(struct chain_topology *topo, struct block *b) @@ -484,6 +488,7 @@ struct txlocator *locate_tx(const void *ctx, const struct chain_topology *topo, return tal_free(loc); } +#if DEVELOPER void json_dev_broadcast(struct command *cmd, struct chain_topology *topo, const char *buffer, const jsmntok_t *params) @@ -534,6 +539,7 @@ static const struct json_command dev_blockheight = { "Returns { blockheight: u32 } on success" }; AUTODATA(json_command, &dev_blockheight); +#endif /* DEVELOPER */ /* On shutdown, peers get deleted last. That frees from our list, so * do it now instead. */ @@ -556,8 +562,10 @@ struct chain_topology *new_topology(const tal_t *ctx, struct log *log) topo->log = log; topo->default_fee_rate = 40000; topo->override_fee_rate = 0; - topo->dev_no_broadcast = false; topo->bitcoind = new_bitcoind(topo, log); +#if DEVELOPER + topo->dev_no_broadcast = false; +#endif return topo; } diff --git a/lightningd/chaintopology.h b/lightningd/chaintopology.h index 4847093fb272..5e69ce7341a9 100644 --- a/lightningd/chaintopology.h +++ b/lightningd/chaintopology.h @@ -113,8 +113,10 @@ struct chain_topology { struct txwatch_hash txwatches; struct txowatch_hash txowatches; +#if DEVELOPER /* Suppress broadcast (for testing) */ bool dev_no_broadcast; +#endif }; /* Information relevant to locating a TX in a blockchain. */ @@ -156,8 +158,9 @@ struct txlocator *locate_tx(const void *ctx, const struct chain_topology *topo, void notify_new_block(struct chain_topology *topo, unsigned int height); +#if DEVELOPER void json_dev_broadcast(struct command *cmd, struct chain_topology *topo, const char *buffer, const jsmntok_t *params); - +#endif #endif /* LIGHTNING_LIGHTNINGD_CHAINTOPOLOGY_H */ diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index ec6647e2c743..596c495c2dfd 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -186,6 +186,7 @@ static const struct json_command getlog_command = { }; AUTODATA(json_command, &getlog_command); +#if DEVELOPER static void json_rhash(struct command *cmd, const char *buffer, const jsmntok_t *params) { @@ -238,6 +239,7 @@ static const struct json_command dev_crash_command = { "Simple crash test for developers" }; AUTODATA(json_command, &dev_crash_command); +#endif /* DEVELOPER */ static void json_getinfo(struct command *cmd, const char *buffer, const jsmntok_t *params) diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 97a7cd62de9a..42305dd8d467 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -71,11 +71,8 @@ static struct lightningd *new_lightningd(const tal_t *ctx, struct lightningd *ld = tal(ctx, struct lightningd); list_head_init(&ld->peers); - ld->dev_debug_subdaemon = NULL; htlc_in_map_init(&ld->htlcs_in); htlc_out_map_init(&ld->htlcs_out); - ld->dev_disconnect_fd = -1; - ld->dev_hsm_seed = NULL; ld->log_book = log_book; ld->log = new_log(log_book, log_book, "lightningd(%u):", (int)getpid()); ld->alias = NULL; @@ -88,6 +85,13 @@ static struct lightningd *new_lightningd(const tal_t *ctx, /* FIXME: Move into invoice daemon. */ ld->invoices = invoices_init(ld); + +#if DEVELOPER + ld->dev_debug_subdaemon = NULL; + ld->dev_disconnect_fd = -1; + ld->dev_hsm_seed = NULL; +#endif + return ld; } diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 9303e982b8de..4bd37cd4b1ee 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -114,18 +114,6 @@ struct lightningd { /* Our chain topology. */ struct chain_topology *topology; - /* If we want to debug a subdaemon. */ - const char *dev_debug_subdaemon; - - /* If we want to set a specific non-random HSM seed. */ - const u8 *dev_hsm_seed; - - /* If we have a --dev-disconnect file */ - int dev_disconnect_fd; - - /* If we have --dev-fail-on-subdaemon-fail */ - bool dev_subdaemon_fail; - /* HTLCs in flight. */ struct htlc_in_map htlcs_in; struct htlc_out_map htlcs_out; @@ -139,6 +127,20 @@ struct lightningd { /* Any outstanding "pay" commands. */ struct list_head pay_commands; + +#if DEVELOPER + /* If we want to debug a subdaemon. */ + const char *dev_debug_subdaemon; + + /* If we want to set a specific non-random HSM seed. */ + const u8 *dev_hsm_seed; + + /* If we have a --dev-disconnect file */ + int dev_disconnect_fd; + + /* If we have --dev-fail-on-subdaemon-fail */ + bool dev_subdaemon_fail; +#endif /* DEVELOPER */ }; /** diff --git a/lightningd/options.c b/lightningd/options.c index 9f2e39696f05..92d44b17c776 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -285,6 +285,7 @@ static void config_register_opts(struct lightningd *ld) " regtest, or litecoin)"); } +#if DEVELOPER static char *opt_set_hsm_seed(const char *arg, struct lightningd *ld) { ld->dev_hsm_seed = tal_hexdata(ld, arg, strlen(arg)); @@ -310,6 +311,7 @@ static void dev_register_opts(struct lightningd *ld) opt_register_arg("--dev-hsm-seed=", opt_set_hsm_seed, NULL, ld, "Hex-encoded seed for HSM"); } +#endif static const struct config testnet_config = { /* 6 blocks to catch cheating attempts. */ @@ -560,7 +562,9 @@ void register_opts(struct lightningd *ld) configdir_register_opts(ld, &ld->config_dir, &ld->rpc_filename); config_register_opts(ld); +#if DEVELOPER dev_register_opts(ld); +#endif } /* Names stolen from https://github.com/ternus/nsaproductgenerator/blob/master/nsa.js */ @@ -634,6 +638,7 @@ bool handle_opts(struct lightningd *ld, int argc, char *argv[]) check_config(ld); +#if DEVELOPER if (ld->dev_hsm_seed) { int fd; unlink("hsm_secret"); @@ -645,6 +650,7 @@ bool handle_opts(struct lightningd *ld, int argc, char *argv[]) strerror(errno)); close(fd); } +#endif return newdir; } diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 2540d55eeec2..a0bcb8163e87 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -204,10 +204,12 @@ void peer_fail_transient(struct peer *peer, const char *fmt, ...) logv_add(peer->log, fmt, ap); va_end(ap); +#if DEVELOPER if (dev_disconnect_permanent(peer->ld)) { peer_internal_error(peer, "dev_disconnect permfail"); return; } +#endif peer_set_owner(peer, NULL); @@ -759,88 +761,6 @@ static const struct json_command connect_command = { }; AUTODATA(json_command, &connect_command); -static void json_dev_fail(struct command *cmd, - const char *buffer, const jsmntok_t *params) -{ - jsmntok_t *peertok; - struct peer *peer; - - if (!json_get_params(buffer, params, - "id", &peertok, - NULL)) { - command_fail(cmd, "Need id"); - return; - } - - peer = peer_from_json(cmd->ld, buffer, peertok); - if (!peer) { - command_fail(cmd, "Could not find peer with that id"); - return; - } - - peer_internal_error(peer, "Failing due to dev-fail command"); - command_success(cmd, null_response(cmd)); -} - -static const struct json_command dev_fail_command = { - "dev-fail", - json_dev_fail, - "Fail with peer {id}", - "Returns {} on success" -}; -AUTODATA(json_command, &dev_fail_command); - -static void dev_reenable_commit_finished(struct subd *channeld, - const u8 *resp, - const int *fds, - struct command *cmd) -{ - command_success(cmd, null_response(cmd)); -} - -static void json_dev_reenable_commit(struct command *cmd, - const char *buffer, const jsmntok_t *params) -{ - jsmntok_t *peertok; - struct peer *peer; - u8 *msg; - - if (!json_get_params(buffer, params, - "id", &peertok, - NULL)) { - command_fail(cmd, "Need id"); - return; - } - - peer = peer_from_json(cmd->ld, buffer, peertok); - if (!peer) { - command_fail(cmd, "Could not find peer with that id"); - return; - } - - if (!peer->owner) { - command_fail(cmd, "Peer has no owner"); - return; - } - - if (!streq(peer->owner->name, "lightning_channeld")) { - command_fail(cmd, "Peer owned by %s", peer->owner->name); - return; - } - - msg = towire_channel_dev_reenable_commit(peer); - subd_req(peer, peer->owner, take(msg), -1, 0, - dev_reenable_commit_finished, cmd); -} - -static const struct json_command dev_reenable_commit = { - "dev-reenable-commit", - json_dev_reenable_commit, - "Reenable the commit timer on peer {id}", - "Returns {} on success" -}; -AUTODATA(json_command, &dev_reenable_commit); - struct log_info { enum log_level level; struct json_result *response; @@ -1700,52 +1620,6 @@ void peer_last_tx(struct peer *peer, struct bitcoin_tx *tx, peer->last_tx = tal_steal(peer, tx); } -/* FIXME: Guard with heavy dev-only #ifdefs! */ -static void json_sign_last_tx(struct command *cmd, - const char *buffer, const jsmntok_t *params) -{ - jsmntok_t *peertok; - struct peer *peer; - struct json_result *response = new_json_result(cmd); - u8 *linear; - - if (!json_get_params(buffer, params, - "id", &peertok, - NULL)) { - command_fail(cmd, "Need id"); - return; - } - - peer = peer_from_json(cmd->ld, buffer, peertok); - if (!peer) { - command_fail(cmd, "Could not find peer with that id"); - return; - } - if (!peer->last_tx) { - command_fail(cmd, "Peer has no final transaction"); - return; - } - - log_debug(peer->log, "dev-sign-last-tx: signing tx with %zu outputs", - tal_count(peer->last_tx->output)); - sign_last_tx(peer); - linear = linearize_tx(cmd, peer->last_tx); - - json_object_start(response, NULL); - json_add_hex(response, "tx", linear, tal_len(linear)); - json_object_end(response); - command_success(cmd, response); -} - -static const struct json_command dev_sign_last_tx = { - "dev-sign-last-tx", - json_sign_last_tx, - "Sign and return the last commitment transaction", - "Sign last transaction with peer @id, return as @tx." - " This should never be called outside testing!" -}; -AUTODATA(json_command, &dev_sign_last_tx); - /* Is this better than the last tx we were holding? */ static bool better_closing_fee(struct peer *peer, const struct bitcoin_tx *tx) { @@ -2673,3 +2547,132 @@ const char *peer_state_name(enum peer_state state) return enum_peer_state_names[i].name; return "unknown"; } + +#if DEVELOPER +static void json_sign_last_tx(struct command *cmd, + const char *buffer, const jsmntok_t *params) +{ + jsmntok_t *peertok; + struct peer *peer; + struct json_result *response = new_json_result(cmd); + u8 *linear; + + if (!json_get_params(buffer, params, + "id", &peertok, + NULL)) { + command_fail(cmd, "Need id"); + return; + } + + peer = peer_from_json(cmd->ld, buffer, peertok); + if (!peer) { + command_fail(cmd, "Could not find peer with that id"); + return; + } + if (!peer->last_tx) { + command_fail(cmd, "Peer has no final transaction"); + return; + } + + log_debug(peer->log, "dev-sign-last-tx: signing tx with %zu outputs", + tal_count(peer->last_tx->output)); + sign_last_tx(peer); + linear = linearize_tx(cmd, peer->last_tx); + + json_object_start(response, NULL); + json_add_hex(response, "tx", linear, tal_len(linear)); + json_object_end(response); + command_success(cmd, response); +} + +static const struct json_command dev_sign_last_tx = { + "dev-sign-last-tx", + json_sign_last_tx, + "Sign and return the last commitment transaction", + "Sign last transaction with peer @id, return as @tx." + " This should never be called outside testing!" +}; +AUTODATA(json_command, &dev_sign_last_tx); + +static void json_dev_fail(struct command *cmd, + const char *buffer, const jsmntok_t *params) +{ + jsmntok_t *peertok; + struct peer *peer; + + if (!json_get_params(buffer, params, + "id", &peertok, + NULL)) { + command_fail(cmd, "Need id"); + return; + } + + peer = peer_from_json(cmd->ld, buffer, peertok); + if (!peer) { + command_fail(cmd, "Could not find peer with that id"); + return; + } + + peer_internal_error(peer, "Failing due to dev-fail command"); + command_success(cmd, null_response(cmd)); +} + +static const struct json_command dev_fail_command = { + "dev-fail", + json_dev_fail, + "Fail with peer {id}", + "Returns {} on success" +}; +AUTODATA(json_command, &dev_fail_command); + +static void dev_reenable_commit_finished(struct subd *channeld, + const u8 *resp, + const int *fds, + struct command *cmd) +{ + command_success(cmd, null_response(cmd)); +} + +static void json_dev_reenable_commit(struct command *cmd, + const char *buffer, const jsmntok_t *params) +{ + jsmntok_t *peertok; + struct peer *peer; + u8 *msg; + + if (!json_get_params(buffer, params, + "id", &peertok, + NULL)) { + command_fail(cmd, "Need id"); + return; + } + + peer = peer_from_json(cmd->ld, buffer, peertok); + if (!peer) { + command_fail(cmd, "Could not find peer with that id"); + return; + } + + if (!peer->owner) { + command_fail(cmd, "Peer has no owner"); + return; + } + + if (!streq(peer->owner->name, "lightning_channeld")) { + command_fail(cmd, "Peer owned by %s", peer->owner->name); + return; + } + + msg = towire_channel_dev_reenable_commit(peer); + subd_req(peer, peer->owner, take(msg), -1, 0, + dev_reenable_commit_finished, cmd); +} + +static const struct json_command dev_reenable_commit = { + "dev-reenable-commit", + json_dev_reenable_commit, + "Reenable the commit timer on peer {id}", + "Returns {} on success" +}; +AUTODATA(json_command, &dev_reenable_commit); +#endif /* DEVELOPER */ diff --git a/lightningd/subd.c b/lightningd/subd.c index 16f2030e463b..09c659f0b6b7 100644 --- a/lightningd/subd.c +++ b/lightningd/subd.c @@ -134,7 +134,6 @@ static int subd(const char *dir, const char *name, const char *debug_subdaemon, int childmsg[2], execfail[2]; pid_t childpid; int err, *fd; - bool debug = debug_subdaemon && strends(name, debug_subdaemon); if (socketpair(AF_LOCAL, SOCK_STREAM, 0, childmsg) != 0) goto fail; @@ -188,10 +187,12 @@ static int subd(const char *dir, const char *name, const char *debug_subdaemon, if (i != dev_disconnect_fd) close(i); +#if DEVELOPER if (dev_disconnect_fd != -1) debug_arg[0] = tal_fmt(NULL, "--dev-disconnect=%i", dev_disconnect_fd); - if (debug) + if (debug_subdaemon && strends(name, debug_subdaemon)) debug_arg[debug_arg[0] ? 1 : 0] = "--debugger"; +#endif execl(path_join(NULL, dir, name), name, debug_arg[0], debug_arg[1], NULL); child_errno_fail: @@ -236,9 +237,16 @@ int subd_raw(struct lightningd *ld, const char *name) { pid_t pid; int msg_fd; + const char *debug_subd = NULL; + int disconnect_fd = -1; - pid = subd(ld->daemon_dir, name, ld->dev_debug_subdaemon, - &msg_fd, ld->dev_disconnect_fd, NULL); +#if DEVELOPER + debug_subd = ld->dev_debug_subdaemon; + disconnect_fd = ld->dev_disconnect_fd; +#endif /* DEVELOPER */ + + pid = subd(ld->daemon_dir, name, debug_subd, &msg_fd, disconnect_fd, + NULL); if (pid == (pid_t)-1) { log_unusual(ld->log, "subd %s failed: %s", name, strerror(errno)); @@ -342,8 +350,10 @@ static void subdaemon_malformed_msg(struct subd *sd, const u8 *msg) msg + sizeof(be16), tal_count(msg) - sizeof(be16))); +#if DEVELOPER if (sd->ld->dev_subdaemon_fail) fatal("Subdaemon %s sent malformed message", sd->name); +#endif } /* Returns true if logged, false if malformed. */ @@ -388,8 +398,10 @@ static bool log_status_fail(struct subd *sd, log_str_broken: log_broken(sd->log, "%s: %.*s", name, str_len, str); +#if DEVELOPER if (sd->ld->dev_subdaemon_fail) fatal("Subdaemon %s hit error", sd->name); +#endif return true; } @@ -491,7 +503,11 @@ static struct io_plan *sd_msg_read(struct io_conn *conn, struct subd *sd) static void destroy_subd(struct subd *sd) { int status; - bool fail_if_subd_fails = sd->ld->dev_subdaemon_fail; + bool fail_if_subd_fails = false; + +#if DEVELOPER + fail_if_subd_fails = sd->ld->dev_subdaemon_fail; +#endif switch (waitpid(sd->pid, &status, WNOHANG)) { case 0: @@ -568,9 +584,16 @@ static struct subd *new_subd(struct lightningd *ld, { struct subd *sd = tal(ld, struct subd); int msg_fd; + const char *debug_subd = NULL; + int disconnect_fd = -1; + +#if DEVELOPER + debug_subd = ld->dev_debug_subdaemon; + disconnect_fd = ld->dev_disconnect_fd; +#endif /* DEVELOPER */ - sd->pid = subd(ld->daemon_dir, name, ld->dev_debug_subdaemon, - &msg_fd, ld->dev_disconnect_fd, ap); + sd->pid = subd(ld->daemon_dir, name, debug_subd, &msg_fd, disconnect_fd, + ap); if (sd->pid == (pid_t)-1) { log_unusual(ld->log, "subd %s failed: %s", name, strerror(errno)); @@ -696,6 +719,7 @@ void subd_release_peer(struct subd *owner, struct peer *peer) } } +#if DEVELOPER char *opt_subd_debug(const char *optarg, struct lightningd *ld) { ld->dev_debug_subdaemon = optarg; @@ -731,3 +755,4 @@ bool dev_disconnect_permanent(struct lightningd *ld) lseek(ld->dev_disconnect_fd, -r, SEEK_CUR); return false; } +#endif /* DEVELOPER */ diff --git a/lightningd/subd.h b/lightningd/subd.h index 62cee7dc22fe..ac24353c2800 100644 --- a/lightningd/subd.h +++ b/lightningd/subd.h @@ -163,8 +163,10 @@ void subd_release_peer(struct subd *owner, struct peer *peer); */ void subd_shutdown(struct subd *subd, unsigned int seconds); +#if DEVELOPER char *opt_subd_debug(const char *optarg, struct lightningd *ld); char *opt_subd_dev_disconnect(const char *optarg, struct lightningd *ld); bool dev_disconnect_permanent(struct lightningd *ld); +#endif /* DEVELOPER */ #endif /* LIGHTNING_LIGHTNINGD_SUBD_H */ diff --git a/lightningd/test/run-cryptomsg.c b/lightningd/test/run-cryptomsg.c index 15b0de13d74c..35126ede92c7 100644 --- a/lightningd/test/run-cryptomsg.c +++ b/lightningd/test/run-cryptomsg.c @@ -39,6 +39,7 @@ static void do_write(const void *buf, size_t len) #define status_trace(fmt, ...) \ printf(fmt "\n", __VA_ARGS__) +#if DEVELOPER /* AUTOGENERATED MOCKS START */ /* Generated stub for dev_blackhole_fd */ void dev_blackhole_fd(int fd UNNEEDED) @@ -52,6 +53,7 @@ enum dev_disconnect dev_disconnect(int pkt_type) { return DEV_DISCONNECT_NORMAL; } +#endif /* DEVELOPER */ /* We test what look like unknown messages. */ #define is_unknown_msg_discardable(x) 0 diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 364091dbb194..64c6ebc1c5df 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -88,6 +88,16 @@ struct wallet *wallet_new(const tal_t *ctx UNNEEDED, struct log *log UNNEEDED) { fprintf(stderr, "wallet_new called!\n"); abort(); } /* AUTOGENERATED MOCKS END */ +/* We only need these in developer mode */ +#if DEVELOPER +/* Generated stub for opt_subd_debug */ +char *opt_subd_debug(const char *optarg UNNEEDED, struct lightningd *ld UNNEEDED) +{ fprintf(stderr, "opt_subd_debug called!\n"); abort(); } +/* Generated stub for opt_subd_dev_disconnect */ +char *opt_subd_dev_disconnect(const char *optarg UNNEEDED, struct lightningd *ld UNNEEDED) +{ fprintf(stderr, "opt_subd_dev_disconnect called!\n"); abort(); } +#endif + #undef main int main(int argc, char *argv[]) { diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index ec13486d8cb6..15fef25deaa6 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -23,6 +23,7 @@ bitcoind = None TEST_DIR = tempfile.mkdtemp(prefix='lightning-') VALGRIND = os.getenv("NO_VALGRIND", "0") == "0" +DEVELOPER = os.getenv("DEVELOPER", "0") == "1" TEST_DEBUG = os.getenv("TEST_DEBUG", "0") == "1" print("Testing results are in {}".format(TEST_DIR)) @@ -104,7 +105,8 @@ def get_node(self, disconnect=None, options=None, may_fail=False): with open(os.path.join(lightning_dir, "dev_disconnect"), "w") as f: f.write("\n".join(disconnect)) daemon.cmd_line.append("--dev-disconnect=dev_disconnect") - daemon.cmd_line.append("--dev-fail-on-subdaemon-fail") + if DEVELOPER: + daemon.cmd_line.append("--dev-fail-on-subdaemon-fail") opts = [] if options is None else options for opt in opts: daemon.cmd_line.append(opt) @@ -441,6 +443,7 @@ def test_closing(self): l2.daemon.wait_for_log('sendrawtx exit 0') assert l1.bitcoin.rpc.getmempoolinfo()['size'] == 1 + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_permfail(self): l1,l2 = self.connect() @@ -485,6 +488,7 @@ def test_permfail(self): bitcoind.rpc.generate(6) l2.daemon.wait_for_log('onchaind complete, forgetting peer') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_onchain_first_commit(self): """Onchain handling where funder immediately drops to chain""" @@ -527,6 +531,7 @@ def test_onchain_first_commit(self): bitcoind.rpc.generate(6) l1.daemon.wait_for_log('onchaind complete, forgetting peer') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_onchain_dust_out(self): """Onchain handling of outgoing dust htlcs (they should fail)""" # HTLC 1->2, 1 fails after it's irrevocably committed @@ -579,6 +584,7 @@ def test_onchain_dust_out(self): # Payment failed, BTW assert l2.rpc.listinvoice('onchain_dust_out')[0]['complete'] == False + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_onchain_timeout(self): """Onchain handling of outgoing failed htlcs""" # HTLC 1->2, 1 fails just after it's irrevocably committed @@ -634,6 +640,7 @@ def test_onchain_timeout(self): # Payment failed, BTW assert l2.rpc.listinvoice('onchain_timeout')[0]['complete'] == False + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_onchain_middleman(self): # HTLC 1->2->3, 1->2 goes down after 2 gets preimage from 3. disconnects = ['-WIRE_UPDATE_FULFILL_HTLC', 'permfail'] @@ -708,6 +715,7 @@ def try_pay(): l1.bitcoin.rpc.generate(100) l2.daemon.wait_for_log('onchaind complete, forgetting peer') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_penalty_inhtlc(self): """Test penalty transaction with an incoming HTLC""" # We suppress each one after first commit; HTLC gets added not fulfilled. @@ -829,6 +837,7 @@ def test_penalty_outhtlc(self): # FIXME: Test wallet balance... l2.daemon.wait_for_log('onchaind complete, forgetting peer') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_permfail_new_commit(self): # Test case where we have two possible commits: it will use new one. disconnects = ['-WIRE_REVOKE_AND_ACK', 'permfail'] @@ -864,6 +873,7 @@ def test_permfail_new_commit(self): l1.daemon.wait_for_log('onchaind complete, forgetting peer') l2.daemon.wait_for_log('onchaind complete, forgetting peer') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_permfail_htlc_in(self): # Test case where we fail with unsettled incoming HTLC. disconnects = ['-WIRE_UPDATE_FULFILL_HTLC', 'permfail'] @@ -905,6 +915,7 @@ def test_permfail_htlc_in(self): bitcoind.rpc.generate(6) l2.daemon.wait_for_log('onchaind complete, forgetting peer') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_permfail_htlc_out(self): # Test case where we fail with unsettled outgoing HTLC. disconnects = ['+WIRE_REVOKE_AND_ACK', 'permfail'] @@ -1003,6 +1014,7 @@ def ping_tests(self, l1, l2): ret = l1.rpc.dev_ping(l2.info['id'], 1000, s) assert ret['totlen'] == 0 + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_ping(self): l1,l2 = self.connect() @@ -1014,6 +1026,7 @@ def test_ping(self): # channeld pinging self.ping_tests(l1, l2) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_routing_gossip_reconnect(self): # Connect two peers, reconnect and then see if we resume the # gossip. @@ -1042,6 +1055,7 @@ def test_second_channel(self): self.fund_channel(l1, l2, 10**6) self.fund_channel(l1, l3, 10**6) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1 for --dev-broadcast-interval") def test_routing_gossip(self): nodes = [self.node_factory.get_node() for _ in range(5)] l1 = nodes[0] @@ -1142,6 +1156,7 @@ def test_forward(self): route = copy.deepcopy(baseroute) l1.rpc.sendpay(to_json(route), rhash) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_disconnect(self): # These should all make us fail, and retry. # FIXME: Configure short timeout for reconnect! @@ -1157,6 +1172,7 @@ def test_disconnect(self): l1.daemon.wait_for_log('Failed connected out for {}, will try again' .format(l2.info['id'])) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_disconnect_funder(self): # Now error on funder side duringchannel open. disconnects = ['-WIRE_OPEN_CHANNEL', @@ -1177,6 +1193,7 @@ def test_disconnect_funder(self): self.assertRaises(ValueError, l1.rpc.fundchannel, l2.info['id'], 20000) assert l1.rpc.getpeer(l2.info['id']) == None + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_disconnect_fundee(self): # Now error on fundee side during channel open. disconnects = ['-WIRE_ACCEPT_CHANNEL', @@ -1195,6 +1212,7 @@ def test_disconnect_fundee(self): self.assertRaises(ValueError, l1.rpc.fundchannel, l2.info['id'], 20000) assert l1.rpc.getpeer(l2.info['id']) == None + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_disconnect_half_signed(self): # Now, these are the corner cases. Fundee sends funding_signed, # but funder doesn't receive it. @@ -1214,6 +1232,7 @@ def test_disconnect_half_signed(self): assert l1.rpc.getpeer(l2.info['id']) == None assert l2.rpc.getpeer(l1.info['id'])['peerid'] == l1.info['id'] + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_reconnect_signed(self): # This will fail *after* both sides consider channel opening. disconnects = ['+WIRE_FUNDING_SIGNED'] @@ -1243,6 +1262,7 @@ def test_reconnect_signed(self): l1.daemon.wait_for_log('-> CHANNELD_NORMAL') l2.daemon.wait_for_log('-> CHANNELD_NORMAL') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_reconnect_openingd(self): # Openingd thinks we're still opening; funder reconnects.. disconnects = ['0WIRE_ACCEPT_CHANNEL'] @@ -1272,6 +1292,7 @@ def test_reconnect_openingd(self): # Just to be sure, second openingd hand over to channeld. l2.daemon.wait_for_log('lightning_openingd.*REPLY WIRE_OPENING_FUNDEE_REPLY with 2 fds') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_reconnect_normal(self): # Should reconnect fine even if locked message gets lost. disconnects = ['-WIRE_FUNDING_LOCKED', @@ -1283,6 +1304,7 @@ def test_reconnect_normal(self): self.fund_channel(l1, l2, 10**6) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_reconnect_sender_add1(self): # Fail after add is OK, will cause payment failure though. disconnects = ['-WIRE_UPDATE_ADD_HTLC-nocommit', @@ -1309,6 +1331,7 @@ def test_reconnect_sender_add1(self): # This will send commit, so will reconnect as required. l1.rpc.sendpay(to_json(route), rhash) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_reconnect_sender_add(self): disconnects = ['-WIRE_COMMITMENT_SIGNED', '@WIRE_COMMITMENT_SIGNED', @@ -1334,6 +1357,7 @@ def test_reconnect_sender_add(self): for i in range(0,len(disconnects)): l1.daemon.wait_for_log('Already have funding locked in') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_reconnect_receiver_add(self): disconnects = ['-WIRE_COMMITMENT_SIGNED', '@WIRE_COMMITMENT_SIGNED', @@ -1357,6 +1381,7 @@ def test_reconnect_receiver_add(self): l1.daemon.wait_for_log('Already have funding locked in') assert l2.rpc.listinvoice('testpayment2')[0]['complete'] == True + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_reconnect_receiver_fulfill(self): # Ordering matters: after +WIRE_UPDATE_FULFILL_HTLC, channeld # will continue and try to send WIRE_COMMITMENT_SIGNED: if @@ -1386,6 +1411,7 @@ def test_reconnect_receiver_fulfill(self): l1.daemon.wait_for_log('Already have funding locked in') assert l2.rpc.listinvoice('testpayment2')[0]['complete'] == True + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_shutdown_reconnect(self): disconnects = ['-WIRE_SHUTDOWN', '@WIRE_SHUTDOWN', @@ -1413,6 +1439,7 @@ def test_shutdown_reconnect(self): l2.daemon.wait_for_logs(['sendrawtx exit 0', '-> CLOSINGD_COMPLETE']) assert l1.bitcoin.rpc.getmempoolinfo()['size'] == 1 + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_closing_negotiation_reconnect(self): disconnects = ['-WIRE_CLOSING_SIGNED', '@WIRE_CLOSING_SIGNED', @@ -1511,6 +1538,7 @@ def test_funding_change(self): assert outputs[0] > 8990000 assert outputs[2] == 10000000 + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_channel_persistence(self): # Start two nodes and open a channel (to remember). l2 will # mysteriously die while committing the first HTLC so we can diff --git a/tests/utils.py b/tests/utils.py index bd4ec1914a52..690e898b5780 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -25,6 +25,7 @@ "locktime-blocks": 6, } +DEVELOPER = os.getenv("DEVELOPER", "0") == "1" def write_config(filename, opts): with open(filename, 'w') as f: @@ -237,11 +238,11 @@ def __init__(self, lightning_dir, bitcoin_dir, port=9735): '--bitcoin-datadir={}'.format(bitcoin_dir), '--lightning-dir={}'.format(lightning_dir), '--port={}'.format(port), - '--network=regtest', - '--dev-broadcast-interval=1000', - '--dev-hsm-seed={}'.format(seed.hex()) + '--network=regtest' ] - + if DEVELOPER: + self.cmd_line += ['--dev-broadcast-interval=1000', + '--dev-hsm-seed={}'.format(seed.hex())] self.cmd_line += ["--{}={}".format(k, v) for k, v in LIGHTNINGD_CONFIG.items()] self.prefix = 'lightningd(%d)' % (port) From 11b43a422b5d291d4978a398987456e9dbc896ab Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 24 Oct 2017 12:41:59 +1030 Subject: [PATCH 0019/1428] lightningd: close one possibly-reachable abort. There are others, but they really are casued by bad failure. We need a parachute system for these. Closes: #176 Signed-off-by: Rusty Russell --- lightningd/peer_htlcs.c | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 1c30d6484447..2f555f674f8b 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -119,7 +119,9 @@ static void fail_in_htlc(struct htlc_in *hin, } } -static u8 *make_failmsg(const tal_t *ctx, u64 msatoshi, +static u8 *make_failmsg(const tal_t *ctx, + struct log *log, + u64 msatoshi, enum onion_type failcode, const u8 *channel_update) { @@ -167,7 +169,10 @@ static u8 *make_failmsg(const tal_t *ctx, u64 msatoshi, case WIRE_INVALID_ONION_KEY: fatal("Bad failmsg for %s", onion_type_name(failcode)); } - abort(); + + log_broken(log, "Asked to create unknown failmsg %u:" + " using temp node failure instead", failcode); + return towire_temporary_node_failure(ctx); } /* This is used for cases where we can immediately fail the HTLC. */ @@ -185,7 +190,8 @@ static void local_fail_htlc(struct htlc_in *hin, enum onion_type failcode) /* FIXME: Ask gossip daemon for channel_update. */ } - msg = make_failmsg(hin, hin->msatoshi, failcode, NULL); + msg = make_failmsg(hin, hin->key.peer->log, + hin->msatoshi, failcode, NULL); fail_in_htlc(hin, 0, take(create_onionreply(hin, &hin->shared_secret, msg))); tal_free(msg); } @@ -387,7 +393,8 @@ static void hout_subd_died(struct htlc_out *hout) "Failing HTLC %"PRIu64" due to peer death", hout->key.id); - hout->failuremsg = make_failmsg(hout, hout->msatoshi, + hout->failuremsg = make_failmsg(hout, hout->key.peer->log, + hout->msatoshi, WIRE_TEMPORARY_CHANNEL_FAILURE, NULL); fail_out_htlc(hout, "Outgoing subdaemon died"); @@ -808,7 +815,7 @@ void onchain_failed_our_htlc(const struct peer *peer, if (hout->failuremsg) return; - hout->failuremsg = make_failmsg(hout, hout->msatoshi, + hout->failuremsg = make_failmsg(hout, peer->log, hout->msatoshi, WIRE_PERMANENT_CHANNEL_FAILURE, NULL); From 81db5896e134b8bd50ddfde372e4040917f1e1fe Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 24 Oct 2017 12:53:43 +1030 Subject: [PATCH 0020/1428] common/json: remove asserts() which may trigger from user input. They don't currently, since callers check, but be safe. In addition, handle NULL returns from these in the bitcoind code. Signed-off-by: Rusty Russell --- common/json.c | 10 ++++++---- lightningd/bitcoind.c | 8 ++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/common/json.c b/common/json.c index 0995af3ffb3f..afb237ee3597 100644 --- a/common/json.c +++ b/common/json.c @@ -157,7 +157,8 @@ const jsmntok_t *json_get_member(const char *buffer, const jsmntok_t tok[], { const jsmntok_t *t, *end; - assert(tok->type == JSMN_OBJECT); + if (tok->type != JSMN_OBJECT) + return NULL; end = json_next(tok); for (t = tok + 1; t < end; t = json_next(t+1)) @@ -171,7 +172,8 @@ const jsmntok_t *json_get_arr(const jsmntok_t tok[], size_t index) { const jsmntok_t *t, *end; - assert(tok->type == JSMN_ARRAY); + if (tok->type != JSMN_ARRAY) + return NULL; end = json_next(tok); for (t = tok + 1; t < end; t = json_next(t)) { @@ -235,8 +237,8 @@ bool json_get_params(const char *buffer, const jsmntok_t param[], ...) else p = param + 1; end = json_next(param); - } else - assert(param->type == JSMN_OBJECT); + } else if (param->type != JSMN_OBJECT) + return false; va_start(ap, param); while ((name = va_arg(ap, const char *)) != NULL) { diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index 7ef13564b6e8..fdaff76b7a78 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -342,6 +342,14 @@ static void process_chaintips(struct bitcoin_cli *bcli) const jsmntok_t *status = json_get_member(bcli->output, t, "status"); const jsmntok_t *hash = json_get_member(bcli->output, t, "hash"); + if (!status || !hash) { + log_broken(bcli->bitcoind->log, + "%s: No status & hash: %.*s", + bcli_args(bcli), + (int)bcli->output_bytes, bcli->output); + continue; + } + if (!json_tok_streq(bcli->output, status, "active")) { log_debug(bcli->bitcoind->log, "Ignoring chaintip %.*s status %.*s", From ac9213860398b48660840e26dc90533b7f5667ae Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 24 Oct 2017 12:54:43 +1030 Subject: [PATCH 0021/1428] common: remove unused assert() headers. Auditing for assert/abort in common/ code used by lightningd, this is all that showed up. Signed-off-by: Rusty Russell --- common/derive_basepoints.c | 1 - common/funding_tx.c | 1 - lightningd/test/run-funding_tx.c | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) diff --git a/common/derive_basepoints.c b/common/derive_basepoints.c index 75041e4c9305..6db8b2d43ec4 100644 --- a/common/derive_basepoints.c +++ b/common/derive_basepoints.c @@ -1,4 +1,3 @@ -#include #include #include #include diff --git a/common/funding_tx.c b/common/funding_tx.c index 6f3b8699b7f9..35ce457ee5ed 100644 --- a/common/funding_tx.c +++ b/common/funding_tx.c @@ -1,5 +1,4 @@ #include "funding_tx.h" -#include #include #include #include diff --git a/lightningd/test/run-funding_tx.c b/lightningd/test/run-funding_tx.c index d5942c52b98d..e6f3f5aac6be 100644 --- a/lightningd/test/run-funding_tx.c +++ b/lightningd/test/run-funding_tx.c @@ -1,3 +1,4 @@ +#include #include #include #include From 3f84ca10527caf84e5fed22b9300512397494153 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:16:02 +1030 Subject: [PATCH 0022/1428] gossipd: really fix peer handoff. 954a3990fa0399522eb75de7adc58a40de6ebdd8 had two errors: 1) We created the handoff message *before* we sent the final packet, meaning that the cryptostate was out-of-sync. 2) We called io_wait() on the output side of a duplex connection: it has to be io_wait_out(). This time, stress testing for 2 hours revealed no more problems. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 77 +++++++++++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 34 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index c93622365a31..750281ed350c 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -128,7 +128,10 @@ struct peer { bool reach_again; /* Waiting to send_peer_with_fds to master? */ - const u8 *send_to_master; + bool return_to_master; + + /* If we're exiting due to non-gossip msg, otherwise release */ + u8 *nongossip_msg; }; struct addrhint { @@ -194,7 +197,7 @@ static struct peer *new_peer(const tal_t *ctx, peer->daemon = daemon; peer->local = true; peer->reach_again = false; - peer->send_to_master = NULL; + peer->return_to_master = false; peer->num_pings_outstanding = 0; peer->broadcast_index = 0; msg_queue_init(&peer->peer_out, peer); @@ -412,23 +415,28 @@ static void fail_release(struct peer *peer) daemon_conn_send(&peer->daemon->master, take(msg)); } -static struct io_plan *wait_until_ready_for_master(struct io_conn *conn, - struct peer *peer) +static struct io_plan *ready_for_master(struct io_conn *conn, struct peer *peer) { - /* One of these is always true, since we've just finished read/write */ - if (!peer_in_started(conn, &peer->pcs) - && !peer_out_started(conn, &peer->pcs)) { - if (send_peer_with_fds(peer, take(peer->send_to_master))) { - /* In case we set this earlier. */ - tal_del_destructor(peer, fail_release); - peer->send_to_master = NULL; - return io_close_taken_fd(conn); - } else - return io_close(conn); - } + u8 *msg; + if (peer->nongossip_msg) + msg = towire_gossip_peer_nongossip(peer, &peer->id, + &peer->pcs.cs, + peer->gfeatures, + peer->lfeatures, + peer->nongossip_msg); + else + msg = towire_gossipctl_release_peer_reply(peer, + &peer->pcs.cs, + peer->gfeatures, + peer->lfeatures); - /* Don't do any more I/O. */ - return io_wait(conn, peer, wait_until_ready_for_master, peer); + if (send_peer_with_fds(peer, take(msg))) { + /* In case we set this earlier. */ + tal_del_destructor(peer, fail_release); + peer->return_to_master = false; + return io_close_taken_fd(conn); + } else + return io_close(conn); } static struct io_plan *peer_msgin(struct io_conn *conn, @@ -438,8 +446,12 @@ static struct io_plan *peer_msgin(struct io_conn *conn, * pass up to master */ static struct io_plan *peer_next_in(struct io_conn *conn, struct peer *peer) { - if (peer->send_to_master) - return wait_until_ready_for_master(conn, peer); + if (peer->return_to_master) { + assert(!peer_in_started(conn, &peer->pcs)); + if (!peer_out_started(conn, &peer->pcs)) + return ready_for_master(conn, peer); + return io_wait(conn, peer, peer_next_in, peer); + } return peer_read_message(conn, &peer->pcs, peer_msgin); } @@ -488,12 +500,8 @@ static struct io_plan *peer_msgin(struct io_conn *conn, case WIRE_REVOKE_AND_ACK: case WIRE_INIT: /* Not our place to handle this, so we punt */ - peer->send_to_master - = towire_gossip_peer_nongossip(peer, &peer->id, - &peer->pcs.cs, - peer->gfeatures, - peer->lfeatures, - msg); + peer->return_to_master = true; + peer->nongossip_msg = tal_steal(peer, msg); /* This will wait. */ return peer_next_in(conn, peer); @@ -543,8 +551,12 @@ static struct io_plan *peer_pkt_out(struct io_conn *conn, struct peer *peer) } /* Do we want to send this peer to the master daemon? */ - if (peer->send_to_master) - return wait_until_ready_for_master(conn, peer); + if (peer->return_to_master) { + assert(!peer_out_started(conn, &peer->pcs)); + if (!peer_in_started(conn, &peer->pcs)) + return ready_for_master(conn, peer); + return io_out_wait(conn, peer, peer_pkt_out, peer); + } /* If we're supposed to be sending gossip, do so now. */ if (peer->gossip_sync) { @@ -738,21 +750,18 @@ static struct io_plan *release_peer(struct io_conn *conn, struct daemon *daemon, master_badmsg(WIRE_GOSSIPCTL_RELEASE_PEER, msg); peer = find_peer(daemon, &id); - if (!peer || !peer->local || peer->send_to_master) { + if (!peer || !peer->local || peer->return_to_master) { /* This can happen with dying peers, or reconnect */ status_trace("release_peer: peer %s %s", type_to_string(trc, struct pubkey, &id), !peer ? "not found" - : !peer->send_to_master ? "already releasing" + : peer->return_to_master ? "already releasing" : "not local"); msg = towire_gossipctl_release_peer_replyfail(msg); daemon_conn_send(&daemon->master, take(msg)); } else { - msg = towire_gossipctl_release_peer_reply(peer, - &peer->pcs.cs, - peer->gfeatures, - peer->lfeatures); - peer->send_to_master = msg; + peer->return_to_master = true; + peer->nongossip_msg = NULL; /* Wake output, in case it's idle. */ msg_wake(&peer->peer_out); From 33bfc2326a2583c397e043a14df58121d5b9dc0c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:42:38 +1030 Subject: [PATCH 0023/1428] gossipd: pass addr of peer though handshake. We need to derive this from the fd when they connect in, but we already know it if we're connecting out. We want this so we can tell (in next few patches) master the peer's address. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 69 ++++++++++++++++++++-------- gossipd/handshake.c | 16 ++++++- gossipd/handshake.h | 15 ++++-- gossipd/test/run-initiator-success.c | 4 +- gossipd/test/run-responder-success.c | 4 +- 5 files changed, 83 insertions(+), 25 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 750281ed350c..19dc43fbfdc5 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -1,3 +1,4 @@ +#include #include #include #include @@ -81,6 +82,9 @@ struct reaching { /* The ID of the peer (not necessarily unique, in transit!) */ struct pubkey id; + /* Where I'm reaching to. */ + struct ipaddr addr; + /* Did we succeed? */ bool succeeded; }; @@ -94,6 +98,9 @@ struct peer { /* The ID of the peer (not necessarily unique, in transit!) */ struct pubkey id; + /* Where it's connected to. */ + struct ipaddr addr; + /* Feature bitmaps. */ u8 *gfeatures, *lfeatures; @@ -332,6 +339,7 @@ static struct io_plan *read_init(struct io_conn *conn, struct peer *peer) * we have the features. */ static struct io_plan *init_new_peer(struct io_conn *conn, const struct pubkey *their_id, + const struct ipaddr *addr, const struct crypto_state *cs, struct daemon *daemon) { @@ -339,6 +347,7 @@ static struct io_plan *init_new_peer(struct io_conn *conn, u8 *initmsg; peer->fd = io_conn_fd(conn); + peer->addr = *addr; /* BOLT #1: * @@ -935,8 +944,38 @@ static int make_listen_fd(int domain, void *addr, socklen_t len) static struct io_plan *connection_in(struct io_conn *conn, struct daemon *daemon) { + struct ipaddr addr; + struct sockaddr_storage s; + socklen_t len = sizeof(s); + + if (getpeername(io_conn_fd(conn), (struct sockaddr *)&s, &len) != 0) { + status_trace("Failed to get peername for incoming conn"); + return io_close(conn); + } + + if (s.ss_family == AF_INET6) { + struct sockaddr_in6 *s6 = (void *)&s; + addr.type = ADDR_TYPE_IPV6; + addr.addrlen = sizeof(s6->sin6_addr); + BUILD_ASSERT(sizeof(s6->sin6_addr) <= sizeof(addr.addr)); + memcpy(addr.addr, &s6->sin6_addr, addr.addrlen); + addr.port = ntohs(s6->sin6_port); + } else if (s.ss_family == AF_INET) { + struct sockaddr_in *s4 = (void *)&s; + addr.type = ADDR_TYPE_IPV4; + addr.addrlen = sizeof(s4->sin_addr); + BUILD_ASSERT(sizeof(s4->sin_addr) <= sizeof(addr.addr)); + memcpy(addr.addr, &s4->sin_addr, addr.addrlen); + addr.port = ntohs(s4->sin_port); + } else { + status_trace("Unknown socket type %i for incoming conn", + s.ss_family); + return io_close(conn); + } + /* FIXME: Timeout */ - return responder_handshake(conn, &daemon->id, init_new_peer, daemon); + return responder_handshake(conn, &daemon->id, &addr, + init_new_peer, daemon); } static void setup_listeners(struct daemon *daemon, u16 portnum) @@ -1068,10 +1107,11 @@ static void handle_forwarded_msg(struct io_conn *conn, struct daemon *daemon, co static struct io_plan *handshake_out_success(struct io_conn *conn, const struct pubkey *id, + const struct ipaddr *addr, const struct crypto_state *cs, struct reaching *reach) { - return init_new_peer(conn, id, cs, reach->daemon); + return init_new_peer(conn, id, addr, cs, reach->daemon); } @@ -1083,6 +1123,7 @@ static struct io_plan *connection_out(struct io_conn *conn, type_to_string(trc, struct pubkey, &reach->id)); return initiator_handshake(conn, &reach->daemon->id, &reach->id, + &reach->addr, handshake_out_success, reach); } @@ -1099,14 +1140,8 @@ static void connect_failed(struct io_conn *conn, struct reaching *reach) try_connect, reach); } -struct reach_addr { - struct reaching *reach; - struct ipaddr addr; -}; - -static struct io_plan *conn_init(struct io_conn *conn, struct reach_addr *r) +static struct io_plan *conn_init(struct io_conn *conn, struct reaching *reach) { - struct reaching *reach = r->reach; struct addrinfo ai; struct sockaddr_in sin; struct sockaddr_in6 sin6; @@ -1118,12 +1153,12 @@ static struct io_plan *conn_init(struct io_conn *conn, struct reach_addr *r) ai.ai_canonname = NULL; ai.ai_next = NULL; - switch (r->addr.type) { + switch (reach->addr.type) { case ADDR_TYPE_IPV4: ai.ai_family = AF_INET; sin.sin_family = AF_INET; - sin.sin_port = htons(r->addr.port); - memcpy(&sin.sin_addr, r->addr.addr, sizeof(sin.sin_addr)); + sin.sin_port = htons(reach->addr.port); + memcpy(&sin.sin_addr, reach->addr.addr, sizeof(sin.sin_addr)); ai.ai_addrlen = sizeof(sin); ai.ai_addr = (struct sockaddr *)&sin; break; @@ -1131,8 +1166,8 @@ static struct io_plan *conn_init(struct io_conn *conn, struct reach_addr *r) ai.ai_family = AF_INET6; memset(&sin6, 0, sizeof(sin6)); sin6.sin6_family = AF_INET6; - sin6.sin6_port = htons(r->addr.port); - memcpy(&sin6.sin6_addr, r->addr.addr, sizeof(sin6.sin6_addr)); + sin6.sin6_port = htons(reach->addr.port); + memcpy(&sin6.sin6_addr, reach->addr.addr, sizeof(sin6.sin6_addr)); ai.ai_addrlen = sizeof(sin6); ai.ai_addr = (struct sockaddr *)&sin6; break; @@ -1148,7 +1183,6 @@ static struct io_plan *conn_init(struct io_conn *conn, struct reach_addr *r) static void try_connect(struct reaching *reach) { struct addrhint *a; - struct reach_addr r; int fd; /* Already succeeded somehow? */ @@ -1192,9 +1226,8 @@ static void try_connect(struct reaching *reach) return; } - r.reach = reach; - r.addr = a->addr; - io_new_conn(reach, fd, conn_init, &r); + reach->addr = a->addr; + io_new_conn(reach, fd, conn_init, reach); } static void try_reach_peer(struct daemon *daemon, const struct pubkey *id) diff --git a/gossipd/handshake.c b/gossipd/handshake.c index 247a2ceda0b2..7cefb2df97ee 100644 --- a/gossipd/handshake.c +++ b/gossipd/handshake.c @@ -19,6 +19,7 @@ #include #include #include +#include #define HSM_FD 3 @@ -169,6 +170,9 @@ struct handshake { struct act_two act2; struct act_three act3; + /* Where is connection from/to */ + struct ipaddr addr; + /* Who we are */ struct pubkey my_id; /* Who they are: set already if we're initiator. */ @@ -180,6 +184,7 @@ struct handshake { /* Function to call once handshake complete. */ struct io_plan *(*cb)(struct io_conn *conn, const struct pubkey *their_id, + const struct ipaddr *ipaddr, const struct crypto_state *cs, void *cbarg); void *cbarg; @@ -348,10 +353,12 @@ static struct io_plan *handshake_succeeded(struct io_conn *conn, struct crypto_state cs; struct io_plan *(*cb)(struct io_conn *conn, const struct pubkey *their_id, + const struct ipaddr *addr, const struct crypto_state *cs, void *cbarg); void *cbarg; struct pubkey their_id; + struct ipaddr addr; /* BOLT #8: * @@ -376,9 +383,10 @@ static struct io_plan *handshake_succeeded(struct io_conn *conn, cb = h->cb; cbarg = h->cbarg; their_id = h->their_id; + addr = h->addr; tal_free(h); - return cb(conn, &their_id, &cs, cbarg); + return cb(conn, &their_id, &addr, &cs, cbarg); } static struct handshake *new_handshake(const tal_t *ctx, @@ -953,8 +961,10 @@ static struct io_plan *act_one_responder(struct io_conn *conn, struct io_plan *responder_handshake_(struct io_conn *conn, const struct pubkey *my_id, + const struct ipaddr *addr, struct io_plan *(*cb)(struct io_conn *, const struct pubkey *, + const struct ipaddr *, const struct crypto_state *, void *cbarg), void *cbarg) @@ -963,6 +973,7 @@ struct io_plan *responder_handshake_(struct io_conn *conn, h->side = RESPONDER; h->my_id = *my_id; + h->addr = *addr; h->cbarg = cbarg; h->cb = cb; @@ -972,8 +983,10 @@ struct io_plan *responder_handshake_(struct io_conn *conn, struct io_plan *initiator_handshake_(struct io_conn *conn, const struct pubkey *my_id, const struct pubkey *their_id, + const struct ipaddr *addr, struct io_plan *(*cb)(struct io_conn *, const struct pubkey *, + const struct ipaddr *, const struct crypto_state *, void *cbarg), void *cbarg) @@ -983,6 +996,7 @@ struct io_plan *initiator_handshake_(struct io_conn *conn, h->side = INITIATOR; h->my_id = *my_id; h->their_id = *their_id; + h->addr = *addr; h->cbarg = cbarg; h->cb = cb; diff --git a/gossipd/handshake.h b/gossipd/handshake.h index 41167ae7331d..343dc501b161 100644 --- a/gossipd/handshake.h +++ b/gossipd/handshake.h @@ -5,14 +5,16 @@ struct crypto_state; struct io_conn; +struct ipaddr; struct pubkey; -#define initiator_handshake(conn, my_id, their_id, cb, cbarg) \ - initiator_handshake_((conn), (my_id), (their_id), \ +#define initiator_handshake(conn, my_id, their_id, addr, cb, cbarg) \ + initiator_handshake_((conn), (my_id), (their_id), (addr), \ typesafe_cb_preargs(struct io_plan *, void *, \ (cb), (cbarg), \ struct io_conn *, \ const struct pubkey *, \ + const struct ipaddr *, \ const struct crypto_state *), \ (cbarg)) @@ -20,26 +22,31 @@ struct pubkey; struct io_plan *initiator_handshake_(struct io_conn *conn, const struct pubkey *my_id, const struct pubkey *their_id, + const struct ipaddr *addr, struct io_plan *(*cb)(struct io_conn *, const struct pubkey *, + const struct ipaddr *, const struct crypto_state *, void *cbarg), void *cbarg); -#define responder_handshake(conn, my_id, cb, cbarg) \ - responder_handshake_((conn), (my_id), \ +#define responder_handshake(conn, my_id, addr, cb, cbarg) \ + responder_handshake_((conn), (my_id), (addr), \ typesafe_cb_preargs(struct io_plan *, void *, \ (cb), (cbarg), \ struct io_conn *, \ const struct pubkey *, \ + const struct ipaddr *, \ const struct crypto_state *), \ (cbarg)) struct io_plan *responder_handshake_(struct io_conn *conn, const struct pubkey *my_id, + const struct ipaddr *addr, struct io_plan *(*cb)(struct io_conn *, const struct pubkey *, + const struct ipaddr *, const struct crypto_state *, void *cbarg), void *cbarg); diff --git a/gossipd/test/run-initiator-success.c b/gossipd/test/run-initiator-success.c index 7bb32eb0d380..8f87df8967c3 100644 --- a/gossipd/test/run-initiator-success.c +++ b/gossipd/test/run-initiator-success.c @@ -176,6 +176,7 @@ static struct io_plan *test_read(struct io_conn *conn, static struct io_plan *success(struct io_conn *conn, const struct pubkey *them, + const struct ipaddr *addr, const struct crypto_state *cs, void *ctx) { @@ -199,6 +200,7 @@ bool hsm_do_ecdh(struct secret *ss, const struct pubkey *point) int main(void) { tal_t *ctx = tal_tmpctx(NULL); + struct ipaddr dummy; trc = tal_tmpctx(ctx); @@ -221,7 +223,7 @@ int main(void) e_priv = privkey("1212121212121212121212121212121212121212121212121212121212121212"); e_pub = pubkey("036360e856310ce5d294e8be33fc807077dc56ac80d95d9cd4ddbd21325eff73f7"); - initiator_handshake(ctx, &ls_pub, &rs_pub, success, ctx); + initiator_handshake(ctx, &ls_pub, &rs_pub, &dummy, success, ctx); /* Should not exit! */ abort(); } diff --git a/gossipd/test/run-responder-success.c b/gossipd/test/run-responder-success.c index e50816803040..ab3799d9865b 100644 --- a/gossipd/test/run-responder-success.c +++ b/gossipd/test/run-responder-success.c @@ -176,6 +176,7 @@ static struct io_plan *test_read(struct io_conn *conn, static struct io_plan *success(struct io_conn *conn, const struct pubkey *them, + const struct ipaddr *addr, const struct crypto_state *cs, void *ctx) { @@ -197,6 +198,7 @@ bool hsm_do_ecdh(struct secret *ss, const struct pubkey *point) int main(void) { tal_t *ctx = tal_tmpctx(NULL); + struct ipaddr dummy; trc = tal_tmpctx(ctx); @@ -217,7 +219,7 @@ int main(void) e_priv = privkey("2222222222222222222222222222222222222222222222222222222222222222"); e_pub = pubkey("02466d7fcae563e5cb09a0d1870bb580344804617879a14949cf22285f1bae3f27"); - responder_handshake(ctx, &ls_pub, success, ctx); + responder_handshake(ctx, &ls_pub, &dummy, success, ctx); /* Should not exit! */ abort(); } From dfd60a204796e77d8667294fee1eda505076a463 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:43:38 +1030 Subject: [PATCH 0024/1428] gossipd: tell the master the peer's address. This will let us remove peer->netaddr. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 15 ++++++++++----- gossipd/gossip_wire.csv | 4 ++++ lightningd/gossip_control.c | 5 +++-- lightningd/peer_control.c | 34 ++++++++++++++++++++++++---------- lightningd/peer_control.h | 2 ++ 5 files changed, 43 insertions(+), 17 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index 19dc43fbfdc5..ac9f1fe58689 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -194,6 +194,7 @@ static struct addrhint *find_addrhint(struct daemon *daemon, static struct peer *new_peer(const tal_t *ctx, struct daemon *daemon, const struct pubkey *their_id, + const struct ipaddr *addr, const struct crypto_state *cs) { struct peer *peer = tal(ctx, struct peer); @@ -201,6 +202,7 @@ static struct peer *new_peer(const tal_t *ctx, init_peer_crypto_state(peer, &peer->pcs); peer->pcs.cs = *cs; peer->id = *their_id; + peer->addr = *addr; peer->daemon = daemon; peer->local = true; peer->reach_again = false; @@ -310,7 +312,8 @@ static struct io_plan *peer_init_received(struct io_conn *conn, peer_finalized(peer); /* We will not have anything queued, since we're not duplex. */ - msg = towire_gossip_peer_connected(peer, &peer->id, &peer->pcs.cs, + msg = towire_gossip_peer_connected(peer, &peer->id, &peer->addr, + &peer->pcs.cs, peer->gfeatures, peer->lfeatures); if (!send_peer_with_fds(peer, msg)) return io_close(conn); @@ -343,11 +346,10 @@ static struct io_plan *init_new_peer(struct io_conn *conn, const struct crypto_state *cs, struct daemon *daemon) { - struct peer *peer = new_peer(conn, daemon, their_id, cs); + struct peer *peer = new_peer(conn, daemon, their_id, addr, cs); u8 *initmsg; peer->fd = io_conn_fd(conn); - peer->addr = *addr; /* BOLT #1: * @@ -429,12 +431,14 @@ static struct io_plan *ready_for_master(struct io_conn *conn, struct peer *peer) u8 *msg; if (peer->nongossip_msg) msg = towire_gossip_peer_nongossip(peer, &peer->id, + &peer->addr, &peer->pcs.cs, peer->gfeatures, peer->lfeatures, peer->nongossip_msg); else msg = towire_gossipctl_release_peer_reply(peer, + &peer->addr, &peer->pcs.cs, peer->gfeatures, peer->lfeatures); @@ -714,10 +718,11 @@ static struct io_plan *handle_peer(struct io_conn *conn, struct daemon *daemon, struct peer *peer; struct crypto_state cs; struct pubkey id; + struct ipaddr addr; u8 *gfeatures, *lfeatures; u8 *inner_msg; - if (!fromwire_gossipctl_handle_peer(msg, msg, NULL, &id, &cs, + if (!fromwire_gossipctl_handle_peer(msg, msg, NULL, &id, &addr, &cs, &gfeatures, &lfeatures, &inner_msg)) master_badmsg(WIRE_GOSSIPCTL_HANDLE_PEER, msg); @@ -738,7 +743,7 @@ static struct io_plan *handle_peer(struct io_conn *conn, struct daemon *daemon, status_trace("handle_peer %s: new peer", type_to_string(trc, struct pubkey, &id)); - peer = new_peer(daemon, daemon, &id, &cs); + peer = new_peer(daemon, daemon, &id, &addr, &cs); peer->gfeatures = tal_steal(peer, gfeatures); peer->lfeatures = tal_steal(peer, lfeatures); peer_finalized(peer); diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index 2a2ce4c30e99..eafdc721791d 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -24,6 +24,7 @@ gossipctl_reach_peer,,id,struct pubkey # Gossipd -> master: we got a peer. Two fds: peer and gossip gossip_peer_connected,3002 gossip_peer_connected,,id,struct pubkey +gossip_peer_connected,,addr,struct ipaddr gossip_peer_connected,,crypto_state,struct crypto_state gossip_peer_connected,,gflen,u16 gossip_peer_connected,,gfeatures,gflen*u8 @@ -33,6 +34,7 @@ gossip_peer_connected,,lfeatures,lflen*u8 # Gossipd -> master: peer sent non-gossip packet. Two fds: peer and gossip gossip_peer_nongossip,3003 gossip_peer_nongossip,,id,struct pubkey +gossip_peer_nongossip,,addr,struct ipaddr gossip_peer_nongossip,,crypto_state,struct crypto_state gossip_peer_nongossip,,gflen,u16 gossip_peer_nongossip,,gfeatures,gflen*u8 @@ -47,6 +49,7 @@ gossipctl_release_peer,,id,struct pubkey # Gossipd -> master: reply to gossip_release_peer. Two fds: peer and gossip. gossipctl_release_peer_reply,3104 +gossipctl_release_peer_reply,,addr,struct ipaddr gossipctl_release_peer_reply,,crypto_state,struct crypto_state gossipctl_release_peer_reply,,gflen,u16 gossipctl_release_peer_reply,,gfeatures,gflen*u8 @@ -59,6 +62,7 @@ gossipctl_release_peer_replyfail,3204 # Gossipd -> master: take over peer, with optional msg. (+peer fd) gossipctl_handle_peer,3013 gossipctl_handle_peer,,id,struct pubkey +gossipctl_handle_peer,,addr,struct ipaddr gossipctl_handle_peer,,crypto_state,struct crypto_state gossipctl_handle_peer,,gflen,u16 gossipctl_handle_peer,,gfeatures,gflen*u8 diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index d7df9d35d4ac..91efb8593689 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -24,10 +24,11 @@ static void peer_nongossip(struct subd *gossip, const u8 *msg, { struct pubkey id; struct crypto_state cs; + struct ipaddr addr; u8 *gfeatures, *lfeatures, *in_pkt; if (!fromwire_gossip_peer_nongossip(msg, msg, NULL, - &id, &cs, + &id, &addr, &cs, &gfeatures, &lfeatures, &in_pkt)) @@ -45,7 +46,7 @@ static void peer_nongossip(struct subd *gossip, const u8 *msg, return; } - peer_sent_nongossip(gossip->ld, &id, &cs, gfeatures, lfeatures, + peer_sent_nongossip(gossip->ld, &id, &addr, &cs, gfeatures, lfeatures, peer_fd, gossip_fd, in_pkt); } diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index a0bcb8163e87..ad1b2dffa346 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -56,6 +56,7 @@ struct connect { struct funding_channel; static void peer_offer_channel(struct lightningd *ld, struct funding_channel *fc, + const struct ipaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, int peer_fd, int gossip_fd); @@ -70,6 +71,7 @@ static void peer_start_closingd(struct peer *peer, bool reconnected); static void peer_accept_channel(struct lightningd *ld, const struct pubkey *peer_id, + const struct ipaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, int peer_fd, int gossip_fd, @@ -305,6 +307,7 @@ static void connect_failed(struct lightningd *ld, const struct pubkey *id, static struct peer *new_peer(struct lightningd *ld, const struct pubkey *id, + const struct ipaddr *addr, const u8 *gfeatures, const u8 *lfeatures, int peer_fd) { @@ -314,6 +317,7 @@ static struct peer *new_peer(struct lightningd *ld, peer = talz(ld, struct peer); peer->error = NULL; peer->id = *id; + peer->addr = *addr; peer->funding_txid = NULL; peer->remote_funding_locked = false; peer->scid = NULL; @@ -506,9 +510,11 @@ void peer_connected(struct lightningd *ld, const u8 *msg, u8 *gfeatures, *lfeatures; u8 *error; struct peer *peer; + struct ipaddr addr; if (!fromwire_gossip_peer_connected(msg, msg, NULL, - &id, &cs, &gfeatures, &lfeatures)) + &id, &addr, &cs, + &gfeatures, &lfeatures)) fatal("Gossip gave bad GOSSIP_PEER_CONNECTED message %s", tal_hex(msg, msg)); @@ -567,6 +573,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, * on this peer. */ peer_set_owner(peer, NULL); + peer->addr = addr; peer_start_channeld(peer, &cs, peer_fd, gossip_fd, NULL, true); return; @@ -577,6 +584,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, * on this peer. */ peer_set_owner(peer, NULL); + peer->addr = addr; peer_start_closingd(peer, &cs, peer_fd, gossip_fd, true); return; @@ -586,7 +594,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, return_to_gossipd: /* Otherwise, we hand back to gossipd, to continue. */ - msg = towire_gossipctl_handle_peer(msg, &id, &cs, + msg = towire_gossipctl_handle_peer(msg, &id, &addr, &cs, gfeatures, lfeatures, NULL); subd_send_msg(ld->gossip, take(msg)); subd_send_fd(ld->gossip, peer_fd); @@ -599,7 +607,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, send_error: /* Hand back to gossipd, with an error packet. */ connect_failed(ld, &id, sanitize_error(msg, error, NULL)); - msg = towire_gossipctl_handle_peer(msg, &id, &cs, + msg = towire_gossipctl_handle_peer(msg, &id, &addr, &cs, gfeatures, lfeatures, error); subd_send_msg(ld->gossip, take(msg)); subd_send_fd(ld->gossip, peer_fd); @@ -608,6 +616,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, void peer_sent_nongossip(struct lightningd *ld, const struct pubkey *id, + const struct ipaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, @@ -636,7 +645,7 @@ void peer_sent_nongossip(struct lightningd *ld, /* Open request? */ if (fromwire_peektype(in_msg) == WIRE_OPEN_CHANNEL) { - peer_accept_channel(ld, id, cs, gfeatures, lfeatures, + peer_accept_channel(ld, id, addr, cs, gfeatures, lfeatures, peer_fd, gossip_fd, in_msg); return; } @@ -649,7 +658,7 @@ void peer_sent_nongossip(struct lightningd *ld, send_error: /* Hand back to gossipd, with an error packet. */ connect_failed(ld, id, sanitize_error(error, error, NULL)); - msg = towire_gossipctl_handle_peer(error, id, cs, + msg = towire_gossipctl_handle_peer(error, id, addr, cs, gfeatures, lfeatures, error); subd_send_msg(ld->gossip, take(msg)); subd_send_fd(ld->gossip, peer_fd); @@ -2204,6 +2213,7 @@ static void opening_fundee_finished(struct subd *opening, /* Peer has spontaneously exited from gossip due to open msg */ static void peer_accept_channel(struct lightningd *ld, const struct pubkey *peer_id, + const struct ipaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, int peer_fd, int gossip_fd, @@ -2218,7 +2228,7 @@ static void peer_accept_channel(struct lightningd *ld, assert(fromwire_peektype(open_msg) == WIRE_OPEN_CHANNEL); /* We make a new peer. */ - peer = new_peer(ld, peer_id, gfeatures, lfeatures, peer_fd); + peer = new_peer(ld, peer_id, addr, gfeatures, lfeatures, peer_fd); /* FIXME: Only happens due to netaddr fail. */ if (!peer) { @@ -2279,7 +2289,7 @@ static void peer_accept_channel(struct lightningd *ld, peer_to_gossipd: /* Return to gossipd, with optional error msg to send. */ - msg = towire_gossipctl_handle_peer(ld, peer_id, cs, + msg = towire_gossipctl_handle_peer(ld, peer_id, addr, cs, gfeatures, lfeatures, errmsg); subd_send_msg(ld->gossip, take(msg)); subd_send_fd(ld->gossip, peer_fd); @@ -2290,6 +2300,7 @@ static void peer_accept_channel(struct lightningd *ld, static void peer_offer_channel(struct lightningd *ld, struct funding_channel *fc, + const struct ipaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, int peer_fd, int gossip_fd) @@ -2300,7 +2311,8 @@ static void peer_offer_channel(struct lightningd *ld, struct utxo *utxos; /* We make a new peer. */ - fc->peer = new_peer(ld, &fc->peerid, gfeatures, lfeatures, peer_fd); + fc->peer = new_peer(ld, &fc->peerid, addr, + gfeatures, lfeatures, peer_fd); /* FIXME: Only happens due to netaddr fail. */ if (!fc->peer) { @@ -2389,11 +2401,12 @@ static void gossip_peer_released(struct subd *gossip, struct lightningd *ld = gossip->ld; struct crypto_state cs; u8 *gfeatures, *lfeatures; + struct ipaddr addr; /* We could have raced with peer doing something else. */ fc->peer = peer_by_id(ld, &fc->peerid); - if (!fromwire_gossipctl_release_peer_reply(fc, resp, NULL, &cs, + if (!fromwire_gossipctl_release_peer_reply(fc, resp, NULL, &addr, &cs, &gfeatures, &lfeatures)) { if (!fromwire_gossipctl_release_peer_replyfail(resp, NULL)) { fatal("Gossip daemon gave invalid reply %s", @@ -2419,7 +2432,8 @@ static void gossip_peer_released(struct subd *gossip, } /* OK, offer peer a channel. */ - peer_offer_channel(ld, fc, &cs, gfeatures, lfeatures, fds[0], fds[1]); + peer_offer_channel(ld, fc, &addr, &cs, gfeatures, lfeatures, + fds[0], fds[1]); } static void json_fund_channel(struct command *cmd, diff --git a/lightningd/peer_control.h b/lightningd/peer_control.h index f89d4a83dbf1..6c1971b294c7 100644 --- a/lightningd/peer_control.h +++ b/lightningd/peer_control.h @@ -56,6 +56,7 @@ struct peer { /* Where we connected to, or it connected from. */ struct netaddr netaddr; + struct ipaddr addr; /* Our channel config. */ struct channel_config our_config; @@ -168,6 +169,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, void peer_sent_nongossip(struct lightningd *ld, const struct pubkey *id, + const struct ipaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, From bd1cac34ce01a399ecb8094a7cf3d9ae408e66b7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:44:38 +1030 Subject: [PATCH 0025/1428] netaddr: remove. We use ipaddr everywhere now, so we can remove this. Signed-off-by: Rusty Russell --- common/type_to_string.h | 1 - gossipd/Makefile | 3 +- lightningd/Makefile | 1 - lightningd/jsonrpc.c | 2 +- lightningd/netaddr.c | 90 --------------------------------------- lightningd/netaddr.h | 35 --------------- lightningd/peer_control.c | 60 +++++++++++--------------- lightningd/peer_control.h | 2 - 8 files changed, 26 insertions(+), 168 deletions(-) delete mode 100644 lightningd/netaddr.c delete mode 100644 lightningd/netaddr.h diff --git a/common/type_to_string.h b/common/type_to_string.h index 02f63e14a1df..87b929105273 100644 --- a/common/type_to_string.h +++ b/common/type_to_string.h @@ -18,7 +18,6 @@ union printable_types { const struct preimage *preimage; const struct channel_state *channel_state; const struct channel_oneside *channel_oneside; - const struct netaddr *netaddr; const secp256k1_pubkey *secp256k1_pubkey; const struct channel_id *channel_id; const struct short_channel_id *short_channel_id; diff --git a/gossipd/Makefile b/gossipd/Makefile index 7f91f04072bb..26b8e60e72c1 100644 --- a/gossipd/Makefile +++ b/gossipd/Makefile @@ -52,8 +52,7 @@ GOSSIPD_COMMON_OBJS := \ common/wire_error.o \ hsmd/client.o \ hsmd/gen_hsm_client_wire.o \ - lightningd/gossip_msg.o \ - lightningd/netaddr.o + lightningd/gossip_msg.o $(LIGHTNINGD_GOSSIP_OBJS) $(LIGHTNINGD_GOSSIP_CLIENT_OBJS): $(LIGHTNINGD_HEADERS) diff --git a/lightningd/Makefile b/lightningd/Makefile index 5668a37290a3..e4401f53da67 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -49,7 +49,6 @@ LIGHTNINGD_SRC := \ lightningd/jsonrpc.c \ lightningd/lightningd.c \ lightningd/log.c \ - lightningd/netaddr.c \ lightningd/opt_time.c \ lightningd/options.c \ lightningd/pay.c \ diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index 596c495c2dfd..ef7c718d2802 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -248,7 +248,7 @@ static void json_getinfo(struct command *cmd, json_object_start(response, NULL); json_add_pubkey(response, "id", &cmd->ld->id); - /* FIXME: Keep netaddrs and list them all. */ + /* FIXME: Keep ipaddr and list them all. */ if (cmd->ld->portnum) json_add_num(response, "port", cmd->ld->portnum); json_add_string(response, "network", diff --git a/lightningd/netaddr.c b/lightningd/netaddr.c deleted file mode 100644 index 9ce429dfd034..000000000000 --- a/lightningd/netaddr.c +++ /dev/null @@ -1,90 +0,0 @@ -#include "bitcoin/pullpush.h" -#include "netaddr.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -void netaddr_to_addrinfo(struct addrinfo *ai, const struct netaddr *a) -{ - ai->ai_flags = 0; - ai->ai_family = a->saddr.s.sa_family; - ai->ai_socktype = a->type; - ai->ai_protocol = a->protocol; - ai->ai_addrlen = a->addrlen; - ai->ai_addr = cast_const(struct sockaddr *, &a->saddr.s); - ai->ai_canonname = NULL; - ai->ai_next = NULL; -} - -char *netaddr_name(const tal_t *ctx, const struct netaddr *a) -{ - char name[INET6_ADDRSTRLEN]; - const void *sockaddr; - uint16_t port; - - switch (a->saddr.s.sa_family) { - case AF_INET: - sockaddr = &a->saddr.ipv4.sin_addr; - port = ntohs(a->saddr.ipv4.sin_port); - break; - case AF_INET6: - sockaddr = &a->saddr.ipv6.sin6_addr; - port = ntohs(a->saddr.ipv6.sin6_port); - break; - default: - return tal_fmt(ctx, "Unknown protocol %u", a->saddr.s.sa_family); - } - - if (!inet_ntop(a->saddr.s.sa_family, sockaddr, name, sizeof(name))) - sprintf(name, "Unprintable-%u-address", a->saddr.s.sa_family); - - return tal_fmt(ctx, "%s:%u", name, port); -} - -char *netaddr_to_hex(const tal_t *ctx, const struct netaddr *a) -{ - u8 *blob = tal_arr(ctx, u8, 0); - char *hex; - - push_le32(a->type, push, &blob); - push_le32(a->protocol, push, &blob); - push_le32(a->addrlen, push, &blob); - assert(a->addrlen <= sizeof(a->saddr)); - push(&a->saddr, a->addrlen, &blob); - - hex = tal_hex(ctx, blob); - tal_free(blob); - return hex; -} - -bool netaddr_from_blob(const void *linear, size_t len, struct netaddr *a) -{ - const u8 *p = linear; - - a->type = pull_le32(&p, &len); - a->protocol = pull_le32(&p, &len); - a->addrlen = pull_le32(&p, &len); - if (a->addrlen > sizeof(a->saddr)) - return false; - pull(&p, &len, &a->saddr, a->addrlen); - return p != NULL && len == 0; -} - -bool netaddr_from_fd(int fd, int type, int protocol, struct netaddr *a) -{ - a->type = type; - a->protocol = protocol; - a->addrlen = sizeof(a->saddr); - return getpeername(fd, &a->saddr.s, &a->addrlen) == 0; -} - -REGISTER_TYPE_TO_STRING(netaddr, netaddr_name); diff --git a/lightningd/netaddr.h b/lightningd/netaddr.h deleted file mode 100644 index 342bbe132b65..000000000000 --- a/lightningd/netaddr.h +++ /dev/null @@ -1,35 +0,0 @@ -/* FIXME: We should deprecate this in favor of BOLT7 address descriptor */ -#ifndef LIGHTNING_LIGHTNINGD_NETADDR_H -#define LIGHTNING_LIGHTNINGD_NETADDR_H -#include "config.h" -#include -#include -#include -#include - -struct addrinfo; - -/* This can be extended to support other protocols in future. */ -struct netaddr { - int type; /* See socket(2): SOCK_STREAM currently */ - int protocol; /* See socket(2): 0 currently */ - socklen_t addrlen; - union { - struct sockaddr s; - struct sockaddr_in ipv4; - struct sockaddr_in6 ipv6; - } saddr; -}; - -/* Get the name for this netaddr. */ -char *netaddr_name(const tal_t *ctx, const struct netaddr *a); - -/* Create a addrinfo (as wanted by io_connect) for this address. */ -void netaddr_to_addrinfo(struct addrinfo *ai, const struct netaddr *a); - -bool netaddr_from_fd(int fd, int type, int protocol, struct netaddr *a); - -bool netaddr_from_blob(const void *linear, size_t len, struct netaddr *a); -char *netaddr_to_hex(const tal_t *ctx, const struct netaddr *a); - -#endif /* LIGHTNING_LIGHTNINGD_NETADDR_H */ diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index ad1b2dffa346..566b264445f2 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -1,6 +1,7 @@ #include "lightningd.h" #include "peer_control.h" #include "subd.h" +#include #include #include #include @@ -347,12 +348,6 @@ static struct peer *new_peer(struct lightningd *ld, /* peer->channel gets populated as soon as we start opening a channel */ peer->channel = NULL; - /* FIXME: Don't assume protocol here! */ - if (!netaddr_from_fd(peer_fd, SOCK_STREAM, IPPROTO_TCP, &peer->netaddr)) { - log_unusual(ld->log, "Failed to get netaddr for outgoing: %s", - strerror(errno)); - return tal_free(peer); - } list_add_tail(&ld->peers, &peer->list); populate_peer(ld, peer); @@ -790,6 +785,28 @@ static void log_to_json(unsigned int skipped, json_add_string(info->response, NULL, log); } +static const char *ipaddr_name(const tal_t *ctx, const struct ipaddr *a) +{ + char name[INET6_ADDRSTRLEN]; + int af; + + switch (a->type) { + case ADDR_TYPE_IPV4: + af = AF_INET; + break; + case ADDR_TYPE_IPV6: + af = AF_INET6; + break; + default: + return tal_fmt(ctx, "Unknown type %u", a->type); + } + + if (!inet_ntop(af, a->addr, name, sizeof(name))) + sprintf(name, "Unprintable-%u-address", a->type); + + return tal_fmt(ctx, "%s:%u", name, a->port); +} + static void json_getpeers(struct command *cmd, const char *buffer, const jsmntok_t *params) { @@ -821,7 +838,7 @@ static void json_getpeers(struct command *cmd, json_object_start(response, NULL); json_add_string(response, "state", peer_state_name(p->state)); json_add_string(response, "netaddr", - netaddr_name(response, &p->netaddr)); + ipaddr_name(response, &p->addr)); json_add_pubkey(response, "peerid", &p->id); json_add_bool(response, "connected", p->owner != NULL); if (p->owner) @@ -2221,7 +2238,6 @@ static void peer_accept_channel(struct lightningd *ld, { u32 max_to_self_delay, max_minimum_depth; u64 min_effective_htlc_capacity_msat; - u8 *errmsg; u8 *msg; struct peer *peer; @@ -2229,13 +2245,6 @@ static void peer_accept_channel(struct lightningd *ld, /* We make a new peer. */ peer = new_peer(ld, peer_id, addr, gfeatures, lfeatures, peer_fd); - - /* FIXME: Only happens due to netaddr fail. */ - if (!peer) { - errmsg = towire_errorfmt(ld, NULL, "Can't resolve your address"); - goto peer_to_gossipd; - } - peer_set_condition(peer, UNINITIALIZED, OPENINGD); peer_set_owner(peer, new_peer_subd(ld, "lightning_openingd", peer, @@ -2285,17 +2294,6 @@ static void peer_accept_channel(struct lightningd *ld, subd_req(peer, peer->owner, take(msg), -1, 2, opening_fundee_finished, peer); - return; - -peer_to_gossipd: - /* Return to gossipd, with optional error msg to send. */ - msg = towire_gossipctl_handle_peer(ld, peer_id, addr, cs, - gfeatures, lfeatures, errmsg); - subd_send_msg(ld->gossip, take(msg)); - subd_send_fd(ld->gossip, peer_fd); - close(gossip_fd); - tal_free(errmsg); - return; } static void peer_offer_channel(struct lightningd *ld, @@ -2313,16 +2311,6 @@ static void peer_offer_channel(struct lightningd *ld, /* We make a new peer. */ fc->peer = new_peer(ld, &fc->peerid, addr, gfeatures, lfeatures, peer_fd); - - /* FIXME: Only happens due to netaddr fail. */ - if (!fc->peer) { - command_fail(fc->cmd, - "Failed to make peer: Can't resolve address: %s", - strerror(errno)); - close(peer_fd); - close(gossip_fd); - return; - } fc->peer->funding_satoshi = fc->funding_satoshi; fc->peer->push_msat = fc->push_msat; diff --git a/lightningd/peer_control.h b/lightningd/peer_control.h index 6c1971b294c7..6eb00ae1304f 100644 --- a/lightningd/peer_control.h +++ b/lightningd/peer_control.h @@ -7,7 +7,6 @@ #include #include #include -#include #include #include #include @@ -55,7 +54,6 @@ struct peer { u8 channel_flags; /* Where we connected to, or it connected from. */ - struct netaddr netaddr; struct ipaddr addr; /* Our channel config. */ From 329269d9d052f559efc37d0d525af6063ff6d2d8 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:45:38 +1030 Subject: [PATCH 0026/1428] lightningd: support multiple addresses. Currently only ipv4 and ipv6. Signed-off-by: Rusty Russell --- lightningd/jsonrpc.c | 10 ++++++---- lightningd/lightningd.c | 1 + lightningd/lightningd.h | 6 +++--- lightningd/options.c | 25 ++++++++++++------------- lightningd/options.h | 2 +- lightningd/peer_control.c | 9 +++++---- 6 files changed, 28 insertions(+), 25 deletions(-) diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index ef7c718d2802..10396c5d4baf 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -248,11 +248,13 @@ static void json_getinfo(struct command *cmd, json_object_start(response, NULL); json_add_pubkey(response, "id", &cmd->ld->id); - /* FIXME: Keep ipaddr and list them all. */ - if (cmd->ld->portnum) + if (cmd->ld->portnum) { json_add_num(response, "port", cmd->ld->portnum); - json_add_string(response, "network", - get_chainparams(cmd->ld)->network_name); + json_array_start(response, "address"); + for (size_t i = 0; i < tal_count(cmd->ld->wireaddrs); i++) + json_add_address(response, NULL, cmd->ld->wireaddrs+i); + json_array_end(response); + } json_add_string(response, "version", version()); json_add_num(response, "blockheight", get_block_height(cmd->ld->topology)); json_object_end(response); diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 42305dd8d467..8cb5af3e3348 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -79,6 +79,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx, ld->rgb = NULL; list_head_init(&ld->pay_commands); list_head_init(&ld->connects); + ld->wireaddrs = tal_arr(ld, struct ipaddr, 0); ld->portnum = DEFAULT_PORT; timers_init(&ld->timers, time_mono()); ld->topology = new_topology(ld, ld->log); diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 4bd37cd4b1ee..89c4b5408c57 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -62,9 +62,6 @@ struct config { /* How long between changing commit and sending COMMIT message. */ struct timerel commit_time; - /* IPv4 or IPv6 address to announce to the network */ - struct ipaddr ipaddr; - /* Disable automatic reconnects */ bool no_reconnect; }; @@ -97,6 +94,9 @@ struct lightningd { /* Port we're listening on */ u16 portnum; + /* Addresses to announce to the network (tal_count()) */ + struct ipaddr *wireaddrs; + /* Bearer of all my secrets. */ int hsm_fd; diff --git a/lightningd/options.c b/lightningd/options.c index 92d44b17c776..bc726d6873ae 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -118,7 +118,7 @@ static char *opt_set_s32(const char *arg, s32 *u) } /* FIXME: Rename ipaddr and hoist up */ -bool parse_ipaddr(const char *arg, struct ipaddr *addr) +bool parse_ipaddr(const char *arg, struct ipaddr *addr, u16 port) { struct in6_addr v6; struct in_addr v4; @@ -133,20 +133,26 @@ bool parse_ipaddr(const char *arg, struct ipaddr *addr) if (inet_pton(AF_INET, arg, &v4) == 1) { addr->type = ADDR_TYPE_IPV4; addr->addrlen = 4; + addr->port = port; memcpy(&addr->addr, &v4, addr->addrlen); return true; } else if (inet_pton(AF_INET6, arg, &v6) == 1) { addr->type = ADDR_TYPE_IPV6; addr->addrlen = 16; + addr->port = port; memcpy(&addr->addr, &v6, addr->addrlen); return true; } return false; } -static char *opt_set_ipaddr(const char *arg, struct ipaddr *addr) +static char *opt_add_ipaddr(const char *arg, struct lightningd *ld) { - if (parse_ipaddr(arg, addr)) + size_t n = tal_count(ld->wireaddrs); + + tal_resize(&ld->wireaddrs, n+1); + + if (parse_ipaddr(arg, &ld->wireaddrs[n], ld->portnum)) return NULL; return tal_fmt(NULL, "Unable to parse IP address '%s'", arg); @@ -275,9 +281,9 @@ static void config_register_opts(struct lightningd *ld) opt_register_noarg("--no-reconnect", opt_set_bool, &ld->config.no_reconnect, "Disable automatic reconnect attempts"); - opt_register_arg("--ipaddr", opt_set_ipaddr, NULL, - &ld->config.ipaddr, - "Set the IP address (v4 or v6) to announce to the network for incoming connections"); + opt_register_arg("--ipaddr", opt_add_ipaddr, NULL, + ld, + "Set the IP address (v4 or v6) to announce to the network for incoming connections"); opt_register_early_arg("--network", opt_set_network, opt_show_network, ld, @@ -366,9 +372,6 @@ static const struct config testnet_config = { /* Take 0.001% */ .fee_per_satoshi = 10, - /* Do not advertise any IP */ - .ipaddr.type = 0, - /* Automatically reconnect */ .no_reconnect = false, }; @@ -427,9 +430,6 @@ static const struct config mainnet_config = { /* Take 0.001% */ .fee_per_satoshi = 10, - /* Do not advertise any IP */ - .ipaddr.type = 0, - /* Automatically reconnect */ .no_reconnect = false, }; @@ -631,7 +631,6 @@ bool handle_opts(struct lightningd *ld, int argc, char *argv[]) /* Now look for config file */ opt_parse_from_config(ld); - ld->config.ipaddr.port = ld->portnum; opt_parse(&argc, argv, opt_log_stderr_exit); if (argc != 1) errx(1, "no arguments accepted"); diff --git a/lightningd/options.h b/lightningd/options.h index dab0e38188ca..2519a8c53128 100644 --- a/lightningd/options.h +++ b/lightningd/options.h @@ -13,7 +13,7 @@ void register_opts(struct lightningd *ld); */ bool handle_opts(struct lightningd *ld, int argc, char *argv[]); -bool parse_ipaddr(const char *arg, struct ipaddr *addr); +bool parse_ipaddr(const char *arg, struct ipaddr *addr, u16 port); /* Derive default color and alias from the pubkey. */ void setup_color_and_alias(struct lightningd *ld); diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 566b264445f2..2d0458c31083 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -741,7 +741,7 @@ static void json_connect(struct command *cmd, port = tal_strdup(cmd, stringify(DEFAULT_PORT)); } addr.port = atoi(port); - if (!parse_ipaddr(name, &addr) || !addr.port) + if (!parse_ipaddr(name, &addr, addr.port) || !addr.port) command_fail(cmd, "host %s:%s not valid", name, port); /* Tell it about the address. */ @@ -1472,13 +1472,14 @@ static u8 *create_node_announcement(const tal_t *ctx, struct lightningd *ld, u8 *features = NULL; u8 *addresses = tal_arr(ctx, u8, 0); u8 *announcement; + size_t i; if (!sig) { sig = tal(ctx, secp256k1_ecdsa_signature); memset(sig, 0, sizeof(*sig)); } - if (ld->config.ipaddr.type != ADDR_TYPE_PADDING) { - towire_ipaddr(&addresses, &ld->config.ipaddr); - } + for (i = 0; i < tal_count(ld->wireaddrs); i++) + towire_ipaddr(&addresses, ld->wireaddrs+i); + announcement = towire_node_announcement(ctx, sig, features, timestamp, &ld->id, ld->rgb, (u8 *)ld->alias, From 4bd03529511e3828735d7bbcd742a2e0cdd7685a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:46:38 +1030 Subject: [PATCH 0027/1428] lightningd: try to figure out our own IP automatically. Most of the code is from bitcoind, to handle the weird different non-public IP ranges. Signed-off-by: Rusty Russell --- lightningd/Makefile | 1 + lightningd/netaddress.c | 275 ++++++++++++++++++++++++++++++++++++++++ lightningd/netaddress.h | 9 ++ lightningd/options.c | 5 + 4 files changed, 290 insertions(+) create mode 100644 lightningd/netaddress.c create mode 100644 lightningd/netaddress.h diff --git a/lightningd/Makefile b/lightningd/Makefile index e4401f53da67..42a844247488 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -49,6 +49,7 @@ LIGHTNINGD_SRC := \ lightningd/jsonrpc.c \ lightningd/lightningd.c \ lightningd/log.c \ + lightningd/netaddress.c \ lightningd/opt_time.c \ lightningd/options.c \ lightningd/pay.c \ diff --git a/lightningd/netaddress.c b/lightningd/netaddress.c new file mode 100644 index 000000000000..b4328b1b135e --- /dev/null +++ b/lightningd/netaddress.c @@ -0,0 +1,275 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* Based on bitcoin's src/netaddress.cpp, hence different naming and styling! + version 7f31762cb6261806542cc6d1188ca07db98a6950: + + Copyright (c) 2009-2010 Satoshi Nakamoto + Copyright (c) 2009-2016 The Bitcoin Core developers + Distributed under the MIT software license, see the accompanying + file COPYING or http://www.opensource.org/licenses/mit-license.php. +*/ + +/* The common IPv4-in-IPv6 prefix */ +static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; + +static bool IsRFC6145(const struct ipaddr *addr) +{ + static const unsigned char pchRFC6145[] = {0,0,0,0,0,0,0,0,0xFF,0xFF,0,0}; + return addr->type == ADDR_TYPE_IPV6 + && memcmp(addr->addr, pchRFC6145, sizeof(pchRFC6145)) == 0; +} + +static bool IsRFC6052(const struct ipaddr *addr) +{ + static const unsigned char pchRFC6052[] = {0,0x64,0xFF,0x9B,0,0,0,0,0,0,0,0}; + return addr->type == ADDR_TYPE_IPV6 + && memcmp(addr->addr, pchRFC6052, sizeof(pchRFC6052)) == 0; +} + +static bool IsRFC3964(const struct ipaddr *addr) +{ + return addr->type == ADDR_TYPE_IPV6 + && addr->addr[0] == 0x20 && addr->addr[1] == 0x02; +} + +/* Return offset of IPv4 address, or 0 == not an IPv4 */ +static size_t IPv4In6(const struct ipaddr *addr) +{ + if (addr->type != ADDR_TYPE_IPV6) + return 0; + if (memcmp(addr->addr, pchIPv4, sizeof(pchIPv4)) == 0) + return sizeof(pchIPv4); + if (IsRFC6052(addr)) + return 12; + if (IsRFC6145(addr)) + return 12; + if (IsRFC3964(addr)) + return 2; + return 0; +} + +/* Is this an IPv4 address, or an IPv6-wrapped IPv4 */ +static bool IsIPv4(const struct ipaddr *addr) +{ + return addr->type == ADDR_TYPE_IPV4 || IPv4In6(addr) != 0; +} + +static bool IsIPv6(const struct ipaddr *addr) +{ + return addr->type == ADDR_TYPE_IPV6 && IPv4In6(addr) == 0; +} + +static bool RawEq(const struct ipaddr *addr, const void *cmp, size_t len) +{ + size_t off = IPv4In6(addr); + + assert(off + len <= addr->addrlen); + return memcmp(addr->addr + off, cmp, len) == 0; +} + +/* The bitcoin code packs addresses backwards, so we map it here. */ +static unsigned int GetByte(const struct ipaddr *addr, int n) +{ + size_t off = IPv4In6(addr); + assert(off + n < addr->addrlen); + return addr->addr[addr->addrlen - 1 - off - n]; +} + +static bool IsRFC1918(const struct ipaddr *addr) +{ + return IsIPv4(addr) && ( + GetByte(addr, 3) == 10 || + (GetByte(addr, 3) == 192 && GetByte(addr, 2) == 168) || + (GetByte(addr, 3) == 172 && (GetByte(addr, 2) >= 16 && GetByte(addr, 2) <= 31))); +} + +static bool IsRFC2544(const struct ipaddr *addr) +{ + return IsIPv4(addr) && GetByte(addr, 3) == 198 && (GetByte(addr, 2) == 18 || GetByte(addr, 2) == 19); +} + +static bool IsRFC3927(const struct ipaddr *addr) +{ + return IsIPv4(addr) && (GetByte(addr, 3) == 169 && GetByte(addr, 2) == 254); +} + +static bool IsRFC6598(const struct ipaddr *addr) +{ + return IsIPv4(addr) && GetByte(addr, 3) == 100 && GetByte(addr, 2) >= 64 && GetByte(addr, 2) <= 127; +} + +static bool IsRFC5737(const struct ipaddr *addr) +{ + return IsIPv4(addr) && ((GetByte(addr, 3) == 192 && GetByte(addr, 2) == 0 && GetByte(addr, 1) == 2) || + (GetByte(addr, 3) == 198 && GetByte(addr, 2) == 51 && GetByte(addr, 1) == 100) || + (GetByte(addr, 3) == 203 && GetByte(addr, 2) == 0 && GetByte(addr, 1) == 113)); +} + +static bool IsRFC3849(const struct ipaddr *addr) +{ + return IsIPv6(addr) && GetByte(addr, 15) == 0x20 && GetByte(addr, 14) == 0x01 && GetByte(addr, 13) == 0x0D && GetByte(addr, 12) == 0xB8; +} + +static bool IsRFC4862(const struct ipaddr *addr) +{ + static const unsigned char pchRFC4862[] = {0xFE,0x80,0,0,0,0,0,0}; + return IsIPv6(addr) && RawEq(addr, pchRFC4862, sizeof(pchRFC4862)); +} + +static bool IsRFC4193(const struct ipaddr *addr) +{ + return IsIPv6(addr) && ((GetByte(addr, 15) & 0xFE) == 0xFC); +} + +static bool IsRFC4843(const struct ipaddr *addr) +{ + return IsIPv6(addr) && (GetByte(addr, 15) == 0x20 && GetByte(addr, 14) == 0x01 && GetByte(addr, 13) == 0x00 && (GetByte(addr, 12) & 0xF0) == 0x10); +} + +static bool IsTor(const struct ipaddr *addr) +{ + /* FIXME */ + return false; +} + +static bool IsLocal(const struct ipaddr *addr) +{ + // IPv4 loopback + if (IsIPv4(addr) && (GetByte(addr, 3) == 127 || GetByte(addr, 3) == 0)) + return true; + + // IPv6 loopback (::1/128) + static const unsigned char pchLocal[16] = {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1}; + if (IsIPv6(addr) && RawEq(addr, pchLocal, sizeof(pchLocal))) + return true; + + return false; +} + +static bool IsInternal(const struct ipaddr *addr) +{ + return addr->type == ADDR_TYPE_PADDING; +} + +static bool IsValid(const struct ipaddr *addr) +{ + // unspecified IPv6 address (::/128) + unsigned char ipNone6[16] = {}; + if (IsIPv6(addr) && RawEq(addr, ipNone6, sizeof(ipNone6))) + return false; + + // documentation IPv6 address + if (IsRFC3849(addr)) + return false; + + if (IsInternal(addr)) + return false; + + if (IsIPv4(addr)) + { + // INADDR_NONE + uint32_t ipNone = INADDR_NONE; + if (RawEq(addr, &ipNone, sizeof(ipNone))) + return false; + + // 0 + ipNone = 0; + if (RawEq(addr, &ipNone, sizeof(ipNone))) + return false; + } + + return true; +} + +static bool IsRoutable(const struct ipaddr *addr) +{ + return IsValid(addr) && !(IsRFC1918(addr) || IsRFC2544(addr) || IsRFC3927(addr) || IsRFC4862(addr) || IsRFC6598(addr) || IsRFC5737(addr) || (IsRFC4193(addr) && !IsTor(addr)) || IsRFC4843(addr) || IsLocal(addr) || IsInternal(addr)); +} + +/* Trick I learned from Harald Welte: create UDP socket, connect() and + * then query address. */ +static bool get_local_sockname(int af, void *saddr, socklen_t saddrlen) +{ + int fd = socket(af, SOCK_DGRAM, 0); + if (fd < 0) + return false; + + if (connect(fd, saddr, saddrlen) != 0) { + close(fd); + return false; + } + + if (getsockname(fd, saddr, &saddrlen) != 0) { + close(fd); + return false; + } + + close(fd); + return true; +} + +/* Return an ipaddr without port filled in */ +static bool guess_one_address(struct ipaddr *addr, u16 portnum, + enum wire_addr_type type) +{ + addr->type = type; + addr->port = portnum; + + /* We point to Google nameservers, works unless you're inside Google :) */ + switch (type) { + case ADDR_TYPE_IPV4: { + struct sockaddr_in sin; + sin.sin_port = htons(53); + /* 8.8.8.8 */ + sin.sin_addr.s_addr = 0x08080808; + if (!get_local_sockname(AF_INET, &sin, sizeof(sin))) + return false; + addr->addrlen = sizeof(sin.sin_addr); + memcpy(addr->addr, &sin.sin_addr, addr->addrlen); + break; + } + case ADDR_TYPE_IPV6: { + struct sockaddr_in6 sin6; + /* 2001:4860:4860::8888 */ + static const unsigned char pchGoogle[16] + = {0x20,0x01,0x48,0x60,0x48,0x60,0,0,0,0,0,0,8,8,8,8}; + memset(&sin6, 0, sizeof(sin6)); + sin6.sin6_port = htons(53); + memcpy(sin6.sin6_addr.s6_addr, pchGoogle, sizeof(pchGoogle)); + if (!get_local_sockname(AF_INET6, &sin6, sizeof(sin6))) + return false; + addr->addrlen = sizeof(sin6.sin6_addr); + memcpy(addr->addr, &sin6.sin6_addr, addr->addrlen); + break; + } + case ADDR_TYPE_PADDING: + return false; + } + + if (!IsRoutable(addr)) + return false; + + return true; +} + +void guess_addresses(struct lightningd *ld) +{ + size_t n = tal_count(ld->wireaddrs); + + /* We allocate an extra, then remove if it's not needed. */ + tal_resize(&ld->wireaddrs, n+1); + if (guess_one_address(&ld->wireaddrs[n], ld->portnum, ADDR_TYPE_IPV4)) { + n++; + tal_resize(&ld->wireaddrs, n+1); + } + if (!guess_one_address(&ld->wireaddrs[n], ld->portnum, ADDR_TYPE_IPV6)) + tal_resize(&ld->wireaddrs, n); +} diff --git a/lightningd/netaddress.h b/lightningd/netaddress.h new file mode 100644 index 000000000000..7af7f779d586 --- /dev/null +++ b/lightningd/netaddress.h @@ -0,0 +1,9 @@ +#ifndef LIGHTNING_LIGHTNINGD_NETADDRESS_H +#define LIGHTNING_LIGHTNINGD_NETADDRESS_H +#include "config.h" + +struct lightningd; + +void guess_addresses(struct lightningd *ld); + +#endif /* LIGHTNING_LIGHTNINGD_NETADDRESS_H */ diff --git a/lightningd/options.c b/lightningd/options.c index bc726d6873ae..248a340ad5a0 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -537,6 +538,10 @@ static void opt_parse_from_config(struct lightningd *ld) setup_default_config(ld); opt_parse(&argc, argv, config_log_stderr_exit); + + if (ld->portnum && tal_count(ld->wireaddrs) == 0) + guess_addresses(ld); + tal_free(contents); } From 78cd25d62073d1cd5fbf821e28bc0b6999ef81ed Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:47:38 +1030 Subject: [PATCH 0028/1428] ipaddr: rename to wireaddr. In future it will have TOR support, so the name will be awkward. We collect the to/fromwire functions in common/wireaddr.c, and the parsing functions in lightningd/netaddress.c. Signed-off-by: Rusty Russell --- common/Makefile | 1 + common/json.h | 1 - common/wireaddr.c | 30 ++++++++++ common/wireaddr.h | 41 ++++++++++++++ gossipd/Makefile | 1 + gossipd/gossip.c | 17 +++--- gossipd/gossip_wire.csv | 11 ++-- gossipd/handshake.c | 17 +++--- gossipd/handshake.h | 14 ++--- gossipd/routing.c | 27 ++++----- gossipd/routing.h | 2 +- gossipd/test/run-find_route-specific.c | 6 +- gossipd/test/run-find_route.c | 6 +- gossipd/test/run-initiator-success.c | 4 +- gossipd/test/run-responder-success.c | 4 +- lightningd/Makefile | 1 + lightningd/gossip_control.c | 2 +- lightningd/gossip_msg.c | 7 ++- lightningd/gossip_msg.h | 2 +- lightningd/jsonrpc.c | 3 +- lightningd/jsonrpc.h | 4 +- lightningd/lightningd.c | 3 +- lightningd/lightningd.h | 2 +- lightningd/netaddress.c | 78 ++++++++++++++++++-------- lightningd/netaddress.h | 3 + lightningd/options.c | 33 +---------- lightningd/options.h | 2 - lightningd/peer_control.c | 27 ++++----- lightningd/peer_control.h | 5 +- wire/fromwire.c | 31 ---------- wire/towire.c | 7 --- wire/wire.h | 21 ------- 32 files changed, 220 insertions(+), 193 deletions(-) create mode 100644 common/wireaddr.c create mode 100644 common/wireaddr.h diff --git a/common/Makefile b/common/Makefile index 9ea159cb8476..5290afe192cd 100644 --- a/common/Makefile +++ b/common/Makefile @@ -32,6 +32,7 @@ COMMON_SRC := \ common/utils.c \ common/utxo.c \ common/version.c \ + common/wireaddr.c \ common/wire_error.c \ common/withdraw_tx.c diff --git a/common/json.h b/common/json.h index badf3bf93fda..b0501b6d02ed 100644 --- a/common/json.h +++ b/common/json.h @@ -10,7 +10,6 @@ #define JSMN_STRICT 1 # include -struct ipaddr; struct json_result; struct short_channel_id; diff --git a/common/wireaddr.c b/common/wireaddr.c new file mode 100644 index 000000000000..784f304f0b70 --- /dev/null +++ b/common/wireaddr.c @@ -0,0 +1,30 @@ +#include +#include + +/* Returns false if we didn't parse it, and *cursor == NULL if malformed. */ +bool fromwire_wireaddr(const u8 **cursor, size_t *max, struct wireaddr *addr) +{ + addr->type = fromwire_u8(cursor, max); + + switch (addr->type) { + case ADDR_TYPE_IPV4: + addr->addrlen = 4; + break; + case ADDR_TYPE_IPV6: + addr->addrlen = 16; + break; + default: + return false; + } + fromwire(cursor, max, addr->addr, addr->addrlen); + addr->port = fromwire_u16(cursor, max); + + return *cursor != NULL; +} + +void towire_wireaddr(u8 **pptr, const struct wireaddr *addr) +{ + towire_u8(pptr, addr->type); + towire(pptr, addr->addr, addr->addrlen); + towire_u16(pptr, addr->port); +} diff --git a/common/wireaddr.h b/common/wireaddr.h new file mode 100644 index 000000000000..ae196b9ccaf4 --- /dev/null +++ b/common/wireaddr.h @@ -0,0 +1,41 @@ +#ifndef LIGHTNING_COMMON_WIREADDR_H +#define LIGHTNING_COMMON_WIREADDR_H +#include "config.h" +#include +#include +#include + +/* BOLT #7: + * + * The following `address descriptor` types are defined: + * + * * `0`: padding. data = none (length 0). + * * `1`: ipv4. data = `[4:ipv4_addr][2:port]` (length 6) + * * `2`: ipv6. data = `[16:ipv6_addr][2:port]` (length 18) + * * `3`: tor v2 onion service. data = `[10:onion_addr][2:port]` (length 12) + * * Version 2 onion service addresses. Encodes an 80-bit truncated `SHA-1` hash + * of a 1024-bit `RSA` public key for the onion service. + * * `4`: tor v3 onion service. data `[35:onion_addr][2:port]` (length 37) + * * Version 3 ([prop224](https://gitweb.torproject.org/torspec.git/tree/proposals/224-rend-spec-ng.txt)) + * onion service addresses. Encodes: `[32:32_byte_ed25519_pubkey] || [2:checksum] || [1:version]`. + * where `checksum = sha3(".onion checksum" | pubkey || version)[:2]` + */ + +enum wire_addr_type { + ADDR_TYPE_PADDING = 0, + ADDR_TYPE_IPV4 = 1, + ADDR_TYPE_IPV6 = 2, +}; + +/* FIXME(cdecker) Extend this once we have defined how TOR addresses + * should look like */ +struct wireaddr { + enum wire_addr_type type; + u8 addrlen; + u8 addr[16]; + u16 port; +}; + +void towire_wireaddr(u8 **pptr, const struct wireaddr *addr); +bool fromwire_wireaddr(const u8 **cursor, size_t *max, struct wireaddr *addr); +#endif /* LIGHTNING_COMMON_WIREADDR_H */ diff --git a/gossipd/Makefile b/gossipd/Makefile index 26b8e60e72c1..2111ed346f19 100644 --- a/gossipd/Makefile +++ b/gossipd/Makefile @@ -49,6 +49,7 @@ GOSSIPD_COMMON_OBJS := \ common/type_to_string.o \ common/utils.o \ common/version.o \ + common/wireaddr.o \ common/wire_error.o \ hsmd/client.o \ hsmd/gen_hsm_client_wire.o \ diff --git a/gossipd/gossip.c b/gossipd/gossip.c index ac9f1fe58689..b48edd43b8eb 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -23,6 +23,7 @@ #include #include #include +#include #include #include #include @@ -83,7 +84,7 @@ struct reaching { struct pubkey id; /* Where I'm reaching to. */ - struct ipaddr addr; + struct wireaddr addr; /* Did we succeed? */ bool succeeded; @@ -99,7 +100,7 @@ struct peer { struct pubkey id; /* Where it's connected to. */ - struct ipaddr addr; + struct wireaddr addr; /* Feature bitmaps. */ u8 *gfeatures, *lfeatures; @@ -147,7 +148,7 @@ struct addrhint { struct pubkey id; /* FIXME: use array... */ - struct ipaddr addr; + struct wireaddr addr; }; /* FIXME: Reorder */ @@ -194,7 +195,7 @@ static struct addrhint *find_addrhint(struct daemon *daemon, static struct peer *new_peer(const tal_t *ctx, struct daemon *daemon, const struct pubkey *their_id, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs) { struct peer *peer = tal(ctx, struct peer); @@ -342,7 +343,7 @@ static struct io_plan *read_init(struct io_conn *conn, struct peer *peer) * we have the features. */ static struct io_plan *init_new_peer(struct io_conn *conn, const struct pubkey *their_id, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, struct daemon *daemon) { @@ -718,7 +719,7 @@ static struct io_plan *handle_peer(struct io_conn *conn, struct daemon *daemon, struct peer *peer; struct crypto_state cs; struct pubkey id; - struct ipaddr addr; + struct wireaddr addr; u8 *gfeatures, *lfeatures; u8 *inner_msg; @@ -949,7 +950,7 @@ static int make_listen_fd(int domain, void *addr, socklen_t len) static struct io_plan *connection_in(struct io_conn *conn, struct daemon *daemon) { - struct ipaddr addr; + struct wireaddr addr; struct sockaddr_storage s; socklen_t len = sizeof(s); @@ -1112,7 +1113,7 @@ static void handle_forwarded_msg(struct io_conn *conn, struct daemon *daemon, co static struct io_plan *handshake_out_success(struct io_conn *conn, const struct pubkey *id, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, struct reaching *reach) { diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index eafdc721791d..020d1cdc647d 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -1,4 +1,5 @@ #include +#include # Initialize the gossip daemon. gossipctl_init,3000 @@ -15,7 +16,7 @@ gossipctl_init,,lfeatures,lflen*u8 # Master -> gossipd: Optional hint for where to find peer. gossipctl_peer_addrhint,3014 gossipctl_peer_addrhint,,id,struct pubkey -gossipctl_peer_addrhint,,addr,struct ipaddr +gossipctl_peer_addrhint,,addr,struct wireaddr # Master -> gossipd: connect to a peer. We may get a peer_connected. gossipctl_reach_peer,3001 @@ -24,7 +25,7 @@ gossipctl_reach_peer,,id,struct pubkey # Gossipd -> master: we got a peer. Two fds: peer and gossip gossip_peer_connected,3002 gossip_peer_connected,,id,struct pubkey -gossip_peer_connected,,addr,struct ipaddr +gossip_peer_connected,,addr,struct wireaddr gossip_peer_connected,,crypto_state,struct crypto_state gossip_peer_connected,,gflen,u16 gossip_peer_connected,,gfeatures,gflen*u8 @@ -34,7 +35,7 @@ gossip_peer_connected,,lfeatures,lflen*u8 # Gossipd -> master: peer sent non-gossip packet. Two fds: peer and gossip gossip_peer_nongossip,3003 gossip_peer_nongossip,,id,struct pubkey -gossip_peer_nongossip,,addr,struct ipaddr +gossip_peer_nongossip,,addr,struct wireaddr gossip_peer_nongossip,,crypto_state,struct crypto_state gossip_peer_nongossip,,gflen,u16 gossip_peer_nongossip,,gfeatures,gflen*u8 @@ -49,7 +50,7 @@ gossipctl_release_peer,,id,struct pubkey # Gossipd -> master: reply to gossip_release_peer. Two fds: peer and gossip. gossipctl_release_peer_reply,3104 -gossipctl_release_peer_reply,,addr,struct ipaddr +gossipctl_release_peer_reply,,addr,struct wireaddr gossipctl_release_peer_reply,,crypto_state,struct crypto_state gossipctl_release_peer_reply,,gflen,u16 gossipctl_release_peer_reply,,gfeatures,gflen*u8 @@ -62,7 +63,7 @@ gossipctl_release_peer_replyfail,3204 # Gossipd -> master: take over peer, with optional msg. (+peer fd) gossipctl_handle_peer,3013 gossipctl_handle_peer,,id,struct pubkey -gossipctl_handle_peer,,addr,struct ipaddr +gossipctl_handle_peer,,addr,struct wireaddr gossipctl_handle_peer,,crypto_state,struct crypto_state gossipctl_handle_peer,,gflen,u16 gossipctl_handle_peer,,gfeatures,gflen*u8 diff --git a/gossipd/handshake.c b/gossipd/handshake.c index 7cefb2df97ee..7a61e5d85b17 100644 --- a/gossipd/handshake.c +++ b/gossipd/handshake.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -171,7 +172,7 @@ struct handshake { struct act_three act3; /* Where is connection from/to */ - struct ipaddr addr; + struct wireaddr addr; /* Who we are */ struct pubkey my_id; @@ -184,7 +185,7 @@ struct handshake { /* Function to call once handshake complete. */ struct io_plan *(*cb)(struct io_conn *conn, const struct pubkey *their_id, - const struct ipaddr *ipaddr, + const struct wireaddr *wireaddr, const struct crypto_state *cs, void *cbarg); void *cbarg; @@ -353,12 +354,12 @@ static struct io_plan *handshake_succeeded(struct io_conn *conn, struct crypto_state cs; struct io_plan *(*cb)(struct io_conn *conn, const struct pubkey *their_id, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, void *cbarg); void *cbarg; struct pubkey their_id; - struct ipaddr addr; + struct wireaddr addr; /* BOLT #8: * @@ -961,10 +962,10 @@ static struct io_plan *act_one_responder(struct io_conn *conn, struct io_plan *responder_handshake_(struct io_conn *conn, const struct pubkey *my_id, - const struct ipaddr *addr, + const struct wireaddr *addr, struct io_plan *(*cb)(struct io_conn *, const struct pubkey *, - const struct ipaddr *, + const struct wireaddr *, const struct crypto_state *, void *cbarg), void *cbarg) @@ -983,10 +984,10 @@ struct io_plan *responder_handshake_(struct io_conn *conn, struct io_plan *initiator_handshake_(struct io_conn *conn, const struct pubkey *my_id, const struct pubkey *their_id, - const struct ipaddr *addr, + const struct wireaddr *addr, struct io_plan *(*cb)(struct io_conn *, const struct pubkey *, - const struct ipaddr *, + const struct wireaddr *, const struct crypto_state *, void *cbarg), void *cbarg) diff --git a/gossipd/handshake.h b/gossipd/handshake.h index 343dc501b161..4ecc988e0a4a 100644 --- a/gossipd/handshake.h +++ b/gossipd/handshake.h @@ -5,7 +5,7 @@ struct crypto_state; struct io_conn; -struct ipaddr; +struct wireaddr; struct pubkey; #define initiator_handshake(conn, my_id, their_id, addr, cb, cbarg) \ @@ -14,7 +14,7 @@ struct pubkey; (cb), (cbarg), \ struct io_conn *, \ const struct pubkey *, \ - const struct ipaddr *, \ + const struct wireaddr *, \ const struct crypto_state *), \ (cbarg)) @@ -22,10 +22,10 @@ struct pubkey; struct io_plan *initiator_handshake_(struct io_conn *conn, const struct pubkey *my_id, const struct pubkey *their_id, - const struct ipaddr *addr, + const struct wireaddr *addr, struct io_plan *(*cb)(struct io_conn *, const struct pubkey *, - const struct ipaddr *, + const struct wireaddr *, const struct crypto_state *, void *cbarg), void *cbarg); @@ -37,16 +37,16 @@ struct io_plan *initiator_handshake_(struct io_conn *conn, (cb), (cbarg), \ struct io_conn *, \ const struct pubkey *, \ - const struct ipaddr *, \ + const struct wireaddr *, \ const struct crypto_state *), \ (cbarg)) struct io_plan *responder_handshake_(struct io_conn *conn, const struct pubkey *my_id, - const struct ipaddr *addr, + const struct wireaddr *addr, struct io_plan *(*cb)(struct io_conn *, const struct pubkey *, - const struct ipaddr *, + const struct wireaddr *, const struct crypto_state *, void *cbarg), void *cbarg); diff --git a/gossipd/routing.c b/gossipd/routing.c index e51ef4992ec9..1661e67d1fc6 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -79,7 +80,7 @@ static struct node *new_node(struct routing_state *rstate, n->alias = NULL; n->node_announcement = NULL; n->last_timestamp = -1; - n->addresses = tal_arr(n, struct ipaddr, 0); + n->addresses = tal_arr(n, struct wireaddr, 0); node_map_add(rstate->nodes, n); tal_add_destructor(n, destroy_node); @@ -628,14 +629,14 @@ void handle_channel_update(struct routing_state *rstate, const u8 *update, size_ tal_free(tmpctx); } -static struct ipaddr *read_addresses(const tal_t *ctx, const u8 *ser) +static struct wireaddr *read_addresses(const tal_t *ctx, const u8 *ser) { const u8 *cursor = ser; size_t max = tal_len(ser); - struct ipaddr *ipaddrs = tal_arr(ctx, struct ipaddr, 0); + struct wireaddr *wireaddrs = tal_arr(ctx, struct wireaddr, 0); int numaddrs = 0; while (cursor && cursor < ser + max) { - struct ipaddr ipaddr; + struct wireaddr wireaddr; /* Skip any padding */ while (max && cursor[0] == ADDR_TYPE_PADDING) @@ -647,19 +648,19 @@ static struct ipaddr *read_addresses(const tal_t *ctx, const u8 *ser) * descriptor` which does not match the types defined * above. */ - if (!fromwire_ipaddr(&cursor, &max, &ipaddr)) { + if (!fromwire_wireaddr(&cursor, &max, &wireaddr)) { if (!cursor) /* Parsing address failed */ - return tal_free(ipaddrs); + return tal_free(wireaddrs); /* Unknown type, stop there. */ break; } - tal_resize(&ipaddrs, numaddrs+1); - ipaddrs[numaddrs] = ipaddr; + tal_resize(&wireaddrs, numaddrs+1); + wireaddrs[numaddrs] = wireaddr; numaddrs++; } - return ipaddrs; + return wireaddrs; } void handle_node_announcement( @@ -675,7 +676,7 @@ void handle_node_announcement( u8 alias[32]; u8 *features, *addresses; const tal_t *tmpctx = tal_tmpctx(rstate); - struct ipaddr *ipaddrs; + struct wireaddr *wireaddrs; serialized = tal_dup_arr(tmpctx, u8, node_ann, len, 0); if (!fromwire_node_announcement(tmpctx, serialized, NULL, @@ -708,14 +709,14 @@ void handle_node_announcement( return; } - ipaddrs = read_addresses(tmpctx, addresses); - if (!ipaddrs) { + wireaddrs = read_addresses(tmpctx, addresses); + if (!wireaddrs) { status_trace("Unable to parse addresses."); tal_free(serialized); return; } tal_free(node->addresses); - node->addresses = tal_steal(node, ipaddrs); + node->addresses = tal_steal(node, wireaddrs); node->last_timestamp = timestamp; diff --git a/gossipd/routing.h b/gossipd/routing.h index b3e562b25bec..6139a946aea1 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -46,7 +46,7 @@ struct node { s64 last_timestamp; /* IP/Hostname and port of this node (may be NULL) */ - struct ipaddr *addresses; + struct wireaddr *addresses; /* Routes connecting to us, from us. */ struct node_connection **in, **out; diff --git a/gossipd/test/run-find_route-specific.c b/gossipd/test/run-find_route-specific.c index 247376db6d19..3581a543a913 100644 --- a/gossipd/test/run-find_route-specific.c +++ b/gossipd/test/run-find_route-specific.c @@ -25,15 +25,15 @@ bool fromwire_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNE /* Generated stub for fromwire_channel_update */ bool fromwire_channel_update(const void *p UNNEEDED, size_t *plen UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, struct sha256_double *chain_hash UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, u32 *timestamp UNNEEDED, u16 *flags UNNEEDED, u16 *cltv_expiry_delta UNNEEDED, u64 *htlc_minimum_msat UNNEEDED, u32 *fee_base_msat UNNEEDED, u32 *fee_proportional_millionths UNNEEDED) { fprintf(stderr, "fromwire_channel_update called!\n"); abort(); } -/* Generated stub for fromwire_ipaddr */ -bool fromwire_ipaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct ipaddr *addr UNNEEDED) -{ fprintf(stderr, "fromwire_ipaddr called!\n"); abort(); } /* Generated stub for fromwire_node_announcement */ bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, size_t *plen UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct pubkey *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED) { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); } /* Generated stub for fromwire_u8 */ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) { fprintf(stderr, "fromwire_u8 called!\n"); abort(); } +/* Generated stub for fromwire_wireaddr */ +bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) +{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); } /* Generated stub for queue_broadcast */ void queue_broadcast(struct broadcast_state *bstate UNNEEDED, const int type UNNEEDED, diff --git a/gossipd/test/run-find_route.c b/gossipd/test/run-find_route.c index b286e78a9be6..236f3e8e2aa8 100644 --- a/gossipd/test/run-find_route.c +++ b/gossipd/test/run-find_route.c @@ -18,15 +18,15 @@ bool fromwire_channel_announcement(const tal_t *ctx UNNEEDED, const void *p UNNE /* Generated stub for fromwire_channel_update */ bool fromwire_channel_update(const void *p UNNEEDED, size_t *plen UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, struct sha256_double *chain_hash UNNEEDED, struct short_channel_id *short_channel_id UNNEEDED, u32 *timestamp UNNEEDED, u16 *flags UNNEEDED, u16 *cltv_expiry_delta UNNEEDED, u64 *htlc_minimum_msat UNNEEDED, u32 *fee_base_msat UNNEEDED, u32 *fee_proportional_millionths UNNEEDED) { fprintf(stderr, "fromwire_channel_update called!\n"); abort(); } -/* Generated stub for fromwire_ipaddr */ -bool fromwire_ipaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct ipaddr *addr UNNEEDED) -{ fprintf(stderr, "fromwire_ipaddr called!\n"); abort(); } /* Generated stub for fromwire_node_announcement */ bool fromwire_node_announcement(const tal_t *ctx UNNEEDED, const void *p UNNEEDED, size_t *plen UNNEEDED, secp256k1_ecdsa_signature *signature UNNEEDED, u8 **features UNNEEDED, u32 *timestamp UNNEEDED, struct pubkey *node_id UNNEEDED, u8 rgb_color[3] UNNEEDED, u8 alias[32] UNNEEDED, u8 **addresses UNNEEDED) { fprintf(stderr, "fromwire_node_announcement called!\n"); abort(); } /* Generated stub for fromwire_u8 */ u8 fromwire_u8(const u8 **cursor UNNEEDED, size_t *max UNNEEDED) { fprintf(stderr, "fromwire_u8 called!\n"); abort(); } +/* Generated stub for fromwire_wireaddr */ +bool fromwire_wireaddr(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct wireaddr *addr UNNEEDED) +{ fprintf(stderr, "fromwire_wireaddr called!\n"); abort(); } /* Generated stub for queue_broadcast */ void queue_broadcast(struct broadcast_state *bstate UNNEEDED, const int type UNNEEDED, diff --git a/gossipd/test/run-initiator-success.c b/gossipd/test/run-initiator-success.c index 8f87df8967c3..05367f33e93e 100644 --- a/gossipd/test/run-initiator-success.c +++ b/gossipd/test/run-initiator-success.c @@ -176,7 +176,7 @@ static struct io_plan *test_read(struct io_conn *conn, static struct io_plan *success(struct io_conn *conn, const struct pubkey *them, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, void *ctx) { @@ -200,7 +200,7 @@ bool hsm_do_ecdh(struct secret *ss, const struct pubkey *point) int main(void) { tal_t *ctx = tal_tmpctx(NULL); - struct ipaddr dummy; + struct wireaddr dummy; trc = tal_tmpctx(ctx); diff --git a/gossipd/test/run-responder-success.c b/gossipd/test/run-responder-success.c index ab3799d9865b..d23e5c3bccb1 100644 --- a/gossipd/test/run-responder-success.c +++ b/gossipd/test/run-responder-success.c @@ -176,7 +176,7 @@ static struct io_plan *test_read(struct io_conn *conn, static struct io_plan *success(struct io_conn *conn, const struct pubkey *them, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, void *ctx) { @@ -198,7 +198,7 @@ bool hsm_do_ecdh(struct secret *ss, const struct pubkey *point) int main(void) { tal_t *ctx = tal_tmpctx(NULL); - struct ipaddr dummy; + struct wireaddr dummy; trc = tal_tmpctx(ctx); diff --git a/lightningd/Makefile b/lightningd/Makefile index 42a844247488..061c9233f9e2 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -35,6 +35,7 @@ LIGHTNINGD_COMMON_OBJS := \ common/utxo.o \ common/version.o \ common/wire_error.o \ + common/wireaddr.o \ common/withdraw_tx.o LIGHTNINGD_SRC := \ diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 91efb8593689..6de52a94b8d2 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -24,7 +24,7 @@ static void peer_nongossip(struct subd *gossip, const u8 *msg, { struct pubkey id; struct crypto_state cs; - struct ipaddr addr; + struct wireaddr addr; u8 *gfeatures, *lfeatures, *in_pkt; if (!fromwire_gossip_peer_nongossip(msg, msg, NULL, diff --git a/lightningd/gossip_msg.c b/lightningd/gossip_msg.c index 4d95ca8dd7d9..6ad59863a206 100644 --- a/lightningd/gossip_msg.c +++ b/lightningd/gossip_msg.c @@ -1,3 +1,4 @@ +#include #include #include @@ -7,10 +8,10 @@ void fromwire_gossip_getnodes_entry(const tal_t *ctx, const u8 **pptr, size_t *m fromwire_pubkey(pptr, max, &entry->nodeid); numaddresses = fromwire_u8(pptr, max); - entry->addresses = tal_arr(ctx, struct ipaddr, numaddresses); + entry->addresses = tal_arr(ctx, struct wireaddr, numaddresses); for (i=0; iaddresses)) { + if (!fromwire_wireaddr(pptr, max, entry->addresses)) { fromwire_fail(pptr, max); return; } @@ -23,7 +24,7 @@ void towire_gossip_getnodes_entry(u8 **pptr, const struct gossip_getnodes_entry towire_u8(pptr, numaddresses); for (i=0; iaddresses[i]); + towire_wireaddr(pptr, &entry->addresses[i]); } } diff --git a/lightningd/gossip_msg.h b/lightningd/gossip_msg.h index 652303423216..d594f444a8d9 100644 --- a/lightningd/gossip_msg.h +++ b/lightningd/gossip_msg.h @@ -6,7 +6,7 @@ struct gossip_getnodes_entry { struct pubkey nodeid; - struct ipaddr *addresses; + struct wireaddr *addresses; }; struct gossip_getchannels_entry { diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index 10396c5d4baf..318814b35fdc 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -372,7 +373,7 @@ void json_add_short_channel_id(struct json_result *response, } void json_add_address(struct json_result *response, const char *fieldname, - const struct ipaddr *addr) + const struct wireaddr *addr) { json_object_start(response, fieldname); char *addrstr = tal_arr(response, char, INET6_ADDRSTRLEN); diff --git a/lightningd/jsonrpc.h b/lightningd/jsonrpc.h index 22507d4be955..3627a397f33e 100644 --- a/lightningd/jsonrpc.h +++ b/lightningd/jsonrpc.h @@ -5,6 +5,8 @@ #include #include +struct wireaddr; + /* Context for a command (from JSON, but might outlive the connection!) * You can allocate off this for temporary objects. */ struct command { @@ -70,7 +72,7 @@ void json_add_short_channel_id(struct json_result *response, /* JSON serialize a network address for a node */ void json_add_address(struct json_result *response, const char *fieldname, - const struct ipaddr *addr); + const struct wireaddr *addr); /* For initialization */ diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 8cb5af3e3348..3a3e1336e777 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -79,7 +80,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx, ld->rgb = NULL; list_head_init(&ld->pay_commands); list_head_init(&ld->connects); - ld->wireaddrs = tal_arr(ld, struct ipaddr, 0); + ld->wireaddrs = tal_arr(ld, struct wireaddr, 0); ld->portnum = DEFAULT_PORT; timers_init(&ld->timers, time_mono()); ld->topology = new_topology(ld, ld->log); diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 89c4b5408c57..7b1af26ac8c7 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -95,7 +95,7 @@ struct lightningd { u16 portnum; /* Addresses to announce to the network (tal_count()) */ - struct ipaddr *wireaddrs; + struct wireaddr *wireaddrs; /* Bearer of all my secrets. */ int hsm_fd; diff --git a/lightningd/netaddress.c b/lightningd/netaddress.c index b4328b1b135e..8b04012dea92 100644 --- a/lightningd/netaddress.c +++ b/lightningd/netaddress.c @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -20,28 +22,28 @@ /* The common IPv4-in-IPv6 prefix */ static const unsigned char pchIPv4[12] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0xff, 0xff }; -static bool IsRFC6145(const struct ipaddr *addr) +static bool IsRFC6145(const struct wireaddr *addr) { static const unsigned char pchRFC6145[] = {0,0,0,0,0,0,0,0,0xFF,0xFF,0,0}; return addr->type == ADDR_TYPE_IPV6 && memcmp(addr->addr, pchRFC6145, sizeof(pchRFC6145)) == 0; } -static bool IsRFC6052(const struct ipaddr *addr) +static bool IsRFC6052(const struct wireaddr *addr) { static const unsigned char pchRFC6052[] = {0,0x64,0xFF,0x9B,0,0,0,0,0,0,0,0}; return addr->type == ADDR_TYPE_IPV6 && memcmp(addr->addr, pchRFC6052, sizeof(pchRFC6052)) == 0; } -static bool IsRFC3964(const struct ipaddr *addr) +static bool IsRFC3964(const struct wireaddr *addr) { return addr->type == ADDR_TYPE_IPV6 && addr->addr[0] == 0x20 && addr->addr[1] == 0x02; } /* Return offset of IPv4 address, or 0 == not an IPv4 */ -static size_t IPv4In6(const struct ipaddr *addr) +static size_t IPv4In6(const struct wireaddr *addr) { if (addr->type != ADDR_TYPE_IPV6) return 0; @@ -57,17 +59,17 @@ static size_t IPv4In6(const struct ipaddr *addr) } /* Is this an IPv4 address, or an IPv6-wrapped IPv4 */ -static bool IsIPv4(const struct ipaddr *addr) +static bool IsIPv4(const struct wireaddr *addr) { return addr->type == ADDR_TYPE_IPV4 || IPv4In6(addr) != 0; } -static bool IsIPv6(const struct ipaddr *addr) +static bool IsIPv6(const struct wireaddr *addr) { return addr->type == ADDR_TYPE_IPV6 && IPv4In6(addr) == 0; } -static bool RawEq(const struct ipaddr *addr, const void *cmp, size_t len) +static bool RawEq(const struct wireaddr *addr, const void *cmp, size_t len) { size_t off = IPv4In6(addr); @@ -76,14 +78,14 @@ static bool RawEq(const struct ipaddr *addr, const void *cmp, size_t len) } /* The bitcoin code packs addresses backwards, so we map it here. */ -static unsigned int GetByte(const struct ipaddr *addr, int n) +static unsigned int GetByte(const struct wireaddr *addr, int n) { size_t off = IPv4In6(addr); assert(off + n < addr->addrlen); return addr->addr[addr->addrlen - 1 - off - n]; } -static bool IsRFC1918(const struct ipaddr *addr) +static bool IsRFC1918(const struct wireaddr *addr) { return IsIPv4(addr) && ( GetByte(addr, 3) == 10 || @@ -91,56 +93,56 @@ static bool IsRFC1918(const struct ipaddr *addr) (GetByte(addr, 3) == 172 && (GetByte(addr, 2) >= 16 && GetByte(addr, 2) <= 31))); } -static bool IsRFC2544(const struct ipaddr *addr) +static bool IsRFC2544(const struct wireaddr *addr) { return IsIPv4(addr) && GetByte(addr, 3) == 198 && (GetByte(addr, 2) == 18 || GetByte(addr, 2) == 19); } -static bool IsRFC3927(const struct ipaddr *addr) +static bool IsRFC3927(const struct wireaddr *addr) { return IsIPv4(addr) && (GetByte(addr, 3) == 169 && GetByte(addr, 2) == 254); } -static bool IsRFC6598(const struct ipaddr *addr) +static bool IsRFC6598(const struct wireaddr *addr) { return IsIPv4(addr) && GetByte(addr, 3) == 100 && GetByte(addr, 2) >= 64 && GetByte(addr, 2) <= 127; } -static bool IsRFC5737(const struct ipaddr *addr) +static bool IsRFC5737(const struct wireaddr *addr) { return IsIPv4(addr) && ((GetByte(addr, 3) == 192 && GetByte(addr, 2) == 0 && GetByte(addr, 1) == 2) || (GetByte(addr, 3) == 198 && GetByte(addr, 2) == 51 && GetByte(addr, 1) == 100) || (GetByte(addr, 3) == 203 && GetByte(addr, 2) == 0 && GetByte(addr, 1) == 113)); } -static bool IsRFC3849(const struct ipaddr *addr) +static bool IsRFC3849(const struct wireaddr *addr) { return IsIPv6(addr) && GetByte(addr, 15) == 0x20 && GetByte(addr, 14) == 0x01 && GetByte(addr, 13) == 0x0D && GetByte(addr, 12) == 0xB8; } -static bool IsRFC4862(const struct ipaddr *addr) +static bool IsRFC4862(const struct wireaddr *addr) { static const unsigned char pchRFC4862[] = {0xFE,0x80,0,0,0,0,0,0}; return IsIPv6(addr) && RawEq(addr, pchRFC4862, sizeof(pchRFC4862)); } -static bool IsRFC4193(const struct ipaddr *addr) +static bool IsRFC4193(const struct wireaddr *addr) { return IsIPv6(addr) && ((GetByte(addr, 15) & 0xFE) == 0xFC); } -static bool IsRFC4843(const struct ipaddr *addr) +static bool IsRFC4843(const struct wireaddr *addr) { return IsIPv6(addr) && (GetByte(addr, 15) == 0x20 && GetByte(addr, 14) == 0x01 && GetByte(addr, 13) == 0x00 && (GetByte(addr, 12) & 0xF0) == 0x10); } -static bool IsTor(const struct ipaddr *addr) +static bool IsTor(const struct wireaddr *addr) { /* FIXME */ return false; } -static bool IsLocal(const struct ipaddr *addr) +static bool IsLocal(const struct wireaddr *addr) { // IPv4 loopback if (IsIPv4(addr) && (GetByte(addr, 3) == 127 || GetByte(addr, 3) == 0)) @@ -154,12 +156,12 @@ static bool IsLocal(const struct ipaddr *addr) return false; } -static bool IsInternal(const struct ipaddr *addr) +static bool IsInternal(const struct wireaddr *addr) { return addr->type == ADDR_TYPE_PADDING; } -static bool IsValid(const struct ipaddr *addr) +static bool IsValid(const struct wireaddr *addr) { // unspecified IPv6 address (::/128) unsigned char ipNone6[16] = {}; @@ -189,7 +191,7 @@ static bool IsValid(const struct ipaddr *addr) return true; } -static bool IsRoutable(const struct ipaddr *addr) +static bool IsRoutable(const struct wireaddr *addr) { return IsValid(addr) && !(IsRFC1918(addr) || IsRFC2544(addr) || IsRFC3927(addr) || IsRFC4862(addr) || IsRFC6598(addr) || IsRFC5737(addr) || (IsRFC4193(addr) && !IsTor(addr)) || IsRFC4843(addr) || IsLocal(addr) || IsInternal(addr)); } @@ -216,8 +218,8 @@ static bool get_local_sockname(int af, void *saddr, socklen_t saddrlen) return true; } -/* Return an ipaddr without port filled in */ -static bool guess_one_address(struct ipaddr *addr, u16 portnum, +/* Return an wireaddr without port filled in */ +static bool guess_one_address(struct wireaddr *addr, u16 portnum, enum wire_addr_type type) { addr->type = type; @@ -273,3 +275,31 @@ void guess_addresses(struct lightningd *ld) if (!guess_one_address(&ld->wireaddrs[n], ld->portnum, ADDR_TYPE_IPV6)) tal_resize(&ld->wireaddrs, n); } + +bool parse_wireaddr(const char *arg, struct wireaddr *addr, u16 port) +{ + struct in6_addr v6; + struct in_addr v4; + + /* FIXME: change arg to addr[:port] and use getaddrinfo? */ + if (streq(arg, "localhost")) + arg = "127.0.0.1"; + else if (streq(arg, "ip6-localhost")) + arg = "::1"; + + memset(&addr->addr, 0, sizeof(addr->addr)); + if (inet_pton(AF_INET, arg, &v4) == 1) { + addr->type = ADDR_TYPE_IPV4; + addr->addrlen = 4; + addr->port = port; + memcpy(&addr->addr, &v4, addr->addrlen); + return true; + } else if (inet_pton(AF_INET6, arg, &v6) == 1) { + addr->type = ADDR_TYPE_IPV6; + addr->addrlen = 16; + addr->port = port; + memcpy(&addr->addr, &v6, addr->addrlen); + return true; + } + return false; +} diff --git a/lightningd/netaddress.h b/lightningd/netaddress.h index 7af7f779d586..17b70e7998b0 100644 --- a/lightningd/netaddress.h +++ b/lightningd/netaddress.h @@ -1,9 +1,12 @@ #ifndef LIGHTNING_LIGHTNINGD_NETADDRESS_H #define LIGHTNING_LIGHTNINGD_NETADDRESS_H #include "config.h" +#include struct lightningd; void guess_addresses(struct lightningd *ld); +bool parse_wireaddr(const char *arg, struct wireaddr *addr, u16 port); + #endif /* LIGHTNING_LIGHTNINGD_NETADDRESS_H */ diff --git a/lightningd/options.c b/lightningd/options.c index 248a340ad5a0..d3fa48a16fb8 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -1,4 +1,3 @@ -#include #include #include #include @@ -9,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -118,42 +118,13 @@ static char *opt_set_s32(const char *arg, s32 *u) return NULL; } -/* FIXME: Rename ipaddr and hoist up */ -bool parse_ipaddr(const char *arg, struct ipaddr *addr, u16 port) -{ - struct in6_addr v6; - struct in_addr v4; - - /* FIXME: change arg to addr[:port] and use getaddrinfo? */ - if (streq(arg, "localhost")) - arg = "127.0.0.1"; - else if (streq(arg, "ip6-localhost")) - arg = "::1"; - - memset(&addr->addr, 0, sizeof(addr->addr)); - if (inet_pton(AF_INET, arg, &v4) == 1) { - addr->type = ADDR_TYPE_IPV4; - addr->addrlen = 4; - addr->port = port; - memcpy(&addr->addr, &v4, addr->addrlen); - return true; - } else if (inet_pton(AF_INET6, arg, &v6) == 1) { - addr->type = ADDR_TYPE_IPV6; - addr->addrlen = 16; - addr->port = port; - memcpy(&addr->addr, &v6, addr->addrlen); - return true; - } - return false; -} - static char *opt_add_ipaddr(const char *arg, struct lightningd *ld) { size_t n = tal_count(ld->wireaddrs); tal_resize(&ld->wireaddrs, n+1); - if (parse_ipaddr(arg, &ld->wireaddrs[n], ld->portnum)) + if (parse_wireaddr(arg, &ld->wireaddrs[n], ld->portnum)) return NULL; return tal_fmt(NULL, "Unable to parse IP address '%s'", arg); diff --git a/lightningd/options.h b/lightningd/options.h index 2519a8c53128..8794c7e873af 100644 --- a/lightningd/options.h +++ b/lightningd/options.h @@ -13,8 +13,6 @@ void register_opts(struct lightningd *ld); */ bool handle_opts(struct lightningd *ld, int argc, char *argv[]); -bool parse_ipaddr(const char *arg, struct ipaddr *addr, u16 port); - /* Derive default color and alias from the pubkey. */ void setup_color_and_alias(struct lightningd *ld); #endif /* LIGHTNING_LIGHTNINGD_OPTIONS_H */ diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 2d0458c31083..e90fd5d2e96c 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -57,7 +58,7 @@ struct connect { struct funding_channel; static void peer_offer_channel(struct lightningd *ld, struct funding_channel *fc, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, int peer_fd, int gossip_fd); @@ -72,7 +73,7 @@ static void peer_start_closingd(struct peer *peer, bool reconnected); static void peer_accept_channel(struct lightningd *ld, const struct pubkey *peer_id, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, int peer_fd, int gossip_fd, @@ -308,7 +309,7 @@ static void connect_failed(struct lightningd *ld, const struct pubkey *id, static struct peer *new_peer(struct lightningd *ld, const struct pubkey *id, - const struct ipaddr *addr, + const struct wireaddr *addr, const u8 *gfeatures, const u8 *lfeatures, int peer_fd) { @@ -505,7 +506,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, u8 *gfeatures, *lfeatures; u8 *error; struct peer *peer; - struct ipaddr addr; + struct wireaddr addr; if (!fromwire_gossip_peer_connected(msg, msg, NULL, &id, &addr, &cs, @@ -611,7 +612,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, void peer_sent_nongossip(struct lightningd *ld, const struct pubkey *id, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, @@ -709,7 +710,7 @@ static void json_connect(struct command *cmd, jsmntok_t *hosttok, *idtok; struct pubkey id; const char *name, *port, *colon; - struct ipaddr addr; + struct wireaddr addr; u8 *msg; if (!json_get_params(buffer, params, @@ -741,7 +742,7 @@ static void json_connect(struct command *cmd, port = tal_strdup(cmd, stringify(DEFAULT_PORT)); } addr.port = atoi(port); - if (!parse_ipaddr(name, &addr, addr.port) || !addr.port) + if (!parse_wireaddr(name, &addr, addr.port) || !addr.port) command_fail(cmd, "host %s:%s not valid", name, port); /* Tell it about the address. */ @@ -785,7 +786,7 @@ static void log_to_json(unsigned int skipped, json_add_string(info->response, NULL, log); } -static const char *ipaddr_name(const tal_t *ctx, const struct ipaddr *a) +static const char *wireaddr_name(const tal_t *ctx, const struct wireaddr *a) { char name[INET6_ADDRSTRLEN]; int af; @@ -838,7 +839,7 @@ static void json_getpeers(struct command *cmd, json_object_start(response, NULL); json_add_string(response, "state", peer_state_name(p->state)); json_add_string(response, "netaddr", - ipaddr_name(response, &p->addr)); + wireaddr_name(response, &p->addr)); json_add_pubkey(response, "peerid", &p->id); json_add_bool(response, "connected", p->owner != NULL); if (p->owner) @@ -1478,7 +1479,7 @@ static u8 *create_node_announcement(const tal_t *ctx, struct lightningd *ld, memset(sig, 0, sizeof(*sig)); } for (i = 0; i < tal_count(ld->wireaddrs); i++) - towire_ipaddr(&addresses, ld->wireaddrs+i); + towire_wireaddr(&addresses, ld->wireaddrs+i); announcement = towire_node_announcement(ctx, sig, features, timestamp, @@ -2231,7 +2232,7 @@ static void opening_fundee_finished(struct subd *opening, /* Peer has spontaneously exited from gossip due to open msg */ static void peer_accept_channel(struct lightningd *ld, const struct pubkey *peer_id, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, int peer_fd, int gossip_fd, @@ -2299,7 +2300,7 @@ static void peer_accept_channel(struct lightningd *ld, static void peer_offer_channel(struct lightningd *ld, struct funding_channel *fc, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, int peer_fd, int gossip_fd) @@ -2390,7 +2391,7 @@ static void gossip_peer_released(struct subd *gossip, struct lightningd *ld = gossip->ld; struct crypto_state cs; u8 *gfeatures, *lfeatures; - struct ipaddr addr; + struct wireaddr addr; /* We could have raced with peer doing something else. */ fc->peer = peer_by_id(ld, &fc->peerid); diff --git a/lightningd/peer_control.h b/lightningd/peer_control.h index 6eb00ae1304f..f22911657588 100644 --- a/lightningd/peer_control.h +++ b/lightningd/peer_control.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -54,7 +55,7 @@ struct peer { u8 channel_flags; /* Where we connected to, or it connected from. */ - struct ipaddr addr; + struct wireaddr addr; /* Our channel config. */ struct channel_config our_config; @@ -167,7 +168,7 @@ void peer_connected(struct lightningd *ld, const u8 *msg, void peer_sent_nongossip(struct lightningd *ld, const struct pubkey *id, - const struct ipaddr *addr, + const struct wireaddr *addr, const struct crypto_state *cs, const u8 *gfeatures, const u8 *lfeatures, diff --git a/wire/fromwire.c b/wire/fromwire.c index f9e118c93c85..0925eb6bd7d1 100644 --- a/wire/fromwire.c +++ b/wire/fromwire.c @@ -170,37 +170,6 @@ void fromwire_ripemd160(const u8 **cursor, size_t *max, struct ripemd160 *ripemd fromwire(cursor, max, ripemd, sizeof(*ripemd)); } -/* BOLT #7: - * - * The following `address descriptor` types are defined: - * - * * `0`: padding. data = none (length 0). - * * `1`: ipv4. data = `[4:ipv4_addr][2:port]` (length 6) - * * `2`: ipv6. data = `[16:ipv6_addr][2:port]` (length 18) - */ -/* FIXME: Tor addresses! */ - -/* Returns false if we didn't parse it, and *cursor == NULL if malformed. */ -bool fromwire_ipaddr(const u8 **cursor, size_t *max, struct ipaddr *addr) -{ - addr->type = fromwire_u8(cursor, max); - - switch (addr->type) { - case ADDR_TYPE_IPV4: - addr->addrlen = 4; - break; - case ADDR_TYPE_IPV6: - addr->addrlen = 16; - break; - default: - return false; - } - fromwire(cursor, max, addr->addr, addr->addrlen); - addr->port = fromwire_u16(cursor, max); - - return *cursor != NULL; -} - void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num) { fromwire(cursor, max, arr, num); diff --git a/wire/towire.c b/wire/towire.c index 62ddca0f116a..87272169883f 100644 --- a/wire/towire.c +++ b/wire/towire.c @@ -115,13 +115,6 @@ void towire_ripemd160(u8 **pptr, const struct ripemd160 *ripemd) towire(pptr, ripemd, sizeof(*ripemd)); } -void towire_ipaddr(u8 **pptr, const struct ipaddr *addr) -{ - towire_u8(pptr, addr->type); - towire(pptr, addr->addr, addr->addrlen); - towire_u16(pptr, addr->port); -} - void towire_u8_array(u8 **pptr, const u8 *arr, size_t num) { towire(pptr, arr, num); diff --git a/wire/wire.h b/wire/wire.h index b7514385449d..7d2c49f20b69 100644 --- a/wire/wire.h +++ b/wire/wire.h @@ -14,25 +14,6 @@ struct channel_id { u8 id[32]; }; -/* Address types as defined in the lightning specification. Does not - * match AF_INET and AF_INET6 so we just define our - * own. ADDR_TYPE_PADDING is also used to signal in the config whether - * an address is defined at all. */ -enum wire_addr_type { - ADDR_TYPE_PADDING = 0, - ADDR_TYPE_IPV4 = 1, - ADDR_TYPE_IPV6 = 2, -}; - -/* FIXME(cdecker) Extend this once we have defined how TOR addresses - * should look like */ -struct ipaddr { - enum wire_addr_type type; - u8 addrlen; - u8 addr[16]; - u16 port; -}; - struct preimage; struct ripemd160; @@ -56,7 +37,6 @@ void towire_sha256(u8 **pptr, const struct sha256 *sha256); void towire_sha256_double(u8 **pptr, const struct sha256_double *sha256d); void towire_preimage(u8 **pptr, const struct preimage *preimage); void towire_ripemd160(u8 **pptr, const struct ripemd160 *ripemd); -void towire_ipaddr(u8 **pptr, const struct ipaddr *addr); void towire_u8(u8 **pptr, u8 v); void towire_u16(u8 **pptr, u16 v); void towire_u32(u8 **pptr, u32 v); @@ -86,7 +66,6 @@ void fromwire_sha256_double(const u8 **cursor, size_t *max, struct sha256_double *sha256d); void fromwire_preimage(const u8 **cursor, size_t *max, struct preimage *preimage); void fromwire_ripemd160(const u8 **cursor, size_t *max, struct ripemd160 *ripemd); -bool fromwire_ipaddr(const u8 **cursor, size_t *max, struct ipaddr *addr); void fromwire_pad(const u8 **cursor, size_t *max, size_t num); void fromwire_u8_array(const u8 **cursor, size_t *max, u8 *arr, size_t num); From a7d6326bef9cf943665aafca3474376df0d6e178 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:48:38 +1030 Subject: [PATCH 0029/1428] type_to_string: format wireaddr. Good for printing, and removes some code from peer_control. Signed-off-by: Rusty Russell --- common/type_to_string.h | 1 + common/wireaddr.c | 29 +++++++++++++++++++++++++++++ lightningd/netaddress.c | 37 +++++++++++++++++++++++++++++-------- lightningd/options.c | 10 ++++++---- lightningd/peer_control.c | 25 ++----------------------- 5 files changed, 67 insertions(+), 35 deletions(-) diff --git a/common/type_to_string.h b/common/type_to_string.h index 87b929105273..c12258283359 100644 --- a/common/type_to_string.h +++ b/common/type_to_string.h @@ -18,6 +18,7 @@ union printable_types { const struct preimage *preimage; const struct channel_state *channel_state; const struct channel_oneside *channel_oneside; + const struct wireaddr *wireaddr; const secp256k1_pubkey *secp256k1_pubkey; const struct channel_id *channel_id; const struct short_channel_id *short_channel_id; diff --git a/common/wireaddr.c b/common/wireaddr.c index 784f304f0b70..ce4bea332121 100644 --- a/common/wireaddr.c +++ b/common/wireaddr.c @@ -1,3 +1,7 @@ +#include +#include +#include +#include #include #include @@ -28,3 +32,28 @@ void towire_wireaddr(u8 **pptr, const struct wireaddr *addr) towire(pptr, addr->addr, addr->addrlen); towire_u16(pptr, addr->port); } + +static char *fmt_wireaddr(const tal_t *ctx, const struct wireaddr *a) +{ + char addrstr[INET6_ADDRSTRLEN]; + char *ret, *hex; + + switch (a->type) { + case ADDR_TYPE_IPV4: + if (!inet_ntop(AF_INET, a->addr, addrstr, INET_ADDRSTRLEN)) + return "Unprintable-ipv4-address"; + return tal_fmt(ctx, "%s:%u", addrstr, a->port); + case ADDR_TYPE_IPV6: + if (!inet_ntop(AF_INET6, a->addr, addrstr, INET6_ADDRSTRLEN)) + return "Unprintable-ipv6-address"; + return tal_fmt(ctx, "%s:%u", addrstr, a->port); + case ADDR_TYPE_PADDING: + break; + } + + hex = tal_hexstr(ctx, a->addr, a->addrlen); + ret = tal_fmt(ctx, "Unknown type %u %s:%u", a->type, hex, a->port); + tal_free(hex); + return ret; +} +REGISTER_TYPE_TO_STRING(wireaddr, fmt_wireaddr); diff --git a/lightningd/netaddress.c b/lightningd/netaddress.c index 8b04012dea92..c838e7b17da6 100644 --- a/lightningd/netaddress.c +++ b/lightningd/netaddress.c @@ -1,7 +1,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -198,18 +200,26 @@ static bool IsRoutable(const struct wireaddr *addr) /* Trick I learned from Harald Welte: create UDP socket, connect() and * then query address. */ -static bool get_local_sockname(int af, void *saddr, socklen_t saddrlen) +static bool get_local_sockname(struct lightningd *ld, + int af, void *saddr, socklen_t saddrlen) { int fd = socket(af, SOCK_DGRAM, 0); - if (fd < 0) + if (fd < 0) { + log_debug(ld->log, "Failed to create %u socket: %s", + af, strerror(errno)); return false; + } if (connect(fd, saddr, saddrlen) != 0) { + log_debug(ld->log, "Failed to connect %u socket: %s", + af, strerror(errno)); close(fd); return false; } if (getsockname(fd, saddr, &saddrlen) != 0) { + log_debug(ld->log, "Failed to get %u socket name: %s", + af, strerror(errno)); close(fd); return false; } @@ -219,7 +229,8 @@ static bool get_local_sockname(int af, void *saddr, socklen_t saddrlen) } /* Return an wireaddr without port filled in */ -static bool guess_one_address(struct wireaddr *addr, u16 portnum, +static bool guess_one_address(struct lightningd *ld, + struct wireaddr *addr, u16 portnum, enum wire_addr_type type) { addr->type = type; @@ -232,7 +243,8 @@ static bool guess_one_address(struct wireaddr *addr, u16 portnum, sin.sin_port = htons(53); /* 8.8.8.8 */ sin.sin_addr.s_addr = 0x08080808; - if (!get_local_sockname(AF_INET, &sin, sizeof(sin))) + sin.sin_family = AF_INET; + if (!get_local_sockname(ld, AF_INET, &sin, sizeof(sin))) return false; addr->addrlen = sizeof(sin.sin_addr); memcpy(addr->addr, &sin.sin_addr, addr->addrlen); @@ -245,20 +257,27 @@ static bool guess_one_address(struct wireaddr *addr, u16 portnum, = {0x20,0x01,0x48,0x60,0x48,0x60,0,0,0,0,0,0,8,8,8,8}; memset(&sin6, 0, sizeof(sin6)); sin6.sin6_port = htons(53); + sin6.sin6_family = AF_INET6; memcpy(sin6.sin6_addr.s6_addr, pchGoogle, sizeof(pchGoogle)); - if (!get_local_sockname(AF_INET6, &sin6, sizeof(sin6))) + if (!get_local_sockname(ld, AF_INET6, &sin6, sizeof(sin6))) return false; addr->addrlen = sizeof(sin6.sin6_addr); memcpy(addr->addr, &sin6.sin6_addr, addr->addrlen); break; } case ADDR_TYPE_PADDING: + log_debug(ld->log, "Padding address, ignoring"); return false; } - if (!IsRoutable(addr)) + if (!IsRoutable(addr)) { + log_debug(ld->log, "Address %s is not routable", + type_to_string(ltmp, struct wireaddr, addr)); return false; + } + log_debug(ld->log, "Public address %s", + type_to_string(ltmp, struct wireaddr, addr)); return true; } @@ -266,13 +285,15 @@ void guess_addresses(struct lightningd *ld) { size_t n = tal_count(ld->wireaddrs); + log_debug(ld->log, "Trying to guess public addresses..."); + /* We allocate an extra, then remove if it's not needed. */ tal_resize(&ld->wireaddrs, n+1); - if (guess_one_address(&ld->wireaddrs[n], ld->portnum, ADDR_TYPE_IPV4)) { + if (guess_one_address(ld, &ld->wireaddrs[n], ld->portnum, ADDR_TYPE_IPV4)) { n++; tal_resize(&ld->wireaddrs, n+1); } - if (!guess_one_address(&ld->wireaddrs[n], ld->portnum, ADDR_TYPE_IPV6)) + if (!guess_one_address(ld, &ld->wireaddrs[n], ld->portnum, ADDR_TYPE_IPV6)) tal_resize(&ld->wireaddrs, n); } diff --git a/lightningd/options.c b/lightningd/options.c index d3fa48a16fb8..dcfd2bfd723d 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -509,10 +509,6 @@ static void opt_parse_from_config(struct lightningd *ld) setup_default_config(ld); opt_parse(&argc, argv, config_log_stderr_exit); - - if (ld->portnum && tal_count(ld->wireaddrs) == 0) - guess_addresses(ld); - tal_free(contents); } @@ -613,6 +609,12 @@ bool handle_opts(struct lightningd *ld, int argc, char *argv[]) check_config(ld); + if (ld->portnum && tal_count(ld->wireaddrs) == 0) + guess_addresses(ld); + else + log_debug(ld->log, "Not guessing addresses: %s", + ld->portnum ? "manually set" : "port set to zero"); + #if DEVELOPER if (ld->dev_hsm_seed) { int fd; diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index e90fd5d2e96c..cc6796a8cf0e 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -786,28 +786,6 @@ static void log_to_json(unsigned int skipped, json_add_string(info->response, NULL, log); } -static const char *wireaddr_name(const tal_t *ctx, const struct wireaddr *a) -{ - char name[INET6_ADDRSTRLEN]; - int af; - - switch (a->type) { - case ADDR_TYPE_IPV4: - af = AF_INET; - break; - case ADDR_TYPE_IPV6: - af = AF_INET6; - break; - default: - return tal_fmt(ctx, "Unknown type %u", a->type); - } - - if (!inet_ntop(af, a->addr, name, sizeof(name))) - sprintf(name, "Unprintable-%u-address", a->type); - - return tal_fmt(ctx, "%s:%u", name, a->port); -} - static void json_getpeers(struct command *cmd, const char *buffer, const jsmntok_t *params) { @@ -839,7 +817,8 @@ static void json_getpeers(struct command *cmd, json_object_start(response, NULL); json_add_string(response, "state", peer_state_name(p->state)); json_add_string(response, "netaddr", - wireaddr_name(response, &p->addr)); + type_to_string(response, struct wireaddr, + &p->addr)); json_add_pubkey(response, "peerid", &p->id); json_add_bool(response, "connected", p->owner != NULL); if (p->owner) From 0c7ca9ab7cb8f930c60314f69c4d7f5963207863 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:49:38 +1030 Subject: [PATCH 0030/1428] gossipd: call to return all connected peers. And we report these through the getpeers JSON RPC again (carefully: in our reconnect tests we can get duplicates which this patch now filters out). Signed-off-by: Rusty Russell --- gossipd/gossip.c | 29 ++++++++++++ gossipd/gossip_wire.csv | 7 +++ lightningd/gossip_control.c | 2 + lightningd/peer_control.c | 94 ++++++++++++++++++++++++++++--------- tests/test_lightningd.py | 6 +-- 5 files changed, 113 insertions(+), 25 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index b48edd43b8eb..dd3f3758d2dc 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -1299,6 +1299,31 @@ static struct io_plan *addr_hint(struct io_conn *conn, return daemon_conn_read_next(conn, &daemon->master); } +static struct io_plan *get_peers(struct io_conn *conn, + struct daemon *daemon, const u8 *msg) +{ + struct peer *peer; + size_t n = 0; + struct pubkey *id = tal_arr(conn, struct pubkey, n); + struct wireaddr *wireaddr = tal_arr(conn, struct wireaddr, n); + + if (!fromwire_gossip_getpeers_request(msg, NULL)) + master_badmsg(WIRE_GOSSIPCTL_PEER_ADDRHINT, msg); + + list_for_each(&daemon->peers, peer, list) { + tal_resize(&id, n+1); + tal_resize(&wireaddr, n+1); + + id[n] = peer->id; + wireaddr[n] = peer->addr; + n++; + } + + daemon_conn_send(&daemon->master, + take(towire_gossip_getpeers_reply(conn, id, wireaddr))); + return daemon_conn_read_next(conn, &daemon->master); +} + static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master) { struct daemon *daemon = container_of(master, struct daemon, master); @@ -1342,11 +1367,15 @@ static struct io_plan *recv_req(struct io_conn *conn, struct daemon_conn *master case WIRE_GOSSIPCTL_PEER_ADDRHINT: return addr_hint(conn, daemon, master->msg_in); + case WIRE_GOSSIP_GETPEERS_REQUEST: + return get_peers(conn, daemon, master->msg_in); + case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY: case WIRE_GOSSIPCTL_RELEASE_PEER_REPLYFAIL: case WIRE_GOSSIP_GETNODES_REPLY: case WIRE_GOSSIP_GETROUTE_REPLY: case WIRE_GOSSIP_GETCHANNELS_REPLY: + case WIRE_GOSSIP_GETPEERS_REPLY: case WIRE_GOSSIP_PING_REPLY: case WIRE_GOSSIP_RESOLVE_CHANNEL_REPLY: case WIRE_GOSSIP_PEER_CONNECTED: diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index 020d1cdc647d..3ded7872c50b 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -123,3 +123,10 @@ gossip_forwarded_msg,3010 gossip_forwarded_msg,,msglen,u16 gossip_forwarded_msg,,msg,msglen*u8 +# The main daemon asks for peers +gossip_getpeers_request,3011 + +gossip_getpeers_reply,3111 +gossip_getpeers_reply,,num,u16 +gossip_getpeers_reply,,id,num*struct pubkey +gossip_getpeers_reply,,addr,num*struct wireaddr diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index 6de52a94b8d2..b41137e42b65 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -60,6 +60,7 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_GETNODES_REQUEST: case WIRE_GOSSIP_GETROUTE_REQUEST: case WIRE_GOSSIP_GETCHANNELS_REQUEST: + case WIRE_GOSSIP_GETPEERS_REQUEST: case WIRE_GOSSIP_PING: case WIRE_GOSSIP_RESOLVE_CHANNEL_REQUEST: case WIRE_GOSSIP_FORWARDED_MSG: @@ -71,6 +72,7 @@ static unsigned gossip_msg(struct subd *gossip, const u8 *msg, const int *fds) case WIRE_GOSSIP_GETNODES_REPLY: case WIRE_GOSSIP_GETROUTE_REPLY: case WIRE_GOSSIP_GETCHANNELS_REPLY: + case WIRE_GOSSIP_GETPEERS_REPLY: case WIRE_GOSSIP_PING_REPLY: case WIRE_GOSSIP_RESOLVE_CHANNEL_REPLY: case WIRE_GOSSIPCTL_RELEASE_PEER_REPLY: diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index cc6796a8cf0e..fb00699fcc7d 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -786,34 +786,31 @@ static void log_to_json(unsigned int skipped, json_add_string(info->response, NULL, log); } -static void json_getpeers(struct command *cmd, - const char *buffer, const jsmntok_t *params) +struct getpeers_args { + struct command *cmd; + /* If non-NULL, they want logs too */ + enum log_level *ll; +}; + +static void gossipd_getpeers_complete(struct subd *gossip, const u8 *msg, + const int *fds, + struct getpeers_args *gpa) { + /* This is a little sneaky... */ + struct pubkey *ids; + struct wireaddr *addrs; + struct json_result *response = new_json_result(gpa->cmd); struct peer *p; - struct json_result *response = new_json_result(cmd); - jsmntok_t *leveltok; - struct log_info info; - - json_get_params(buffer, params, "?level", &leveltok, NULL); - if (!leveltok) - ; - else if (json_tok_streq(buffer, leveltok, "debug")) - info.level = LOG_DBG; - else if (json_tok_streq(buffer, leveltok, "info")) - info.level = LOG_INFORM; - else if (json_tok_streq(buffer, leveltok, "unusual")) - info.level = LOG_UNUSUAL; - else if (json_tok_streq(buffer, leveltok, "broken")) - info.level = LOG_BROKEN; - else { - command_fail(cmd, "Invalid level param"); + if (!fromwire_gossip_getpeers_reply(msg, msg, NULL, &ids, &addrs)) { + command_fail(gpa->cmd, "Bad response from gossipd"); return; } + /* First the peers not just gossiping. */ json_object_start(response, NULL); json_array_start(response, "peers"); - list_for_each(&cmd->ld->peers, p, list) { + list_for_each(&gpa->cmd->ld->peers, p, list) { json_object_start(response, NULL); json_add_string(response, "state", peer_state_name(p->state)); json_add_string(response, "netaddr", @@ -832,7 +829,9 @@ static void json_getpeers(struct command *cmd, p->funding_satoshi * 1000); } - if (leveltok) { + if (gpa->ll) { + struct log_info info; + info.level = *gpa->ll; info.response = response; json_array_start(response, "log"); log_each_line(p->log_book, log_to_json, &info); @@ -840,9 +839,60 @@ static void json_getpeers(struct command *cmd, } json_object_end(response); } + + for (size_t i = 0; i < tal_count(ids); i++) { + /* Don't report peers in both, which can happen if they're + * reconnecting */ + if (peer_by_id(gpa->cmd->ld, ids + i)) + continue; + + json_object_start(response, NULL); + /* Fake state. */ + json_add_string(response, "state", "GOSSIPING"); + json_add_pubkey(response, "peerid", ids+i); + json_add_string(response, "netaddr", + type_to_string(response, struct wireaddr, + addrs + i)); + json_add_bool(response, "connected", "true"); + json_add_string(response, "owner", gossip->name); + json_object_end(response); + } + json_array_end(response); json_object_end(response); - command_success(cmd, response); + command_success(gpa->cmd, response); +} + +static void json_getpeers(struct command *cmd, + const char *buffer, const jsmntok_t *params) +{ + jsmntok_t *leveltok; + struct getpeers_args *gpa = tal(cmd, struct getpeers_args); + + gpa->cmd = cmd; + json_get_params(buffer, params, "?level", &leveltok, NULL); + + if (leveltok) { + gpa->ll = tal(gpa, enum log_level); + if (json_tok_streq(buffer, leveltok, "debug")) + *gpa->ll = LOG_DBG; + else if (json_tok_streq(buffer, leveltok, "info")) + *gpa->ll = LOG_INFORM; + else if (json_tok_streq(buffer, leveltok, "unusual")) + *gpa->ll = LOG_UNUSUAL; + else if (json_tok_streq(buffer, leveltok, "broken")) + *gpa->ll = LOG_BROKEN; + else { + command_fail(cmd, "Invalid level param"); + return; + } + } else + gpa->ll = NULL; + + /* Get peers from gossipd. */ + subd_req(cmd, cmd->ld->gossip, + take(towire_gossip_getpeers_request(cmd)), + -1, 0, gossipd_getpeers_complete, gpa); } static const struct json_command getpeers_command = { diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 15fef25deaa6..c4f22500e4bf 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -279,9 +279,9 @@ def test_shutdown(self): def test_connect(self): l1,l2 = self.connect() - # Main daemon has no idea about these peers; they're in gossipd. - assert l1.rpc.getpeer(l2.info['id'], 'info') == None - assert l2.rpc.getpeer(l1.info['id'], 'info') == None + # These should be in gossipd. + assert l1.rpc.getpeer(l2.info['id'])['state'] == 'GOSSIPING' + assert l2.rpc.getpeer(l1.info['id'])['state'] == 'GOSSIPING' # Both gossipds will have them as new peers once handed back. l1.daemon.wait_for_log('handle_peer {}: new peer'.format(l2.info['id'])) From 2aadd351f8d0af6308dcc483a5137a5afd0f562c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 27 Oct 2017 20:38:27 +1030 Subject: [PATCH 0031/1428] test_lightningd.py: more information when we fail to find payment. Travis has been hitting this intermittantly, can't reproduce here. Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index c4f22500e4bf..e3698c7b797e 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -237,12 +237,13 @@ def fund_channel(self, l1, l2, amount): l2.daemon.wait_for_log('-> CHANNELD_NORMAL') # Hacky way to find our output. - for out in bitcoind.rpc.decoderawtransaction(tx)['vout']: + decoded=bitcoind.rpc.decoderawtransaction(tx)['vout'] + for out in decoded: # Sometimes a float? Sometimes a decimal? WTF Python?! if out['scriptPubKey']['type'] == 'witness_v0_scripthash': if out['value'] == Decimal(amount) / 10**8 or out['value'] * 10**8 == amount: return "{}:1:{}".format(bitcoind.rpc.getblockcount(), out['n']) - raise ValueError("Can't find {} payment in {}".format(amount, tx)) + raise ValueError("Can't find {} payment in {} ({})".format(amount, tx, decoded)) def pay(self, lsrc, ldst, amt, label=None, async=False): if not label: From 82f252c79a036ae3c9159fbe89c39456257b7375 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 28 Oct 2017 12:49:10 +1030 Subject: [PATCH 0032/1428] test_permfail_new_commit: fix intermittant failure. Normally, we get an error as soon as we send WIRE_REVOKE_AND_ACK. But if the commit timer goes off, we get some extra cycles, during which the other side can reconnect. In this case, we simply kill the channeld before it fails, and never check for the permfail string. b'lightning_channeld(18613): TRACE: dev_disconnect: -WIRE_REVOKE_AND_ACK' b'lightning_channeld(18613): TRACE: Trying commit' b'lightning_channeld(18613): TRACE: htlc 0: SENT_ADD_REVOCATION->SENT_ADD_ACK_COMMIT' b'lightning_channeld(18613): TRACE: htlc added REMOTE: local +0 remote -200000000' b'lightning_channeld(18613): TRACE: sending_commit: HTLC REMOTE 0 = SENT_ADD_ACK_COMMIT/RCVD_ADD_ACK_COMMIT' b'lightning_gossipd(18590): TRACE: Responder: Act 1' b'lightning_channeld(18613): TRACE: Derived key 034aab0b5cb755de836cffb34c053ba115fba6fe75414e8f56261e23c80eabb1fe from basepoint 03e0a7bb422b254f54bc954be05bd6823a7b7a4b996ff8d3079ca211590fb5df39, point 02f3bf525b6ca595bf85d63e89c95fc59c0fde3ae434b55c8093bbb5c64849da37' b'lightningd(18465): Connected json input' b'lightningd(18465):jcon fd 16: Success' b'lightningd(18465):jcon fd 16: Closing (Bad file descriptor)' b'lightning_gossipd(18590): TRACE: Responder: Act 2' b'lightning_gossipd(18590): TRACE: Responder: Act 3' b'lightning_gossipd(18590): UPDATE WIRE_GOSSIP_PEER_CONNECTED' b'lightning_gossipd(18590): UPDATE WIRE_GOSSIP_PEER_CONNECTED' b'lightningd(18465): peer 0266e4598d1d3c415f572a8488830b60f7e744ed9235eb0b1ba93283b315c03518: Peer has reconnected, state CHANNELD_NORMAL' b'lightning_channeld(18613): Status closed, but not exited. Killing' Signed-off-by: Rusty Russell --- lightningd/peer_control.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index fb00699fcc7d..e3e76c77f810 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -543,6 +543,14 @@ void peer_connected(struct lightningd *ld, const u8 *msg, goto send_error; } +#if DEVELOPER + if (dev_disconnect_permanent(ld)) { + peer_internal_error(peer, "dev_disconnect permfail"); + error = peer->error; + goto send_error; + } +#endif + switch (peer->state) { /* This can't happen. */ case UNINITIALIZED: From ee1fb07c8ebd84207a09122d876bfd3f475c6ff6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 29 Oct 2017 21:29:04 +1030 Subject: [PATCH 0033/1428] update-mocks: fix stubs generation for fatal(). We're going to need this, and the PRINTF_FMT(1,2) in front of it caused mockup.sh to miss the declaration. We also eliminate the obviously-unused fallback case (which referred to daemon/*.h). Signed-off-by: Rusty Russell --- tools/mockup.sh | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/tools/mockup.sh b/tools/mockup.sh index 1fd293ab432a..f8a0fc95ebb2 100755 --- a/tools/mockup.sh +++ b/tools/mockup.sh @@ -15,17 +15,12 @@ if [ $# -eq 0 ]; then fi for SYMBOL; do - WHERE=$(grep -nH "^[a-z0-9_ ]* [*]*$SYMBOL(" */*.h ) + WHERE=$(grep -nH "^[a-zA-Z0-9_ (),]* [*]*$SYMBOL(" */*.h ) if [ x"$WHERE" != x ]; then STUB='\n{ fprintf(stderr, "'$SYMBOL' called!\\n"); abort(); }' else - WHERE=$(grep -nH "^extern \(const \)\?struct [a-zA-Z0-9_]* $SYMBOL;$" daemon/*.h) - if [ x"$WHERE" != x ]; then - STUB=';' - else - echo "/* Could not find declaration for $SYMBOL */" - continue - fi + echo "/* Could not find declaration for $SYMBOL */" + continue fi echo "/* Generated stub for $SYMBOL */" @@ -35,5 +30,5 @@ for SYMBOL; do END=$(tail -n +$LINE < $FILE | grep -n ';$'); NUM=${END%%:*} - tail -n +$LINE < $FILE | head -n $NUM | sed 's/^extern *//' | sed 's/PRINTF_FMT([^)]*)//' | sed 's/,/ UNNEEDED,/g' | sed 's/\([a-z0-9A-Z*_]* [a-z0-9A-Z*_]*\));/\1 UNNEEDED);/' | sed "s/;\$/$STUB/" | sed 's/\s*$//' + tail -n +$LINE < $FILE | head -n $NUM | sed 's/^extern *//' | sed 's/PRINTF_FMT([^)]*)//' | sed 's/NORETURN//g' | sed 's/,/ UNNEEDED,/g' | sed 's/\([a-z0-9A-Z*_]* [a-z0-9A-Z*_]*\));/\1 UNNEEDED);/' | sed "s/;\$/$STUB/" | sed 's/\s*$//' done From 45b92dc2b08e62d6b8619b68ecbe39aaa26fb2e4 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 29 Oct 2017 21:34:02 +1030 Subject: [PATCH 0034/1428] pytest: pass DEVELOPER flag through. Signed-off-by: Rusty Russell --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index de8a917762c5..79f53736a287 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,7 @@ check: $(MAKE) pytest pytest: $(ALL_PROGRAMS) - PYTHONPATH=contrib/pylightning python3 tests/test_lightningd.py -f + PYTHONPATH=contrib/pylightning DEVELOPER=$(DEVELOPER) python3 tests/test_lightningd.py -f # Keep includes in alpha order. check-src-include-order/%: % From 21305c0d28e41df7f6db68fd71cb0e6c78a6323c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 29 Oct 2017 21:48:13 +1030 Subject: [PATCH 0035/1428] fatal: cause a backtrace. Much nicer for debugging. Signed-off-by: Rusty Russell --- lightningd/log.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lightningd/log.c b/lightningd/log.c index 83fd883a8c97..15601c0b3d38 100644 --- a/lightningd/log.c +++ b/lightningd/log.c @@ -521,5 +521,5 @@ void fatal(const char *fmt, ...) va_end(ap); log_crash(0); } - exit(1); + abort(); } From 4a06da8f78883a79183a0a5f76bec1f7a878ab26 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 30 Oct 2017 10:04:26 +1030 Subject: [PATCH 0036/1428] wallet: fix wallet_update_output_status where oldstatus == output_state_any "near \"AND\": syntax error" This was caught by the "always keep errors for db_commit_transaction". Signed-off-by: Rusty Russell --- wallet/wallet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wallet/wallet.c b/wallet/wallet.c index 247c3195af8c..032136b87c37 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -73,7 +73,7 @@ bool wallet_update_output_status(struct wallet *w, } else { db_exec(__func__, w->db, "UPDATE outputs SET status=%d WHERE " - "AND prev_out_tx = '%s' AND prev_out_index = " + "prev_out_tx = '%s' AND prev_out_index = " "%d;", newstatus, hextxid, outnum); } From bc9918ad460ff200b62e06282e0ce4c542bbf9f1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 31 Oct 2017 10:34:59 +1030 Subject: [PATCH 0037/1428] dev: option not to do backtracing. It crashes under valgrind, causing a valgrind error: valgrind gives us a backtrace anyway, so we don't need it. Signed-off-by: Rusty Russell --- lightningd/log.c | 15 ++++++++++++--- lightningd/log.h | 4 ++++ lightningd/options.c | 3 +++ tests/test_lightningd.py | 2 ++ 4 files changed, 21 insertions(+), 3 deletions(-) diff --git a/lightningd/log.c b/lightningd/log.c index 15601c0b3d38..efc3f8389492 100644 --- a/lightningd/log.c +++ b/lightningd/log.c @@ -18,6 +18,10 @@ #include #include +#if DEVELOPER +bool dev_no_backtrace; +#endif + static struct backtrace_state *backtrace_state; struct log_entry { @@ -432,8 +436,9 @@ static void log_crash(int sig) if (sig) { log_broken(crashlog, "FATAL SIGNAL %i RECEIVED", sig); - backtrace_full(backtrace_state, 0, log_backtrace, NULL, - crashlog); + if (backtrace_state) + backtrace_full(backtrace_state, 0, log_backtrace, NULL, + crashlog); } if (crashlog->lr->print == log_default_print) { @@ -457,7 +462,8 @@ static void log_crash(int sig) if (sig) { fprintf(stderr, "Fatal signal %u. ", sig); - backtrace_print(backtrace_state, 0, stderr); + if (backtrace_state) + backtrace_print(backtrace_state, 0, stderr); } if (logfile) fprintf(stderr, "Log dumped in %s", logfile); @@ -469,6 +475,9 @@ void crashlog_activate(const char *argv0, struct log *log) struct sigaction sa; crashlog = log; +#if DEVELOPER + if (!dev_no_backtrace) +#endif backtrace_state = backtrace_create_state(argv0, 0, NULL, NULL); sa.sa_handler = log_crash; sigemptyset(&sa.sa_mask); diff --git a/lightningd/log.h b/lightningd/log.h index c0d788a2887a..46aa53734c5e 100644 --- a/lightningd/log.h +++ b/lightningd/log.h @@ -94,4 +94,8 @@ const tal_t *ltmp; /* Before the crashlog is activated, just prints to stderr. */ void NORETURN PRINTF_FMT(1,2) fatal(const char *fmt, ...); + +#if DEVELOPER +extern bool dev_no_backtrace; +#endif #endif /* LIGHTNING_LIGHTNINGD_LOG_H */ diff --git a/lightningd/options.c b/lightningd/options.c index dcfd2bfd723d..42ba6657b3e1 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -288,6 +288,9 @@ static void dev_register_opts(struct lightningd *ld) NULL, ld, "File containing disconnection points"); opt_register_arg("--dev-hsm-seed=", opt_set_hsm_seed, NULL, ld, "Hex-encoded seed for HSM"); + opt_register_noarg("--dev-no-backtrace", opt_set_bool, + &dev_no_backtrace, + "Disable backtrace (crashes under valgrind)"); } #endif diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index e3698c7b797e..dd9db76fe084 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -107,6 +107,8 @@ def get_node(self, disconnect=None, options=None, may_fail=False): daemon.cmd_line.append("--dev-disconnect=dev_disconnect") if DEVELOPER: daemon.cmd_line.append("--dev-fail-on-subdaemon-fail") + if VALGRIND: + daemon.cmd_line.append("--dev-no-backtrace") opts = [] if options is None else options for opt in opts: daemon.cmd_line.append(opt) From 390bf6359e3cef2dc5bb2e376a7389d917d1d8d3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 31 Oct 2017 10:35:08 +1030 Subject: [PATCH 0038/1428] test_lightning.py: don't get confused by valgrind core files. I run with ulimit -c unlimited, and valgrind leaves core files like valgrind-errors.22114.core.22114 which test_lightning.py tries to parse as log files. Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index dd9db76fe084..ce020f45960e 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -153,7 +153,7 @@ def setUp(self): def getValgrindErrors(self, node): for error_file in os.listdir(node.daemon.lightning_dir): - if not re.match("valgrind-errors.\d+", error_file): + if not re.fullmatch("valgrind-errors.\d+", error_file): continue; with open(os.path.join(node.daemon.lightning_dir, error_file), 'r') as f: errors = f.read().strip() From b257b8960ba66673d267952e3a90f6ce313b3ee3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 31 Oct 2017 12:36:36 +1030 Subject: [PATCH 0039/1428] test_lightning.py: more debugging for the tx decoding fail under travis. Signed-off-by: Rusty Russell --- tests/test_lightningd.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index ce020f45960e..452e324953b0 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -239,13 +239,15 @@ def fund_channel(self, l1, l2, amount): l2.daemon.wait_for_log('-> CHANNELD_NORMAL') # Hacky way to find our output. - decoded=bitcoind.rpc.decoderawtransaction(tx)['vout'] - for out in decoded: + decoded=bitcoind.rpc.decoderawtransaction(tx) + for out in decoded['vout']: # Sometimes a float? Sometimes a decimal? WTF Python?! if out['scriptPubKey']['type'] == 'witness_v0_scripthash': if out['value'] == Decimal(amount) / 10**8 or out['value'] * 10**8 == amount: return "{}:1:{}".format(bitcoind.rpc.getblockcount(), out['n']) - raise ValueError("Can't find {} payment in {} ({})".format(amount, tx, decoded)) + # Intermittant decoding failure. See if it decodes badly twice? + decoded2=bitcoind.rpc.decoderawtransaction(tx) + raise ValueError("Can't find {} payment in {} (1={} 2={})".format(amount, tx, decoded, decoded2)) def pay(self, lsrc, ldst, amt, label=None, async=False): if not label: From 8fdf33416824ff8c83ca38eeb1d922945ff8a4ef Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 31 Oct 2017 12:37:53 +1030 Subject: [PATCH 0040/1428] Travis: turn off email notifications. Believe me Travis, I notice... Signed-off-by: Rusty Russell --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 721cbc62bf54..7433aa307c0f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: c dist: trusty sudo: true +notifications: + email: false + env: - NO_VALGRIND=1 ARCH=32 DEVELOPER=1 - NO_VALGRIND=1 ARCH=64 DEVELOPER=1 From 68dae5648d9b06f09a143a31c5f9344cd52c472a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:46:57 +1030 Subject: [PATCH 0041/1428] gossipd: route correctly using final CLTV value. Signed-off-by: Rusty Russell --- gossipd/gossip.c | 6 +++--- gossipd/gossip_wire.csv | 1 + gossipd/routing.c | 13 +++++++------ gossipd/routing.h | 3 ++- lightningd/gossip_control.c | 13 ++++++++++--- 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/gossipd/gossip.c b/gossipd/gossip.c index dd3f3758d2dc..64f6e019f544 100644 --- a/gossipd/gossip.c +++ b/gossipd/gossip.c @@ -789,19 +789,19 @@ static struct io_plan *getroute_req(struct io_conn *conn, struct daemon *daemon, { tal_t *tmpctx = tal_tmpctx(msg); struct pubkey source, destination; - u32 msatoshi; + u32 msatoshi, final_cltv; u16 riskfactor; u8 *out; struct route_hop *hops; fromwire_gossip_getroute_request(msg, NULL, &source, &destination, - &msatoshi, &riskfactor); + &msatoshi, &riskfactor, &final_cltv); status_trace("Trying to find a route from %s to %s for %d msatoshi", pubkey_to_hexstr(tmpctx, &source), pubkey_to_hexstr(tmpctx, &destination), msatoshi); hops = get_route(tmpctx, daemon->rstate, &source, &destination, - msatoshi, 1); + msatoshi, 1, final_cltv); out = towire_gossip_getroute_reply(msg, hops); tal_free(tmpctx); diff --git a/gossipd/gossip_wire.csv b/gossipd/gossip_wire.csv index 3ded7872c50b..f151ecf9d694 100644 --- a/gossipd/gossip_wire.csv +++ b/gossipd/gossip_wire.csv @@ -86,6 +86,7 @@ gossip_getroute_request,,source,struct pubkey gossip_getroute_request,,destination,struct pubkey gossip_getroute_request,,msatoshi,u32 gossip_getroute_request,,riskfactor,u16 +gossip_getroute_request,,final_cltv,u32 gossip_getroute_reply,3106 gossip_getroute_reply,,num_hops,u16 diff --git a/gossipd/routing.c b/gossipd/routing.c index 1661e67d1fc6..fc57423fcec0 100644 --- a/gossipd/routing.c +++ b/gossipd/routing.c @@ -736,7 +736,8 @@ void handle_node_announcement( struct route_hop *get_route(tal_t *ctx, struct routing_state *rstate, const struct pubkey *source, const struct pubkey *destination, - const u32 msatoshi, double riskfactor) + const u32 msatoshi, double riskfactor, + u32 final_cltv) { struct node_connection **route; u64 total_amount; @@ -756,7 +757,7 @@ struct route_hop *get_route(tal_t *ctx, struct routing_state *rstate, /* Fees, delays need to be calculated backwards along route. */ hops = tal_arr(ctx, struct route_hop, tal_count(route) + 1); total_amount = msatoshi; - total_delay = 0; + total_delay = final_cltv; for (i = tal_count(route) - 1; i >= 0; i--) { hops[i + 1].channel_id = route[i]->short_channel_id; @@ -764,16 +765,16 @@ struct route_hop *get_route(tal_t *ctx, struct routing_state *rstate, hops[i + 1].amount = total_amount; total_amount += connection_fee(route[i], total_amount); - total_delay += route[i]->delay; hops[i + 1].delay = total_delay; + total_delay += route[i]->delay; } /* Backfill the first hop manually */ hops[0].channel_id = first_conn->short_channel_id; hops[0].nodeid = first_conn->dst->id; - /* We don't charge ourselves any fees. */ + /* We don't charge ourselves any fees, nor require delay */ hops[0].amount = total_amount; - /* We do require delay though. */ - total_delay += first_conn->delay; hops[0].delay = total_delay; + + /* FIXME: Shadow route! */ return hops; } diff --git a/gossipd/routing.h b/gossipd/routing.h index 6139a946aea1..658ebfd4476a 100644 --- a/gossipd/routing.h +++ b/gossipd/routing.h @@ -124,7 +124,8 @@ void handle_node_announcement(struct routing_state *rstate, const u8 *node, size struct route_hop *get_route(tal_t *ctx, struct routing_state *rstate, const struct pubkey *source, const struct pubkey *destination, - const u32 msatoshi, double riskfactor); + const u32 msatoshi, double riskfactor, + u32 final_cltv); /* Utility function that, given a source and a destination, gives us * the direction bit the matching channel should get */ diff --git a/lightningd/gossip_control.c b/lightningd/gossip_control.c index b41137e42b65..c2e5c006b2ee 100644 --- a/lightningd/gossip_control.c +++ b/lightningd/gossip_control.c @@ -204,8 +204,9 @@ static void json_getroute_reply(struct subd *gossip, const u8 *reply, const int static void json_getroute(struct command *cmd, const char *buffer, const jsmntok_t *params) { struct pubkey id; - jsmntok_t *idtok, *msatoshitok, *riskfactortok; + jsmntok_t *idtok, *msatoshitok, *riskfactortok, *cltvtok; u64 msatoshi; + unsigned cltv = 9; double riskfactor; struct lightningd *ld = cmd->ld; @@ -213,6 +214,7 @@ static void json_getroute(struct command *cmd, const char *buffer, const jsmntok "id", &idtok, "msatoshi", &msatoshitok, "riskfactor", &riskfactortok, + "?cltv", &cltvtok, NULL)) { command_fail(cmd, "Need id, msatoshi and riskfactor"); return; @@ -223,6 +225,11 @@ static void json_getroute(struct command *cmd, const char *buffer, const jsmntok return; } + if (cltvtok && !json_tok_number(buffer, cltvtok, &cltv)) { + command_fail(cmd, "Invalid cltv"); + return; + } + if (!json_tok_u64(buffer, msatoshitok, &msatoshi)) { command_fail(cmd, "'%.*s' is not a valid number", (int)(msatoshitok->end - msatoshitok->start), @@ -236,13 +243,13 @@ static void json_getroute(struct command *cmd, const char *buffer, const jsmntok buffer + riskfactortok->start); return; } - u8 *req = towire_gossip_getroute_request(cmd, &ld->id, &id, msatoshi, riskfactor*1000); + u8 *req = towire_gossip_getroute_request(cmd, &ld->id, &id, msatoshi, riskfactor*1000, cltv); subd_req(ld->gossip, ld->gossip, req, -1, 0, json_getroute_reply, cmd); } static const struct json_command getroute_command = { "getroute", json_getroute, - "Return route to {id} for {msatoshi}, using {riskfactor}", + "Return route to {id} for {msatoshi}, using {riskfactor} and optional {cltv} (default 9)", "Returns a {route} array of {id} {msatoshi} {delay}: msatoshi and delay (in blocks) is cumulative." }; AUTODATA(json_command, &getroute_command); From 71e794a046774ae54110cf959326229353f2e73a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:46:57 +1030 Subject: [PATCH 0042/1428] lightningd: split ctlv_expiry and final_cltv. These need to be different for testing the example in BOLT 11. We also use the cltv_final instead of deadline_blocks in the final hop: various tests assumed 5 was OK, so we tweak utils.py. Signed-off-by: Rusty Russell --- channeld/channel.c | 3 +- lightningd/lightningd.h | 10 ++- lightningd/options.c | 36 ++++++--- lightningd/peer_control.c | 2 +- lightningd/peer_htlcs.c | 10 ++- tests/test_lightningd.py | 165 +++++++++++++++++++++++++++++++++----- tests/utils.py | 7 +- 7 files changed, 194 insertions(+), 39 deletions(-) diff --git a/channeld/channel.c b/channeld/channel.c index 648b6570c192..76156317f3f6 100644 --- a/channeld/channel.c +++ b/channeld/channel.c @@ -1733,7 +1733,8 @@ static void handle_offer_htlc(struct peer *peer, const u8 *inmsg) e = channel_add_htlc(peer->channel, LOCAL, peer->htlc_id, amount_msat, cltv_expiry, &payment_hash, onion_routing_packet); - status_trace("Adding HTLC %"PRIu64" gave %i", peer->htlc_id, e); + status_trace("Adding HTLC %"PRIu64" msat=%"PRIu64" cltv=%u gave %i", + peer->htlc_id, amount_msat, cltv_expiry, e); switch (e) { case CHANNEL_ERR_ADD_OK: diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 7b1af26ac8c7..543d3d5f295c 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -46,8 +46,14 @@ struct config { /* Percent of fee rate we'll use. */ u32 commitment_fee_percent; - /* Minimum/maximum time for an expiring HTLC (blocks). */ - u32 min_htlc_expiry, max_htlc_expiry; + /* Minimum CLTV to subtract from incoming HTLCs to outgoing */ + u32 cltv_expiry_delta; + + /* Minimum CLTV if we're the final hop.*/ + u32 cltv_final; + + /* Maximum time for an expiring HTLC (blocks). */ + u32 max_htlc_expiry; /* How many blocks before upstream HTLC expiry do we panic and dump? */ u32 deadline_blocks; diff --git a/lightningd/options.c b/lightningd/options.c index 42ba6657b3e1..47a6bb212fc1 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -229,9 +229,12 @@ static void config_register_opts(struct lightningd *ld) opt_register_arg("--default-fee-rate", opt_set_u64, opt_show_u64, &ld->topology->default_fee_rate, "Satoshis per kb if can't estimate fees"); - opt_register_arg("--min-htlc-expiry", opt_set_u32, opt_show_u32, - &ld->config.min_htlc_expiry, - "Minimum number of blocks to accept an HTLC before expiry"); + opt_register_arg("--cltv-delta", opt_set_u32, opt_show_u32, + &ld->config.cltv_expiry_delta, + "Number of blocks for ctlv_expiry_delta"); + opt_register_arg("--cltv-final", opt_set_u32, opt_show_u32, + &ld->config.cltv_final, + "Number of blocks for final ctlv_expiry"); opt_register_arg("--max-htlc-expiry", opt_set_u32, opt_show_u32, &ld->config.max_htlc_expiry, "Maximum number of blocks to accept an HTLC before expiry"); @@ -328,8 +331,10 @@ static const struct config testnet_config = { /* We offer to pay 5 times 2-block fee */ .commitment_fee_percent = 500, - /* Don't bother me unless I have 6 hours to collect. */ - .min_htlc_expiry = 6 * 6, + /* Be aggressive on testnet. */ + .cltv_expiry_delta = 6, + .cltv_final = 6, + /* Don't lock up channel for more than 5 days. */ .max_htlc_expiry = 5 * 6 * 24, @@ -386,8 +391,18 @@ static const struct config mainnet_config = { /* We offer to pay 5 times 2-block fee */ .commitment_fee_percent = 500, - /* Don't bother me unless I have 6 hours to collect. */ - .min_htlc_expiry = 6 * 6, + /* BOLT #2: + * + * The `cltv_expiry_delta` for channels. `3R+2G+2S` */ + /* R = 2, G = 1, S = 3 */ + .cltv_expiry_delta = 14, + + /* BOLT #2: + * + * The minimum `cltv_expiry` we will accept for terminal payments: the + * worst case for the terminal node C lower at `2R+G+S` blocks */ + .cltv_final = 8, + /* Don't lock up channel for more than 5 days. */ .max_htlc_expiry = 5 * 6 * 24, @@ -432,10 +447,11 @@ static void check_config(struct lightningd *ld) * a node MUST estimate the deadline for successful redemption * for each HTLC it offers. A node MUST NOT offer a HTLC * after this deadline */ - if (ld->config.deadline_blocks >= ld->config.min_htlc_expiry) - fatal("Deadline %u can't be more than minimum expiry %u", + if (ld->config.deadline_blocks >= ld->config.cltv_final + || ld->config.deadline_blocks >= ld->config.cltv_expiry_delta) + fatal("Deadline %u can't be more than final/expiry %u/%u", ld->config.deadline_blocks, - ld->config.min_htlc_expiry); + ld->config.cltv_final, ld->config.cltv_expiry_delta); } static void setup_default_config(struct lightningd *ld) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index e3e76c77f810..fcb4fdc48c10 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -2047,7 +2047,7 @@ static bool peer_start_channeld(struct peer *peer, &peer->ld->id, &peer->id, time_to_msec(cfg->commit_time), - cfg->min_htlc_expiry, + cfg->cltv_expiry_delta, peer->last_was_revoke, peer->last_sent_commit, peer->next_index[LOCAL], diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 2f555f674f8b..4536b3243d6c 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -361,19 +361,21 @@ static void handle_localpay(struct htlc_in *hin, * * If the `cltv_expiry` is too low, the final node MUST fail the HTLC: */ - if (get_block_height(ld->topology) + ld->config.deadline_blocks + if (get_block_height(ld->topology) + ld->config.cltv_final >= cltv_expiry) { log_debug(hin->key.peer->log, - "Expiry cltv %u too close to current %u + deadline %u", + "Expiry cltv %u too close to current %u + %u", cltv_expiry, get_block_height(ld->topology), - ld->config.deadline_blocks); + ld->config.cltv_final); failcode = WIRE_FINAL_EXPIRY_TOO_SOON; goto fail; } log_info(ld->log, "Resolving invoice '%s' with HTLC %"PRIu64, invoice->label, hin->key.id); + log_debug(ld->log, "%s: Actual amount %"PRIu64"msat, HTLC expiry %u", + invoice->label, hin->msatoshi, cltv_expiry); fulfill_htlc(hin, &invoice->r); resolve_invoice(ld, invoice); return; @@ -518,7 +520,7 @@ static void forward_htlc(struct htlc_in *hin, } if (!check_cltv(hin, cltv_expiry, outgoing_cltv_value, - ld->config.min_htlc_expiry)) { + ld->config.cltv_expiry_delta)) { failcode = WIRE_INCORRECT_CLTV_EXPIRY; goto fail; } diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 452e324953b0..9c116852b750 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -476,21 +476,21 @@ def test_permfail(self): l1.daemon.wait_for_log('Their unilateral tx, old commit point') l1.daemon.wait_for_log('-> ONCHAIND_THEIR_UNILATERAL') l2.daemon.wait_for_log('-> ONCHAIND_OUR_UNILATERAL') - l2.daemon.wait_for_log('Propose handling OUR_UNILATERAL/DELAYED_OUTPUT_TO_US by OUR_DELAYED_RETURN_TO_WALLET (.*) in 6 blocks') + l2.daemon.wait_for_log('Propose handling OUR_UNILATERAL/DELAYED_OUTPUT_TO_US by OUR_DELAYED_RETURN_TO_WALLET (.*) in 5 blocks') - # Now, mine 6 blocks so it sends out the spending tx. - bitcoind.rpc.generate(6) + # Now, mine 5 blocks so it sends out the spending tx. + bitcoind.rpc.generate(5) # It should send the to-wallet tx. l2.daemon.wait_for_log('Broadcasting OUR_DELAYED_RETURN_TO_WALLET') l2.daemon.wait_for_log('sendrawtx exit 0') # 100 after l1 sees tx, it should be done. - bitcoind.rpc.generate(94) + bitcoind.rpc.generate(95) l1.daemon.wait_for_log('onchaind complete, forgetting peer') # Now, 100 blocks l2 should be done. - bitcoind.rpc.generate(6) + bitcoind.rpc.generate(5) l2.daemon.wait_for_log('onchaind complete, forgetting peer') @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") @@ -619,8 +619,8 @@ def test_onchain_timeout(self): l2.daemon.wait_for_log('-> ONCHAIND_THEIR_UNILATERAL') # Wait for timeout. - l1.daemon.wait_for_log('Propose handling OUR_UNILATERAL/DELAYED_OUTPUT_TO_US by OUR_DELAYED_RETURN_TO_WALLET .* in 6 blocks') - bitcoind.rpc.generate(6) + l1.daemon.wait_for_log('Propose handling OUR_UNILATERAL/DELAYED_OUTPUT_TO_US by OUR_DELAYED_RETURN_TO_WALLET .* in 5 blocks') + bitcoind.rpc.generate(5) # (l1 will also collect its to-self payment.) l1.daemon.wait_for_log('sendrawtx exit 0') @@ -706,12 +706,12 @@ def try_pay(): t.join(timeout=1) assert not t.isAlive() - # After 4 more blocks, l2 can spend to-us. - l1.bitcoin.rpc.generate(4) + # Three more, l2 can spend to-us. + bitcoind.rpc.generate(3) l2.daemon.wait_for_log('Broadcasting OUR_DELAYED_RETURN_TO_WALLET .* to resolve OUR_UNILATERAL/DELAYED_OUTPUT_TO_US') l2.daemon.wait_for_log('sendrawtx exit 0') - # One more, HTLC tx is now spentable. + # One more block, HTLC tx is now spentable. l1.bitcoin.rpc.generate(1) l2.daemon.wait_for_log('Broadcasting OUR_DELAYED_RETURN_TO_WALLET .* to resolve OUR_HTLC_SUCCESS_TX/DELAYED_OUTPUT_TO_US') l2.daemon.wait_for_log('sendrawtx exit 0') @@ -906,18 +906,18 @@ def test_permfail_htlc_in(self): # OK, l1 sees l2 fulfill htlc. l1.daemon.wait_for_log('THEIR_UNILATERAL/OUR_HTLC gave us preimage') - l2.daemon.wait_for_log('Propose handling OUR_HTLC_SUCCESS_TX/DELAYED_OUTPUT_TO_US by OUR_DELAYED_RETURN_TO_WALLET .* in 6 blocks') - bitcoind.rpc.generate(6) + l2.daemon.wait_for_log('Propose handling OUR_HTLC_SUCCESS_TX/DELAYED_OUTPUT_TO_US by OUR_DELAYED_RETURN_TO_WALLET .* in 5 blocks') + bitcoind.rpc.generate(5) l2.daemon.wait_for_log('sendrawtx exit 0') t.cancel() # Now, 100 blocks it should be done. - bitcoind.rpc.generate(94) + bitcoind.rpc.generate(95) l1.daemon.wait_for_log('onchaind complete, forgetting peer') assert not l2.daemon.is_in_log('onchaind complete, forgetting peer') - bitcoind.rpc.generate(6) + bitcoind.rpc.generate(5) l2.daemon.wait_for_log('onchaind complete, forgetting peer') @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") @@ -940,7 +940,7 @@ def test_permfail_htlc_out(self): l1.daemon.wait_for_log('-> ONCHAIND_THEIR_UNILATERAL') l2.daemon.wait_for_log('-> ONCHAIND_OUR_UNILATERAL') l2.daemon.wait_for_logs(['Propose handling OUR_UNILATERAL/OUR_HTLC by OUR_HTLC_TIMEOUT_TO_US \\(.*\\) in 5 blocks', - 'Propose handling OUR_UNILATERAL/DELAYED_OUTPUT_TO_US by OUR_DELAYED_RETURN_TO_WALLET .* in 6 blocks']) + 'Propose handling OUR_UNILATERAL/DELAYED_OUTPUT_TO_US by OUR_DELAYED_RETURN_TO_WALLET .* in 5 blocks']) l1.daemon.wait_for_log('Propose handling THEIR_UNILATERAL/THEIR_HTLC by THEIR_HTLC_TIMEOUT_TO_THEM \\(IGNORING\\) in 5 blocks') # l1 then gets preimage, uses it instead of ignoring @@ -953,13 +953,13 @@ def test_permfail_htlc_out(self): l2.daemon.wait_for_log('OUR_UNILATERAL/OUR_HTLC gave us preimage') t.cancel() - # l2 can send OUR_DELAYED_RETURN_TO_WALLET after 5 more blocks. - bitcoind.rpc.generate(5) + # l2 can send OUR_DELAYED_RETURN_TO_WALLET after 4 more blocks. + bitcoind.rpc.generate(4) l2.daemon.wait_for_log('Broadcasting OUR_DELAYED_RETURN_TO_WALLET .* to resolve OUR_UNILATERAL/DELAYED_OUTPUT_TO_US') l2.daemon.wait_for_log('sendrawtx exit 0') # Now, 100 blocks they should be done. - bitcoind.rpc.generate(93) + bitcoind.rpc.generate(94) assert not l1.daemon.is_in_log('onchaind complete, forgetting peer') assert not l2.daemon.is_in_log('onchaind complete, forgetting peer') bitcoind.rpc.generate(1) @@ -1161,6 +1161,135 @@ def test_forward(self): route = copy.deepcopy(baseroute) l1.rpc.sendpay(to_json(route), rhash) + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1 for --dev-broadcast-interval") + def test_forward_different_fees_and_cltv(self): + # FIXME: Check BOLT quotes here too + # BOLT #7: + # ``` + # B + # / \ + # / \ + # A C + # \ / + # \ / + # D + # ``` + # + # Each advertises the following `cltv_expiry_delta` on its end of every + # channel: + # + # 1. A: 10 blocks + # 2. B: 20 blocks + # 3. C: 30 blocks + # 4. D: 40 blocks + # + # C also uses a minimum `cltv_expiry` of 9 (the default) when requesting + # payments. + # + # Also, each node has the same fee scheme which it uses for each of its + # channels: + # + # 1. A: 100 base + 1000 millionths + # 1. B: 200 base + 2000 millionths + # 1. C: 300 base + 3000 millionths + # 1. D: 400 base + 4000 millionths + + # We don't do D yet. + l1 = self.node_factory.get_node(options=['--cltv-delta=10', '--fee-base=100', '--fee-per-satoshi=1000']) + l2 = self.node_factory.get_node(options=['--cltv-delta=20', '--fee-base=200', '--fee-per-satoshi=2000']) + l3 = self.node_factory.get_node(options=['--cltv-delta=30', '--cltv-final=9', '--fee-base=300', '--fee-per-satoshi=3000']) + + ret = l1.rpc.connect(l2.info['id'], 'localhost:{}'.format(l2.info['port'])) + assert ret['id'] == l2.info['id'] + + l1.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') + l2.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') + + ret = l2.rpc.connect(l3.info['id'], 'localhost:{}'.format(l3.info['port'])) + assert ret['id'] == l3.info['id'] + + l2.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') + l3.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') + + c1 = self.fund_channel(l1, l2, 10**6) + c2 = self.fund_channel(l2, l3, 10**6) + + # Allow announce messages. + l1.bitcoin.rpc.generate(5) + + # Make sure l1 has seen announce for all channels. + l1.daemon.wait_for_logs([ + 'Received channel_update for channel {}\\(0\\)'.format(c1), + 'Received channel_update for channel {}\\(1\\)'.format(c1), + 'Received channel_update for channel {}\\(0\\)'.format(c2), + 'Received channel_update for channel {}\\(1\\)'.format(c2)]) + + # BOLT #7: + # + # If B were to send 4,999,999 millisatoshi directly to C, it wouldn't + # charge itself a fee nor add its own `cltv_expiry_delta`, so it would + # use C's requested `cltv_expiry` of 9. We also assume it adds a + # "shadow route" to give an extra CLTV of 42. It could also add extra + # cltv deltas at other hops, as these values are a minimum, but we don't + # here for simplicity: + + # FIXME: Add shadow route + shadow_route=0 + route = l2.rpc.getroute(l3.info['id'], 4999999, 1)["route"] + assert len(route) == 1 + + # BOLT #7: + # + # * `amount_msat`: 4999999 + # * `cltv_expiry`: current-block-height + 9 + 42 + # * `onion_routing_packet`: + # * `amt_to_forward` = 4999999 + # * `outgoing_cltv_value` = current-block-height + 9 + 42 + # + assert route[0]['msatoshi'] == 4999999 + assert route[0]['delay'] == 9 + shadow_route + + # BOLT #7: + # If A were to send an 4,999,999 millisatoshi to C via B, it needs to + # pay B the fee it specified in the B->C `channel_update`, calculated as + # per [HTLC Fees](#htlc_fees): + # + # 200 + 4999999 * 2000 / 1000000 = 10199 + # + # Similarly, it would need to add the `cltv_expiry` from B->C's + # `channel_update` (20), plus C's requested minimum (9), plus 42 for the + # "shadow route". Thus the `update_add_htlc` message from A to B would + # be: + # + # * `amount_msat`: 5010198 + # * `cltv_expiry`: current-block-height + 20 + 9 + 42 + # * `onion_routing_packet`: + # * `amt_to_forward` = 4999999 + # * `outgoing_cltv_value` = current-block-height + 9 + 42 + route = l1.rpc.getroute(l3.info['id'], 4999999, 1)["route"] + assert len(route) == 2 + + assert route[0]['msatoshi'] == 5010198 + assert route[0]['delay'] == 20 + 9 + shadow_route + assert route[1]['msatoshi'] == 4999999 + assert route[1]['delay'] == 9 + shadow_route + + rhash = l3.rpc.invoice(4999999, 'test_forward_different_fees_and_cltv')['rhash'] + assert l3.rpc.listinvoice('test_forward_different_fees_and_cltv')[0]['complete'] == False + + # This should work. + l1.rpc.sendpay(to_json(route), rhash) + + # We add one to the blockcount for a bit of fuzz (FIXME: Shadowroute would fix this!) + shadow_route = 1 + l1.daemon.wait_for_log("Adding HTLC 0 msat=5010198 cltv={} gave 0" + .format(bitcoind.rpc.getblockcount() + 20 + 9 + shadow_route)) + l2.daemon.wait_for_log("Adding HTLC 0 msat=4999999 cltv={} gave 0" + .format(bitcoind.rpc.getblockcount() + 9 + shadow_route)) + l3.daemon.wait_for_log("test_forward_different_fees_and_cltv: Actual amount 4999999msat, HTLC expiry {}" + .format(bitcoind.rpc.getblockcount() + 9 + shadow_route)) + assert l3.rpc.listinvoice('test_forward_different_fees_and_cltv')[0]['complete'] == True + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_disconnect(self): # These should all make us fail, and retry. diff --git a/tests/utils.py b/tests/utils.py index 690e898b5780..1785506551a1 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -20,9 +20,10 @@ LIGHTNINGD_CONFIG = { "bitcoind-poll": "1s", "log-level": "debug", - "deadline-blocks": 5, - "min-htlc-expiry": 6, - "locktime-blocks": 6, + "deadline-blocks": 4, + "cltv-delta": 6, + "cltv-final": 5, + "locktime-blocks": 5, } DEVELOPER = os.getenv("DEVELOPER", "0") == "1" From 0000dc5908836a976a9e78c25d92a9764bcd39ae Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:46:57 +1030 Subject: [PATCH 0043/1428] lightningd: Allow in excess of ctlv_expiry. We add a test which both overpays and over-sets the delta on the intermediate and final nodes. Signed-off-by: Rusty Russell --- lightningd/peer_htlcs.c | 4 ++-- tests/test_lightningd.py | 52 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 4536b3243d6c..e6063f5da9e0 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -243,7 +243,7 @@ static bool check_amount(struct htlc_in *hin, * * `outgoing_cltv_value` - The CLTV value that the _outgoing_ HTLC carrying * the packet should have. * - * cltv_expiry - cltv_expiry_delta = outgoing_cltv_value + * cltv_expiry - cltv_expiry_delta >= outgoing_cltv_value * * Inclusion of this field allows a node to both authenticate the information * specified by the original sender and the parameters of the HTLC forwarded, @@ -259,7 +259,7 @@ static bool check_amount(struct htlc_in *hin, static bool check_cltv(struct htlc_in *hin, u32 cltv_expiry, u32 outgoing_cltv_value, u32 delta) { - if (cltv_expiry - delta == outgoing_cltv_value) + if (cltv_expiry - delta >= outgoing_cltv_value) return true; log_debug(hin->key.peer->ld->log, "HTLC %"PRIu64" incorrect CLTV:" " %u in, %u out, delta reqd %u", diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 9c116852b750..28b9a6c5abe8 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -1290,6 +1290,58 @@ def test_forward_different_fees_and_cltv(self): .format(bitcoind.rpc.getblockcount() + 9 + shadow_route)) assert l3.rpc.listinvoice('test_forward_different_fees_and_cltv')[0]['complete'] == True + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1 for --dev-broadcast-interval") + def test_forward_pad_fees_and_cltv(self): + """Test that we are allowed extra locktime delta, and fees""" + + l1 = self.node_factory.get_node(options=['--cltv-delta=10', '--fee-base=100', '--fee-per-satoshi=1000']) + l2 = self.node_factory.get_node(options=['--cltv-delta=20', '--fee-base=200', '--fee-per-satoshi=2000']) + l3 = self.node_factory.get_node(options=['--cltv-delta=30', '--cltv-final=9', '--fee-base=300', '--fee-per-satoshi=3000']) + + ret = l1.rpc.connect(l2.info['id'], 'localhost:{}'.format(l2.info['port'])) + assert ret['id'] == l2.info['id'] + + l1.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') + l2.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') + + ret = l2.rpc.connect(l3.info['id'], 'localhost:{}'.format(l3.info['port'])) + assert ret['id'] == l3.info['id'] + + l2.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') + l3.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') + + c1 = self.fund_channel(l1, l2, 10**6) + c2 = self.fund_channel(l2, l3, 10**6) + + # Allow announce messages. + l1.bitcoin.rpc.generate(5) + + # Make sure l1 has seen announce for all channels. + l1.daemon.wait_for_logs([ + 'Received channel_update for channel {}\\(0\\)'.format(c1), + 'Received channel_update for channel {}\\(1\\)'.format(c1), + 'Received channel_update for channel {}\\(0\\)'.format(c2), + 'Received channel_update for channel {}\\(1\\)'.format(c2)]) + + route = l1.rpc.getroute(l3.info['id'], 4999999, 1)["route"] + assert len(route) == 2 + + assert route[0]['msatoshi'] == 5010198 + assert route[0]['delay'] == 20 + 9 + assert route[1]['msatoshi'] == 4999999 + assert route[1]['delay'] == 9 + + # Modify so we overpay, overdo the cltv. + route[0]['msatoshi'] += 2000 + route[0]['delay'] += 20 + route[1]['msatoshi'] += 1000 + route[1]['delay'] += 10 + + # This should work. + rhash = l3.rpc.invoice(4999999, 'test_forward_pad_fees_and_cltv')['rhash'] + l1.rpc.sendpay(to_json(route), rhash) + assert l3.rpc.listinvoice('test_forward_pad_fees_and_cltv')[0]['complete'] == True + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_disconnect(self): # These should all make us fail, and retry. From c2a0c51c30ff143537bdd7b68d6ee704067ed4c6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 23 Oct 2017 14:46:57 +1030 Subject: [PATCH 0044/1428] lightningd: check for excessive HTLC locktimes. Signed-off-by: Rusty Russell --- lightningd/peer_htlcs.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index e6063f5da9e0..b6f4011c7782 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -545,6 +545,19 @@ static void forward_htlc(struct htlc_in *hin, goto fail; } + /* FIXME: Add this to BOLT! */ + if (get_block_height(next->ld->topology) + + next->ld->config.max_htlc_expiry < outgoing_cltv_value) { + log_debug(hin->key.peer->log, + "Expiry cltv %u too far from current %u + max %u", + outgoing_cltv_value, + get_block_height(next->ld->topology), + next->ld->config.max_htlc_expiry); + /* FIXME: WIRE_EXPIRY_TOO_FAR? */ + failcode = WIRE_TEMPORARY_CHANNEL_FAILURE; + goto fail; + } + failcode = send_htlc_out(next, amt_to_forward, outgoing_cltv_value, &hin->payment_hash, next_onion, hin, NULL, NULL); From 847df2eb1df6cf89b7986432264feea829036807 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 31 Oct 2017 12:27:13 +0100 Subject: [PATCH 0045/1428] contrib: Updated dockerfile to use bitcoind v0.15.1 This was causing intermittent `rawtransactiondecode` errors see ElementsProject/lightning#332 Reported-by: @achow101 Signed-off-by: Christian Decker --- contrib/Dockerfile.builder | 18 +++++++--- contrib/Dockerfile.builder.i386 | 61 +++++++++++++++++---------------- 2 files changed, 44 insertions(+), 35 deletions(-) diff --git a/contrib/Dockerfile.builder b/contrib/Dockerfile.builder index 66042b45cb5b..11950f4a59ff 100644 --- a/contrib/Dockerfile.builder +++ b/contrib/Dockerfile.builder @@ -1,12 +1,14 @@ FROM ubuntu:16.04 +MAINTAINER Christian Decker ENV DEBIAN_FRONTEND noninteractive WORKDIR /build -RUN echo "deb http://ppa.launchpad.net/bitcoin/bitcoin/ubuntu xenial main" | tee -a /etc/apt/sources.list.d/bitcoin.list RUN apt-get -qq update && \ - apt-get -qq install --allow-unauthenticated -yy \ - eatmydata \ + apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ + autoconf \ + automake \ + eatmydata \ software-properties-common \ build-essential \ autoconf \ @@ -20,8 +22,14 @@ RUN apt-get -qq update && \ python3 \ valgrind \ net-tools \ - bitcoind \ python3-pip \ - && rm -rf /var/lib/apt/lists/* + wget && \ + rm -rf /var/lib/apt/lists/* + +RUN cd /tmp/ && \ + wget https://bitcoin.org/bin/bitcoin-core-0.15.0.1/bitcoin-0.15.0.1-x86_64-linux-gnu.tar.gz -O bitcoin.tar.gz && \ + tar -xvzf bitcoin.tar.gz && \ + mv /tmp/bitcoin-0.15.0/bin/bitcoin* /usr/local/bin/ && \ + rm -rf bitcoin.tar.gz /tmp/bitcoin-0.15.0 RUN pip3 install python-bitcoinlib==0.7.0 diff --git a/contrib/Dockerfile.builder.i386 b/contrib/Dockerfile.builder.i386 index 45910b360e24..b71fc7bbf824 100644 --- a/contrib/Dockerfile.builder.i386 +++ b/contrib/Dockerfile.builder.i386 @@ -1,34 +1,35 @@ FROM i386/ubuntu:16.04 MAINTAINER Christian Decker -RUN echo deb http://ppa.launchpad.net/bitcoin/bitcoin/ubuntu xenial main > /etc/apt/sources.list.d/bitcoin-bitcoin-xenial.list -RUN apt-get update -qq -RUN apt-get install -qq -y --no-install-recommends --allow-unauthenticated \ - asciidoc \ - curl \ - git \ - make \ - automake \ - autoconf \ - libtool \ - bitcoind \ - build-essential \ - libprotobuf-c-dev \ - libsodium-dev \ - libbase58-dev \ - libsqlite3-dev \ - libgmp-dev \ - libsqlite3-dev \ - git \ - net-tools \ - valgrind \ - ca-certificates \ - python \ - python3 \ - python3-pip \ - python3-setuptools \ - && rm -rf /var/lib/apt/lists/* - -RUN pip3 install python-bitcoinlib==0.7. -RUN mkdir /build +ENV DEBIAN_FRONTEND noninteractive WORKDIR /build + +RUN apt-get -qq update && \ + apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ + autoconf \ + automake \ + eatmydata \ + software-properties-common \ + build-essential \ + autoconf \ + libtool \ + libprotobuf-c-dev \ + libsqlite3-dev \ + libgmp-dev \ + libsqlite3-dev \ + git \ + python \ + python3 \ + valgrind \ + net-tools \ + python3-pip \ + wget && \ + rm -rf /var/lib/apt/lists/* + +RUN cd /tmp/ && \ + wget https://bitcoin.org/bin/bitcoin-core-0.15.0.1/bitcoin-0.15.0.1-i686-pc-linux-gnu.tar.gz -O bitcoin.tar.gz && \ + tar -xvzf bitcoin.tar.gz && \ + mv /tmp/bitcoin-0.15.0/bin/bitcoin* /usr/local/bin/ && \ + rm -rf bitcoin.tar.gz /tmp/bitcoin-0.15.0 + +RUN pip3 install python-bitcoinlib==0.7.0 From fa28bf4ec9dba2b479f7a42df203bb6510f40023 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 31 Oct 2017 12:29:22 +0100 Subject: [PATCH 0046/1428] travis: use tee while pulling docker images Shows us progress while downloading images and we can verify that the images match what we expect. `tee` is mainly used to disable the pull animation that was spamming the logs otherwise. Signed-off-by: Christian Decker --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7433aa307c0f..937988940f3f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ env: # Trusty (aka 14.04) is way way too old, so run in docker... script: - - docker pull cdecker/lightning-ci:${ARCH}bit > /dev/null + - docker pull cdecker/lightning-ci:${ARCH}bit | tee - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make -j3 DEVELOPER=${DEVELOPER} - docker run --rm=true -e NO_VALGRIND=${NO_VALGRIND:-0} -e TEST_DEBUG=${TEST_DEBUG:-0} -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check DEVELOPER=${DEVELOPER} - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check-source From f78205f30f90ff2da5785497e4689cfbe5be4a95 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 31 Oct 2017 12:59:37 +0100 Subject: [PATCH 0047/1428] travis: Fold build output Signed-off-by: Christian Decker --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 937988940f3f..4503e07d6fca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,9 @@ env: # Trusty (aka 14.04) is way way too old, so run in docker... script: - docker pull cdecker/lightning-ci:${ARCH}bit | tee + - echo "Building..." + - echo "travis_fold:start:SCRIPT folding starts" - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make -j3 DEVELOPER=${DEVELOPER} + - echo "travis_fold:end:SCRIPT folding ends" - docker run --rm=true -e NO_VALGRIND=${NO_VALGRIND:-0} -e TEST_DEBUG=${TEST_DEBUG:-0} -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check DEVELOPER=${DEVELOPER} - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check-source From b267b24c082d1995a15a260c171eef855cff8d7f Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 30 Oct 2017 16:05:00 +0100 Subject: [PATCH 0048/1428] db: db_exec_prepared takes ownership of the statement Technically it's the caller that'll own the statement, but it is nice to have db_exec_prepared dispose of it. Signed-off-by: Christian Decker --- wallet/db.c | 16 +++++++++++----- wallet/db.h | 3 ++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index f7c5f5ffa2b1..b7c56ba0540f 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -150,17 +150,23 @@ sqlite3_stmt *db_prepare_(const char *caller, struct db *db, const char *query) bool db_exec_prepared_(const char *caller, struct db *db, sqlite3_stmt *stmt) { - if (db->in_transaction && db->err) - return false; + if (db->in_transaction && db->err) { + goto fail; + } + db_clear_error(db); if (sqlite3_step(stmt) != SQLITE_DONE) { db->err = tal_fmt(db, "%s: %s", caller, sqlite3_errmsg(db->sql)); - return false; - } else { - return true; + goto fail; } + + sqlite3_finalize(stmt); + return true; +fail: + sqlite3_finalize(stmt); + return false; } bool PRINTF_FMT(3, 4) diff --git a/wallet/db.h b/wallet/db.h index 4217f87bfa29..86015d117c2b 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -102,7 +102,8 @@ sqlite3_stmt *db_prepare_(const char *caller, struct db *db, const char *query); * all non-null variables using the `sqlite3_bind_*` functions, it can * be executed with this function. It is a small, transaction-aware, * wrapper around `sqlite3_step`, that also sets `db->err` if the - * execution fails. + * execution fails. This will take ownership of `stmt` and will free + * it before returning. * * @db: The database to execute on * @stmt: The prepared statement to execute From 33da7f50c74472bb47e8471ab5847baafc3ed96a Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 30 Oct 2017 19:21:41 +0100 Subject: [PATCH 0049/1428] db: Added short_channel_id, tx, pubkey and signature primitives We use these quite often and it is cumbersome having to do these simple conversions inline, so just expose pseudo-sqlite3 methods to bind and extract from/to a stmt. Signed-off-by: Christian Decker --- wallet/db.c | 101 ++++++++++++++++++++++++++++++++++++++++++++++++ wallet/db.h | 24 ++++++++++++ wallet/wallet.c | 24 ------------ 3 files changed, 125 insertions(+), 24 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index b7c56ba0540f..009fd7452ed5 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -419,3 +419,104 @@ bool sqlite3_column_hexval(sqlite3_stmt *s, int col, void *dest, size_t destlen) return false; return hex_decode(source, sourcelen, dest, destlen); } + +bool sqlite3_bind_short_channel_id(sqlite3_stmt *stmt, int col, + const struct short_channel_id *id) +{ + char *ser = short_channel_id_to_str(id, id); + sqlite3_bind_blob(stmt, col, ser, strlen(ser), SQLITE_TRANSIENT); + tal_free(ser); + return true; +} + +bool sqlite3_column_short_channel_id(sqlite3_stmt *stmt, int col, + struct short_channel_id *dest) +{ + const char *source = sqlite3_column_blob(stmt, col); + size_t sourcelen = sqlite3_column_bytes(stmt, col); + return short_channel_id_from_str(source, sourcelen, dest); +} + +bool sqlite3_bind_tx(sqlite3_stmt *stmt, int col, const struct bitcoin_tx *tx) +{ + u8 *ser = linearize_tx(NULL, tx); + sqlite3_bind_blob(stmt, col, tal_hex(ser, ser), 2*tal_len(ser), SQLITE_TRANSIENT); + tal_free(ser); + return true; +} + +struct bitcoin_tx *sqlite3_column_tx(const tal_t *ctx, sqlite3_stmt *stmt, + int col) +{ + return bitcoin_tx_from_hex( + ctx, + sqlite3_column_blob(stmt, col), + sqlite3_column_bytes(stmt, col)); +} + +bool sqlite3_bind_signature(sqlite3_stmt *stmt, int col, + const secp256k1_ecdsa_signature *sig) +{ + bool ok; + u8 buf[64]; + ok = secp256k1_ecdsa_signature_serialize_compact(secp256k1_ctx, buf, + sig) == 1; + sqlite3_bind_blob(stmt, col, buf, sizeof(buf), SQLITE_TRANSIENT); + return ok; +} + +bool sqlite3_column_signature(sqlite3_stmt *stmt, int col, + secp256k1_ecdsa_signature *sig) +{ + assert(sqlite3_column_bytes(stmt, col) == 64); + return secp256k1_ecdsa_signature_parse_compact( + secp256k1_ctx, sig, sqlite3_column_blob(stmt, col)) == 1; +} + +bool sqlite3_column_pubkey(sqlite3_stmt *stmt, int col, struct pubkey *dest) +{ + u8 buf[PUBKEY_DER_LEN]; + + if (sqlite3_column_bytes(stmt, col) == 2*PUBKEY_DER_LEN) { + /* FIXME: Remove the legacy path for hex-values */ + if (!sqlite3_column_hexval(stmt, col, buf, sizeof(buf))) + return false; + } else { + assert(sqlite3_column_bytes(stmt, col) == PUBKEY_DER_LEN); + memcpy(buf, sqlite3_column_blob(stmt, col), PUBKEY_DER_LEN); + } + + return pubkey_from_der(buf, sizeof(buf), dest); +} + +bool sqlite3_bind_pubkey(sqlite3_stmt *stmt, int col, const struct pubkey *pk) +{ + u8 der[PUBKEY_DER_LEN]; + pubkey_to_der(der, pk); + sqlite3_bind_blob(stmt, col, der, sizeof(der), SQLITE_TRANSIENT); + return true; +} + +bool sqlite3_column_preimage(sqlite3_stmt *stmt, int col, struct preimage *dest) +{ + assert(sqlite3_column_bytes(stmt, col) == sizeof(struct preimage)); + return memcpy(dest, sqlite3_column_blob(stmt, col), sizeof(struct preimage)); +} + +bool sqlite3_bind_preimage(sqlite3_stmt *stmt, int col, const struct preimage *p) +{ + sqlite3_bind_blob(stmt, col, p, sizeof(struct preimage), SQLITE_TRANSIENT); + return true; +} + +bool sqlite3_column_sha256(sqlite3_stmt *stmt, int col, struct sha256 *dest) +{ + assert(sqlite3_column_bytes(stmt, col) == sizeof(struct sha256)); + return memcpy(dest, sqlite3_column_blob(stmt, col), sizeof(struct sha256)); +} + +bool sqlite3_bind_sha256(sqlite3_stmt *stmt, int col, const struct sha256 *p) +{ + sqlite3_bind_blob(stmt, col, p, sizeof(struct sha256), SQLITE_TRANSIENT); + return true; +} diff --git a/wallet/db.h b/wallet/db.h index 86015d117c2b..5641f25c5014 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -2,9 +2,14 @@ #define WALLET_DB_H #include "config.h" +#include +#include +#include +#include #include #include +#include #include #include @@ -111,4 +116,23 @@ sqlite3_stmt *db_prepare_(const char *caller, struct db *db, const char *query); #define db_exec_prepared(db,stmt) db_exec_prepared_(__func__,db,stmt) bool db_exec_prepared_(const char *caller, struct db *db, sqlite3_stmt *stmt); +bool sqlite3_bind_short_channel_id(sqlite3_stmt *stmt, int col, + const struct short_channel_id *id); +bool sqlite3_column_short_channel_id(sqlite3_stmt *stmt, int col, + struct short_channel_id *dest); +bool sqlite3_bind_tx(sqlite3_stmt *stmt, int col, const struct bitcoin_tx *tx); +struct bitcoin_tx *sqlite3_column_tx(const tal_t *ctx, sqlite3_stmt *stmt, + int col); +bool sqlite3_bind_signature(sqlite3_stmt *stmt, int col, const secp256k1_ecdsa_signature *sig); +bool sqlite3_column_signature(sqlite3_stmt *stmt, int col, secp256k1_ecdsa_signature *sig); + +bool sqlite3_column_pubkey(sqlite3_stmt *stmt, int col, struct pubkey *dest); +bool sqlite3_bind_pubkey(sqlite3_stmt *stmt, int col, const struct pubkey *pk); + +bool sqlite3_column_preimage(sqlite3_stmt *stmt, int col, struct preimage *dest); +bool sqlite3_bind_preimage(sqlite3_stmt *stmt, int col, const struct preimage *p); + +bool sqlite3_column_sha256(sqlite3_stmt *stmt, int col, struct sha256 *dest); +bool sqlite3_bind_sha256(sqlite3_stmt *stmt, int col, const struct sha256 *p); + #endif /* WALLET_DB_H */ diff --git a/wallet/wallet.c b/wallet/wallet.c index 032136b87c37..8f0efc304c4a 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -351,14 +351,6 @@ bool wallet_shachain_load(struct wallet *wallet, u64 id, return true; } -static bool sqlite3_column_short_channel_id(sqlite3_stmt *stmt, int col, - struct short_channel_id *dest) -{ - const char *source = sqlite3_column_blob(stmt, col); - size_t sourcelen = sqlite3_column_bytes(stmt, col); - return short_channel_id_from_str(source, sourcelen, dest); -} - static bool sqlite3_column_sig(sqlite3_stmt *stmt, int col, secp256k1_ecdsa_signature *sig) { u8 buf[64]; @@ -367,14 +359,6 @@ static bool sqlite3_column_sig(sqlite3_stmt *stmt, int col, secp256k1_ecdsa_sign return secp256k1_ecdsa_signature_parse_compact(secp256k1_ctx, sig, buf) == 1; } -static bool sqlite3_column_pubkey(sqlite3_stmt *stmt, int col, struct pubkey *dest) -{ - u8 buf[PUBKEY_DER_LEN]; - if (!sqlite3_column_hexval(stmt, col, buf, sizeof(buf))) - return false; - return pubkey_from_der(buf, sizeof(buf), dest); -} - static u8 *sqlite3_column_varhexblob(tal_t *ctx, sqlite3_stmt *stmt, int col) { const u8 *source = sqlite3_column_blob(stmt, col); @@ -382,14 +366,6 @@ static u8 *sqlite3_column_varhexblob(tal_t *ctx, sqlite3_stmt *stmt, int col) return tal_hexdata(ctx, source, sourcelen); } -static struct bitcoin_tx *sqlite3_column_tx(const tal_t *ctx, - sqlite3_stmt *stmt, int col) -{ - return bitcoin_tx_from_hex(ctx, - sqlite3_column_blob(stmt, col), - sqlite3_column_bytes(stmt, col)); -} - static bool wallet_peer_load(struct wallet *w, const u64 id, struct peer *peer) { bool ok = true; From e9cfa65a12fcbb07fcfcaea27f83a60c08e730cc Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 30 Oct 2017 16:49:25 +0100 Subject: [PATCH 0050/1428] wallet: Migrate HTLC persistence to native sqlite3 binding This is a preparatory step before we get rid of the hex encoding of blob values. Signed-off-by: Christian Decker --- wallet/wallet.c | 128 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 42 deletions(-) diff --git a/wallet/wallet.c b/wallet/wallet.c index 8f0efc304c4a..1c99ad39bf8a 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -825,21 +825,47 @@ bool wallet_htlc_save_in(struct wallet *wallet, { bool ok = true; tal_t *tmpctx = tal_tmpctx(wallet); + sqlite3_stmt *stmt; - ok &= db_exec( - __func__, wallet->db, - "INSERT INTO channel_htlcs " - "(channel_id, channel_htlc_id, direction, origin_htlc, msatoshi, cltv_expiry, payment_hash, payment_key, hstate, shared_secret, routing_onion) VALUES " - "(%" PRIu64 ", %"PRIu64", %d, NULL, %"PRIu64", %d, '%s', %s, %d, '%s', '%s');", - chan->id, in->key.id, DIRECTION_INCOMING, in->msatoshi, in->cltv_expiry, - tal_hexstr(tmpctx, &in->payment_hash, sizeof(struct sha256)), - in->preimage == NULL ? "NULL" : tal_hexstr(tmpctx, &in->preimage, - sizeof(struct preimage)), - in->hstate, - tal_hexstr(tmpctx, &in->shared_secret, sizeof(struct secret)), - tal_hexstr(tmpctx, &in->onion_routing_packet, sizeof(in->onion_routing_packet)) - ); + stmt = db_prepare( + wallet->db, + "INSERT INTO channel_htlcs (" + " channel_id," + " channel_htlc_id, " + " direction," + " msatoshi," + " cltv_expiry," + " payment_hash, " + " payment_key," + " hstate," + " shared_secret," + " routing_onion) VALUES " + "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"); + sqlite3_bind_int64(stmt, 1, chan->id); + sqlite3_bind_int64(stmt, 2, in->key.id); + sqlite3_bind_int(stmt, 3, DIRECTION_INCOMING); + sqlite3_bind_int64(stmt, 4, in->msatoshi); + sqlite3_bind_int(stmt, 5, in->cltv_expiry); + sqlite3_bind_blob(stmt, 6, tal_hexstr(tmpctx, &in->payment_hash, + sizeof(struct sha256)), + 2 * sizeof(struct sha256), SQLITE_TRANSIENT); + if (in->preimage) + sqlite3_bind_blob( + stmt, 7, + tal_hexstr(tmpctx, in->preimage, sizeof(struct preimage)), + 2 * sizeof(struct preimage), SQLITE_TRANSIENT); + sqlite3_bind_int(stmt, 8, in->hstate); + sqlite3_bind_blob(stmt, 9, tal_hexstr(tmpctx, &in->shared_secret, + sizeof(struct secret)), + 2 * sizeof(struct secret), SQLITE_TRANSIENT); + + sqlite3_bind_blob( + stmt, 10, tal_hexstr(tmpctx, &in->onion_routing_packet, + sizeof(in->onion_routing_packet)), + 2 * sizeof(in->onion_routing_packet), SQLITE_TRANSIENT); + + ok = db_exec_prepared(wallet->db, stmt); tal_free(tmpctx); if (ok) { in->dbid = sqlite3_last_insert_rowid(wallet->db->sql); @@ -853,29 +879,49 @@ bool wallet_htlc_save_out(struct wallet *wallet, { bool ok = true; tal_t *tmpctx = tal_tmpctx(wallet); + sqlite3_stmt *stmt; /* We absolutely need the incoming HTLC to be persisted before * we can persist it's dependent */ assert(out->in == NULL || out->in->dbid != 0); out->origin_htlc_id = out->in?out->in->dbid:0; - ok &= db_exec( - __func__, wallet->db, - "INSERT INTO channel_htlcs " - "(channel_id, channel_htlc_id, direction, origin_htlc, msatoshi, cltv_expiry, " - "payment_hash, payment_key, hstate, shared_secret, routing_onion) VALUES " - "(%" PRIu64 ", %" PRIu64 ", %d, %s, %" PRIu64", %d, '%s', %s, %d, NULL, '%s');", - chan->id, - out->key.id, - DIRECTION_OUTGOING, - out->in ? tal_fmt(tmpctx, "%" PRIu64, out->in->dbid) : "NULL", - out->msatoshi, - out->cltv_expiry, - tal_hexstr(tmpctx, &out->payment_hash, sizeof(struct sha256)), - out->preimage ? tal_hexstr(tmpctx, &out->preimage, sizeof(struct preimage)) : "NULL", - out->hstate, - tal_hexstr(tmpctx, &out->onion_routing_packet, sizeof(out->onion_routing_packet)) - ); + stmt = db_prepare( + wallet->db, + "INSERT INTO channel_htlcs (" + " channel_id," + " channel_htlc_id," + " direction," + " origin_htlc," + " msatoshi," + " cltv_expiry," + " payment_hash," + " payment_key," + " hstate," + " routing_onion) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"); + + sqlite3_bind_int64(stmt, 1, chan->id); + sqlite3_bind_int64(stmt, 2, out->key.id); + sqlite3_bind_int(stmt, 3, DIRECTION_OUTGOING); + if (out->in) + sqlite3_bind_int64(stmt, 4, out->in->dbid); + sqlite3_bind_int64(stmt, 5, out->msatoshi); + sqlite3_bind_int(stmt, 6, out->cltv_expiry); + sqlite3_bind_blob(stmt, 7, tal_hexstr(tmpctx, &out->payment_hash, + sizeof(struct sha256)), + 2 * sizeof(struct sha256), SQLITE_TRANSIENT); + if (out->preimage) + sqlite3_bind_blob( + stmt, 8, + tal_hexstr(tmpctx, out->preimage, sizeof(struct preimage)), + 2 * sizeof(struct preimage), SQLITE_TRANSIENT); + sqlite3_bind_int(stmt, 9, out->hstate); + sqlite3_bind_blob( + stmt, 10, tal_hexstr(tmpctx, &out->onion_routing_packet, + sizeof(out->onion_routing_packet)), + 2 * sizeof(out->onion_routing_packet), SQLITE_TRANSIENT); + + ok = db_exec_prepared(wallet->db, stmt); tal_free(tmpctx); if (ok) { @@ -890,24 +936,22 @@ bool wallet_htlc_update(struct wallet *wallet, const u64 htlc_dbid, { bool ok = true; tal_t *tmpctx = tal_tmpctx(wallet); + sqlite3_stmt *stmt; /* The database ID must be set by a previous call to * `wallet_htlc_save_*` */ assert(htlc_dbid); - if (payment_key) { - ok &= db_exec( - __func__, wallet->db, "UPDATE channel_htlcs SET hstate=%d, " - "payment_key='%s' WHERE id=%" PRIu64, - new_state, + stmt = db_prepare( + wallet->db, "UPDATE channel_htlcs SET hstate=?, payment_key=? WHERE id=?"); + sqlite3_bind_int(stmt, 1, new_state); + if (payment_key) + sqlite3_bind_blob( + stmt, 2, tal_hexstr(tmpctx, payment_key, sizeof(struct preimage)), - htlc_dbid); + 2 * sizeof(struct preimage), SQLITE_TRANSIENT); + sqlite3_bind_int64(stmt, 3, htlc_dbid); - } else { - ok &= db_exec( - __func__, wallet->db, - "UPDATE channel_htlcs SET hstate = %d WHERE id=%" PRIu64, - new_state, htlc_dbid); - } + ok = db_exec_prepared(wallet->db, stmt); tal_free(tmpctx); return ok; } From abad23b33955fe123ef030c4e4ae33ae8790d48b Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Mon, 30 Oct 2017 22:44:00 +0100 Subject: [PATCH 0051/1428] wallet: Removing hex fields from HTLCS --- wallet/wallet.c | 81 +++++++++++++++++++++---------------------------- 1 file changed, 35 insertions(+), 46 deletions(-) diff --git a/wallet/wallet.c b/wallet/wallet.c index 1c99ad39bf8a..5a3edda466c0 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -847,23 +847,17 @@ bool wallet_htlc_save_in(struct wallet *wallet, sqlite3_bind_int(stmt, 3, DIRECTION_INCOMING); sqlite3_bind_int64(stmt, 4, in->msatoshi); sqlite3_bind_int(stmt, 5, in->cltv_expiry); - sqlite3_bind_blob(stmt, 6, tal_hexstr(tmpctx, &in->payment_hash, - sizeof(struct sha256)), - 2 * sizeof(struct sha256), SQLITE_TRANSIENT); + sqlite3_bind_sha256(stmt, 6, &in->payment_hash); + if (in->preimage) - sqlite3_bind_blob( - stmt, 7, - tal_hexstr(tmpctx, in->preimage, sizeof(struct preimage)), - 2 * sizeof(struct preimage), SQLITE_TRANSIENT); + sqlite3_bind_preimage(stmt, 7, in->preimage); sqlite3_bind_int(stmt, 8, in->hstate); - sqlite3_bind_blob(stmt, 9, tal_hexstr(tmpctx, &in->shared_secret, - sizeof(struct secret)), - 2 * sizeof(struct secret), SQLITE_TRANSIENT); - sqlite3_bind_blob( - stmt, 10, tal_hexstr(tmpctx, &in->onion_routing_packet, - sizeof(in->onion_routing_packet)), - 2 * sizeof(in->onion_routing_packet), SQLITE_TRANSIENT); + sqlite3_bind_blob(stmt, 9, &in->shared_secret, + sizeof(in->shared_secret), SQLITE_TRANSIENT); + + sqlite3_bind_blob(stmt, 10, &in->onion_routing_packet, + sizeof(in->onion_routing_packet), SQLITE_TRANSIENT); ok = db_exec_prepared(wallet->db, stmt); tal_free(tmpctx); @@ -907,19 +901,14 @@ bool wallet_htlc_save_out(struct wallet *wallet, sqlite3_bind_int64(stmt, 4, out->in->dbid); sqlite3_bind_int64(stmt, 5, out->msatoshi); sqlite3_bind_int(stmt, 6, out->cltv_expiry); - sqlite3_bind_blob(stmt, 7, tal_hexstr(tmpctx, &out->payment_hash, - sizeof(struct sha256)), - 2 * sizeof(struct sha256), SQLITE_TRANSIENT); + sqlite3_bind_sha256(stmt, 7, &out->payment_hash); + if (out->preimage) - sqlite3_bind_blob( - stmt, 8, - tal_hexstr(tmpctx, out->preimage, sizeof(struct preimage)), - 2 * sizeof(struct preimage), SQLITE_TRANSIENT); + sqlite3_bind_preimage(stmt, 8,out->preimage); sqlite3_bind_int(stmt, 9, out->hstate); - sqlite3_bind_blob( - stmt, 10, tal_hexstr(tmpctx, &out->onion_routing_packet, - sizeof(out->onion_routing_packet)), - 2 * sizeof(out->onion_routing_packet), SQLITE_TRANSIENT); + + sqlite3_bind_blob(stmt, 10, &out->onion_routing_packet, + sizeof(out->onion_routing_packet), SQLITE_TRANSIENT); ok = db_exec_prepared(wallet->db, stmt); @@ -935,24 +924,22 @@ bool wallet_htlc_update(struct wallet *wallet, const u64 htlc_dbid, const struct preimage *payment_key) { bool ok = true; - tal_t *tmpctx = tal_tmpctx(wallet); sqlite3_stmt *stmt; /* The database ID must be set by a previous call to * `wallet_htlc_save_*` */ assert(htlc_dbid); stmt = db_prepare( - wallet->db, "UPDATE channel_htlcs SET hstate=?, payment_key=? WHERE id=?"); + wallet->db, + "UPDATE channel_htlcs SET hstate=?, payment_key=? WHERE id=?"); + sqlite3_bind_int(stmt, 1, new_state); - if (payment_key) - sqlite3_bind_blob( - stmt, 2, - tal_hexstr(tmpctx, payment_key, sizeof(struct preimage)), - 2 * sizeof(struct preimage), SQLITE_TRANSIENT); sqlite3_bind_int64(stmt, 3, htlc_dbid); + if (payment_key) + sqlite3_bind_preimage(stmt, 2, payment_key); + ok = db_exec_prepared(wallet->db, stmt); - tal_free(tmpctx); return ok; } @@ -967,20 +954,22 @@ static bool wallet_stmt2htlc_in(const struct wallet_channel *channel, in->cltv_expiry = sqlite3_column_int(stmt, 3); in->hstate = sqlite3_column_int(stmt, 4); - ok &= sqlite3_column_hexval(stmt, 5, &in->payment_hash, - sizeof(in->payment_hash)); - ok &= sqlite3_column_hexval(stmt, 6, &in->shared_secret, - sizeof(in->shared_secret)); + sqlite3_column_sha256(stmt, 5, &in->payment_hash); + + assert(sqlite3_column_bytes(stmt, 6) == sizeof(struct secret)); + memcpy(&in->shared_secret, sqlite3_column_blob(stmt, 6), + sizeof(struct secret)); if (sqlite3_column_type(stmt, 7) != SQLITE_NULL) { in->preimage = tal(in, struct preimage); - ok &= sqlite3_column_hexval(stmt, 7, in->preimage, sizeof(*in->preimage)); + sqlite3_column_preimage(stmt, 7, in->preimage); } else { in->preimage = NULL; } - sqlite3_column_hexval(stmt, 8, &in->onion_routing_packet, - sizeof(in->onion_routing_packet)); + assert(sqlite3_column_bytes(stmt, 8) == sizeof(in->onion_routing_packet)); + memcpy(&in->onion_routing_packet, sqlite3_column_blob(stmt, 8), + sizeof(in->onion_routing_packet)); in->failuremsg = NULL; in->malformed = 0; @@ -997,8 +986,7 @@ static bool wallet_stmt2htlc_out(const struct wallet_channel *channel, out->msatoshi = sqlite3_column_int64(stmt, 2); out->cltv_expiry = sqlite3_column_int(stmt, 3); out->hstate = sqlite3_column_int(stmt, 4); - ok &= sqlite3_column_hexval(stmt, 5, &out->payment_hash, - sizeof(out->payment_hash)); + sqlite3_column_sha256(stmt, 5, &out->payment_hash); if (sqlite3_column_type(stmt, 6) != SQLITE_NULL) { out->origin_htlc_id = sqlite3_column_int64(stmt, 6); @@ -1008,13 +996,14 @@ static bool wallet_stmt2htlc_out(const struct wallet_channel *channel, if (sqlite3_column_type(stmt, 7) != SQLITE_NULL) { out->preimage = tal(out, struct preimage); - ok &= sqlite3_column_hexval(stmt, 7, &out->preimage, sizeof(struct preimage)); + sqlite3_column_preimage(stmt, 7, out->preimage); } else { out->preimage = NULL; } - sqlite3_column_hexval(stmt, 8, &out->onion_routing_packet, - sizeof(out->onion_routing_packet)); + assert(sqlite3_column_bytes(stmt, 8) == sizeof(out->onion_routing_packet)); + memcpy(&out->onion_routing_packet, sqlite3_column_blob(stmt, 8), + sizeof(out->onion_routing_packet)); out->failuremsg = NULL; out->malformed = 0; @@ -1244,7 +1233,7 @@ struct htlc_stub *wallet_htlc_stubs(tal_t *ctx, struct wallet *wallet, stubs[n].owner = sqlite3_column_int(stmt, 1)==DIRECTION_INCOMING?REMOTE:LOCAL; stubs[n].cltv_expiry = sqlite3_column_int(stmt, 2); - sqlite3_column_hexval(stmt, 3, &payment_hash, sizeof(payment_hash)); + sqlite3_column_sha256(stmt, 3, &payment_hash); ripemd160(&stubs[n].ripemd, payment_hash.u.u8, sizeof(payment_hash.u)); } sqlite3_finalize(stmt); From 8f198f37462e32acc8c978cdaaff4894865247ee Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Wed, 18 Oct 2017 14:54:33 +0200 Subject: [PATCH 0052/1428] wallet: Migrate output tracking to native sqlite3 binding Signed-off-by: Christian Decker --- wallet/wallet.c | 62 +++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/wallet/wallet.c b/wallet/wallet.c index 5a3edda466c0..467760d3a688 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -28,16 +28,16 @@ struct wallet *wallet_new(const tal_t *ctx, struct log *log) bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, enum wallet_output_type type) { - tal_t *tmpctx = tal_tmpctx(w); - char *hextxid = tal_hexstr(tmpctx, &utxo->txid, 32); - bool result = db_exec( - __func__, w->db, - "INSERT INTO outputs (prev_out_tx, prev_out_index, value, type, " - "status, keyindex) VALUES ('%s', %d, %"PRIu64", %d, %d, %d);", - hextxid, utxo->outnum, utxo->amount, type, output_state_available, - utxo->keyindex); - tal_free(tmpctx); - return result; + sqlite3_stmt *stmt; + + stmt = db_prepare(w->db, "INSERT INTO outputs (prev_out_tx, prev_out_index, value, type, status, keyindex) VALUES (?, ?, ?, ?, ?, ?);"); + sqlite3_bind_blob(stmt, 1, &utxo->txid, sizeof(utxo->txid), SQLITE_TRANSIENT); + sqlite3_bind_int(stmt, 2, utxo->outnum); + sqlite3_bind_int64(stmt, 3, utxo->amount); + sqlite3_bind_int(stmt, 4, type); + sqlite3_bind_int(stmt, 5, output_state_available); + sqlite3_bind_int(stmt, 6, utxo->keyindex); + return db_exec_prepared(w->db, stmt); } /** @@ -47,8 +47,7 @@ bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, */ static bool wallet_stmt2output(sqlite3_stmt *stmt, struct utxo *utxo) { - const unsigned char *hextxid = sqlite3_column_text(stmt, 0); - hex_decode((const char*)hextxid, sizeof(utxo->txid) * 2, &utxo->txid, sizeof(utxo->txid)); + memcpy(&utxo->txid, sqlite3_column_blob(stmt, 0), sqlite3_column_bytes(stmt, 0)); utxo->outnum = sqlite3_column_int(stmt, 1); utxo->amount = sqlite3_column_int(stmt, 2); utxo->is_p2sh = sqlite3_column_int(stmt, 3) == p2sh_wpkh; @@ -62,22 +61,22 @@ bool wallet_update_output_status(struct wallet *w, const u32 outnum, enum output_status oldstatus, enum output_status newstatus) { - tal_t *tmpctx = tal_tmpctx(w); - char *hextxid = tal_hexstr(tmpctx, txid, sizeof(*txid)); + sqlite3_stmt *stmt; if (oldstatus != output_state_any) { - db_exec(__func__, w->db, - "UPDATE outputs SET status=%d WHERE status=%d " - "AND prev_out_tx = '%s' AND prev_out_index = " - "%d;", - newstatus, oldstatus, hextxid, outnum); + stmt = db_prepare( + w->db, "UPDATE outputs SET status=? WHERE status=? AND prev_out_tx=? AND prev_out_index=?"); + sqlite3_bind_int(stmt, 1, newstatus); + sqlite3_bind_int(stmt, 2, oldstatus); + sqlite3_bind_blob(stmt, 3, txid, sizeof(*txid), SQLITE_TRANSIENT); + sqlite3_bind_int(stmt, 4, outnum); } else { - db_exec(__func__, w->db, - "UPDATE outputs SET status=%d WHERE " - "prev_out_tx = '%s' AND prev_out_index = " - "%d;", - newstatus, hextxid, outnum); + stmt = db_prepare( + w->db, "UPDATE outputs SET status=? WHERE prev_out_tx=? AND prev_out_index=?"); + sqlite3_bind_int(stmt, 1, newstatus); + sqlite3_bind_blob(stmt, 2, txid, sizeof(*txid), SQLITE_TRANSIENT); + sqlite3_bind_int(stmt, 3, outnum); } - tal_free(tmpctx); + db_exec_prepared(w->db, stmt); return sqlite3_changes(w->db->sql) > 0; } @@ -85,16 +84,13 @@ struct utxo **wallet_get_utxos(const tal_t *ctx, struct wallet *w, const enum ou { struct utxo **results; int i; - sqlite3_stmt *stmt = - db_query(__func__, w->db, "SELECT prev_out_tx, prev_out_index, " - "value, type, status, keyindex FROM " - "outputs WHERE status=%d OR %d=255", - state, state); - if (!stmt) - return NULL; + sqlite3_stmt *stmt = db_prepare( + w->db, "SELECT prev_out_tx, prev_out_index, value, type, status, keyindex " + "FROM outputs WHERE status=?1 OR ?1=255"); + sqlite3_bind_int(stmt, 1, state); - results = tal_arr(ctx, struct utxo*, 0); + results = tal_arr(ctx, struct utxo*, 0); for (i=0; sqlite3_step(stmt) == SQLITE_ROW; i++) { tal_resize(&results, i+1); results[i] = tal(results, struct utxo); From 9c12c807d1e061844acd50c68e42ff1140828311 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Wed, 18 Oct 2017 15:34:51 +0200 Subject: [PATCH 0053/1428] wallet: Migrate shachain persistence to native sqlite3 binding Signed-off-by: Christian Decker --- wallet/wallet.c | 57 ++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 31 deletions(-) diff --git a/wallet/wallet.c b/wallet/wallet.c index 467760d3a688..db9200ae2f99 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -240,14 +240,15 @@ s64 wallet_get_newindex(struct lightningd *ld) bool wallet_shachain_init(struct wallet *wallet, struct wallet_shachain *chain) { + sqlite3_stmt *stmt; /* Create shachain */ shachain_init(&chain->chain); - if (!db_exec( - __func__, wallet->db, - "INSERT INTO shachains (min_index, num_valid) VALUES (%"PRIu64",0);", - chain->chain.min_index)) { + stmt = db_prepare(wallet->db, "INSERT INTO shachains (min_index, num_valid) VALUES (?, 0);"); + sqlite3_bind_int64(stmt, 1, chain->chain.min_index); + if (!db_exec_prepared(wallet->db, stmt)) { return false; } + chain->id = sqlite3_last_insert_rowid(wallet->db->sql); return true; } @@ -273,34 +274,35 @@ bool wallet_shachain_add_hash(struct wallet *wallet, uint64_t index, const struct sha256 *hash) { - tal_t *tmpctx = tal_tmpctx(wallet); + sqlite3_stmt *stmt; bool ok = true; u32 pos = count_trailing_zeroes(index); assert(index < SQLITE_MAX_UINT); - char *hexhash = tal_hexstr(tmpctx, hash, sizeof(struct sha256)); if (!shachain_add_hash(&chain->chain, index, hash)) { - tal_free(tmpctx); return false; } db_begin_transaction(wallet->db); - ok &= db_exec(__func__, wallet->db, - "UPDATE shachains SET num_valid=%d, min_index=%" PRIu64 - " WHERE id=%" PRIu64, - chain->chain.num_valid, index, chain->id); + stmt = db_prepare(wallet->db, "UPDATE shachains SET num_valid=?, min_index=? WHERE id=?"); + sqlite3_bind_int(stmt, 1, chain->chain.num_valid); + sqlite3_bind_int64(stmt, 2, index); + sqlite3_bind_int64(stmt, 3, chain->id); + ok &= db_exec_prepared(wallet->db, stmt); - ok &= db_exec(__func__, wallet->db, - "REPLACE INTO shachain_known " - "(shachain_id, pos, idx, hash) VALUES " - "(%" PRIu64 ", %d, %" PRIu64 ", '%s');", - chain->id, pos, index, hexhash); + stmt = db_prepare( + wallet->db, + "REPLACE INTO shachain_known (shachain_id, pos, idx, hash) VALUES (?, ?, ?, ?);"); + sqlite3_bind_int64(stmt, 1, chain->id); + sqlite3_bind_int(stmt, 2, pos); + sqlite3_bind_int64(stmt, 3, index); + sqlite3_bind_blob(stmt, 4, hash, sizeof(*hash), SQLITE_TRANSIENT); + ok &= db_exec_prepared(wallet->db, stmt); if (ok) ok &= db_commit_transaction(wallet->db); else db_rollback_transaction(wallet->db); - tal_free(tmpctx); return ok; } @@ -313,11 +315,9 @@ bool wallet_shachain_load(struct wallet *wallet, u64 id, shachain_init(&chain->chain); /* Load shachain metadata */ - stmt = db_query( - __func__, wallet->db, - "SELECT min_index, num_valid FROM shachains WHERE id=%" PRIu64, id); - if (!stmt) - return false; + stmt = db_prepare(wallet->db, "SELECT min_index, num_valid FROM shachains WHERE id=?"); + sqlite3_bind_int64(stmt, 1, id); + err = sqlite3_step(stmt); if (err != SQLITE_ROW) { sqlite3_finalize(stmt); @@ -329,21 +329,16 @@ bool wallet_shachain_load(struct wallet *wallet, u64 id, sqlite3_finalize(stmt); /* Load shachain known entries */ - stmt = db_query( - __func__, wallet->db, - "SELECT idx, hash, pos FROM shachain_known WHERE shachain_id=%" PRIu64, - id); + stmt = db_prepare(wallet->db, "SELECT idx, hash, pos FROM shachain_known WHERE shachain_id=?"); + sqlite3_bind_int64(stmt, 1, id); - if (!stmt) - return false; while (sqlite3_step(stmt) == SQLITE_ROW) { int pos = sqlite3_column_int(stmt, 2); chain->chain.known[pos].index = sqlite3_column_int64(stmt, 0); - sqlite3_column_hexval(stmt, 1, &chain->chain.known[pos].hash, - sizeof(struct sha256)); + memcpy(&chain->chain.known[pos].hash, sqlite3_column_blob(stmt, 1), sqlite3_column_bytes(stmt, 1)); } - sqlite3_finalize(stmt); + sqlite3_finalize(stmt); return true; } From c1d364c5fb3648adf2504020eeb6c90f50909771 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 20 Oct 2017 19:04:43 +0200 Subject: [PATCH 0054/1428] wallet: Migrate channel persistence to native sqlite3 binding Signed-off-by: Christian Decker --- wallet/db.c | 24 ++--- wallet/wallet.c | 269 +++++++++++++++++++++++------------------------- 2 files changed, 133 insertions(+), 160 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index 009fd7452ed5..225089adaa9e 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -440,7 +440,7 @@ bool sqlite3_column_short_channel_id(sqlite3_stmt *stmt, int col, bool sqlite3_bind_tx(sqlite3_stmt *stmt, int col, const struct bitcoin_tx *tx) { u8 *ser = linearize_tx(NULL, tx); - sqlite3_bind_blob(stmt, col, tal_hex(ser, ser), 2*tal_len(ser), SQLITE_TRANSIENT); + sqlite3_bind_blob(stmt, col, ser, tal_len(ser), SQLITE_TRANSIENT); tal_free(ser); return true; } @@ -448,12 +448,10 @@ bool sqlite3_bind_tx(sqlite3_stmt *stmt, int col, const struct bitcoin_tx *tx) struct bitcoin_tx *sqlite3_column_tx(const tal_t *ctx, sqlite3_stmt *stmt, int col) { - return bitcoin_tx_from_hex( - ctx, - sqlite3_column_blob(stmt, col), - sqlite3_column_bytes(stmt, col)); + const u8 *src = sqlite3_column_blob(stmt, col); + size_t len = sqlite3_column_bytes(stmt, col); + return pull_bitcoin_tx(ctx, &src, &len); } - bool sqlite3_bind_signature(sqlite3_stmt *stmt, int col, const secp256k1_ecdsa_signature *sig) { @@ -475,18 +473,8 @@ bool sqlite3_column_signature(sqlite3_stmt *stmt, int col, bool sqlite3_column_pubkey(sqlite3_stmt *stmt, int col, struct pubkey *dest) { - u8 buf[PUBKEY_DER_LEN]; - - if (sqlite3_column_bytes(stmt, col) == 2*PUBKEY_DER_LEN) { - /* FIXME: Remove the legacy path for hex-values */ - if (!sqlite3_column_hexval(stmt, col, buf, sizeof(buf))) - return false; - } else { - assert(sqlite3_column_bytes(stmt, col) == PUBKEY_DER_LEN); - memcpy(buf, sqlite3_column_blob(stmt, col), PUBKEY_DER_LEN); - } - - return pubkey_from_der(buf, sizeof(buf), dest); + assert(sqlite3_column_bytes(stmt, col) == PUBKEY_DER_LEN); + return pubkey_from_der(sqlite3_column_blob(stmt, col), PUBKEY_DER_LEN, dest); } bool sqlite3_bind_pubkey(sqlite3_stmt *stmt, int col, const struct pubkey *pk) diff --git a/wallet/wallet.c b/wallet/wallet.c index db9200ae2f99..4c790ccd451f 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -342,21 +342,6 @@ bool wallet_shachain_load(struct wallet *wallet, u64 id, return true; } -static bool sqlite3_column_sig(sqlite3_stmt *stmt, int col, secp256k1_ecdsa_signature *sig) -{ - u8 buf[64]; - if (!sqlite3_column_hexval(stmt, col, buf, sizeof(buf))) - return false; - return secp256k1_ecdsa_signature_parse_compact(secp256k1_ctx, sig, buf) == 1; -} - -static u8 *sqlite3_column_varhexblob(tal_t *ctx, sqlite3_stmt *stmt, int col) -{ - const u8 *source = sqlite3_column_blob(stmt, col); - size_t sourcelen = sqlite3_column_bytes(stmt, col); - return tal_hexdata(ctx, source, sourcelen); -} - static bool wallet_peer_load(struct wallet *w, const u64 id, struct peer *peer) { bool ok = true; @@ -376,9 +361,8 @@ bool wallet_peer_by_nodeid(struct wallet *w, const struct pubkey *nodeid, { bool ok; tal_t *tmpctx = tal_tmpctx(w); - sqlite3_stmt *stmt = db_query( - __func__, w->db, "SELECT id, node_id FROM peers WHERE node_id='%s';", - pubkey_to_hexstr(tmpctx, nodeid)); + sqlite3_stmt *stmt = db_prepare(w->db, "SELECT id, node_id FROM peers WHERE node_id=?;"); + sqlite3_bind_pubkey(stmt, 1, nodeid); ok = stmt != NULL && sqlite3_step(stmt) == SQLITE_ROW; if (ok) { @@ -404,8 +388,6 @@ static bool wallet_stmt2channel(struct wallet *w, sqlite3_stmt *stmt, bool ok = true; int col = 0; struct channel_info *channel_info; - struct sha256_double temphash; - struct short_channel_id scid; u64 remote_config_id; if (!chan->peer) { @@ -415,11 +397,12 @@ static bool wallet_stmt2channel(struct wallet *w, sqlite3_stmt *stmt, chan->peer->dbid = sqlite3_column_int64(stmt, col++); wallet_peer_load(w, chan->peer->dbid, chan->peer); - if (sqlite3_column_short_channel_id(stmt, col++, &scid)) { + if (sqlite3_column_type(stmt, col) != SQLITE_NULL) { chan->peer->scid = tal(chan->peer, struct short_channel_id); - *chan->peer->scid = scid; + sqlite3_column_short_channel_id(stmt, col++, chan->peer->scid); } else { chan->peer->scid = NULL; + col++; } chan->peer->our_config.id = sqlite3_column_int64(stmt, col++); @@ -434,12 +417,14 @@ static bool wallet_stmt2channel(struct wallet *w, sqlite3_stmt *stmt, chan->peer->next_index[REMOTE] = sqlite3_column_int64(stmt, col++); chan->peer->next_htlc_id = sqlite3_column_int64(stmt, col++); - if (sqlite3_column_hexval(stmt, col++, &temphash, sizeof(temphash))) { + if (sqlite3_column_type(stmt, col) != SQLITE_NULL) { + assert(sqlite3_column_bytes(stmt, col) == 32); chan->peer->funding_txid = tal(chan->peer, struct sha256_double); - *chan->peer->funding_txid = temphash; + memcpy(chan->peer->funding_txid, sqlite3_column_blob(stmt, col), 32); } else { chan->peer->funding_txid = NULL; } + col++; chan->peer->funding_outnum = sqlite3_column_int(stmt, col++); chan->peer->funding_satoshi = sqlite3_column_int64(stmt, col++); @@ -484,7 +469,8 @@ static bool wallet_stmt2channel(struct wallet *w, sqlite3_stmt *stmt, /* Do we have a non-null remote_shutdown_scriptpubkey? */ if (sqlite3_column_type(stmt, col) != SQLITE_NULL) { - chan->peer->remote_shutdown_scriptpubkey = sqlite3_column_varhexblob(chan->peer, stmt, col++); + chan->peer->remote_shutdown_scriptpubkey = tal_arr(chan->peer, u8, sqlite3_column_bytes(stmt, col)); + memcpy(chan->peer->remote_shutdown_scriptpubkey, sqlite3_column_blob(stmt, col), sqlite3_column_bytes(stmt, col)); chan->peer->local_shutdown_idx = sqlite3_column_int64(stmt, col++); } else { chan->peer->remote_shutdown_scriptpubkey = tal_free(chan->peer->remote_shutdown_scriptpubkey); @@ -508,7 +494,7 @@ static bool wallet_stmt2channel(struct wallet *w, sqlite3_stmt *stmt, if (sqlite3_column_type(stmt, col) != SQLITE_NULL) { chan->peer->last_tx = sqlite3_column_tx(chan->peer, stmt, col++); chan->peer->last_sig = tal(chan->peer, secp256k1_ecdsa_signature); - sqlite3_column_sig(stmt, col++, chan->peer->last_sig); + sqlite3_column_signature(stmt, col++, chan->peer->last_sig); } else { chan->peer->last_tx = tal_free(chan->peer->last_tx); chan->peer->last_sig = tal_free(chan->peer->last_sig); @@ -581,54 +567,34 @@ bool wallet_channels_load_active(struct wallet *w, struct list_head *peers) return ok; } -static char* db_serialize_signature(const tal_t *ctx, secp256k1_ecdsa_signature* sig) -{ - u8 buf[64]; - if (!sig || secp256k1_ecdsa_signature_serialize_compact(secp256k1_ctx, buf, sig) != 1) - return "null"; - return tal_fmt(ctx, "'%s'", tal_hexstr(ctx, buf, sizeof(buf))); -} - -static char* db_serialize_pubkey(const tal_t *ctx, struct pubkey *pk) -{ - u8 *der; - if (!pk) - return "NULL"; - der = tal_arr(ctx, u8, PUBKEY_DER_LEN); - pubkey_to_der(der, pk); - return tal_hex(ctx, der); -} - -static char* db_serialize_tx(const tal_t *ctx, const struct bitcoin_tx *tx) -{ - if (!tx) - return "NULL"; - - return tal_fmt(ctx, "'%s'", tal_hex(ctx, linearize_tx(ctx, tx))); -} - bool wallet_channel_config_save(struct wallet *w, struct channel_config *cc) { bool ok = true; + sqlite3_stmt *stmt; /* Is this an update? If not insert a stub first */ if (!cc->id) { - ok &= db_exec(__func__, w->db, - "INSERT INTO channel_configs DEFAULT VALUES;"); + stmt = db_prepare( + w->db,"INSERT INTO channel_configs DEFAULT VALUES;"); + ok &= db_exec_prepared(w->db, stmt); cc->id = sqlite3_last_insert_rowid(w->db->sql); } - ok &= db_exec( - __func__, w->db, "UPDATE channel_configs SET" - " dust_limit_satoshis=%" PRIu64 "," - " max_htlc_value_in_flight_msat=%" PRIu64 "," - " channel_reserve_satoshis=%" PRIu64 "," - " htlc_minimum_msat=%" PRIu64 "," - " to_self_delay=%d," - " max_accepted_htlcs=%d" - " WHERE id=%" PRIu64 ";", - cc->dust_limit_satoshis, cc->max_htlc_value_in_flight_msat, - cc->channel_reserve_satoshis, cc->htlc_minimum_msat, - cc->to_self_delay, cc->max_accepted_htlcs, cc->id); + stmt = db_prepare(w->db, "UPDATE channel_configs SET" + " dust_limit_satoshis=?," + " max_htlc_value_in_flight_msat=?," + " channel_reserve_satoshis=?," + " htlc_minimum_msat=?," + " to_self_delay=?," + " max_accepted_htlcs=?" + " WHERE id=?;"); + sqlite3_bind_int64(stmt, 1, cc->dust_limit_satoshis); + sqlite3_bind_int64(stmt, 2, cc->max_htlc_value_in_flight_msat); + sqlite3_bind_int64(stmt, 3, cc->channel_reserve_satoshis); + sqlite3_bind_int64(stmt, 4, cc->htlc_minimum_msat); + sqlite3_bind_int(stmt, 5, cc->to_self_delay); + sqlite3_bind_int(stmt, 6, cc->max_accepted_htlcs); + sqlite3_bind_int64(stmt, 7, cc->id); + ok &= db_exec_prepared(w->db, stmt); return ok; } @@ -663,20 +629,23 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ bool ok = true; struct peer *p = chan->peer; tal_t *tmpctx = tal_tmpctx(w); + sqlite3_stmt *stmt; + + db_begin_transaction(w->db); if (p->dbid == 0) { /* Need to store the peer first */ - ok &= db_exec(__func__, w->db, - "INSERT INTO peers (node_id) VALUES ('%s');", - db_serialize_pubkey(tmpctx, &chan->peer->id)); + stmt = db_prepare(w->db, "INSERT INTO peers (node_id) VALUES (?);"); + sqlite3_bind_pubkey(stmt, 1, &chan->peer->id); + db_exec_prepared(w->db, stmt); p->dbid = sqlite3_last_insert_rowid(w->db->sql); } - db_begin_transaction(w->db); - /* Insert a stub, that we can update, unifies INSERT and UPDATE paths */ if (chan->id == 0) { - ok &= db_exec(__func__, w->db, "INSERT INTO channels (peer_id) VALUES (%"PRIu64");", p->dbid); + stmt = db_prepare(w->db, "INSERT INTO channels (peer_id) VALUES (?);"); + sqlite3_bind_int64(stmt, 1, p->dbid); + db_exec_prepared(w->db, stmt); chan->id = sqlite3_last_insert_rowid(w->db->sql); } @@ -688,83 +657,99 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ ok &= wallet_channel_config_save(w, &p->our_config); /* Now do the real update */ - ok &= db_exec(__func__, w->db, "UPDATE channels SET" - " shachain_remote_id=%"PRIu64"," - " short_channel_id=%s," - " state=%d," - " funder=%d," - " channel_flags=%d," - " minimum_depth=%d," - " next_index_local=%"PRIu64"," - " next_index_remote=%"PRIu64"," - " next_htlc_id=%"PRIu64"," - " funding_tx_id=%s," - " funding_tx_outnum=%d," - " funding_satoshi=%"PRIu64"," - " funding_locked_remote=%d," - " push_msatoshi=%"PRIu64"," - " msatoshi_local=%s," - " shutdown_scriptpubkey_remote='%s'," - " shutdown_keyidx_local=%"PRId64"," - " channel_config_local=%"PRIu64"," - " last_tx=%s, last_sig=%s" - " WHERE id=%"PRIu64, - p->their_shachain.id, - p->scid?tal_fmt(tmpctx,"'%s'", short_channel_id_to_str(tmpctx, p->scid)):"null", - p->state, - p->funder, - p->channel_flags, - p->minimum_depth, - p->next_index[LOCAL], - p->next_index[REMOTE], - p->next_htlc_id, - p->funding_txid?tal_fmt(tmpctx, "'%s'", tal_hexstr(tmpctx, p->funding_txid, sizeof(struct sha256_double))):"null", - p->funding_outnum, - p->funding_satoshi, - p->remote_funding_locked, - p->push_msat, - p->our_msatoshi?tal_fmt(tmpctx, "%"PRIu64, *p->our_msatoshi):"NULL", - p->remote_shutdown_scriptpubkey?tal_hex(tmpctx, p->remote_shutdown_scriptpubkey):"", - p->local_shutdown_idx, - p->our_config.id, - db_serialize_tx(tmpctx, p->last_tx), - db_serialize_signature(tmpctx, p->last_sig), - chan->id); + stmt = db_prepare(w->db, tal_fmt(w, "UPDATE channels SET" + " shachain_remote_id=?," + " short_channel_id=?," + " state=?," + " funder=?," + " channel_flags=?," + " minimum_depth=?," + " next_index_local=?," + " next_index_remote=?," + " next_htlc_id=?," + " funding_tx_id=?," + " funding_tx_outnum=?," + " funding_satoshi=?," + " funding_locked_remote=?," + " push_msatoshi=?," + " msatoshi_local=?," + " shutdown_scriptpubkey_remote=?," + " shutdown_keyidx_local=?," + " channel_config_local=?," + " last_tx=?, last_sig=?" + " WHERE id=?")); + sqlite3_bind_int64(stmt, 1, p->their_shachain.id); + if (p->scid) + sqlite3_bind_short_channel_id(stmt, 2, p->scid); + sqlite3_bind_int(stmt, 3, p->state); + sqlite3_bind_int(stmt, 4, p->funder); + sqlite3_bind_int(stmt, 5, p->channel_flags); + sqlite3_bind_int(stmt, 6, p->minimum_depth); + + sqlite3_bind_int64(stmt, 7, p->next_index[LOCAL]); + sqlite3_bind_int64(stmt, 8, p->next_index[REMOTE]); + sqlite3_bind_int64(stmt, 9, p->next_htlc_id); + + if (p->funding_txid) + sqlite3_bind_blob(stmt, 10, p->funding_txid, sizeof(*p->funding_txid), SQLITE_TRANSIENT); + + sqlite3_bind_int(stmt, 11, p->funding_outnum); + sqlite3_bind_int64(stmt, 12, p->funding_satoshi); + sqlite3_bind_int(stmt, 13, p->remote_funding_locked); + sqlite3_bind_int64(stmt, 14, p->push_msat); + + if (p->our_msatoshi) + sqlite3_bind_int64(stmt, 15, *p->our_msatoshi); + + if (p->remote_shutdown_scriptpubkey) + sqlite3_bind_blob(stmt, 16, p->remote_shutdown_scriptpubkey, + tal_len(p->remote_shutdown_scriptpubkey), + SQLITE_TRANSIENT); + + sqlite3_bind_int64(stmt, 17, p->local_shutdown_idx); + sqlite3_bind_int64(stmt, 18, p->our_config.id); + if (p->last_tx) + sqlite3_bind_tx(stmt, 19, p->last_tx); + if (p->last_sig) + sqlite3_bind_signature(stmt, 20, p->last_sig); + sqlite3_bind_int64(stmt, 21, chan->id); + db_exec_prepared(w->db, stmt); if (chan->peer->channel_info) { ok &= wallet_channel_config_save(w, &p->channel_info->their_config); - ok &= db_exec(__func__, w->db, - "UPDATE channels SET" - " fundingkey_remote='%s'," - " revocation_basepoint_remote='%s'," - " payment_basepoint_remote='%s'," - " delayed_payment_basepoint_remote='%s'," - " per_commit_remote='%s'," - " old_per_commit_remote='%s'," - " feerate_per_kw=%d," - " channel_config_remote=%"PRIu64 - " WHERE id=%"PRIu64, - db_serialize_pubkey(tmpctx, &p->channel_info->remote_fundingkey), - db_serialize_pubkey(tmpctx, &p->channel_info->theirbase.revocation), - db_serialize_pubkey(tmpctx, &p->channel_info->theirbase.payment), - db_serialize_pubkey(tmpctx, &p->channel_info->theirbase.delayed_payment), - db_serialize_pubkey(tmpctx, &p->channel_info->remote_per_commit), - db_serialize_pubkey(tmpctx, &p->channel_info->old_remote_per_commit), - p->channel_info->feerate_per_kw, - p->channel_info->their_config.id, - chan->id); + stmt = db_prepare(w->db, "UPDATE channels SET" + " fundingkey_remote=?," + " revocation_basepoint_remote=?," + " payment_basepoint_remote=?," + " delayed_payment_basepoint_remote=?," + " per_commit_remote=?," + " old_per_commit_remote=?," + " feerate_per_kw=?," + " channel_config_remote=?" + " WHERE id=?"); + sqlite3_bind_pubkey(stmt, 1, &p->channel_info->remote_fundingkey); + sqlite3_bind_pubkey(stmt, 2, &p->channel_info->theirbase.revocation); + sqlite3_bind_pubkey(stmt, 3, &p->channel_info->theirbase.payment); + sqlite3_bind_pubkey(stmt, 4, &p->channel_info->theirbase.delayed_payment); + sqlite3_bind_pubkey(stmt, 5, &p->channel_info->remote_per_commit); + sqlite3_bind_pubkey(stmt, 6, &p->channel_info->old_remote_per_commit); + sqlite3_bind_int(stmt, 7, p->channel_info->feerate_per_kw); + sqlite3_bind_int64(stmt, 8, p->channel_info->their_config.id); + sqlite3_bind_int64(stmt, 9, chan->id); + ok &= db_exec_prepared(w->db, stmt); } /* If we have a last_sent_commit, store it */ if (chan->peer->last_sent_commit) { - ok &= db_exec(__func__, w->db, - "UPDATE channels SET" - " last_sent_commit_state=%d," - " last_sent_commit_id=%"PRIu64 - " WHERE id=%"PRIu64, - p->last_sent_commit->newstate, - p->last_sent_commit->id, - chan->id); + stmt = db_prepare(w->db, + "UPDATE channels SET" + " last_sent_commit_state=?," + " last_sent_commit_id=?" + " WHERE id=?"); + sqlite3_bind_int(stmt, 1, p->last_sent_commit->newstate); + sqlite3_bind_int64(stmt, 2, p->last_sent_commit->id); + sqlite3_bind_int64(stmt, 3, chan->id); + ok &= db_exec_prepared(w->db, stmt); } if (ok) From da183c22a097172a452ffea602563379aa8d12c5 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Tue, 31 Oct 2017 00:11:37 +0100 Subject: [PATCH 0055/1428] db: Cleanup all remaining traces of hex-encoded values In addition we also set some of the test values to a pattern instead of just `memset`ting it to 0, which may hide some crossed lines. Signed-off-by: Christian Decker --- wallet/db.c | 10 ---------- wallet/db.h | 6 ------ wallet/wallet.c | 1 - wallet/wallet_tests.c | 22 ++++++++++++++++++---- 4 files changed, 18 insertions(+), 21 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index 225089adaa9e..e6fe5db4cabf 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -1,6 +1,5 @@ #include "db.h" -#include #include #include #include @@ -411,15 +410,6 @@ bool db_set_intvar(struct db *db, char *varname, s64 val) varname, val); } -bool sqlite3_column_hexval(sqlite3_stmt *s, int col, void *dest, size_t destlen) -{ - const char *source = sqlite3_column_blob(s, col); - size_t sourcelen = sqlite3_column_bytes(s, col); - if (sourcelen / 2 != destlen) - return false; - return hex_decode(source, sourcelen, dest, destlen); -} - bool sqlite3_bind_short_channel_id(sqlite3_stmt *stmt, int col, const struct short_channel_id *id) { diff --git a/wallet/db.h b/wallet/db.h index 5641f25c5014..9064bb21d356 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -79,12 +79,6 @@ bool db_set_intvar(struct db *db, char *varname, s64 val); */ s64 db_get_intvar(struct db *db, char *varname, s64 defval); -/** - * sqlite3_column_hexval - Helper to populate a binary field from a hex value - */ -bool sqlite3_column_hexval(sqlite3_stmt *s, int col, void *dest, - size_t destlen); - /** * db_prepare -- Prepare a DB query/command * diff --git a/wallet/wallet.c b/wallet/wallet.c index 4c790ccd451f..a2637d7a7475 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -1,7 +1,6 @@ #include "wallet.h" #include -#include #include #include #include diff --git a/wallet/wallet_tests.c b/wallet/wallet_tests.c index 709a0d073fc9..bba988e53726 100644 --- a/wallet/wallet_tests.c +++ b/wallet/wallet_tests.c @@ -11,6 +11,20 @@ void invoice_add(struct invoices *invs, struct invoice *inv){} +/** + * mempat -- Set the memory to a pattern + * + * Used mainly to check that we don't mix fields while + * serializing/unserializing. + */ +static void mempat(void *dst, size_t len) +{ + static int n = 0; + u8 *p = (u8*)dst; + for(int i=0 ; i < len; ++i) + p[i] = n % 251; /* Prime */ +} + static struct wallet *create_test_wallet(const tal_t *ctx) { char filename[] = "/tmp/ldb-XXXXXX"; @@ -204,12 +218,12 @@ static bool test_channel_crud(const tal_t *ctx) memset(c2, 0, sizeof(*c2)); memset(&p, 0, sizeof(p)); memset(&ci, 3, sizeof(ci)); - memset(hash, 'B', sizeof(*hash)); - memset(sig, 0, sizeof(*sig)); - memset(&last_commit, 0, sizeof(last_commit)); + mempat(hash, sizeof(*hash)); + mempat(sig, sizeof(*sig)); + mempat(&last_commit, sizeof(last_commit)); pubkey_from_der(tal_hexdata(w, "02a1633cafcc01ebfb6d78e39f687a1f0995c62fc95f51ead10a02ee0be551b5dc", 66), 33, &pk); ci.feerate_per_kw = 31337; - memset(&p.id, 'A', sizeof(p.id)); + mempat(&p.id, sizeof(p.id)); c1.peer = &p; p.id = pk; p.our_msatoshi = NULL; From 79df10f7829674fab7c6d265db1bdd0496b1633a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:24:19 +1030 Subject: [PATCH 0056/1428] bech32: pull in bech32 sample code, almost untouched. Signed-off-by: Rusty Russell --- lightningd/Makefile | 1 + lightningd/bech32.c | 195 ++++++++++++++++++++++++++++++++++++++++++++ lightningd/bech32.h | 122 +++++++++++++++++++++++++++ 3 files changed, 318 insertions(+) create mode 100644 lightningd/bech32.c create mode 100644 lightningd/bech32.h diff --git a/lightningd/Makefile b/lightningd/Makefile index 061c9233f9e2..de0a6c50a9a1 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -39,6 +39,7 @@ LIGHTNINGD_COMMON_OBJS := \ common/withdraw_tx.o LIGHTNINGD_SRC := \ + lightningd/bech32.c \ lightningd/bitcoind.c \ lightningd/build_utxos.c \ lightningd/chaintopology.c \ diff --git a/lightningd/bech32.c b/lightningd/bech32.c new file mode 100644 index 000000000000..0c4f115bbfc5 --- /dev/null +++ b/lightningd/bech32.c @@ -0,0 +1,195 @@ +/* Stolen from https://github.com/sipa/bech32/blob/master/ref/c/segwit_addr.c, + * with only the two ' > 90' checks hoisted, and more internals exposed */ + +/* Copyright (c) 2017 Pieter Wuille + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ +#include "bech32.h" + +#include +#include +#include + +static uint32_t bech32_polymod_step(uint32_t pre) { + uint8_t b = pre >> 25; + return ((pre & 0x1FFFFFF) << 5) ^ + (-((b >> 0) & 1) & 0x3b6a57b2UL) ^ + (-((b >> 1) & 1) & 0x26508e6dUL) ^ + (-((b >> 2) & 1) & 0x1ea119faUL) ^ + (-((b >> 3) & 1) & 0x3d4233ddUL) ^ + (-((b >> 4) & 1) & 0x2a1462b3UL); +} + +const char bech32_charset[32] = "qpzry9x8gf2tvdw0s3jn54khce6mua7l"; + +const int8_t bech32_charset_rev[128] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 15, -1, 10, 17, 21, 20, 26, 30, 7, 5, -1, -1, -1, -1, -1, -1, + -1, 29, -1, 24, 13, 25, 9, 8, 23, -1, 18, 22, 31, 27, 19, -1, + 1, 0, 3, 16, 11, 28, 12, 14, 6, 4, 2, -1, -1, -1, -1, -1, + -1, 29, -1, 24, 13, 25, 9, 8, 23, -1, 18, 22, 31, 27, 19, -1, + 1, 0, 3, 16, 11, 28, 12, 14, 6, 4, 2, -1, -1, -1, -1, -1 +}; + +int bech32_encode(char *output, const char *hrp, const uint8_t *data, size_t data_len, size_t max_input_len) { + uint32_t chk = 1; + size_t i = 0; + while (hrp[i] != 0) { + int ch = hrp[i]; + if (ch < 33 || ch > 126) { + return 0; + } + + if (ch >= 'A' && ch <= 'Z') return 0; + chk = bech32_polymod_step(chk) ^ (ch >> 5); + ++i; + } + if (i + 7 + data_len > max_input_len) return 0; + chk = bech32_polymod_step(chk); + while (*hrp != 0) { + chk = bech32_polymod_step(chk) ^ (*hrp & 0x1f); + *(output++) = *(hrp++); + } + *(output++) = '1'; + for (i = 0; i < data_len; ++i) { + if (*data >> 5) return 0; + chk = bech32_polymod_step(chk) ^ (*data); + *(output++) = bech32_charset[*(data++)]; + } + for (i = 0; i < 6; ++i) { + chk = bech32_polymod_step(chk); + } + chk ^= 1; + for (i = 0; i < 6; ++i) { + *(output++) = bech32_charset[(chk >> ((5 - i) * 5)) & 0x1f]; + } + *output = 0; + return 1; +} + +int bech32_decode(char* hrp, uint8_t *data, size_t *data_len, const char *input, + size_t max_input_len) { + uint32_t chk = 1; + size_t i; + size_t input_len = strlen(input); + size_t hrp_len; + int have_lower = 0, have_upper = 0; + if (input_len < 8 || input_len > max_input_len) { + return 0; + } + *data_len = 0; + while (*data_len < input_len && input[(input_len - 1) - *data_len] != '1') { + ++(*data_len); + } + hrp_len = input_len - (1 + *data_len); + if (hrp_len < 1 || *data_len < 6) { + return 0; + } + *(data_len) -= 6; + for (i = 0; i < hrp_len; ++i) { + int ch = input[i]; + if (ch < 33 || ch > 126) { + return 0; + } + if (ch >= 'a' && ch <= 'z') { + have_lower = 1; + } else if (ch >= 'A' && ch <= 'Z') { + have_upper = 1; + ch = (ch - 'A') + 'a'; + } + hrp[i] = ch; + chk = bech32_polymod_step(chk) ^ (ch >> 5); + } + hrp[i] = 0; + chk = bech32_polymod_step(chk); + for (i = 0; i < hrp_len; ++i) { + chk = bech32_polymod_step(chk) ^ (input[i] & 0x1f); + } + ++i; + while (i < input_len) { + int v = (input[i] & 0x80) ? -1 : bech32_charset_rev[(int)input[i]]; + if (input[i] >= 'a' && input[i] <= 'z') have_lower = 1; + if (input[i] >= 'A' && input[i] <= 'Z') have_upper = 1; + if (v == -1) { + return 0; + } + chk = bech32_polymod_step(chk) ^ v; + if (i + 6 < input_len) { + data[i - (1 + hrp_len)] = v; + } + ++i; + } + if (have_lower && have_upper) { + return 0; + } + return chk == 1; +} + +int bech32_convert_bits(uint8_t* out, size_t* outlen, int outbits, const uint8_t* in, size_t inlen, int inbits, int pad) { + uint32_t val = 0; + int bits = 0; + uint32_t maxv = (((uint32_t)1) << outbits) - 1; + while (inlen--) { + val = (val << inbits) | *(in++); + bits += inbits; + while (bits >= outbits) { + bits -= outbits; + out[(*outlen)++] = (val >> bits) & maxv; + } + } + if (pad) { + if (bits) { + out[(*outlen)++] = (val << (outbits - bits)) & maxv; + } + } else if (((val << (outbits - bits)) & maxv) || bits >= inbits) { + return 0; + } + return 1; +} + +int segwit_addr_encode(char *output, const char *hrp, int witver, const uint8_t *witprog, size_t witprog_len) { + uint8_t data[65]; + size_t datalen = 0; + if (witver > 16) return 0; + if (witver == 0 && witprog_len != 20 && witprog_len != 32) return 0; + if (witprog_len < 2 || witprog_len > 40) return 0; + data[0] = witver; + bech32_convert_bits(data + 1, &datalen, 5, witprog, witprog_len, 8, 1); + ++datalen; + return bech32_encode(output, hrp, data, datalen, 90); +} + +int segwit_addr_decode(int* witver, uint8_t* witdata, size_t* witdata_len, const char* hrp, const char* addr) { + uint8_t data[84]; + char hrp_actual[84]; + size_t data_len; + if (!bech32_decode(hrp_actual, data, &data_len, addr, 90)) return 0; + if (data_len == 0 || data_len > 65) return 0; + if (strncmp(hrp, hrp_actual, 84) != 0) return 0; + if (data[0] > 16) return 0; + *witdata_len = 0; + if (!bech32_convert_bits(witdata, witdata_len, 8, data + 1, data_len - 1, 5, 0)) return 0; + if (*witdata_len < 2 || *witdata_len > 40) return 0; + if (data[0] == 0 && *witdata_len != 20 && *witdata_len != 32) return 0; + *witver = data[0]; + return 1; +} diff --git a/lightningd/bech32.h b/lightningd/bech32.h new file mode 100644 index 000000000000..072b31293623 --- /dev/null +++ b/lightningd/bech32.h @@ -0,0 +1,122 @@ +/* Stolen from https://github.com/sipa/bech32/blob/master/ref/c/segwit_addr.h, + * with only the two ' > 90' checks hoisted */ + +/* Copyright (c) 2017 Pieter Wuille + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef _SEGWIT_ADDR_H_ +#define _SEGWIT_ADDR_H_ 1 +#include "config.h" + +#include +#include + +/** Encode a SegWit address + * + * Out: output: Pointer to a buffer of size 73 + strlen(hrp) that will be + * updated to contain the null-terminated address. + * In: hrp: Pointer to the null-terminated human readable part to use + * (chain/network specific). + * ver: Version of the witness program (between 0 and 16 inclusive). + * prog: Data bytes for the witness program (between 2 and 40 bytes). + * prog_len: Number of data bytes in prog. + * Returns 1 if successful. + */ +int segwit_addr_encode( + char *output, + const char *hrp, + int ver, + const uint8_t *prog, + size_t prog_len +); + +/** Decode a SegWit address + * + * Out: ver: Pointer to an int that will be updated to contain the witness + * program version (between 0 and 16 inclusive). + * prog: Pointer to a buffer of size 40 that will be updated to + * contain the witness program bytes. + * prog_len: Pointer to a size_t that will be updated to contain the length + * of bytes in prog. + * hrp: Pointer to the null-terminated human readable part that is + * expected (chain/network specific). + * addr: Pointer to the null-terminated address. + * Returns 1 if successful. + */ +int segwit_addr_decode( + int* ver, + uint8_t* prog, + size_t* prog_len, + const char* hrp, + const char* addr +); + +/** Encode a Bech32 string + * + * Out: output: Pointer to a buffer of size strlen(hrp) + data_len + 8 that + * will be updated to contain the null-terminated Bech32 string. + * In: hrp : Pointer to the null-terminated human readable part. + * data : Pointer to an array of 5-bit values. + * data_len: Length of the data array. + * max_input_len: Maximum valid length of input (90 for segwit usage). + * Returns 1 if successful. + */ +int bech32_encode( + char *output, + const char *hrp, + const uint8_t *data, + size_t data_len, + size_t max_input_len +); + +/** Decode a Bech32 string + * + * Out: hrp: Pointer to a buffer of size strlen(input) - 6. Will be + * updated to contain the null-terminated human readable part. + * data: Pointer to a buffer of size strlen(input) - 8 that will + * hold the encoded 5-bit data values. + * data_len: Pointer to a size_t that will be updated to be the number + * of entries in data. + * In: input: Pointer to a null-terminated Bech32 string. + * max_input_len: Maximum valid length of input (90 for segwit usage). + * Returns 1 if succesful. + */ +int bech32_decode( + char *hrp, + uint8_t *data, + size_t *data_len, + const char *input, + size_t max_input_len +); + +/* Helper from bech32: translates inbits-bit bytes to outbits-bit bytes. + * @outlen is incremented as bytes are added. + * @pad is true if we're to pad, otherwise truncate last byte if necessary + */ +int bech32_convert_bits(uint8_t* out, size_t* outlen, int outbits, + const uint8_t* in, size_t inlen, int inbits, + int pad); + +/* The charset, and reverse mapping */ +extern const char bech32_charset[32]; +extern const int8_t bech32_charset_rev[128]; + +#endif /* LIGHTNING_LIGHTNINGD_BECH32_H */ From f9edbcb4ec2a7c6a4e5951b9570fbd01f2c1ae7f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:25:19 +1030 Subject: [PATCH 0057/1428] script: add p2sh scriptpubkey helper to create from hash. Signed-off-by: Rusty Russell --- bitcoin/script.c | 16 +++++++++++----- bitcoin/script.h | 3 +++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/bitcoin/script.c b/bitcoin/script.c index 397aab53639e..675340471317 100644 --- a/bitcoin/script.c +++ b/bitcoin/script.c @@ -182,19 +182,25 @@ u8 *bitcoin_redeem_2of2(const tal_t *ctx, return script; } -/* Create p2sh for this redeem script. */ -u8 *scriptpubkey_p2sh(const tal_t *ctx, const u8 *redeemscript) +u8 *scriptpubkey_p2sh_hash(const tal_t *ctx, const struct ripemd160 *redeemhash) { - struct ripemd160 redeemhash; u8 *script = tal_arr(ctx, u8, 0); add_op(&script, OP_HASH160); - hash160(&redeemhash, redeemscript, tal_count(redeemscript)); - add_push_bytes(&script, redeemhash.u.u8, sizeof(redeemhash.u.u8)); + add_push_bytes(&script, redeemhash->u.u8, sizeof(redeemhash->u.u8)); add_op(&script, OP_EQUAL); return script; } +/* Create p2sh for this redeem script. */ +u8 *scriptpubkey_p2sh(const tal_t *ctx, const u8 *redeemscript) +{ + struct ripemd160 redeemhash; + + hash160(&redeemhash, redeemscript, tal_count(redeemscript)); + return scriptpubkey_p2sh_hash(ctx, &redeemhash); +} + /* Create an output script using p2pkh */ u8 *scriptpubkey_p2pkh(const tal_t *ctx, const struct bitcoin_address *addr) { diff --git a/bitcoin/script.h b/bitcoin/script.h index 8a11783d8424..5752bc9f617b 100644 --- a/bitcoin/script.h +++ b/bitcoin/script.h @@ -23,6 +23,9 @@ u8 *bitcoin_redeem_2of2(const tal_t *ctx, /* Create an output script using p2sh for this redeem script. */ u8 *scriptpubkey_p2sh(const tal_t *ctx, const u8 *redeemscript); +/* Create an output script using p2sh for this hash. */ +u8 *scriptpubkey_p2sh_hash(const tal_t *ctx, const struct ripemd160 *redeemhash); + /* Create an output script using p2pkh */ u8 *scriptpubkey_p2pkh(const tal_t *ctx, const struct bitcoin_address *addr); From 7ed81cdf7b064cd73cd48e5772d13681cb844fb1 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:26:19 +1030 Subject: [PATCH 0058/1428] pay: factor out actual payment sending from json part of sendpay. We're going to reuse this for the new 'pay' all-in-one command. Signed-off-by: Rusty Russell --- lightningd/pay.c | 236 ++++++++++++++++++++------------------- tests/test_lightningd.py | 2 - 2 files changed, 123 insertions(+), 115 deletions(-) diff --git a/lightningd/pay.c b/lightningd/pay.c index ecdc00ab86cd..38fa52cae603 100644 --- a/lightningd/pay.c +++ b/lightningd/pay.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -143,128 +144,45 @@ static void pay_command_destroyed(struct pay_command *pc) list_del(&pc->list); } -static void json_sendpay(struct command *cmd, - const char *buffer, const jsmntok_t *params) +static void send_payment(struct command *cmd, + const struct sha256 *rhash, + const struct route_hop *route) { - struct pubkey *ids; - jsmntok_t *routetok, *rhashtok; - const jsmntok_t *t, *end; - unsigned int delay, base_expiry; - size_t n_hops; - struct sha256 rhash; - struct peer *peer; struct pay_command *pc; + struct peer *peer; const u8 *onion; u8 sessionkey[32]; - struct hop_data *hop_data; - struct hop_data first_hop_data; - u64 amount, lastamount; + unsigned int base_expiry; struct onionpacket *packet; struct secret *path_secrets; enum onion_type failcode; - - if (!json_get_params(buffer, params, - "route", &routetok, - "rhash", &rhashtok, - NULL)) { - command_fail(cmd, "Need route and rhash"); - return; - } - - if (!hex_decode(buffer + rhashtok->start, - rhashtok->end - rhashtok->start, - &rhash, sizeof(rhash))) { - command_fail(cmd, "'%.*s' is not a valid sha256 hash", - (int)(rhashtok->end - rhashtok->start), - buffer + rhashtok->start); - return; - } - - if (routetok->type != JSMN_ARRAY) { - command_fail(cmd, "'%.*s' is not an array", - (int)(routetok->end - routetok->start), - buffer + routetok->start); - return; - } + size_t i, n_hops = tal_count(route); + struct hop_data *hop_data = tal_arr(cmd, struct hop_data, n_hops); + struct pubkey *ids = tal_arr(cmd, struct pubkey, n_hops); /* Expiry for HTLCs is absolute. And add one to give some margin. */ base_expiry = get_block_height(cmd->ld->topology) + 1; - end = json_next(routetok); - n_hops = 0; - ids = tal_arr(cmd, struct pubkey, n_hops); - - hop_data = tal_arr(cmd, struct hop_data, n_hops); - for (t = routetok + 1; t < end; t = json_next(t)) { - const jsmntok_t *amttok, *idtok, *delaytok, *chantok; + /* Extract IDs for each hop: create_onionpacket wants array. */ + for (i = 0; i < n_hops; i++) + ids[i] = route[i].nodeid; - if (t->type != JSMN_OBJECT) { - command_fail(cmd, "route %zu '%.*s' is not an object", - n_hops, - (int)(t->end - t->start), - buffer + t->start); - return; - } - amttok = json_get_member(buffer, t, "msatoshi"); - idtok = json_get_member(buffer, t, "id"); - delaytok = json_get_member(buffer, t, "delay"); - chantok = json_get_member(buffer, t, "channel"); - if (!amttok || !idtok || !delaytok || !chantok) { - command_fail(cmd, "route %zu needs msatoshi/id/channel/delay", - n_hops); - return; - } - - tal_resize(&hop_data, n_hops + 1); - tal_resize(&ids, n_hops+1); - hop_data[n_hops].realm = 0; - /* What that hop will forward */ - if (!json_tok_u64(buffer, amttok, &amount)) { - command_fail(cmd, "route %zu invalid msatoshi", - n_hops); - return; - } - hop_data[n_hops].amt_forward = amount; - - if (!short_channel_id_from_str(buffer + chantok->start, - chantok->end - chantok->start, - &hop_data[n_hops].channel_id)) { - command_fail(cmd, "route %zu invalid channel_id", n_hops); - return; - } - if (!json_tok_pubkey(buffer, idtok, &ids[n_hops])) { - command_fail(cmd, "route %zu invalid id", n_hops); - return; - } - if (!json_tok_number(buffer, delaytok, &delay)) { - command_fail(cmd, "route %zu invalid delay", n_hops); - return; - } - hop_data[n_hops].outgoing_cltv = base_expiry + delay; - n_hops++; - } - - if (n_hops == 0) { - command_fail(cmd, "Empty route"); - return; + /* Copy hop_data[n] from route[n+1] (ie. where it goes next) */ + for (i = 0; i < n_hops - 1; i++) { + hop_data[i].realm = 0; + hop_data[i].channel_id = route[i+1].channel_id; + hop_data[i].amt_forward = route[i+1].amount; + hop_data[i].outgoing_cltv = base_expiry + route[i+1].delay; } - /* Store some info we'll need for our own HTLC */ - amount = hop_data[0].amt_forward; - lastamount = hop_data[n_hops-1].amt_forward; - first_hop_data = hop_data[0]; - - /* Shift the hop_data down by one, so each hop gets its - * instructions, not how we got there */ - for (size_t i=0; i < n_hops - 1; i++) { - hop_data[i] = hop_data[i+1]; - } /* And finally set the final hop to the special values in * BOLT04 */ - hop_data[n_hops-1].outgoing_cltv = base_expiry + delay; - memset(&hop_data[n_hops-1].channel_id, 0, sizeof(struct short_channel_id)); + hop_data[i].realm = 0; + hop_data[i].outgoing_cltv = base_expiry + route[i].delay; + memset(&hop_data[i].channel_id, 0, sizeof(struct short_channel_id)); + hop_data[i].amt_forward = route[i].amount; - pc = find_pay_command(cmd->ld, &rhash); + pc = find_pay_command(cmd->ld, rhash); if (pc) { log_debug(cmd->ld->log, "json_sendpay: found previous"); if (pc->out) { @@ -276,7 +194,7 @@ static void json_sendpay(struct command *cmd, size_t old_nhops = tal_count(pc->ids); log_add(cmd->ld->log, "... succeeded"); /* Must match successful payment parameters. */ - if (pc->msatoshi != lastamount) { + if (pc->msatoshi != hop_data[n_hops-1].amt_forward) { command_fail(cmd, "already succeeded with amount %" PRIu64, pc->msatoshi); @@ -307,7 +225,7 @@ static void json_sendpay(struct command *cmd, randombytes_buf(&sessionkey, sizeof(sessionkey)); /* Onion will carry us from first peer onwards. */ - packet = create_onionpacket(cmd, ids, hop_data, sessionkey, rhash.u.u8, + packet = create_onionpacket(cmd, ids, hop_data, sessionkey, rhash->u.u8, sizeof(struct sha256), &path_secrets); onion = serialize_onionpacket(cmd, packet); @@ -319,14 +237,14 @@ static void json_sendpay(struct command *cmd, tal_add_destructor(pc, pay_command_destroyed); } pc->cmd = cmd; - pc->rhash = rhash; + pc->rhash = *rhash; pc->rval = NULL; pc->ids = tal_steal(pc, ids); - pc->msatoshi = lastamount; + pc->msatoshi = route[n_hops-1].amount; pc->path_secrets = tal_steal(pc, path_secrets); - log_info(cmd->ld->log, "Sending %"PRIu64" over %zu hops to deliver %"PRIu64, - amount, n_hops, lastamount); + log_info(cmd->ld->log, "Sending %u over %zu hops to deliver %"PRIu64, + route[0].amount, n_hops, pc->msatoshi); /* Wait until we get response. */ tal_add_destructor2(cmd, remove_cmd_from_pc, pc); @@ -336,8 +254,9 @@ static void json_sendpay(struct command *cmd, * remove_cmd_from_pc destructor causes a use-after-free */ tal_steal(pc, cmd); - failcode = send_htlc_out(peer, amount, first_hop_data.outgoing_cltv, - &rhash, onion, NULL, pc, &pc->out); + failcode = send_htlc_out(peer, route[0].amount, + base_expiry + route[0].delay, + rhash, onion, NULL, pc, &pc->out); if (failcode) { command_fail(cmd, "first peer not ready: %s", onion_type_name(failcode)); @@ -345,6 +264,97 @@ static void json_sendpay(struct command *cmd, } } +static void json_sendpay(struct command *cmd, + const char *buffer, const jsmntok_t *params) +{ + jsmntok_t *routetok, *rhashtok; + const jsmntok_t *t, *end; + size_t n_hops; + struct sha256 rhash; + struct route_hop *route; + + if (!json_get_params(buffer, params, + "route", &routetok, + "rhash", &rhashtok, + NULL)) { + command_fail(cmd, "Need route and rhash"); + return; + } + + if (!hex_decode(buffer + rhashtok->start, + rhashtok->end - rhashtok->start, + &rhash, sizeof(rhash))) { + command_fail(cmd, "'%.*s' is not a valid sha256 hash", + (int)(rhashtok->end - rhashtok->start), + buffer + rhashtok->start); + return; + } + + if (routetok->type != JSMN_ARRAY) { + command_fail(cmd, "'%.*s' is not an array", + (int)(routetok->end - routetok->start), + buffer + routetok->start); + return; + } + + end = json_next(routetok); + n_hops = 0; + route = tal_arr(cmd, struct route_hop, n_hops); + + for (t = routetok + 1; t < end; t = json_next(t)) { + const jsmntok_t *amttok, *idtok, *delaytok, *chantok; + + if (t->type != JSMN_OBJECT) { + command_fail(cmd, "route %zu '%.*s' is not an object", + n_hops, + (int)(t->end - t->start), + buffer + t->start); + return; + } + amttok = json_get_member(buffer, t, "msatoshi"); + idtok = json_get_member(buffer, t, "id"); + delaytok = json_get_member(buffer, t, "delay"); + chantok = json_get_member(buffer, t, "channel"); + if (!amttok || !idtok || !delaytok || !chantok) { + command_fail(cmd, "route %zu needs msatoshi/id/channel/delay", + n_hops); + return; + } + + tal_resize(&route, n_hops + 1); + + /* What that hop will forward */ + if (!json_tok_number(buffer, amttok, &route[n_hops].amount)) { + command_fail(cmd, "route %zu invalid msatoshi", + n_hops); + return; + } + + if (!short_channel_id_from_str(buffer + chantok->start, + chantok->end - chantok->start, + &route[n_hops].channel_id)) { + command_fail(cmd, "route %zu invalid channel_id", n_hops); + return; + } + if (!json_tok_pubkey(buffer, idtok, &route[n_hops].nodeid)) { + command_fail(cmd, "route %zu invalid id", n_hops); + return; + } + if (!json_tok_number(buffer, delaytok, &route[n_hops].delay)) { + command_fail(cmd, "route %zu invalid delay", n_hops); + return; + } + n_hops++; + } + + if (n_hops == 0) { + command_fail(cmd, "Empty route"); + return; + } + + send_payment(cmd, &rhash, route); +} + static const struct json_command sendpay_command = { "sendpay", json_sendpay, diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 28b9a6c5abe8..00fefd16e38b 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -309,8 +309,6 @@ def test_sendpay(self): self.fund_channel(l1, l2, 10**6) - time.sleep(5) - amt = 200000000 rhash = l2.rpc.invoice(amt, 'testpayment2')['rhash'] assert l2.rpc.listinvoice('testpayment2')[0]['complete'] == False From 58604a049749c1d032e3a307e7d0a3753a74670f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:27:19 +1030 Subject: [PATCH 0059/1428] chainparams: add bip173 name. Google lead me to a discussion about litecint, it suggested they would use 'ltc' and I don't really care. Signed-off-by: Rusty Russell --- bitcoin/chainparams.c | 14 ++++++++++++++ bitcoin/chainparams.h | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/bitcoin/chainparams.c b/bitcoin/chainparams.c index c9d300aaba06..1e6c8f060801 100644 --- a/bitcoin/chainparams.c +++ b/bitcoin/chainparams.c @@ -6,6 +6,7 @@ const struct chainparams networks[] = { {.index = 0, .network_name = "bitcoin", + .bip173_name = "bc", .genesis_blockhash = {{.u.u8 = {0x6f, 0xe2, 0x8c, 0x0a, 0xb6, 0xf1, 0xb3, 0x72, 0xc1, 0xa6, 0xa2, 0x46, 0xae, 0x63, 0xf7, 0x4f, 0x93, 0x1e, 0x83, 0x65, 0xe1, 0x5a, 0x08, 0x9c, 0x68, 0xd6, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00}}}, .rpc_port = 8332, .cli = "bitcoin-cli", @@ -14,6 +15,7 @@ const struct chainparams networks[] = { .testnet = false}, {.index = 1, .network_name = "regtest", + .bip173_name = "tb", .genesis_blockhash = {{.u.u8 = {0x06, 0x22, 0x6e, 0x46, 0x11, 0x1a, 0x0b, 0x59, 0xca, 0xaf, 0x12, 0x60, 0x43, 0xeb, 0x5b, 0xbf, 0x28, 0xc3, 0x4f, 0x3a, 0x5e, 0x33, 0x2a, 0x1f, 0xc7, 0xb2, 0xb7, 0x3c, 0xf1, 0x88, 0x91, 0x0f}}}, .rpc_port = 18332, .cli = "bitcoin-cli", @@ -22,6 +24,7 @@ const struct chainparams networks[] = { .testnet = true}, {.index = 2, .network_name = "testnet", + .bip173_name = "tb", .genesis_blockhash = {{.u.u8 = {0x43, 0x49, 0x7f, 0xd7, 0xf8, 0x26, 0x95, 0x71, 0x08, 0xf4, 0xa3, 0x0f, 0xd9, 0xce, 0xc3, 0xae, 0xba, 0x79, 0x97, 0x20, 0x84, 0xe9, 0x0e, 0xad, 0x01, 0xea, 0x33, 0x09, 0x00, 0x00, 0x00, 0x00}}}, .rpc_port = 18332, .cli = "bitcoin-cli", @@ -30,6 +33,7 @@ const struct chainparams networks[] = { .testnet = true}, {.index = 3, .network_name = "litecoin", + .bip173_name = "ltc", .genesis_blockhash = {{.u.u8 = {0xe2, 0xbf, 0x04, 0x7e, 0x7e, 0x5a, 0x19, 0x1a, 0xa4, 0xef, 0x34, 0xd3, 0x14, 0x97, 0x9d, 0xc9, 0x98, 0x6e, 0x0f, 0x19, 0x25, 0x1e, 0xda, 0xba, 0x59, 0x40, 0xfd, 0x1f, 0xe3, 0x65, 0xa7, 0x12 }}}, .rpc_port = 18332, .cli = "litecoin-cli", @@ -55,3 +59,13 @@ const struct chainparams *chainparams_by_index(const int index) return &networks[index]; } } + +const struct chainparams *chainparams_by_bip173(const char *bip173_name) +{ + for (size_t i = 0; i < ARRAY_SIZE(networks); i++) { + if (streq(bip173_name, networks[i].bip173_name)) { + return &networks[i]; + } + } + return NULL; +} diff --git a/bitcoin/chainparams.h b/bitcoin/chainparams.h index a0fd5b8136fe..2982a3aa81a9 100644 --- a/bitcoin/chainparams.h +++ b/bitcoin/chainparams.h @@ -9,6 +9,7 @@ struct chainparams { const int index; const char *network_name; + const char *bip173_name; const struct sha256_double genesis_blockhash; const int rpc_port; const char *cli; @@ -31,4 +32,11 @@ const struct chainparams *chainparams_for_network(const char *network_name); * we allows lookup by index. */ const struct chainparams *chainparams_by_index(const int index); + +/** + * chainparams_by_bip173 - Helper to get a network by its bip173 name + * + * This lets us decode BOLT11 addresses. + */ +const struct chainparams *chainparams_by_bip173(const char *bip173_name); #endif /* LIGHTNING_BITCOIN_CHAINPARAMS_H */ From 9ec5cb7ba2b7f46b89a0200124e872995f436f3b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:28:19 +1030 Subject: [PATCH 0060/1428] script: enhance is_p2sh/is_p2pkh/is_p2wsh/is_p2wpkh to extract addr. Signed-off-by: Rusty Russell --- bitcoin/script.c | 16 ++++++++++++---- bitcoin/script.h | 16 ++++++++-------- lightningd/peer_control.c | 4 ++-- onchaind/onchain.c | 2 +- wallet/wallet.c | 4 ++-- 5 files changed, 25 insertions(+), 17 deletions(-) diff --git a/bitcoin/script.c b/bitcoin/script.c index 675340471317..108e17c093fd 100644 --- a/bitcoin/script.c +++ b/bitcoin/script.c @@ -368,7 +368,7 @@ u8 *p2wpkh_scriptcode(const tal_t *ctx, const struct pubkey *key) return script; } -bool is_p2pkh(const u8 *script) +bool is_p2pkh(const u8 *script, struct bitcoin_address *addr) { size_t script_len = tal_len(script); @@ -384,10 +384,12 @@ bool is_p2pkh(const u8 *script) return false; if (script[24] != OP_CHECKSIG) return false; + if (addr) + memcpy(addr, script+3, 20); return true; } -bool is_p2sh(const u8 *script) +bool is_p2sh(const u8 *script, struct ripemd160 *addr) { size_t script_len = tal_len(script); @@ -399,10 +401,12 @@ bool is_p2sh(const u8 *script) return false; if (script[22] != OP_EQUAL) return false; + if (addr) + memcpy(addr, script+2, 20); return true; } -bool is_p2wsh(const u8 *script) +bool is_p2wsh(const u8 *script, struct sha256 *addr) { size_t script_len = tal_len(script); @@ -412,10 +416,12 @@ bool is_p2wsh(const u8 *script) return false; if (script[1] != OP_PUSHBYTES(sizeof(struct sha256))) return false; + if (addr) + memcpy(addr, script+2, sizeof(struct sha256)); return true; } -bool is_p2wpkh(const u8 *script) +bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr) { size_t script_len = tal_len(script); @@ -425,6 +431,8 @@ bool is_p2wpkh(const u8 *script) return false; if (script[1] != OP_PUSHBYTES(sizeof(struct ripemd160))) return false; + if (addr) + memcpy(addr, script+2, sizeof(*addr)); return true; } diff --git a/bitcoin/script.h b/bitcoin/script.h index 5752bc9f617b..d0cc224ec20a 100644 --- a/bitcoin/script.h +++ b/bitcoin/script.h @@ -125,17 +125,17 @@ u8 *bitcoin_wscript_htlc_tx(const tal_t *ctx, const struct pubkey *revocation_pubkey, const struct pubkey *local_delayedkey); -/* Is this a pay to pubkey hash? */ -bool is_p2pkh(const u8 *script); +/* Is this a pay to pubkey hash? (extract addr if not NULL) */ +bool is_p2pkh(const u8 *script, struct bitcoin_address *addr); -/* Is this a pay to script hash? */ -bool is_p2sh(const u8 *script); +/* Is this a pay to script hash? (extract addr if not NULL) */ +bool is_p2sh(const u8 *script, struct ripemd160 *addr); -/* Is this (version 0) pay to witness script hash? */ -bool is_p2wsh(const u8 *script); +/* Is this (version 0) pay to witness script hash? (extract addr if not NULL) */ +bool is_p2wsh(const u8 *script, struct sha256 *addr); -/* Is this (version 0) pay to witness pubkey hash? */ -bool is_p2wpkh(const u8 *script); +/* Is this (version 0) pay to witness pubkey hash? (extract addr if not NULL) */ +bool is_p2wpkh(const u8 *script, struct bitcoin_address *addr); /* Are these two scripts equal? */ bool scripteq(const tal_t *s1, const tal_t *s2); diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index fcb4fdc48c10..c1799ffc3b5c 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -1620,8 +1620,8 @@ static void peer_got_shutdown(struct peer *peer, const u8 *msg) * * A receiving node SHOULD fail the connection if the `scriptpubkey` * is not one of those forms. */ - if (!is_p2pkh(scriptpubkey) && !is_p2sh(scriptpubkey) - && !is_p2wpkh(scriptpubkey) && !is_p2wsh(scriptpubkey)) { + if (!is_p2pkh(scriptpubkey, NULL) && !is_p2sh(scriptpubkey, NULL) + && !is_p2wpkh(scriptpubkey, NULL) && !is_p2wsh(scriptpubkey, NULL)) { char *str = tal_fmt(peer, "Bad shutdown scriptpubkey %s", tal_hex(peer, scriptpubkey)); peer_fail_permanent_str(peer, take(str)); diff --git a/onchaind/onchain.c b/onchaind/onchain.c index c6425a4794e7..89277cd33f6a 100644 --- a/onchaind/onchain.c +++ b/onchaind/onchain.c @@ -1086,7 +1086,7 @@ static int match_htlc_output(const struct bitcoin_tx *tx, u8 **htlc_scripts) { /* Must be a p2wsh output */ - if (!is_p2wsh(tx->output[outnum].script)) + if (!is_p2wsh(tx->output[outnum].script, NULL)) return -1; for (size_t i = 0; i < tal_count(htlc_scripts); i++) { diff --git a/wallet/wallet.c b/wallet/wallet.c index a2637d7a7475..2df4edfe6d96 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -195,9 +195,9 @@ bool wallet_can_spend(struct wallet *w, const u8 *script, u32 i; /* If not one of these, can't be for us. */ - if (is_p2sh(script)) + if (is_p2sh(script, NULL)) *output_is_p2sh = true; - else if (is_p2wpkh(script)) + else if (is_p2wpkh(script, NULL)) *output_is_p2sh = false; else return false; From 20ce829f81275868db130731a2729e3a9256a0c5 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:29:19 +1030 Subject: [PATCH 0061/1428] script: add helper to use a raw witness program. This is for future compatibility. Signed-off-by: Rusty Russell --- bitcoin/script.c | 9 +++++++++ bitcoin/script.h | 4 ++++ 2 files changed, 13 insertions(+) diff --git a/bitcoin/script.c b/bitcoin/script.c index 108e17c093fd..22f5bd24d4e8 100644 --- a/bitcoin/script.c +++ b/bitcoin/script.c @@ -319,6 +319,15 @@ u8 *scriptpubkey_p2wpkh_derkey(const tal_t *ctx, const u8 der[33]) return script; } +u8 *scriptpubkey_witness_raw(const tal_t *ctx, u8 version, + const u8 *wprog, size_t wprog_size) +{ + u8 *script = tal_arr(ctx, u8, 0); + add_number(&script, version); + add_push_bytes(&script, wprog, wprog_size); + return script; +} + /* Create a witness which spends the 2of2. */ u8 **bitcoin_witness_2of2(const tal_t *ctx, const secp256k1_ecdsa_signature *sig1, diff --git a/bitcoin/script.h b/bitcoin/script.h index d0cc224ec20a..f894e9ee6a69 100644 --- a/bitcoin/script.h +++ b/bitcoin/script.h @@ -57,6 +57,10 @@ u8 *scriptpubkey_p2wpkh(const tal_t *ctx, const struct pubkey *key); /* Same as above, but compressed key is already DER-encoded. */ u8 *scriptpubkey_p2wpkh_derkey(const tal_t *ctx, const u8 der[33]); +/* Encode an arbitrary witness as */ +u8 *scriptpubkey_witness_raw(const tal_t *ctx, u8 version, + const u8 *wprog, size_t wprog_size); + /* Create a witness which spends the 2of2. */ u8 **bitcoin_witness_2of2(const tal_t *ctx, const secp256k1_ecdsa_signature *sig1, From 753f15f50348fb9d7de5ca3eafd1ef5f04673f4c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:30:19 +1030 Subject: [PATCH 0062/1428] common/hash_u5: routines to hash 5 bit values. Needed for creating/verifying signatures on bolt11 invoices. Signed-off-by: Rusty Russell --- common/Makefile | 3 ++- common/hash_u5.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ common/hash_u5.h | 21 +++++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) create mode 100644 common/hash_u5.c create mode 100644 common/hash_u5.h diff --git a/common/Makefile b/common/Makefile index 5290afe192cd..de27d5c8f286 100644 --- a/common/Makefile +++ b/common/Makefile @@ -11,6 +11,7 @@ COMMON_SRC := \ common/derive_basepoints.c \ common/dev_disconnect.c \ common/funding_tx.c \ + common/hash_u5.c \ common/htlc_state.c \ common/htlc_tx.c \ common/htlc_wire.c \ @@ -54,7 +55,7 @@ common/gen_htlc_state_names.h: common/htlc_state.h ccan/ccan/cdump/tools/cdump-e check-makefile: check-common-makefile check-common-makefile: - @if [ x"`LC_ALL=C ls common/*.h | grep -v ^common/gen_`" != x"`echo $(COMMON_HEADERS_NOGEN) | tr ' ' '\n' | LC_ALL=C sort`" ]; then echo COMMON_HEADERS_NOGEN incorrect; exit 1; fi + if [ x"`LC_ALL=C ls common/*.h | grep -v ^common/gen_`" != x"`echo $(COMMON_HEADERS_NOGEN) | tr ' ' '\n' | LC_ALL=C sort`" ]; then echo COMMON_HEADERS_NOGEN incorrect; exit 1; fi check-source-bolt: $(COMMON_SRC:%=bolt-check/%) $(COMMON_HEADERS:%=bolt-check/%) check-whitespace: $(COMMON_SRC:%=check-whitespace/%) $(COMMON_HEADERS:%=check-whitespace/%) diff --git a/common/hash_u5.c b/common/hash_u5.c new file mode 100644 index 000000000000..0d3eb91a6bb0 --- /dev/null +++ b/common/hash_u5.c @@ -0,0 +1,47 @@ +#include +#include +#include + +void hash_u5_init(struct hash_u5 *hu5, const char *hrp) +{ + hu5->buf = 0; + hu5->num_bits = 0; + sha256_init(&hu5->hash); + sha256_update(&hu5->hash, hrp, strlen(hrp)); +} + +void hash_u5(struct hash_u5 *hu5, const u8 *u5, size_t len) +{ + size_t bits = len * 5; + + while (bits) { + size_t n = 5; + + if (bits < n) + n = bits; + + hu5->buf <<= n; + hu5->buf |= (*u5 >> (5-n)); + bits -= n; + hu5->num_bits += n; + + if (n == 5) + u5++; + + if (hu5->num_bits >= 32) { + be32 be32 = cpu_to_be32(hu5->buf >> (hu5->num_bits-32)); + sha256_update(&hu5->hash, &be32, sizeof(be32)); + hu5->num_bits -= 32; + } + } +} + +void hash_u5_done(struct hash_u5 *hu5, struct sha256 *res) +{ + if (hu5->num_bits) { + be32 be32 = cpu_to_be32(hu5->buf << (32 - hu5->num_bits)); + + sha256_update(&hu5->hash, &be32, (hu5->num_bits + 7) / 8); + } + sha256_done(&hu5->hash, res); +} diff --git a/common/hash_u5.h b/common/hash_u5.h new file mode 100644 index 000000000000..65ded9a6df55 --- /dev/null +++ b/common/hash_u5.h @@ -0,0 +1,21 @@ +/* bech32 (thus bolt11) deal in 5-bit values */ +#ifndef LIGHTNING_COMMON_HASH_U5_H +#define LIGHTNING_COMMON_HASH_U5_H +#include "config.h" +#include +#include + +/* Type to annotate a 5 bit value. */ +typedef unsigned char u5; + +struct hash_u5 { + u64 buf; + unsigned int num_bits; + struct sha256_ctx hash; +}; + +void hash_u5_init(struct hash_u5 *hu5, const char *hrp); +void hash_u5(struct hash_u5 *hu5, const u5 *u5, size_t len); +void hash_u5_done(struct hash_u5 *hu5, struct sha256 *res); + +#endif /* LIGHTNING_COMMON_HASH_U5_H */ From a02ca46b03870b0a3f1f8ea7452d1b538917cc37 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:31:19 +1030 Subject: [PATCH 0063/1428] secp256k1_ecdsa_recoverable_signature: add support. Signed-off-by: Rusty Russell --- external/Makefile | 2 +- wire/fromwire.c | 16 ++++++++++++++++ wire/towire.c | 14 ++++++++++++++ wire/wire.h | 6 ++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/external/Makefile b/external/Makefile index 0f4a03b2449f..c0e6035ef82e 100644 --- a/external/Makefile +++ b/external/Makefile @@ -37,7 +37,7 @@ external/libsecp256k1.% external/libwallycore.%: external/libwally-core/src/secp $(MAKE) -C external/libwally-core install-exec external/libwally-core/src/libwallycore.% external/libwally-core/src/secp256k1/libsecp256k1.%: $(LIBWALLY_HEADERS) $(LIBSECP_HEADERS) - cd external/libwally-core && ./tools/autogen.sh && ./configure CC="$(CC)" --enable-static=yes --enable-shared=no --libdir=`pwd`/.. && $(MAKE) + cd external/libwally-core && ./tools/autogen.sh && ./configure CC="$(CC)" --enable-static=yes --enable-module-recovery --enable-shared=no --libdir=`pwd`/.. && $(MAKE) # Git submodules are seriously broken. external/jsmn/jsmn.h: diff --git a/wire/fromwire.c b/wire/fromwire.c index 0925eb6bd7d1..4d4badd772e1 100644 --- a/wire/fromwire.c +++ b/wire/fromwire.c @@ -129,6 +129,22 @@ void fromwire_secp256k1_ecdsa_signature(const u8 **cursor, fromwire_fail(cursor, max); } +void fromwire_secp256k1_ecdsa_recoverable_signature(const u8 **cursor, + size_t *max, + secp256k1_ecdsa_recoverable_signature *rsig) +{ + u8 compact[64]; + int recid; + + fromwire(cursor, max, compact, sizeof(compact)); + recid = fromwire_u8(cursor, max); + + if (secp256k1_ecdsa_recoverable_signature_parse_compact(secp256k1_ctx, + rsig, compact, + recid) != 1) + fromwire_fail(cursor, max); +} + void fromwire_channel_id(const u8 **cursor, size_t *max, struct channel_id *channel_id) { diff --git a/wire/towire.c b/wire/towire.c index 87272169883f..4ca3fd7e7bb2 100644 --- a/wire/towire.c +++ b/wire/towire.c @@ -79,6 +79,20 @@ void towire_secp256k1_ecdsa_signature(u8 **pptr, towire(pptr, compact, sizeof(compact)); } +void towire_secp256k1_ecdsa_recoverable_signature(u8 **pptr, + const secp256k1_ecdsa_recoverable_signature *rsig) +{ + u8 compact[64]; + int recid; + + secp256k1_ecdsa_recoverable_signature_serialize_compact(secp256k1_ctx, + compact, + &recid, + rsig); + towire(pptr, compact, sizeof(compact)); + towire_u8(pptr, recid); +} + void towire_channel_id(u8 **pptr, const struct channel_id *channel_id) { towire(pptr, channel_id, sizeof(*channel_id)); diff --git a/wire/wire.h b/wire/wire.h index 7d2c49f20b69..f89abe6e5379 100644 --- a/wire/wire.h +++ b/wire/wire.h @@ -8,6 +8,7 @@ #include #include #include +#include #include struct channel_id { @@ -30,6 +31,8 @@ void towire_privkey(u8 **pptr, const struct privkey *privkey); void towire_secret(u8 **pptr, const struct secret *secret); void towire_secp256k1_ecdsa_signature(u8 **pptr, const secp256k1_ecdsa_signature *signature); +void towire_secp256k1_ecdsa_recoverable_signature(u8 **pptr, + const secp256k1_ecdsa_recoverable_signature *rsig); void towire_channel_id(u8 **pptr, const struct channel_id *channel_id); void towire_short_channel_id(u8 **pptr, const struct short_channel_id *short_channel_id); @@ -57,6 +60,9 @@ void fromwire_privkey(const u8 **cursor, size_t *max, struct privkey *privkey); void fromwire_pubkey(const u8 **cursor, size_t *max, struct pubkey *pubkey); void fromwire_secp256k1_ecdsa_signature(const u8 **cursor, size_t *max, secp256k1_ecdsa_signature *signature); +void fromwire_secp256k1_ecdsa_recoverable_signature(const u8 **cursor, + size_t *max, + secp256k1_ecdsa_recoverable_signature *rsig); void fromwire_channel_id(const u8 **cursor, size_t *max, struct channel_id *channel_id); void fromwire_short_channel_id(const u8 **cursor, size_t *max, From 02c1d10c9fe1d3e66bb5ef8e7e46404e01113383 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:32:19 +1030 Subject: [PATCH 0064/1428] json: escape strings we output in JSON. We're going to output description strings, which are untrusted. Signed-off-by: Rusty Russell --- common/json.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/common/json.c b/common/json.c index afb237ee3597..9f96ca81e6fa 100644 --- a/common/json.c +++ b/common/json.c @@ -412,8 +412,17 @@ void json_add_literal(struct json_result *result, const char *fieldname, void json_add_string(struct json_result *result, const char *fieldname, const char *value) { + char *escaped = tal_arr(result, char, strlen(value) * 2 + 1); + size_t i, n; + json_start_member(result, fieldname); - result_append_fmt(result, "\"%s\"", value); + for (i = n = 0; value[i]; i++) { + if (value[i] == '\\' || value[i] == '"') + escaped[n++] = '\\'; + escaped[n++] = value[i]; + } + escaped[n] = '\0'; + result_append_fmt(result, "\"%s\"", escaped); } void json_add_bool(struct json_result *result, const char *fieldname, bool value) From f9c6f6413fd776754173fcd8fc101493a7c461ab Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:33:19 +1030 Subject: [PATCH 0065/1428] hsmd: invoice signing support. Signed-off-by: Rusty Russell --- hsmd/Makefile | 1 + hsmd/hsm.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++ hsmd/hsm_wire.csv | 10 ++++++++++ 3 files changed, 62 insertions(+) diff --git a/hsmd/Makefile b/hsmd/Makefile index bb2c1ff21e1c..5e580940763a 100644 --- a/hsmd/Makefile +++ b/hsmd/Makefile @@ -29,6 +29,7 @@ HSMD_COMMON_OBJS := \ common/daemon_conn.o \ common/debug.o \ common/funding_tx.o \ + common/hash_u5.o \ common/io_debug.o \ common/key_derive.o \ common/msg_queue.o \ diff --git a/hsmd/hsm.c b/hsmd/hsm.c index 50e5e51ee892..d322931d1c66 100644 --- a/hsmd/hsm.c +++ b/hsmd/hsm.c @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -569,6 +570,51 @@ static void sign_withdrawal_tx(struct daemon_conn *master, const u8 *msg) tal_free(tmpctx); } +/** + * sign_invoice - Sign an invoice with our key. + */ +static void sign_invoice(struct daemon_conn *master, const u8 *msg) +{ + const tal_t *tmpctx = tal_tmpctx(master); + u5 *u5bytes; + u8 *hrpu8; + char *hrp; + struct sha256 sha; + secp256k1_ecdsa_recoverable_signature rsig; + struct hash_u5 hu5; + struct privkey node_pkey; + + if (!fromwire_hsmctl_sign_invoice(tmpctx, msg, NULL, &u5bytes, &hrpu8)) { + status_trace("Failed to parse sign_invoice: %s", + tal_hex(trc, msg)); + return; + } + + /* FIXME: Check invoice! */ + + hrp = tal_dup_arr(tmpctx, char, (char *)hrpu8, tal_len(hrpu8), 1); + hrp[tal_len(hrpu8)] = '\0'; + + hash_u5_init(&hu5, hrp); + hash_u5(&hu5, u5bytes, tal_len(u5bytes)); + hash_u5_done(&hu5, &sha); + + node_key(&node_pkey, NULL); + if (!secp256k1_ecdsa_sign_recoverable(secp256k1_ctx, &rsig, + (const u8 *)&sha, + node_pkey.secret.data, + NULL, NULL)) { + /* FIXME: Now master will freeze... */ + status_trace("Failed to sign invoice: %s", + tal_hex(trc, msg)); + return; + } + + daemon_conn_send(master, + take(towire_hsmctl_sign_invoice_reply(tmpctx, &rsig))); + tal_free(tmpctx); +} + static void sign_node_announcement(struct daemon_conn *master, const u8 *msg) { /* 2 bytes msg type + 64 bytes signature */ @@ -626,6 +672,10 @@ static struct io_plan *control_received_req(struct io_conn *conn, sign_withdrawal_tx(master, master->msg_in); return daemon_conn_read_next(conn, master); + case WIRE_HSMCTL_SIGN_INVOICE: + sign_invoice(master, master->msg_in); + return daemon_conn_read_next(conn, master); + case WIRE_HSMCTL_NODE_ANNOUNCEMENT_SIG_REQ: sign_node_announcement(master, master->msg_in); return daemon_conn_read_next(conn, master); @@ -635,6 +685,7 @@ static struct io_plan *control_received_req(struct io_conn *conn, case WIRE_HSMCTL_HSMFD_CHANNELD_REPLY: case WIRE_HSMCTL_SIGN_FUNDING_REPLY: case WIRE_HSMCTL_SIGN_WITHDRAWAL_REPLY: + case WIRE_HSMCTL_SIGN_INVOICE_REPLY: case WIRE_HSMSTATUS_CLIENT_BAD_REQUEST: case WIRE_HSMCTL_NODE_ANNOUNCEMENT_SIG_REPLY: break; diff --git a/hsmd/hsm_wire.csv b/hsmd/hsm_wire.csv index 3e79a3f96aa5..c9b92d13d096 100644 --- a/hsmd/hsm_wire.csv +++ b/hsmd/hsm_wire.csv @@ -63,3 +63,13 @@ hsmctl_sign_withdrawal,,inputs,num_inputs*struct utxo hsmctl_sign_withdrawal_reply,107 hsmctl_sign_withdrawal_reply,,num_sigs,u16 hsmctl_sign_withdrawal_reply,,sig,num_sigs*secp256k1_ecdsa_signature + +# Sign an invoice +hsmctl_sign_invoice,8 +hsmctl_sign_invoice,,len,u16 +hsmctl_sign_invoice,,u5bytes,len*u8 +hsmctl_sign_invoice,,hrplen,u16 +hsmctl_sign_invoice,,hrp,hrplen*u8 + +hsmctl_sign_invoice_reply,108 +hsmctl_sign_invoice_reply,,sig,secp256k1_ecdsa_recoverable_signature From 15e1e4b099579535a114db5c53e5fc6347e41932 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:34:19 +1030 Subject: [PATCH 0066/1428] bolt11: support for encoding/decoding and checking. Signed-off-by: Rusty Russell --- lightningd/Makefile | 2 + lightningd/bolt11.c | 1002 +++++++++++++++++++++++++++++++++++++++++++ lightningd/bolt11.h | 93 ++++ 3 files changed, 1097 insertions(+) create mode 100644 lightningd/bolt11.c create mode 100644 lightningd/bolt11.h diff --git a/lightningd/Makefile b/lightningd/Makefile index de0a6c50a9a1..5aaade82c1c7 100644 --- a/lightningd/Makefile +++ b/lightningd/Makefile @@ -20,6 +20,7 @@ LIGHTNINGD_COMMON_OBJS := \ common/crypto_state.o \ common/derive_basepoints.o \ common/funding_tx.o \ + common/hash_u5.o \ common/htlc_state.o \ common/htlc_wire.o \ common/io_debug.o \ @@ -41,6 +42,7 @@ LIGHTNINGD_COMMON_OBJS := \ LIGHTNINGD_SRC := \ lightningd/bech32.c \ lightningd/bitcoind.c \ + lightningd/bolt11.c \ lightningd/build_utxos.c \ lightningd/chaintopology.c \ lightningd/gossip_control.c \ diff --git a/lightningd/bolt11.c b/lightningd/bolt11.c new file mode 100644 index 000000000000..fed18ded0632 --- /dev/null +++ b/lightningd/bolt11.c @@ -0,0 +1,1002 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* 1000 * 10^8 millisatoshi == 1 bitcoin */ +#define MSAT_PER_BTC 100000000000ULL + +struct multiplier { + const char letter; + /* We can't represent p postfix to msat, so we multiply this by 10 */ + u64 m10; +}; + +/* BOLT #11: + * + * The following `multiplier` letters are defined: + * + * * `m` (milli): multiply by 0.001 + * * `u` (micro): multiply by 0.000001 + * * `n` (nano): multiply by 0.000000001 + * * `p` (pico): multiply by 0.000000000001 + */ +static struct multiplier multipliers[] = { + { 'm', 10 * MSAT_PER_BTC / 1000 }, + { 'u', 10 * MSAT_PER_BTC / 1000000 }, + { 'n', 10 * MSAT_PER_BTC / 1000000000 }, + { 'p', 10 * MSAT_PER_BTC / 1000000000000ULL } +}; + +/* If pad is false, we discard any bits which don't fit in the last byte. + * Otherwise we add an extra byte */ +static bool pull_bits(struct hash_u5 *hu5, + u5 **data, size_t *data_len, void *dst, size_t nbits, + bool pad) +{ + size_t n5 = nbits / 5; + size_t len = 0; + + if (nbits % 5) + n5++; + + if (*data_len < n5) + return false; + if (!bech32_convert_bits(dst, &len, 8, *data, n5, 5, pad)) + return false; + if (hu5) + hash_u5(hu5, *data, n5); + *data += n5; + *data_len -= n5; + + return true; +} + +/* For pulling fields where we should have checked it will succeed already. */ +#ifndef NDEBUG +#define pull_bits_certain(hu5, data, data_len, dst, nbits, pad) \ + assert(pull_bits((hu5), (data), (data_len), (dst), (nbits), (pad))) +#else +#define pull_bits_certain pull_bits +#endif + +/* Helper for pulling a variable-length big-endian int. */ +static bool pull_uint(struct hash_u5 *hu5, + u5 **data, size_t *data_len, + u64 *val, size_t databits) +{ + be64 be_val; + + /* Too big. */ + if (databits > sizeof(be_val) * CHAR_BIT) + return false; + if (!pull_bits(hu5, data, data_len, &be_val, databits, true)) + return false; + *val = be64_to_cpu(be_val) >> (sizeof(be_val) * CHAR_BIT - databits); + return true; +} + +static size_t num_u8(size_t num_u5) +{ + return (num_u5 * 5 + 4) / 8; +} + +/* Frees bolt11, returns NULL. */ +static struct bolt11 *decode_fail(struct bolt11 *b11, char **fail, + const char *fmt, ...) + PRINTF_FMT(3,4); + +static struct bolt11 *decode_fail(struct bolt11 *b11, char **fail, + const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + *fail = tal_vfmt(tal_parent(b11), fmt, ap); + va_end(ap); + return tal_free(b11); +} + +/* + * These handle specific fields in the payment request; returning the problem + * if any, or NULL. + */ +static char *unknown_field(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + u5 type, size_t length) +{ + struct bolt11_field *extra = tal(b11, struct bolt11_field); + u8 u8data[num_u8(length)]; + + extra->tag = type; + extra->data = tal_dup_arr(extra, u5, *data, length, 0); + list_add_tail(&b11->extra_fields, &extra->list); + + pull_bits_certain(hu5, data, data_len, u8data, length, false); + (*data) += length; + (*data_len) -= length; + return NULL; +} + +/* BOLT #11: + * + * `p` (1): `data_length` 52. 256-bit SHA256 payment_hash: preimage of this + * provides proof of payment. + */ +static void decode_p(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + size_t data_length, bool *have_p) +{ + /* BOLT #11: + * + * A payer SHOULD use the first `p` field did not skip as the payment + * hash. + */ + if (*have_p) { + unknown_field(b11, hu5, data, data_len, 'p', data_length); + return; + } + + /* BOLT #11: + * + * A reader MUST skip over unknown fields, an `f` field with unknown + * `version`, or a `p`, `h`, or `n` field which does not have + * `data_length` 52, 52, or 53 respectively. */ + if (data_length != 52) { + unknown_field(b11, hu5, data, data_len, 'p', data_length); + return; + } + + pull_bits_certain(hu5, data, data_len, &b11->payment_hash, 256, false); + *have_p = true; +} + +/* BOLT #11: + * + * `d` (13): `data_length` variable. short description of purpose of payment + * (ASCII), e.g. '1 cup of coffee' + */ +static void decode_d(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + size_t data_length, bool *have_d) +{ + if (*have_d) { + unknown_field(b11, hu5, data, data_len, 'd', data_length); + return; + } + + b11->description = tal_arrz(b11, char, num_u8(data_length) + 1); + pull_bits_certain(hu5, data, data_len, (u8 *)b11->description, + data_length*5, false); + *have_d = true; +} + +/* BOLT #11: + * + * `h` (23): `data_length` 52. 256-bit description of purpose of payment + * (SHA256). This is used to commit to an associated description which is too + * long to fit, such as may be contained in a web page. + */ +static void decode_h(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + size_t data_length, bool *have_h) +{ + if (*have_h) { + unknown_field(b11, hu5, data, data_len, 'h', data_length); + return; + } + + /* BOLT #11: + * + * A reader MUST skip over unknown fields, an `f` field with unknown + * `version`, or a `p`, `h`, or `n` field which does not have + * `data_length` 52, 52, or 53 respectively. */ + if (data_length != 52) { + unknown_field(b11, hu5, data, data_len, 'h', data_length); + return; + } + + b11->description_hash = tal(b11, struct sha256); + pull_bits_certain(hu5, data, data_len, b11->description_hash, 256, + false); + *have_h = true; +} + +/* BOLT #11: + * + * `x` (6): `data_length` variable. `expiry` time in seconds + * (big-endian). Default is 3600 (1 hour) if not specified. + */ +#define DEFAULT_X 3600 +static char *decode_x(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + size_t data_length, bool *have_x) +{ + if (*have_x) + return unknown_field(b11, hu5, data, data_len, 'x', + data_length); + + /* FIXME: Put upper limit in bolt 11 */ + if (!pull_uint(hu5, data, data_len, &b11->expiry, data_length * 5)) + return tal_fmt(b11, "x: length %zu chars is excessive", + *data_len); + return NULL; +} + +static char *decode_n(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + size_t data_length, bool *have_n) +{ + u8 der[PUBKEY_DER_LEN]; + + if (*have_n) + return unknown_field(b11, hu5, data, data_len, 'n', + data_length); + + /* BOLT #11: + * + * A reader MUST skip over unknown fields, an `f` field with unknown + * `version`, or a `p`, `h`, or `n` field which does not have + * `data_length` 52, 52, or 53 respectively. */ + if (data_length != 53) + return unknown_field(b11, hu5, data, data_len, 'n', + data_length); + + pull_bits_certain(hu5, data, data_len, der, data_length * 5, false); + if (!pubkey_from_der(der, sizeof(der), &b11->receiver_id)) + return tal_fmt(b11, "n: invalid pubkey %.*s", + (int)sizeof(der), der); + + *have_n = true; + return NULL; +} + +/* BOLT #11: + * + * `f` (9): `data_length` variable, depending on version. Fallback on-chain + * address: for bitcoin, this starts with a 5 bit `version`; a witness program + * or P2PKH or P2SH address. + */ +static char *decode_f(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + size_t data_length, bool *have_f) +{ + u64 version; + + if (*have_f) + return unknown_field(b11, hu5, data, data_len, 'f', + data_length); + + if (!pull_uint(hu5, data, data_len, &version, 5)) + return tal_fmt(b11, "f: data_length %zu short", data_length); + data_length--; + + /* BOLT #11: + * + * For bitcoin payments, a writer MUST set an `f` field to a + * valid witness version and program, or `17` followed by a + * public key hash, or `18` followed by a script hash. */ + if (version == 17) { + /* Pay to pubkey hash (P2PKH) */ + struct bitcoin_address pkhash; + if (num_u8(data_length) != sizeof(pkhash)) + return tal_fmt(b11, "f: pkhash length %zu", + data_length); + + pull_bits_certain(hu5, data, data_len, &pkhash, data_length*5, + false); + b11->fallback = scriptpubkey_p2pkh(b11, &pkhash); + return NULL; + } else if (version == 18) { + /* Pay to pubkey script hash (P2SH) */ + struct ripemd160 shash; + if (num_u8(data_length) != sizeof(shash)) + return tal_fmt(b11, "f: p2sh length %zu", + data_length); + + pull_bits_certain(hu5, data, data_len, &shash, data_length*5, + false); + b11->fallback = scriptpubkey_p2sh_hash(b11, &shash); + } else if (version < 17) { + u8 *f = tal_arr(b11, u8, data_length * 5 / 8); + if (version == 0) { + if (tal_len(f) != 20 && tal_len(f) != 32) + return tal_fmt(b11, + "f: witness v0 bad length %zu", + data_length); + } + pull_bits_certain(hu5, data, data_len, f, data_length * 5, + false); + b11->fallback = scriptpubkey_witness_raw(b11, version, + f, tal_len(f)); + tal_free(f); + } else + return unknown_field(b11, hu5, data, data_len, 'f', + data_length); + + *have_f = true; + return NULL; +} + +static bool fromwire_route_info(const u8 **cursor, size_t *max, + struct route_info *route_info) +{ + fromwire_pubkey(cursor, max, &route_info->pubkey); + fromwire_short_channel_id(cursor, max, &route_info->short_channel_id); + route_info->fee = fromwire_u64(cursor, max); + route_info->cltv_expiry_delta = fromwire_u16(cursor, max); + return *cursor != NULL; +} + +static void towire_route_info(u8 **pptr, const struct route_info *route_info) +{ + towire_pubkey(pptr, &route_info->pubkey); + towire_short_channel_id(pptr, &route_info->short_channel_id); + towire_u64(pptr, route_info->fee); + towire_u16(pptr, route_info->cltv_expiry_delta); +} + +/* BOLT #11: + * + * `r` (3): `data_length` variable. One or more entries containing + * extra routing information for a private route; there may be more + * than one `r` field, too. + * + * * `pubkey` (264 bits) + * * `short_channel_id` (64 bits) + * * `fee` (64 bits, big-endian) + * * `cltv_expiry_delta` (16 bits, big-endian) + */ +static char *decode_r(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + size_t data_length) +{ + tal_t *tmpctx = tal_tmpctx(b11); + size_t rlen = data_length * 5 / 8; + u8 *r8 = tal_arr(tmpctx, u8, rlen); + size_t n = 0; + struct route_info *r = tal_arr(tmpctx, struct route_info, n); + const u8 *cursor = r8; + + /* Route hops don't split in 5 bit boundaries, so convert whole thing */ + pull_bits_certain(hu5, data, data_len, r8, data_length * 5, false); + + do { + tal_resize(&r, n+1); + if (!fromwire_route_info(&cursor, &rlen, &r[n])) { + tal_free(tmpctx); + return tal_fmt(b11, "r: hop %zu truncated", n); + } + n++; + } while (rlen); + + /* Append route */ + n = tal_count(b11->routes); + tal_resize(&b11->routes, n+1); + b11->routes[n] = tal_steal(b11, r); + + tal_free(tmpctx); + return NULL; +} + +struct bolt11 *new_bolt11(const tal_t *ctx, u64 *msatoshi) +{ + struct bolt11 *b11 = tal(ctx, struct bolt11); + + list_head_init(&b11->extra_fields); + b11->description = NULL; + b11->description_hash = NULL; + b11->fallback = NULL; + b11->routes = NULL; + b11->msatoshi = NULL; + + if (msatoshi) + b11->msatoshi = tal_dup(b11, u64, msatoshi); + return b11; +} + +/* Decodes and checks signature; returns NULL on error. */ +struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str, + const char *description, char **fail) +{ + char *hrp, *amountstr, *prefix; + u5 *data; + size_t data_len; + struct bolt11 *b11 = new_bolt11(ctx, NULL); + tal_t *tmpctx = tal_tmpctx(b11); + u8 sig_and_recid[65]; + secp256k1_ecdsa_recoverable_signature sig; + struct hash_u5 hu5; + struct sha256 hash; + bool have_p = false, have_n = false, have_d = false, have_h = false, + have_x = false, have_f = false; + + b11->routes = tal_arr(b11, struct route_info *, 0); + + hrp = tal_arr(tmpctx, char, strlen(str) - 6); + data = tal_arr(tmpctx, u5, strlen(str) - 8); + + if (!bech32_decode(hrp, data, &data_len, str, (size_t)-1)) + return decode_fail(b11, fail, "Bad bech32 string"); + + /* For signature checking at the end. */ + hash_u5_init(&hu5, hrp); + + /* BOLT #11: + * + * The human readable part consists of two sections: + * 1. `prefix`: `ln` + BIP-0173 currency prefix (e.g. `lnbc`, `lntb`) + * 1. `amount`: optional number in that currency, followed by optional + * `multiplier`. + */ + prefix = tal_strndup(tmpctx, hrp, strcspn(hrp, "0123456789")); + + /* BOLT #11: + * + * A reader MUST fail if it does not understand the `prefix`. + */ + if (!strstarts(prefix, "ln")) + return decode_fail(b11, fail, + "Prefix '%s' does not start with ln", prefix); + + b11->chain = chainparams_by_bip173(prefix + 2); + if (!b11->chain) + return decode_fail(b11, fail, "Unknown chain %s", prefix + 2); + + /* BOLT #11: + * + * A reader SHOULD fail if `amount` contains a non-digit, or + * is followed by anything except a `multiplier` in the table + * above. */ + amountstr = tal_strdup(tmpctx, hrp + strlen(prefix)); + if (streq(amountstr, "")) { + /* BOLT #11: + * + * A reader SHOULD indicate if amount is unspecified + */ + b11->msatoshi = NULL; + } else { + u64 m10 = 10; + u64 amount; + char *end; + + /* Gather and trim multiplier */ + end = amountstr + strlen(amountstr)-1; + for (size_t i = 0; i < ARRAY_SIZE(multipliers); i++) { + if (*end == multipliers[i].letter) { + m10 = multipliers[i].m10; + *end = '\0'; + break; + } + } + + /* BOLT #11: + * + * A reader SHOULD fail if `amount` contains a non-digit, or + * is followed by anything except a `multiplier` in the table + * above. + */ + amount = strtoull(amountstr, &end, 10); + if (amount == ULLONG_MAX && errno == ERANGE) + return decode_fail(b11, fail, + "Invalid amount '%s'", amountstr); + if (!*amountstr || *end) + return decode_fail(b11, fail, + "Invalid amount postfix '%s'", end); + + /* Convert to millisatoshis. */ + b11->msatoshi = tal(b11, u64); + *b11->msatoshi = amount * m10 / 10; + } + + /* BOLT #11: + * + * The data part consists of multiple sections: + * + * 1. `timestamp`: seconds-since-1970 (35 bits, big-endian) + * 1. Zero or more tagged parts. + * 1. `signature`: bitcoin-style signature of above. (520 bits) + */ + if (!pull_uint(&hu5, &data, &data_len, &b11->timestamp, 35)) + return decode_fail(b11, fail, "Can't get 35-bit timestamp"); + + b11->expiry = DEFAULT_X; + + while (data_len > 520 / 5) { + const char *problem = NULL; + u64 type, data_length; + + /* BOLT #11: + * + * Each Tagged Field is of format: + * + * 1. `type` (5 bits) + * 1. `data_length` (10 bits, big-endian) + * 1. `data` (`data_length` x 5 bits) + */ + if (!pull_uint(&hu5, &data, &data_len, &type, 5) + || !pull_uint(&hu5, &data, &data_len, &data_length, 10)) + return decode_fail(b11, fail, + "Can't get tag and length"); + + /* Can't exceed total data remaining. */ + if (data_length > data_len) + return decode_fail(b11, fail, "%c: truncated", + bech32_charset[type]); + + switch (bech32_charset[type]) { + case 'p': + decode_p(b11, &hu5, &data, &data_len, data_length, + &have_p); + break; + + case 'd': + decode_d(b11, &hu5, &data, &data_len, data_length, + &have_d); + break; + + case 'h': + decode_h(b11, &hu5, &data, &data_len, data_length, + &have_h); + break; + + case 'n': + problem = decode_n(b11, &hu5, &data, + &data_len, data_length, + &have_n); + break; + + case 'x': + problem = decode_x(b11, &hu5, &data, + &data_len, data_length, + &have_x); + break; + + case 'f': + problem = decode_f(b11, &hu5, &data, + &data_len, data_length, + &have_f); + break; + case 'r': + problem = decode_r(b11, &hu5, &data, &data_len, + data_length); + break; + default: + unknown_field(b11, &hu5, &data, &data_len, + bech32_charset[type], data_length); + } + if (problem) + return decode_fail(b11, fail, "%s", problem); + } + + if (!have_p) + return decode_fail(b11, fail, "No valid 'p' field found"); + + if (have_h) { + struct sha256 sha; + + /* BOLT #11: + * + * A reader MUST check that the SHA-2 256 in the `h` field + * exactly matches the hashed description. + */ + if (!description) + return decode_fail(b11, fail, + "h: no description to check"); + sha256(&sha, description, strlen(description)); + if (!structeq(b11->description_hash, &sha)) + return decode_fail(b11, fail, + "h: does not match description"); + } + + hash_u5_done(&hu5, &hash); + + /* BOLT #11: + * + * A writer MUST set `signature` to a valid 512-bit secp256k1 + * signature of the SHA2 256-bit hash of the Human Readable Part + * concatenated with the Data Part and zero bits appended to the next + * byte boundary, with a trailing byte containing the recovery ID (0, + * 1, 2 or 3). + */ + if (!pull_bits(NULL, &data, &data_len, sig_and_recid, 520, false)) + return decode_fail(b11, fail, "signature truncated"); + + assert(data_len == 0); + + if (!secp256k1_ecdsa_recoverable_signature_parse_compact + (secp256k1_ctx, &sig, sig_and_recid, sig_and_recid[64])) + return decode_fail(b11, fail, "signature invalid"); + + secp256k1_ecdsa_recoverable_signature_convert(secp256k1_ctx, + &b11->sig, &sig); + + /* BOLT #11: + * + * A reader MUST check that the `signature` is valid (see the `n` + * tagged field specified below). + *... + * A reader MUST use the `n` field to validate the signature instead of + * performing signature recovery if a valid `n` field is provided. + */ + if (!have_n) { + if (!secp256k1_ecdsa_recover(secp256k1_ctx, + &b11->receiver_id.pubkey, + &sig, + (const u8 *)&hash)) + return decode_fail(b11, fail, + "signature recovery failed"); + } else { + if (!secp256k1_ecdsa_verify(secp256k1_ctx, &b11->sig, + (const u8 *)&hash, + &b11->receiver_id.pubkey)) + return decode_fail(b11, fail, "invalid signature"); + } + + tal_free(tmpctx); + return b11; +} + +static size_t num_u5(size_t num_u8) +{ + return (num_u8 * 8 + 7) / 5; +} + +static u8 get_bit(const u8 *src, size_t bitoff) +{ + return ((src[bitoff / 8] >> (7 - (bitoff % 8))) & 1); +} + +/* Returns now many u5s were appended. */ +static void push_bits(u5 **data, const void *src, size_t nbits) +{ + size_t i, b; + size_t data_len = tal_len(*data); + + for (i = 0; i < nbits; i += b) { + tal_resize(data, data_len+1); + (*data)[data_len] = 0; + for (b = 0; b < 5; b++) { + (*data)[data_len] <<= 1; + /* If we need bits we don't have, zero */ + if (i+b < nbits) + (*data)[data_len] |= get_bit(src, i+b); + } + data_len++; + } +} + +/* Helper for pushing a variable-length big-endian int. */ +static void push_varlen_uint(u5 **data, u64 val, size_t nbits) +{ + be64 be_val = cpu_to_be64(val << (64 - nbits)); + push_bits(data, &be_val, nbits); +} + +/* BOLT #11: + * + * Each Tagged Field is of format: + * + * 1. `type` (5 bits) + * 1. `data_length` (10 bits, big-endian) + * 1. `data` (`data_length` x 5 bits) + */ +static void push_field(u5 **data, char type, const void *src, size_t nbits) +{ + assert(bech32_charset_rev[(unsigned char)type] >= 0); + push_varlen_uint(data, bech32_charset_rev[(unsigned char)type], 5); + push_varlen_uint(data, (nbits + 4) / 5, 10); + push_bits(data, src, nbits); +} + +/* BOLT #11: + * + * SHOULD use the minimum `data_length` possible. + */ +static void push_varlen_field(u5 **data, char type, u64 val) +{ + assert(bech32_charset_rev[(unsigned char)type] >= 0); + push_varlen_uint(data, bech32_charset_rev[(unsigned char)type], 5); + + for (size_t nbits = 5; nbits < 65; nbits++) { + if ((val >> nbits) == 0) { + push_varlen_uint(data, nbits / 5, 10); + push_varlen_uint(data, val, nbits); + return; + } + } + /* Can't be encoded in <= 60 bits. */ + abort(); +} + +static void encode_p(u5 **data, const struct sha256 *hash) +{ + push_field(data, 'p', hash, 256); +} + +static void encode_d(u5 **data, const char *description) +{ + push_field(data, 'd', description, strlen(description) * CHAR_BIT); +} + +static void encode_h(u5 **data, const struct sha256 *hash) +{ + push_field(data, 'h', hash, 256); +} + +static void encode_n(u5 **data, const struct pubkey *id) +{ + u8 der[PUBKEY_DER_LEN]; + + pubkey_to_der(der, id); + push_field(data, 'n', der, sizeof(der) * CHAR_BIT); +} + +static void encode_x(u5 **data, u64 expiry) +{ + push_varlen_field(data, 'x', expiry); +} + +static void encode_f(u5 **data, const u8 *fallback) +{ + struct bitcoin_address pkh; + struct ripemd160 sh; + struct sha256 wsh; + + /* BOLT #11: + * + * For bitcoin payments, a writer MUST set an `f` field to a valid + * witness version and program, or `17` followed by a public key hash, + * or `18` followed by a script hash. + */ + if (is_p2pkh(fallback, &pkh)) { + u8 v17[1 + sizeof(pkh)]; + v17[0] = 17; + memcpy(v17+1, &pkh, sizeof(pkh)); + push_field(data, 'f', v17, sizeof(v17) * CHAR_BIT); + } else if (is_p2sh(fallback, &sh)) { + u8 v18[1 + sizeof(sh)]; + v18[0] = 18; + memcpy(v18+1, &sh, sizeof(sh)); + push_field(data, 'f', v18, sizeof(v18) * CHAR_BIT); + } else if (is_p2wpkh(fallback, &pkh)) { + u8 v0[1 + sizeof(pkh)]; + v0[0] = 0; + memcpy(v0+1, &pkh, sizeof(pkh)); + push_field(data, 'f', v0, sizeof(v0) * CHAR_BIT); + } else if (is_p2wsh(fallback, &wsh)) { + u8 v0[1 + sizeof(wsh)]; + v0[0] = 0; + memcpy(v0+1, &wsh, sizeof(wsh)); + push_field(data, 'f', v0, sizeof(v0) * CHAR_BIT); + } else if (tal_len(fallback) + && fallback[0] >= 0x50 + && fallback[0] < (0x50+16)) { + /* Other (future) witness versions: turn OP_N into N */ + u8 *f = tal_dup_arr(NULL, u8, + fallback, tal_len(fallback), 0); + f[0] -= 0x50; + push_field(data, 'f', f, tal_len(f) * CHAR_BIT); + tal_free(f); + } else { + /* Copy raw. */ + push_field(data, 'f', + fallback, tal_len(fallback) * CHAR_BIT); + } +} + +static void encode_r(u5 **data, const struct route_info *r) +{ + u8 *rinfo = tal_arr(NULL, u8, 0); + + for (size_t i = 0; i < tal_count(r); i++) + towire_route_info(&rinfo, r); + + push_field(data, 'r', rinfo, tal_len(rinfo) * CHAR_BIT); + tal_free(rinfo); +} + +static void encode_extra(u5 **data, const struct bolt11_field *extra) +{ + size_t len; + + push_varlen_uint(data, extra->tag, 5); + push_varlen_uint(data, tal_count(extra->data), 10); + + /* extra->data is already u5s, so do this raw. */ + len = tal_len(*data); + tal_resize(data, len + tal_count(extra->data)); + memcpy(*data + len, extra->data, tal_count(extra->data)); +} + +/* Encodes, even if it's nonsense. */ +char *bolt11_encode(const tal_t *ctx, + struct lightningd *ld, + const struct bolt11 *b11, bool n_field) +{ + tal_t *tmpctx = tal_tmpctx(ctx); + u5 *data = tal_arr(tmpctx, u5, 0); + char *hrp, *output; + char postfix; + u64 amount; + struct bolt11_field *extra; + secp256k1_ecdsa_recoverable_signature rsig; + u8 sig_and_recid[65]; + u8 *hrpu8, *msg; + int recid; + size_t i; + + /* BOLT #11: + * + * A writer MUST encode `amount` as a positive decimal integer + * with no leading zeroes, SHOULD use the shortest + * representation possible. + */ + if (b11->msatoshi) { + if (*b11->msatoshi % MSAT_PER_BTC == 0) { + postfix = '\0'; + amount = *b11->msatoshi / MSAT_PER_BTC; + } else { + for (i = 0; i < ARRAY_SIZE(multipliers)-1; i++) { + if (!(*b11->msatoshi * 10 % multipliers[i].m10)) + break; + } + postfix = multipliers[i].letter; + amount = *b11->msatoshi * 10 / multipliers[i].m10; + } + hrp = tal_fmt(tmpctx, "ln%s%"PRIu64"%c", + b11->chain->bip173_name, amount, postfix); + } else + hrp = tal_fmt(tmpctx, "ln%s", b11->chain->bip173_name); + + /* BOLT #11: + * + * 1. `timestamp`: seconds-since-1970 (35 bits, big-endian) + * 1. Zero or more tagged parts. + * 1. `signature`: bitcoin-style signature of above. (520 bits) + */ + push_varlen_uint(&data, b11->timestamp, 35); + + /* BOLT #11: + * + * If a writer offers more than one of any field type, it MUST + * specify the most-preferred field first, followed by + * less-preferred fields in order. + */ + /* Thus we do built-in fields, then extras last. */ + encode_p(&data, &b11->payment_hash); + + if (b11->description) + encode_d(&data, b11->description); + + if (b11->description_hash) + encode_h(&data, b11->description_hash); + + if (n_field) + encode_n(&data, &b11->receiver_id); + + if (b11->expiry != DEFAULT_X) + encode_x(&data, b11->expiry); + + if (b11->fallback) + encode_f(&data, b11->fallback); + + for (size_t i = 0; i < tal_count(b11->routes); i++) + encode_r(&data, b11->routes[i]); + + list_for_each(&b11->extra_fields, extra, list) + encode_extra(&data, extra); + + /* FIXME: towire_ should check this? */ + if (tal_len(data) > 65535) + return tal_free(tmpctx); + + /* Need exact length here */ + hrpu8 = tal_dup_arr(tmpctx, u8, (const u8 *)hrp, strlen(hrp), 0); + msg = towire_hsmctl_sign_invoice(tmpctx, data, hrpu8); + + if (!wire_sync_write(ld->hsm_fd, take(msg))) + fatal("Could not write to HSM: %s", strerror(errno)); + + msg = hsm_sync_read(tmpctx, ld); + if (!fromwire_hsmctl_sign_invoice_reply(msg, NULL, &rsig)) + fatal("HSM gave bad sign_invoice_reply %s", + tal_hex(msg, msg)); + + secp256k1_ecdsa_recoverable_signature_serialize_compact( + secp256k1_ctx, + sig_and_recid, + &recid, + &rsig); + sig_and_recid[64] = recid; + + push_bits(&data, sig_and_recid, sizeof(sig_and_recid) * CHAR_BIT); + + output = tal_arr(ctx, char, strlen(hrp) + tal_count(data) + 8); + if (!bech32_encode(output, hrp, data, tal_count(data), (size_t)-1)) + output = tal_free(output); + + tal_free(tmpctx); + return output; +} + +static PRINTF_FMT(2,3) void *bad(const char *abortstr, const char *fmt, ...) +{ + if (abortstr) { + va_list ap; + + va_start(ap, fmt); + fprintf(stderr, "%s: ", abortstr); + vfprintf(stderr, fmt, ap); + fprintf(stderr, "\n"); + abort(); + } + return NULL; +} + +struct bolt11 *bolt11_out_check(const struct bolt11 *b11, const char *abortstr) +{ + struct bolt11_field *extra; + + /* BOLT #11: + * + * For channels with `chain_hash` identifying the Bitcoin blockchain, + * the sending node MUST set the 4 most significant bytes of + * `amount_msat` to zero. + */ + if (*b11->msatoshi >= 1ULL << 32) + return bad(abortstr, "msatoshi %"PRIu64" too large", + *b11->msatoshi); + + if (!b11->description && !b11->description_hash) + return bad(abortstr, "No description or description_hash"); + + if (b11->description && b11->description_hash) + return bad(abortstr, "Both description or description_hash"); + + if (b11->description && num_u5(strlen(b11->description)) > 1024) + return bad(abortstr, "Description too long"); + + /* FIXME: Check fallback is known type. */ + if (b11->fallback && tal_count(b11->fallback) == 0) + return bad(abortstr, "Empty fallback"); + + if (!list_check(&b11->extra_fields, abortstr)) + return bad(abortstr, "Invalid extras list"); + + list_for_each(&b11->extra_fields, extra, list) { + if (bech32_charset_rev[(unsigned char)extra->tag] < 0) + return bad(abortstr, "Invalid extra type %c (0x%02x)", + extra->tag, extra->tag); + + if (tal_len(extra->data) >= 1024) + return bad(abortstr, "Extra %c too long", extra->tag); + } + return cast_const(struct bolt11 *, b11); +} diff --git a/lightningd/bolt11.h b/lightningd/bolt11.h new file mode 100644 index 000000000000..af02962242a2 --- /dev/null +++ b/lightningd/bolt11.h @@ -0,0 +1,93 @@ +#ifndef LIGHTNING_LIGHTNINGD_BOLT11_H +#define LIGHTNING_LIGHTNINGD_BOLT11_H +#include "config.h" + +#include +#include +#include +#include +#include + +struct lightningd; + +/* We only have 10 bits for the field length, meaning < 640 bytes */ +#define BOLT11_FIELD_BYTE_LIMIT ((1 << 10) * 5 / 8) + +struct bolt11_field { + struct list_node list; + + char tag; + u5 *data; +}; + +/* BOLT #11: + * * `pubkey` (264 bits) + * * `short_channel_id` (64 bits) + * * `fee` (64 bits, big-endian) + * * `cltv_expiry_delta` (16 bits, big-endian) + */ + +struct route_info { + struct pubkey pubkey; + struct short_channel_id short_channel_id; + u64 fee; + u16 cltv_expiry_delta; +}; + +struct bolt11 { + const struct chainparams *chain; + u64 timestamp; + u64 *msatoshi; /* NULL if not specified. */ + + struct sha256 payment_hash; + struct pubkey receiver_id; + + /* description_hash valid iff description is NULL. */ + const char *description; + struct sha256 *description_hash; + + /* How many seconds to pay from @timestamp above. */ + u64 expiry; + + /* If non-NULL, indicates a fallback address to pay to. */ + const u8 *fallback; + + /* If non-NULL: array of route arrays */ + struct route_info **routes; + + /* signature of sha256 of entire thing. */ + secp256k1_ecdsa_signature sig; + + struct list_head extra_fields; +}; + +/* Decodes and checks signature; returns NULL on error; description is + * (optional) out-of-band description of payment, for `h` field. */ +struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str, + const char *description, char **fail); + +/* Initialize an empty bolt11 struct with optional amount */ +struct bolt11 *new_bolt11(const tal_t *ctx, u64 *msatoshi); + +/* Encodes and signs, even if it's nonsense. */ +char *bolt11_encode(const tal_t *ctx, + struct lightningd *ld, + const struct bolt11 *b11, bool n_field); + +/** + * bolt11_out_check - check a bolt11 struct for validity and consistency + * @bolt11: the bolt11 + * @abortstr: the location to print on aborting, or NULL. + * + * Note this does not apply to bolt11's we decoded, which may not be spec + * compliant. + * + * If @abortstr is non-NULL, that will be printed in a diagnostic if the bolt11 + * is invalid, and the function will abort. + * + * Returns @bolt11 if all OK, NULL if not (it can never return NULL if + * @abortstr is set). + */ +struct bolt11 *bolt11_out_check(const struct bolt11 *bolt11, + const char *abortstr); +#endif /* LIGHTNING_LIGHTNINGD_BOLT11_H */ From 8d09734536e2cf077daa7f7b1c774ceb6d6af72a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:35:19 +1030 Subject: [PATCH 0067/1428] bolt11: add decodepay RPC helper. Signed-off-by: Rusty Russell --- lightningd/bolt11.c | 150 ++++++++++++++++++++++++++ tests/test_lightningd.py | 224 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 374 insertions(+) diff --git a/lightningd/bolt11.c b/lightningd/bolt11.c index fed18ded0632..d7933f311deb 100644 --- a/lightningd/bolt11.c +++ b/lightningd/bolt11.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -13,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -1000,3 +1002,151 @@ struct bolt11 *bolt11_out_check(const struct bolt11 *b11, const char *abortstr) } return cast_const(struct bolt11 *, b11); } + +static void json_decodepay(struct command *cmd, + const char *buffer, const jsmntok_t *params) +{ + jsmntok_t *bolt11tok, *desctok; + struct bolt11 *b11; + struct json_result *response; + char *str, *desc, *fail; + + if (!json_get_params(buffer, params, + "bolt11", &bolt11tok, + "?description", &desctok, + NULL)) { + command_fail(cmd, "Need bolt11 string"); + return; + } + + str = tal_strndup(cmd, buffer + bolt11tok->start, + bolt11tok->end - bolt11tok->start); + + if (desctok) + desc = tal_strndup(cmd, buffer + desctok->start, + desctok->end - desctok->start); + else + desc = NULL; + + b11 = bolt11_decode(cmd, str, desc, &fail); + + if (!b11) { + command_fail(cmd, "Invalid bolt11: %s", fail); + return; + } + + response = new_json_result(cmd); + json_object_start(response, NULL); + + json_add_string(response, "currency", b11->chain->bip173_name); + json_add_u64(response, "timestamp", b11->timestamp); + json_add_u64(response, "expiry", b11->expiry); + json_add_pubkey(response, "payee", &b11->receiver_id); + if (b11->msatoshi) + json_add_u64(response, "msatoshi", *b11->msatoshi); + if (b11->description) + json_add_string(response, "description", b11->description); + if (b11->description_hash) + json_add_hex(response, "description_hash", + b11->description_hash, + sizeof(*b11->description_hash)); + if (tal_len(b11->fallback)) { + struct bitcoin_address pkh; + struct ripemd160 sh; + struct sha256 wsh; + + json_object_start(response, "fallback"); + if (is_p2pkh(b11->fallback, &pkh)) { + json_add_string(response, "type", "P2PKH"); + json_add_string(response, "addr", + bitcoin_to_base58(cmd, + b11->chain->testnet, + &pkh)); + } else if (is_p2sh(b11->fallback, &sh)) { + json_add_string(response, "type", "P2SH"); + json_add_string(response, "addr", + p2sh_to_base58(cmd, + b11->chain->testnet, + &sh)); + } else if (is_p2wpkh(b11->fallback, &pkh)) { + char out[73 + strlen(b11->chain->bip173_name)]; + json_add_string(response, "type", "P2WPKH"); + if (segwit_addr_encode(out, b11->chain->bip173_name, 0, + (const u8 *)&pkh, sizeof(pkh))) + json_add_string(response, "addr", out); + } else if (is_p2wsh(b11->fallback, &wsh)) { + char out[73 + strlen(b11->chain->bip173_name)]; + json_add_string(response, "type", "P2WSH"); + if (segwit_addr_encode(out, b11->chain->bip173_name, 0, + (const u8 *)&wsh, sizeof(wsh))) + json_add_string(response, "addr", out); + } + json_add_hex(response, "hex", + b11->fallback, tal_len(b11->fallback)); + json_object_end(response); + } + + if (tal_count(b11->routes)) { + size_t i, n; + + json_array_start(response, "routes"); + for (i = 0; i < tal_count(b11->routes); i++) { + json_array_start(response, NULL); + for (n = 0; n < tal_count(b11->routes[i]); n++) { + json_object_start(response, NULL); + json_add_pubkey(response, "pubkey", + &b11->routes[i][n].pubkey); + json_add_short_channel_id(response, + "short_channel_id", + &b11->routes[i][n] + .short_channel_id); + json_add_u64(response, "fee", + b11->routes[i][n].fee); + json_add_num(response, "cltv_expiry_delta", + b11->routes[i][n] + .cltv_expiry_delta); + json_object_end(response); + } + json_array_end(response); + } + json_array_end(response); + } + + if (!list_empty(&b11->extra_fields)) { + struct bolt11_field *extra; + + json_array_start(response, "extra"); + list_for_each(&b11->extra_fields, extra, list) { + char *data = tal_arr(cmd, char, tal_len(extra->data)+1); + size_t i; + + for (i = 0; i < tal_len(extra->data); i++) + data[i] = bech32_charset[extra->data[i]]; + data[i] = '\0'; + json_object_start(response, NULL); + json_add_string(response, "tag", + tal_fmt(data, "%c", extra->tag)); + json_add_string(response, "data", data); + tal_free(data); + json_object_end(response); + } + json_array_end(response); + } + + json_add_hex(response, "payment_hash", + &b11->payment_hash, sizeof(b11->payment_hash)); + + json_add_string(response, "signature", + type_to_string(cmd, secp256k1_ecdsa_signature, + &b11->sig)); + json_object_end(response); + command_success(cmd, response); +} + +static const struct json_command decodepay_command = { + "decodepay", + json_decodepay, + "Parse and decode {bolt11} if possible", + "Returns a verbose description on success" +}; +AUTODATA(json_command, &decodepay_command); diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 00fefd16e38b..1ee5f785ba3a 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -304,6 +304,230 @@ def test_balance(self): assert p2['msatoshi_to_us'] == 0 assert p2['msatoshi_total'] == 10**6 * 1000 + def test_decodepay(self): + l1 = self.node_factory.get_node() + + # BOLT #11: + # > ### Please make a donation of any amount using payment_hash 0001020304050607080900010203040506070809000102030405060708090102 to me @03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad + # > lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdpl2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ecky03ylcqca784w + # + # Breakdown: + # + # * `lnbc`: prefix, lightning on bitcoin mainnet + # * `1`: Bech32 separator + # * `pvjluez`: timestamp (1496314658) + # * `p`: payment hash + # * `p5`: `data_length` (`p` = 1, `5` = 20. 1 * 32 + 20 == 52) + # * `qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypq`: payment hash 0001020304050607080900010203040506070809000102030405060708090102 + # * `d`: short description + # * `pl`: `data_length` (`p` = 1, `l` = 31. 1 * 32 + 31 == 63) + # * `2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq`: 'Please consider supporting this project' + # * `32vjcgqxyuj7nqphl3xmmhls2rkl3t97uan4j0xa87gj5779czc8p0z58zf5wpt9ggem6adl64cvawcxlef9djqwp2jzzfvs272504sp`: signature + # * `0lkg3c`: Bech32 checksum + b11 = l1.rpc.decodepay('lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdpl2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ecky03ylcqca784w') + assert b11['currency'] == 'bc' + assert b11['timestamp'] == 1496314658 + assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102' + assert b11['description'] == 'Please consider supporting this project' + assert b11['expiry'] == 3600 + assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' + + # BOLT #11: + # > ### Please send $3 for a cup of coffee to the same peer, within 1 minute + # > lnbc2500u1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5xysxxatsyp3k7enxv4jsxqzpuaztrnwngzn3kdzw5hydlzf03qdgm2hdq27cqv3agm2awhz5se903vruatfhq77w3ls4evs3ch9zw97j25emudupq63nyw24cg27h2rspfj9srp + # + # Breakdown: + # + # * `lnbc`: prefix, lightning on bitcoin mainnet + # * `2500u`: amount (2500 micro-bitcoin) + # * `1`: Bech32 separator + # * `pvjluez`: timestamp (1496314658) + # * `p`: payment hash... + # * `d`: short description + # * `q5`: `data_length` (`q` = 0, `5` = 20. 0 * 32 + 20 == 20) + # * `xysxxatsyp3k7enxv4js`: '1 cup coffee' + # * `x`: expiry time + # * `qz`: `data_length` (`q` = 0, `z` = 2. 0 * 32 + 2 == 2) + # * `pu`: 60 seconds (`p` = 1, `u` = 28. 1 * 32 + 28 == 60) + # * `azh8qt5w7qeewkmxtv55khqxvdfs9zzradsvj7rcej9knpzdwjykcq8gv4v2dl705pjadhpsc967zhzdpuwn5qzjm0s4hqm2u0vuhhqq`: signature + # * `7vc09u`: Bech32 checksum + b11 = l1.rpc.decodepay('lnbc2500u1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5xysxxatsyp3k7enxv4jsxqzpuaztrnwngzn3kdzw5hydlzf03qdgm2hdq27cqv3agm2awhz5se903vruatfhq77w3ls4evs3ch9zw97j25emudupq63nyw24cg27h2rspfj9srp') + assert b11['currency'] == 'bc' + assert b11['msatoshi'] == 2500 * 10**11 // 1000000 + assert b11['timestamp'] == 1496314658 + assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102' + assert b11['description'] == '1 cup coffee' + assert b11['expiry'] == 60 + assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' + + # BOLT #11: + # > ### Now send $24 for an entire list of things (hashed) + # > lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jrc5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7khhr8wpald05e92xw006sq94mg8v2ndf4sefvf9sygkshp5zfem29trqq2yxxz7 + # + # Breakdown: + # + # * `lnbc`: prefix, lightning on bitcoin mainnet + # * `20m`: amount (20 milli-bitcoin) + # * `1`: Bech32 separator + # * `pvjluez`: timestamp (1496314658) + # * `p`: payment hash... + # * `h`: tagged field: hash of description + # * `p5`: `data_length` (`p` = 1, `5` = 20. 1 * 32 + 20 == 52) + # * `8yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqs`: SHA256 of 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon' + # * `vjfls3ljx9e93jkw0kw40yxn4pevgzflf83qh2852esjddv4xk4z70nehrdcxa4fk0t6hlcc6vrxywke6njenk7yzkzw0quqcwxphkcp`: signature + # * `vam37w`: Bech32 checksum + b11 = l1.rpc.decodepay('lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jrc5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7khhr8wpald05e92xw006sq94mg8v2ndf4sefvf9sygkshp5zfem29trqq2yxxz7', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon') + assert b11['currency'] == 'bc' + assert b11['msatoshi'] == 20 * 10**11 // 1000 + assert b11['timestamp'] == 1496314658 + assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102' + assert b11['expiry'] == 3600 + assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' + + # > ### The same, on testnet, with a fallback address mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP + # > lntb20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfpp3x9et2e20v6pu37c5d9vax37wxq72un98kmzzhznpurw9sgl2v0nklu2g4d0keph5t7tj9tcqd8rexnd07ux4uv2cjvcqwaxgj7v4uwn5wmypjd5n69z2xm3xgksg28nwht7f6zspwp3f9t + # + # Breakdown: + # + # * `lntb`: prefix, lightning on bitcoin testnet + # * `20m`: amount (20 milli-bitcoin) + # * `1`: Bech32 separator + # * `pvjluez`: timestamp (1496314658) + # * `p`: payment hash... + # * `f`: tagged field: fallback address + # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33) + # * `3x9et2e20v6pu37c5d9vax37wxq72un98`: `3` = 17, so P2PKH address + # * `h`: tagged field: hash of description... + # * `qh84fmvn2klvglsjxfy0vq2mz6t9kjfzlxfwgljj35w2kwa60qv49k7jlsgx43yhs9nuutllkhhnt090mmenuhp8ue33pv4klmrzlcqp`: signature + # * `us2s2r`: Bech32 checksum + b11 = l1.rpc.decodepay('lntb20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfpp3x9et2e20v6pu37c5d9vax37wxq72un98kmzzhznpurw9sgl2v0nklu2g4d0keph5t7tj9tcqd8rexnd07ux4uv2cjvcqwaxgj7v4uwn5wmypjd5n69z2xm3xgksg28nwht7f6zspwp3f9t', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon') + assert b11['currency'] == 'tb' + assert b11['msatoshi'] == 20 * 10**11 // 1000 + assert b11['timestamp'] == 1496314658 + assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102' + assert b11['expiry'] == 3600 + assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' + assert b11['fallback']['type'] == 'P2PKH' + assert b11['fallback']['addr'] == 'mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP' + + # > ### On mainnet, with fallback address 1RustyRX2oai4EYYDpQGWvEL62BBGqN9T with extra routing info to go via nodes 029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255 then 039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255 + # > lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqsfpp3qjmp7lwpagxun9pygexvgpjdc4jdj85fr9yq20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqqqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqqqqqqq7qqzqfnlkwydm8rg30gjku7wmxmk06sevjp53fmvrcfegvwy7d5443jvyhxsel0hulkstws7vqv400q4j3wgpk4crg49682hr4scqvmad43cqd5m7tf + # + # Breakdown: + # + # * `lnbc`: prefix, lightning on bitcoin mainnet + # * `20m`: amount (20 milli-bitcoin) + # * `1`: Bech32 separator + # * `pvjluez`: timestamp (1496314658) + # * `p`: payment hash... + # * `h`: tagged field: hash of description... + # * `f`: tagged field: fallback address + # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33) + # * `3qjmp7lwpagxun9pygexvgpjdc4jdj85f`: `3` = 17, so P2PKH address + # * `r`: tagged field: route information + # * `9y`: `data_length` (`9` = 5, `y` = 4. 5 * 32 + 4 = 164) + # `q20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqqqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqqqqqqq7qqzq`: pubkey `029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255`, `short_channel_id` 0102030405060708, `fee` 20 millisatoshi, `cltv_expiry_delta` 3. pubkey `039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255`, `short_channel_id` 030405060708090a, `fee` 30 millisatoshi, `cltv_expiry_delta` 4. + # * `fnlkwydm8rg30gjku7wmxmk06sevjp53fmvrcfegvwy7d5443jvyhxsel0hulkstws7vqv400q4j3wgpk4crg49682hr4scqvmad43cq`: signature + # * `d5m7tf`: Bech32 checksum + b11 = l1.rpc.decodepay('lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqsfpp3qjmp7lwpagxun9pygexvgpjdc4jdj85fr9yq20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqqqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqqqqqqq7qqzqfnlkwydm8rg30gjku7wmxmk06sevjp53fmvrcfegvwy7d5443jvyhxsel0hulkstws7vqv400q4j3wgpk4crg49682hr4scqvmad43cqd5m7tf', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon') + assert b11['currency'] == 'bc' + assert b11['msatoshi'] == 20 * 10**11 // 1000 + assert b11['timestamp'] == 1496314658 + assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102' + assert b11['expiry'] == 3600 + assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' + assert b11['fallback']['type'] == 'P2PKH' + assert b11['fallback']['addr'] == '1RustyRX2oai4EYYDpQGWvEL62BBGqN9T' + assert len(b11['routes']) == 1 + assert len(b11['routes'][0]) == 2 + assert b11['routes'][0][0]['pubkey'] == '029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255' + # 0x010203:0x040506:0x0708 + assert b11['routes'][0][0]['short_channel_id'] == '66051:263430:1800' + assert b11['routes'][0][0]['fee'] == 20 + assert b11['routes'][0][0]['cltv_expiry_delta'] == 3 + + assert b11['routes'][0][1]['pubkey'] == '039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255' + # 0x030405:0x060708:0x090a + assert b11['routes'][0][1]['short_channel_id'] == '197637:395016:2314' + assert b11['routes'][0][1]['fee'] == 30 + assert b11['routes'][0][1]['cltv_expiry_delta'] == 4 + + # > ### On mainnet, with fallback (P2SH) address 3EktnHQD7RiAE6uzMj2ZifT9YgRrkSgzQX + # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppj3a24vwu6r8ejrss3axul8rxldph2q7z9kmrgvr7xlaqm47apw3d48zm203kzcq357a4ls9al2ea73r8jcceyjtya6fu5wzzpe50zrge6ulk4nvjcpxlekvmxl6qcs9j3tz0469gq5g658y + # + # Breakdown: + # + # * `lnbc`: prefix, lightning on bitcoin mainnet + # * `20m`: amount (20 milli-bitcoin) + # * `1`: Bech32 separator + # * `pvjluez`: timestamp (1496314658) + # * `p`: payment hash... + # * `f`: tagged field: fallback address. + # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33) + # * `j3a24vwu6r8ejrss3axul8rxldph2q7z9`: `j` = 18, so P2SH address + # * `h`: tagged field: hash of description... + # * `2jhz8j78lv2jynuzmz6g8ve53he7pheeype33zlja5azae957585uu7x59w0f2l3rugyva6zpu394y4rh093j6wxze0ldsvk757a9msq`: signature + # * `mf9swh`: Bech32 checksum + b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppj3a24vwu6r8ejrss3axul8rxldph2q7z9kmrgvr7xlaqm47apw3d48zm203kzcq357a4ls9al2ea73r8jcceyjtya6fu5wzzpe50zrge6ulk4nvjcpxlekvmxl6qcs9j3tz0469gq5g658y', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon') + assert b11['currency'] == 'bc' + assert b11['msatoshi'] == 20 * 10**11 // 1000 + assert b11['timestamp'] == 1496314658 + assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102' + assert b11['expiry'] == 3600 + assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' + assert b11['fallback']['type'] == 'P2SH' + assert b11['fallback']['addr'] == '3EktnHQD7RiAE6uzMj2ZifT9YgRrkSgzQX' + + # > ### On mainnet, with fallback (P2WPKH) address bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4 + # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kepvrhrm9s57hejg0p662ur5j5cr03890fa7k2pypgttmh4897d3raaq85a293e9jpuqwl0rnfuwzam7yr8e690nd2ypcq9hlkdwdvycqa0qza8 + # + # * `lnbc`: prefix, lightning on bitcoin mainnet + # * `20m`: amount (20 milli-bitcoin) + # * `1`: Bech32 separator + # * `pvjluez`: timestamp (1496314658) + # * `p`: payment hash... + # * `f`: tagged field: fallback address. + # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33) + # * `q`: 0, so witness version 0. + # * `qw508d6qejxtdg4y5r3zarvary0c5xw7k`: 160 bits = P2WPKH. + # * `h`: tagged field: hash of description... + # * `gw6tk8z0p0qdy9ulggx65lvfsg3nxxhqjxuf2fvmkhl9f4jc74gy44d5ua9us509prqz3e7vjxrftn3jnk7nrglvahxf7arye5llphgq`: signature + # * `qdtpa4`: Bech32 checksum + b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kepvrhrm9s57hejg0p662ur5j5cr03890fa7k2pypgttmh4897d3raaq85a293e9jpuqwl0rnfuwzam7yr8e690nd2ypcq9hlkdwdvycqa0qza8', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon') + assert b11['currency'] == 'bc' + assert b11['msatoshi'] == 20 * 10**11 // 1000 + assert b11['timestamp'] == 1496314658 + assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102' + assert b11['expiry'] == 3600 + assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' + assert b11['fallback']['type'] == 'P2WPKH' + assert b11['fallback']['addr'] == 'bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4' + + # > ### On mainnet, with fallback (P2WSH) address bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3 + # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfp4qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q28j0v3rwgy9pvjnd48ee2pl8xrpxysd5g44td63g6xcjcu003j3qe8878hluqlvl3km8rm92f5stamd3jw763n3hck0ct7p8wwj463cql26ava + # + # * `lnbc`: prefix, lightning on bitcoin mainnet + # * `20m`: amount (20 milli-bitcoin) + # * `1`: Bech32 separator + # * `pvjluez`: timestamp (1496314658) + # * `p`: payment hash... + # * `f`: tagged field: fallback address. + # * `p4`: `data_length` (`p` = 1, `4` = 21. 1 * 32 + 21 == 53) + # * `q`: 0, so witness version 0. + # * `rp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q`: 260 bits = P2WSH. + # * `h`: tagged field: hash of description... + # * `5yps56lmsvgcrf476flet6js02m93kgasews8q3jhtp7d6cqckmh70650maq4u65tk53ypszy77v9ng9h2z3q3eqhtc3ewgmmv2grasp`: signature + # * `akvd7y`: Bech32 checksum + b11 = l1.rpc.decodepay('lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfp4qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q28j0v3rwgy9pvjnd48ee2pl8xrpxysd5g44td63g6xcjcu003j3qe8878hluqlvl3km8rm92f5stamd3jw763n3hck0ct7p8wwj463cql26ava', 'One piece of chocolate cake, one icecream cone, one pickle, one slice of swiss cheese, one slice of salami, one lollypop, one piece of cherry pie, one sausage, one cupcake, and one slice of watermelon') + assert b11['currency'] == 'bc' + assert b11['msatoshi'] == 20 * 10**11 // 1000 + assert b11['timestamp'] == 1496314658 + assert b11['payment_hash'] == '0001020304050607080900010203040506070809000102030405060708090102' + assert b11['expiry'] == 3600 + assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' + assert b11['fallback']['type'] == 'P2WSH' + assert b11['fallback']['addr'] == 'bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3' + def test_sendpay(self): l1,l2 = self.connect() From d57776935052ee0f9220f8e5b6b26655519d8421 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:36:19 +1030 Subject: [PATCH 0068/1428] invoice: provide bolt11 invoice. Signed-off-by: Rusty Russell --- lightningd/invoice.c | 34 +++++++++++++++++++++++++++++---- tests/test_lightningd.py | 41 +++++++++++++++++++++++++++------------- 2 files changed, 58 insertions(+), 17 deletions(-) diff --git a/lightningd/invoice.c b/lightningd/invoice.c index 19402db13874..5aa4029fc207 100644 --- a/lightningd/invoice.c +++ b/lightningd/invoice.c @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -109,16 +110,19 @@ static void json_invoice(struct command *cmd, const char *buffer, const jsmntok_t *params) { struct invoice *invoice; - jsmntok_t *msatoshi, *r, *label; + jsmntok_t *msatoshi, *r, *label, *desc; struct json_result *response = new_json_result(cmd); struct invoices *invs = cmd->ld->invoices; + struct bolt11 *b11; + char *b11enc; if (!json_get_params(buffer, params, "amount", &msatoshi, "label", &label, + "description", &desc, "?r", &r, NULL)) { - command_fail(cmd, "Need {amount} and {label}"); + command_fail(cmd, "Need {amount}, {label} and {description}"); return; } @@ -171,6 +175,25 @@ static void json_invoice(struct command *cmd, return; } + /* Construct bolt11 string. */ + b11 = new_bolt11(cmd, &invoice->msatoshi); + b11->chain = get_chainparams(cmd->ld); + b11->timestamp = time_now().ts.tv_sec; + b11->payment_hash = invoice->rhash; + b11->receiver_id = cmd->ld->id; + if (desc->end - desc->start >= BOLT11_FIELD_BYTE_LIMIT) { + b11->description_hash = tal(b11, struct sha256); + sha256(b11->description_hash, buffer + desc->start, + desc->end - desc->start); + } else + b11->description = tal_strndup(b11, buffer + desc->start, + desc->end - desc->start); + /* FIXME: add option to set this */ + b11->expiry = 3600; + + /* FIXME: add private routes if necessary! */ + b11enc = bolt11_encode(cmd, cmd->ld, b11, false); + /* OK, connect it to main state, respond with hash */ tal_steal(invs, invoice); list_add(&invs->invlist, &invoice->list); @@ -178,6 +201,9 @@ static void json_invoice(struct command *cmd, json_object_start(response, NULL); json_add_hex(response, "rhash", &invoice->rhash, sizeof(invoice->rhash)); + json_add_string(response, "bolt11", b11enc); + if (b11->description_hash) + json_add_string(response, "description", b11->description); json_object_end(response); command_success(cmd, response); @@ -186,8 +212,8 @@ static void json_invoice(struct command *cmd, static const struct json_command invoice_command = { "invoice", json_invoice, - "Create invoice for {msatoshi} with {label} (with a set {r}, otherwise generate one)", - "Returns the {rhash} on success. " + "Create invoice for {msatoshi} with {label} and {description} (with a set {r}, otherwise generate one)", + "Returns the {rhash} and {bolt11} on success, and {description} if too alrge for {bolt11}. " }; AUTODATA(json_command, &invoice_command); diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 1ee5f785ba3a..4f753374fabe 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -253,7 +253,7 @@ def pay(self, lsrc, ldst, amt, label=None, async=False): if not label: label = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(20)) - rhash = ldst.rpc.invoice(amt, label)['rhash'] + rhash = ldst.rpc.invoice(amt, label, label)['rhash'] assert ldst.rpc.listinvoice(label)[0]['complete'] == False routestep = { @@ -281,6 +281,21 @@ def test_shutdown(self): l1 = self.node_factory.get_node() l1.rpc.stop() + def test_invoice(self): + l1 = self.node_factory.get_node() + + before = int(time.time()) + inv = l1.rpc.invoice(123000, 'label', 'description') + after = int(time.time()) + b11 = l1.rpc.decodepay(inv['bolt11']) + assert b11['currency'] == 'tb' + assert b11['timestamp'] >= before + assert b11['timestamp'] <= after + assert b11['payment_hash'] == inv['rhash'] + assert b11['description'] == 'description' + assert b11['expiry'] == 3600 + assert b11['payee'] == l1.info['id'] + def test_connect(self): l1,l2 = self.connect() @@ -534,7 +549,7 @@ def test_sendpay(self): self.fund_channel(l1, l2, 10**6) amt = 200000000 - rhash = l2.rpc.invoice(amt, 'testpayment2')['rhash'] + rhash = l2.rpc.invoice(amt, 'testpayment2', 'desc')['rhash'] assert l2.rpc.listinvoice('testpayment2')[0]['complete'] == False routestep = { @@ -596,7 +611,7 @@ def test_sendpay(self): assert l2.rpc.listinvoice('testpayment2')[0]['complete'] == True # Overpaying by "only" a factor of 2 succeeds. - rhash = l2.rpc.invoice(amt, 'testpayment3')['rhash'] + rhash = l2.rpc.invoice(amt, 'testpayment3', 'desc')['rhash'] assert l2.rpc.listinvoice('testpayment3')[0]['complete'] == False routestep = { 'msatoshi' : amt * 2, 'id' : l2.info['id'], 'delay' : 5, 'channel': '1:1:1'} l1.rpc.sendpay(to_json([routestep]), rhash) @@ -770,7 +785,7 @@ def test_onchain_dust_out(self): self.fund_channel(l1, l2, 10**6) # Must be dust! - rhash = l2.rpc.invoice(1, 'onchain_dust_out')['rhash'] + rhash = l2.rpc.invoice(1, 'onchain_dust_out', 'desc')['rhash'] routestep = { 'msatoshi' : 1, 'id' : l2.info['id'], @@ -822,7 +837,7 @@ def test_onchain_timeout(self): l1.rpc.connect(l2.info['id'], 'localhost:{}'.format(l2.info['port'])) self.fund_channel(l1, l2, 10**6) - rhash = l2.rpc.invoice(10**8, 'onchain_timeout')['rhash'] + rhash = l2.rpc.invoice(10**8, 'onchain_timeout', 'desc')['rhash'] # We underpay, so it fails. routestep = { 'msatoshi' : 10**8 - 1, @@ -885,7 +900,7 @@ def test_onchain_middleman(self): self.pay(l2, l1, 2 * 10**8) # Must be bigger than dust! - rhash = l3.rpc.invoice(10**8, 'middleman')['rhash'] + rhash = l3.rpc.invoice(10**8, 'middleman', 'desc')['rhash'] # Wait for route propagation. l1.bitcoin.rpc.generate(5) l1.daemon.wait_for_log('Received node_announcement for node {}' @@ -1348,7 +1363,7 @@ def test_forward(self): assert l2.rpc.getpeer(l1.info['id'])['channel'] == chanid1 assert l3.rpc.getpeer(l2.info['id'])['channel'] == chanid2 - rhash = l3.rpc.invoice(100000000, 'testpayment1')['rhash'] + rhash = l3.rpc.invoice(100000000, 'testpayment1', 'desc')['rhash'] assert l3.rpc.listinvoice('testpayment1')[0]['complete'] == False # Fee for node2 is 10 millionths, plus 1. @@ -1496,7 +1511,7 @@ def test_forward_different_fees_and_cltv(self): assert route[1]['msatoshi'] == 4999999 assert route[1]['delay'] == 9 + shadow_route - rhash = l3.rpc.invoice(4999999, 'test_forward_different_fees_and_cltv')['rhash'] + rhash = l3.rpc.invoice(4999999, 'test_forward_different_fees_and_cltv', 'desc')['rhash'] assert l3.rpc.listinvoice('test_forward_different_fees_and_cltv')[0]['complete'] == False # This should work. @@ -1560,7 +1575,7 @@ def test_forward_pad_fees_and_cltv(self): route[1]['delay'] += 10 # This should work. - rhash = l3.rpc.invoice(4999999, 'test_forward_pad_fees_and_cltv')['rhash'] + rhash = l3.rpc.invoice(4999999, 'test_forward_pad_fees_and_cltv', 'desc')['rhash'] l1.rpc.sendpay(to_json(route), rhash) assert l3.rpc.listinvoice('test_forward_pad_fees_and_cltv')[0]['complete'] == True @@ -1726,7 +1741,7 @@ def test_reconnect_sender_add1(self): self.fund_channel(l1, l2, 10**6) amt = 200000000 - rhash = l2.rpc.invoice(amt, 'test_reconnect_sender_add1')['rhash'] + rhash = l2.rpc.invoice(amt, 'test_reconnect_sender_add1', 'desc')['rhash'] assert l2.rpc.listinvoice('test_reconnect_sender_add1')[0]['complete'] == False route = [ { 'msatoshi' : amt, 'id' : l2.info['id'], 'delay' : 5, 'channel': '1:1:1'} ] @@ -1754,7 +1769,7 @@ def test_reconnect_sender_add(self): self.fund_channel(l1, l2, 10**6) amt = 200000000 - rhash = l2.rpc.invoice(amt, 'testpayment')['rhash'] + rhash = l2.rpc.invoice(amt, 'testpayment', 'desc')['rhash'] assert l2.rpc.listinvoice('testpayment')[0]['complete'] == False route = [ { 'msatoshi' : amt, 'id' : l2.info['id'], 'delay' : 5, 'channel': '1:1:1'} ] @@ -1780,7 +1795,7 @@ def test_reconnect_receiver_add(self): self.fund_channel(l1, l2, 10**6) amt = 200000000 - rhash = l2.rpc.invoice(amt, 'testpayment2')['rhash'] + rhash = l2.rpc.invoice(amt, 'testpayment2', 'desc')['rhash'] assert l2.rpc.listinvoice('testpayment2')[0]['complete'] == False route = [ { 'msatoshi' : amt, 'id' : l2.info['id'], 'delay' : 5, 'channel': '1:1:1'} ] @@ -1810,7 +1825,7 @@ def test_reconnect_receiver_fulfill(self): self.fund_channel(l1, l2, 10**6) amt = 200000000 - rhash = l2.rpc.invoice(amt, 'testpayment2')['rhash'] + rhash = l2.rpc.invoice(amt, 'testpayment2', 'desc')['rhash'] assert l2.rpc.listinvoice('testpayment2')[0]['complete'] == False route = [ { 'msatoshi' : amt, 'id' : l2.info['id'], 'delay' : 5, 'channel': '1:1:1'} ] From e1dc75e954e5e5eb697b4afbdfac1bf3564e6e12 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:37:19 +1030 Subject: [PATCH 0069/1428] pay: new all-in-one RPC command. Closes: #240 Signed-off-by: Rusty Russell --- lightningd/pay.c | 107 +++++++++++++++++++++++++++++++++++++++ tests/test_lightningd.py | 16 ++++++ 2 files changed, 123 insertions(+) diff --git a/lightningd/pay.c b/lightningd/pay.c index 38fa52cae603..6ec0bec8dbe6 100644 --- a/lightningd/pay.c +++ b/lightningd/pay.c @@ -2,9 +2,12 @@ #include #include #include +#include #include +#include #include #include +#include #include #include #include @@ -362,3 +365,107 @@ static const struct json_command sendpay_command = { "Returns the {preimage} on success" }; AUTODATA(json_command, &sendpay_command); + +struct pay { + struct sha256 payment_hash; + struct command *cmd; +}; + +static void json_pay_getroute_reply(struct subd *gossip, + const u8 *reply, const int *fds, + struct pay *pay) +{ + struct route_hop *route; + + fromwire_gossip_getroute_reply(reply, reply, NULL, &route); + + if (tal_count(route) == 0) { + command_fail(pay->cmd, "Could not find a route"); + return; + } + + send_payment(pay->cmd, &pay->payment_hash, route); +} + +static void json_pay(struct command *cmd, + const char *buffer, const jsmntok_t *params) +{ + jsmntok_t *bolt11tok, *msatoshitok, *desctok, *riskfactortok; + double riskfactor = 1.0; + u64 msatoshi; + /* FIXME: add ctlv to bolt11 */ + u32 cltv = 9; + struct pay *pay = tal(cmd, struct pay); + struct bolt11 *b11; + char *fail, *b11str, *desc; + u8 *req; + + if (!json_get_params(buffer, params, + "bolt11", &bolt11tok, + "?msatoshi", &msatoshitok, + "?description", &desctok, + "?riskfactor", &riskfactortok, + NULL)) { + command_fail(cmd, "Need bolt11 string"); + return; + } + + b11str = tal_strndup(cmd, buffer + bolt11tok->start, + bolt11tok->end - bolt11tok->start); + if (desctok) + desc = tal_strndup(cmd, buffer + desctok->start, + desctok->end - desctok->start); + else + desc = NULL; + + b11 = bolt11_decode(pay, b11str, desc, &fail); + if (!b11) { + command_fail(cmd, "Invalid bolt11: %s", fail); + return; + } + + pay->cmd = cmd; + pay->payment_hash = b11->payment_hash; + + if (b11->msatoshi) { + msatoshi = *b11->msatoshi; + if (msatoshitok) { + command_fail(cmd, "msatoshi parameter unnecessary"); + return; + } + } else { + if (!msatoshitok) { + command_fail(cmd, "msatoshi parameter required"); + return; + } + if (!json_tok_u64(buffer, msatoshitok, &msatoshi)) { + command_fail(cmd, + "msatoshi '%.*s' is not a valid number", + (int)(msatoshitok->end-msatoshitok->start), + buffer + msatoshitok->start); + return; + } + } + + if (riskfactortok + && !json_tok_double(buffer, riskfactortok, &riskfactor)) { + command_fail(cmd, "'%.*s' is not a valid double", + (int)(riskfactortok->end - riskfactortok->start), + buffer + riskfactortok->start); + return; + } + + /* FIXME: use b11->routes */ + req = towire_gossip_getroute_request(cmd, &cmd->ld->id, + &b11->receiver_id, + msatoshi, riskfactor*1000, cltv); + subd_req(pay, cmd->ld->gossip, req, -1, 0, json_pay_getroute_reply, pay); +} + +static const struct json_command pay_command = { + "pay", + json_pay, + "Send payment in {bolt11} with optional {msatoshi}, {description} and {riskfactor}", + "Returns the {preimage} on success" +}; +AUTODATA(json_command, &pay_command); diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 4f753374fabe..bf2ca90c6ca0 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -644,6 +644,22 @@ def test_sendpay_cant_afford(self): # But this should work. self.pay(l2, l1, available - reserve*2) + def test_pay(self): + l1,l2 = self.connect() + + chanid = self.fund_channel(l1, l2, 10**6) + + # Wait for route propagation. + bitcoind.rpc.generate(5) + l1.daemon.wait_for_logs(['Received channel_update for channel {}\(0\)' + .format(chanid), + 'Received channel_update for channel {}\(1\)' + .format(chanid)]) + + inv = l2.rpc.invoice(123000, 'test_pay', 'description')['bolt11'] + l1.rpc.pay(inv); + assert l2.rpc.listinvoice('test_pay')[0]['complete'] == True + def test_bad_opening(self): # l1 asks for a too-long locktime l1 = self.node_factory.get_node(options=['--locktime-blocks=100']) From 6c21da69e6d5cb1dccd97e621e53b696d34cc4cc Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:38:19 +1030 Subject: [PATCH 0070/1428] bolt11: 'c' support for min_final_cltv_expiry. Based on latest draft spec, using variable length encoding. Signed-off-by: Rusty Russell --- lightningd/bolt11.c | 47 +++++++++++++++++++++++++++++++++++++++++--- lightningd/bolt11.h | 3 +++ lightningd/invoice.c | 4 ++-- lightningd/pay.c | 5 ++--- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/lightningd/bolt11.c b/lightningd/bolt11.c index d7933f311deb..d7322586887b 100644 --- a/lightningd/bolt11.c +++ b/lightningd/bolt11.c @@ -248,6 +248,33 @@ static char *decode_x(struct bolt11 *b11, return NULL; } +/* BOLT #11: + * + * `c` (24): `data_length` variable. `min_final_cltv_expiry` to use for the + * last HTLC in the route. Default is 9 if not specified. + */ +#define DEFAULT_C 9 +static char *decode_c(struct bolt11 *b11, + struct hash_u5 *hu5, + u5 **data, size_t *data_len, + size_t data_length, bool *have_c) +{ + u64 c; + if (*have_c) + return unknown_field(b11, hu5, data, data_len, 'c', + data_length); + + /* FIXME: Put upper limit in bolt 11 */ + if (!pull_uint(hu5, data, data_len, &c, data_length * 5)) + return tal_fmt(b11, "c: length %zu chars is excessive", + *data_len); + b11->min_final_cltv_expiry = c; + if (b11->min_final_cltv_expiry != c) + return tal_fmt(b11, "c: %"PRIu64" is too large", c); + + return NULL; +} + static char *decode_n(struct bolt11 *b11, struct hash_u5 *hu5, u5 **data, size_t *data_len, @@ -417,6 +444,8 @@ struct bolt11 *new_bolt11(const tal_t *ctx, u64 *msatoshi) b11->fallback = NULL; b11->routes = NULL; b11->msatoshi = NULL; + b11->expiry = DEFAULT_X; + b11->min_final_cltv_expiry = DEFAULT_C; if (msatoshi) b11->msatoshi = tal_dup(b11, u64, msatoshi); @@ -437,7 +466,7 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str, struct hash_u5 hu5; struct sha256 hash; bool have_p = false, have_n = false, have_d = false, have_h = false, - have_x = false, have_f = false; + have_x = false, have_f = false, have_c = false; b11->routes = tal_arr(b11, struct route_info *, 0); @@ -528,8 +557,6 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str, if (!pull_uint(&hu5, &data, &data_len, &b11->timestamp, 35)) return decode_fail(b11, fail, "Can't get 35-bit timestamp"); - b11->expiry = DEFAULT_X; - while (data_len > 520 / 5) { const char *problem = NULL; u64 type, data_length; @@ -580,6 +607,12 @@ struct bolt11 *bolt11_decode(const tal_t *ctx, const char *str, &have_x); break; + case 'c': + problem = decode_c(b11, &hu5, &data, + &data_len, data_length, + &have_c); + break; + case 'f': problem = decode_f(b11, &hu5, &data, &data_len, data_length, @@ -765,6 +798,11 @@ static void encode_x(u5 **data, u64 expiry) push_varlen_field(data, 'x', expiry); } +static void encode_c(u5 **data, u16 min_final_cltv_expiry) +{ + push_varlen_field(data, 'c', min_final_cltv_expiry); +} + static void encode_f(u5 **data, const u8 *fallback) { struct bitcoin_address pkh; @@ -906,6 +944,9 @@ char *bolt11_encode(const tal_t *ctx, if (b11->expiry != DEFAULT_X) encode_x(&data, b11->expiry); + if (b11->min_final_cltv_expiry != DEFAULT_C) + encode_c(&data, b11->min_final_cltv_expiry); + if (b11->fallback) encode_f(&data, b11->fallback); diff --git a/lightningd/bolt11.h b/lightningd/bolt11.h index af02962242a2..73b0a9802289 100644 --- a/lightningd/bolt11.h +++ b/lightningd/bolt11.h @@ -49,6 +49,9 @@ struct bolt11 { /* How many seconds to pay from @timestamp above. */ u64 expiry; + /* How many blocks final hop requires. */ + u32 min_final_cltv_expiry; + /* If non-NULL, indicates a fallback address to pay to. */ const u8 *fallback; diff --git a/lightningd/invoice.c b/lightningd/invoice.c index 5aa4029fc207..955de51803ab 100644 --- a/lightningd/invoice.c +++ b/lightningd/invoice.c @@ -181,6 +181,7 @@ static void json_invoice(struct command *cmd, b11->timestamp = time_now().ts.tv_sec; b11->payment_hash = invoice->rhash; b11->receiver_id = cmd->ld->id; + b11->min_final_cltv_expiry = cmd->ld->config.cltv_final; if (desc->end - desc->start >= BOLT11_FIELD_BYTE_LIMIT) { b11->description_hash = tal(b11, struct sha256); sha256(b11->description_hash, buffer + desc->start, @@ -188,8 +189,7 @@ static void json_invoice(struct command *cmd, } else b11->description = tal_strndup(b11, buffer + desc->start, desc->end - desc->start); - /* FIXME: add option to set this */ - b11->expiry = 3600; + /* FIXME: add option to set expiry */ /* FIXME: add private routes if necessary! */ b11enc = bolt11_encode(cmd, cmd->ld, b11, false); diff --git a/lightningd/pay.c b/lightningd/pay.c index 6ec0bec8dbe6..66e63e4bb9c0 100644 --- a/lightningd/pay.c +++ b/lightningd/pay.c @@ -393,8 +393,6 @@ static void json_pay(struct command *cmd, jsmntok_t *bolt11tok, *msatoshitok, *desctok, *riskfactortok; double riskfactor = 1.0; u64 msatoshi; - /* FIXME: add ctlv to bolt11 */ - u32 cltv = 9; struct pay *pay = tal(cmd, struct pay); struct bolt11 *b11; char *fail, *b11str, *desc; @@ -458,7 +456,8 @@ static void json_pay(struct command *cmd, /* FIXME: use b11->routes */ req = towire_gossip_getroute_request(cmd, &cmd->ld->id, &b11->receiver_id, - msatoshi, riskfactor*1000, cltv); + msatoshi, riskfactor*1000, + b11->min_final_cltv_expiry); subd_req(pay, cmd->ld->gossip, req, -1, 0, json_pay_getroute_reply, pay); } From a9002eac52a43e8d4db2c0649e4abab9e9107661 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:39:19 +1030 Subject: [PATCH 0071/1428] Update to latest BOLT. And nail "make check-source" to that specific version (which is a commit id, not a branch name, so needs a different syntax for git). Signed-off-by: Rusty Russell --- Makefile | 4 ++-- common/sphinx.c | 2 +- gossipd/handshake.c | 2 +- lightningd/bolt11.c | 4 ++-- lightningd/peer_htlcs.c | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 79f53736a287..6ee559fa8233 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ CCANDIR := ccan # Where we keep the BOLT RFCs BOLTDIR := ../lightning-rfc/ -BOLTVERSION := master +BOLTVERSION := 58d4d9bca3ae5896eeea4b46324df27b8ecb0ce1 # If you don't have (working) valgrind. #NO_VALGRIND := 1 @@ -190,7 +190,7 @@ bolt-check/%: % bolt-precheck tools/check-bolt @[ ! -d .tmp.lightningrfc ] || tools/check-bolt .tmp.lightningrfc $< bolt-precheck: - @rm -rf .tmp.lightningrfc; if [ ! -d $(BOLTDIR) ]; then echo Not checking BOLT references: BOLTDIR $(BOLTDIR) does not exist >&2; exit 0; fi; set -e; if [ -n "$(BOLTVERSION)" ]; then git clone -q -b $(BOLTVERSION) $(BOLTDIR) .tmp.lightningrfc; else cp -a $(BOLTDIR) .tmp.lightningrfc; fi + @rm -rf .tmp.lightningrfc; if [ ! -d $(BOLTDIR) ]; then echo Not checking BOLT references: BOLTDIR $(BOLTDIR) does not exist >&2; exit 0; fi; set -e; if [ -n "$(BOLTVERSION)" ]; then git clone -q $(BOLTDIR) .tmp.lightningrfc && cd .tmp.lightningrfc && git checkout -q $(BOLTVERSION); else cp -a $(BOLTDIR) .tmp.lightningrfc; fi check-source-bolt: $(ALL_TEST_PROGRAMS:%=bolt-check/%.c) diff --git a/common/sphinx.c b/common/sphinx.c index 9d8520d12988..4fc0bdab4aeb 100644 --- a/common/sphinx.c +++ b/common/sphinx.c @@ -487,7 +487,7 @@ u8 *create_onionreply(const tal_t *ctx, const struct secret *shared_secret, /* BOLT #4: * * Where `hmac` is an HMAC authenticating the remainder of the packet, - * with a key using the above key generation with key type "_um_" + * with a key using the above key generation with key type `um` */ generate_key(key, "um", 2, shared_secret->data); diff --git a/gossipd/handshake.c b/gossipd/handshake.c index 7a61e5d85b17..f60d3cd3ada3 100644 --- a/gossipd/handshake.c +++ b/gossipd/handshake.c @@ -35,7 +35,7 @@ enum bolt8_side { /* BOLT #8: * - * Act One is sent from initiator tog responder. During `Act One`, the + * Act One is sent from initiator to responder. During `Act One`, the * initiator attempts to satisfy an implicit challenge by the responder. To * complete this challenge, the initiator _must_ know the static public key of * the responder. diff --git a/lightningd/bolt11.c b/lightningd/bolt11.c index d7322586887b..79c03f2bb94f 100644 --- a/lightningd/bolt11.c +++ b/lightningd/bolt11.c @@ -752,7 +752,7 @@ static void push_field(u5 **data, char type, const void *src, size_t nbits) /* BOLT #11: * - * SHOULD use the minimum `data_length` possible. + * SHOULD use the minimum `data_length` possible for `x` and `c` fields. */ static void push_varlen_field(u5 **data, char type, u64 val) { @@ -1007,7 +1007,7 @@ struct bolt11 *bolt11_out_check(const struct bolt11 *b11, const char *abortstr) { struct bolt11_field *extra; - /* BOLT #11: + /* BOLT #2: * * For channels with `chain_hash` identifying the Bitcoin blockchain, * the sending node MUST set the 4 most significant bytes of diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index b6f4011c7782..82b9a0ce64ba 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -527,7 +527,7 @@ static void forward_htlc(struct htlc_in *hin, /* BOLT #4: * - * If the cltv-expiry is too near, we tell them the the current channel + * If the `cltv_expiry` is too near, we tell them the the current channel * setting for the outgoing channel: * 1. type: UPDATE|14 (`expiry_too_soon`) * 2. data: From 1ec19093f7b7df0b6a0f6a93d13ef7cbac0132ad Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 26 Oct 2017 13:40:19 +1030 Subject: [PATCH 0072/1428] Travis: check bolt quotes. Signed-off-by: Rusty Russell --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4503e07d6fca..a4a042bfb62d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,4 +19,5 @@ script: - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make -j3 DEVELOPER=${DEVELOPER} - echo "travis_fold:end:SCRIPT folding ends" - docker run --rm=true -e NO_VALGRIND=${NO_VALGRIND:-0} -e TEST_DEBUG=${TEST_DEBUG:-0} -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check DEVELOPER=${DEVELOPER} - - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check-source + - git clone https://github.com/lightningnetwork/lightning-rfc.git + - docker run --rm=true -v "${TRAVIS_BUILD_DIR}":/build -t cdecker/lightning-ci:${ARCH}bit make check-source BOLTDIR=lightning-rfc From 8f17effeeb604e35501dad7d2213ea62bdb33aaf Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 10:07:33 +1030 Subject: [PATCH 0073/1428] json: just blatt weird characters in string. Don't try to escape them. It's whack-a-mole and they shouldn't do it anyway. Suggested-by: Christian Decker Signed-off-by: Rusty Russell --- common/json.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/common/json.c b/common/json.c index 9f96ca81e6fa..6ae98637d4ed 100644 --- a/common/json.c +++ b/common/json.c @@ -412,16 +412,17 @@ void json_add_literal(struct json_result *result, const char *fieldname, void json_add_string(struct json_result *result, const char *fieldname, const char *value) { - char *escaped = tal_arr(result, char, strlen(value) * 2 + 1); - size_t i, n; + char *escaped = tal_strdup(result, value); + size_t i; json_start_member(result, fieldname); - for (i = n = 0; value[i]; i++) { - if (value[i] == '\\' || value[i] == '"') - escaped[n++] = '\\'; - escaped[n++] = value[i]; + for (i = 0; escaped[i]; i++) { + /* Replace any funny business. Better safe than accurate! */ + if (escaped[i] == '\\' + || escaped[i] == '"' + || !cisprint(escaped[i])) + escaped[i] = '?'; } - escaped[n] = '\0'; result_append_fmt(result, "\"%s\"", escaped); } From b7774fcb0a19d9b0ebecea913ead05ab1f04689c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 10:10:09 +1030 Subject: [PATCH 0074/1428] bolt11: comment on weird assignment-then-check test. Suggested-by: Christian Decker Signed-off-by: Rusty Russell --- lightningd/bolt11.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lightningd/bolt11.c b/lightningd/bolt11.c index 79c03f2bb94f..aab05e09f006 100644 --- a/lightningd/bolt11.c +++ b/lightningd/bolt11.c @@ -269,6 +269,7 @@ static char *decode_c(struct bolt11 *b11, return tal_fmt(b11, "c: length %zu chars is excessive", *data_len); b11->min_final_cltv_expiry = c; + /* Can overflow, since c is 64 bits but value must be < 32 bits */ if (b11->min_final_cltv_expiry != c) return tal_fmt(b11, "c: %"PRIu64" is too large", c); From ab634dfcdd9e53d40c5c8a041e641e20c94545a4 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 12:28:25 +1030 Subject: [PATCH 0075/1428] common/test_sphinx: change to standard run- format. Signed-off-by: Rusty Russell --- common/test/Makefile | 4 +--- common/test/{test_sphinx.c => run-sphinx.c} | 1 + 2 files changed, 2 insertions(+), 3 deletions(-) rename common/test/{test_sphinx.c => run-sphinx.c} (99%) diff --git a/common/test/Makefile b/common/test/Makefile index 8c38dd92c7a2..fb9c01a5ccf2 100644 --- a/common/test/Makefile +++ b/common/test/Makefile @@ -1,10 +1,8 @@ -COMMON_TEST_SRC := \ - common/test/test_sphinx.c +COMMON_TEST_SRC := $(wildcard common/test/run-*.c) COMMON_TEST_OBJS := $(COMMON_TEST_SRC:.c=.o) COMMON_TEST_PROGRAMS := $(COMMON_TEST_OBJS:.o=) COMMON_TEST_COMMON_OBJS := \ - common/sphinx.o \ common/utils.o $(COMMON_TEST_PROGRAMS): $(COMMON_TEST_COMMON_OBJS) $(BITCOIN_OBJS) diff --git a/common/test/test_sphinx.c b/common/test/run-sphinx.c similarity index 99% rename from common/test/test_sphinx.c rename to common/test/run-sphinx.c index e885abcc4085..6917964eaf50 100644 --- a/common/test/test_sphinx.c +++ b/common/test/run-sphinx.c @@ -1,3 +1,4 @@ +#include "../sphinx.c" #include #include #include From 112ae0d0f5bd283bd7dd52317bf29c332ee32684 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 12:28:44 +1030 Subject: [PATCH 0076/1428] common/test/run-json: test JSON escaping. Signed-off-by: Rusty Russell --- common/test/run-json.c | 45 ++++++++++++++++++++++++++++++ lightningd/test/run-cryptomsg.c | 2 ++ lightningd/test/run-find_my_path.c | 2 ++ 3 files changed, 49 insertions(+) create mode 100644 common/test/run-json.c diff --git a/common/test/run-json.c b/common/test/run-json.c new file mode 100644 index 000000000000..e599a0f59cc8 --- /dev/null +++ b/common/test/run-json.c @@ -0,0 +1,45 @@ +#include "../json.c" +#include + +/* AUTOGENERATED MOCKS START */ +/* AUTOGENERATED MOCKS END */ + +int main(void) +{ + struct json_result *result = new_json_result(NULL); + jsmntok_t *toks; + const jsmntok_t *x; + bool valid; + int i; + char *badstr = tal_arr(result, char, 256); + const char *str; + + /* Fill with junk, and nul-terminate (256 -> 0) */ + for (i = 1; i < 257; i++) + badstr[i-1] = i; + + json_object_start(result, NULL); + json_add_string(result, "x", badstr); + json_object_end(result); + + /* Parse back in, make sure nothing crazy. */ + str = json_result_string(result); + + toks = json_parse_input(str, strlen(str), &valid); + assert(valid); + assert(toks); + + assert(toks[0].type == JSMN_OBJECT); + x = json_get_member(str, toks, "x"); + assert(x); + assert(x->type == JSMN_STRING); + assert((x->end - x->start) == 255); + for (i = x->start; i < x->end; i++) { + assert(cisprint(str[i])); + assert(str[i] != '\\'); + assert(str[i] != '"'); + assert(str[i] == '?' || str[i] == badstr[i - x->start]); + } + tal_free(result); + return 0; +} diff --git a/lightningd/test/run-cryptomsg.c b/lightningd/test/run-cryptomsg.c index 35126ede92c7..fead63c04748 100644 --- a/lightningd/test/run-cryptomsg.c +++ b/lightningd/test/run-cryptomsg.c @@ -47,6 +47,8 @@ void dev_blackhole_fd(int fd UNNEEDED) /* Generated stub for dev_sabotage_fd */ void dev_sabotage_fd(int fd UNNEEDED) { fprintf(stderr, "dev_sabotage_fd called!\n"); abort(); } +/* Could not find declaration for secp256k1_ecdsa_recoverable_signature_parse_compact */ +/* Could not find declaration for secp256k1_ecdsa_recoverable_signature_serialize_compact */ /* AUTOGENERATED MOCKS END */ enum dev_disconnect dev_disconnect(int pkt_type) diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 64c6ebc1c5df..13b6736188eb 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -46,6 +46,8 @@ void populate_peer(struct lightningd *ld UNNEEDED, struct peer *peer UNNEEDED) /* Generated stub for register_opts */ void register_opts(struct lightningd *ld UNNEEDED) { fprintf(stderr, "register_opts called!\n"); abort(); } +/* Could not find declaration for secp256k1_ecdsa_recoverable_signature_parse_compact */ +/* Could not find declaration for secp256k1_ecdsa_recoverable_signature_serialize_compact */ /* Generated stub for setup_color_and_alias */ void setup_color_and_alias(struct lightningd *ld UNNEEDED) { fprintf(stderr, "setup_color_and_alias called!\n"); abort(); } From 5502a19d1ecf7517535d995ccbc95d201e67d318 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 12:29:58 +1030 Subject: [PATCH 0077/1428] json: reject incoming JSON which has any unusual characters in tokens. ie. non-printable, quotes or escapes. We allow these outside tokens (we expect tabs and \n for example). This is a big hammer, but if someone really wants we can add support later. Signed-off-by: Rusty Russell --- common/json.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/common/json.c b/common/json.c index 6ae98637d4ed..22f918617ea7 100644 --- a/common/json.c +++ b/common/json.c @@ -272,11 +272,22 @@ bool json_get_params(const char *buffer, const jsmntok_t param[], ...) return true; } +static bool strange_chars(const char *str, size_t len) +{ + for (size_t i = 0; i < len; i++) { + if (!cisprint(str[i]) || str[i] == '"' || str[i] == '\\') + return true; + } + + return false; +} + jsmntok_t *json_parse_input(const char *input, int len, bool *valid) { jsmn_parser parser; jsmntok_t *toks; jsmnerr_t ret; + size_t i; toks = tal_arr(input, jsmntok_t, 10); @@ -303,6 +314,19 @@ jsmntok_t *json_parse_input(const char *input, int len, bool *valid) toks[ret].type = -1; toks[ret].start = toks[ret].end = toks[ret].size = 0; + /* Don't allow tokens to contain weird characters (outside toks ok). */ + for (i = 0; i < ret; i++) { + if (toks[i].type != JSMN_STRING + && toks[i].type != JSMN_PRIMITIVE) + continue; + + if (strange_chars(input + toks[i].start, + toks[i].end - toks[i].start)) { + *valid = false; + return tal_free(toks); + } + } + return toks; } From 1ae57fb72cda34ef9c5e17dbeac863ed6244e43a Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 12:50:51 +1030 Subject: [PATCH 0078/1428] Update to latest BOLT. Nothing changes. Signed-off-by: Rusty Russell --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ee559fa8233..66ae00c555d1 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ CCANDIR := ccan # Where we keep the BOLT RFCs BOLTDIR := ../lightning-rfc/ -BOLTVERSION := 58d4d9bca3ae5896eeea4b46324df27b8ecb0ce1 +BOLTVERSION := c93cd75d880c82fb2ff7173a99d9f49abf75f8bf # If you don't have (working) valgrind. #NO_VALGRIND := 1 From 860a76b1c91420d4dcdb4f5771b38342a95f9703 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 12:51:00 +1030 Subject: [PATCH 0079/1428] lightningd: update to add `wire_expiry_too_far`. From recently-merged BOLT update. Signed-off-by: Rusty Russell --- lightningd/peer_htlcs.c | 12 +++++++++--- wire/gen_onion_wire_csv | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 82b9a0ce64ba..48b5fbcfc54f 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -153,6 +153,8 @@ static u8 *make_failmsg(const tal_t *ctx, return towire_incorrect_cltv_expiry(ctx, 0, channel_update); case WIRE_EXPIRY_TOO_SOON: return towire_expiry_too_soon(ctx, channel_update); + case WIRE_EXPIRY_TOO_FAR: + return towire_expiry_too_far(ctx); case WIRE_UNKNOWN_PAYMENT_HASH: return towire_unknown_payment_hash(ctx); case WIRE_INCORRECT_PAYMENT_AMOUNT: @@ -545,7 +547,12 @@ static void forward_htlc(struct htlc_in *hin, goto fail; } - /* FIXME: Add this to BOLT! */ + /* BOLT #4: + * + * If the `cltv_expiry` is unreasonably far, we can also report an error: + * + * 1. type: 21 (`expiry_too_far`) + */ if (get_block_height(next->ld->topology) + next->ld->config.max_htlc_expiry < outgoing_cltv_value) { log_debug(hin->key.peer->log, @@ -553,8 +560,7 @@ static void forward_htlc(struct htlc_in *hin, outgoing_cltv_value, get_block_height(next->ld->topology), next->ld->config.max_htlc_expiry); - /* FIXME: WIRE_EXPIRY_TOO_FAR? */ - failcode = WIRE_TEMPORARY_CHANNEL_FAILURE; + failcode = WIRE_EXPIRY_TOO_FAR; goto fail; } diff --git a/wire/gen_onion_wire_csv b/wire/gen_onion_wire_csv index fab3c535ff17..f5d07c9bd90e 100644 --- a/wire/gen_onion_wire_csv +++ b/wire/gen_onion_wire_csv @@ -30,6 +30,7 @@ incorrect_cltv_expiry,6,channel_update,len expiry_too_soon,UPDATE|14 expiry_too_soon,0,len,2 expiry_too_soon,2,channel_update,len +expiry_too_far,21 channel_disabled,UPDATE|20 unknown_payment_hash,PERM|15 incorrect_payment_amount,PERM|16 From a55ce607a1c867667647c812e3d94a94c5493432 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 20:50:40 +1030 Subject: [PATCH 0080/1428] bitcoind: contain ld pointer. This is a subset of a "bitcoind: wrap callbacks in transaction." from the everything-in-transaction branch, but we need the ld pointer now. Signed-off-by: Rusty Russell --- lightningd/bitcoind.c | 5 ++++- lightningd/bitcoind.h | 7 ++++++- lightningd/chaintopology.c | 6 +++--- lightningd/chaintopology.h | 2 +- lightningd/test/run-cryptomsg.c | 2 -- lightningd/test/run-find_my_path.c | 4 +--- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index fdaff76b7a78..bab699080ff5 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -543,13 +543,16 @@ void wait_for_bitcoind(struct bitcoind *bitcoind) tal_free(cmd); } -struct bitcoind *new_bitcoind(const tal_t *ctx, struct log *log) +struct bitcoind *new_bitcoind(const tal_t *ctx, + struct lightningd *ld, + struct log *log) { struct bitcoind *bitcoind = tal(ctx, struct bitcoind); /* Use testnet by default, change later if we want another network */ bitcoind->chainparams = chainparams_for_network("testnet"); bitcoind->datadir = NULL; + bitcoind->ld = ld; bitcoind->log = log; bitcoind->req_running = false; bitcoind->shutdown = false; diff --git a/lightningd/bitcoind.h b/lightningd/bitcoind.h index 1fb371e94797..9157b6aea0c7 100644 --- a/lightningd/bitcoind.h +++ b/lightningd/bitcoind.h @@ -29,6 +29,9 @@ struct bitcoind { /* Where to do logging. */ struct log *log; + /* Main lightningd structure */ + struct lightningd *ld; + /* Are we currently running a bitcoind request (it's ratelimited) */ bool req_running; @@ -46,7 +49,9 @@ struct bitcoind { bool shutdown; }; -struct bitcoind *new_bitcoind(const tal_t *ctx, struct log *log); +struct bitcoind *new_bitcoind(const tal_t *ctx, + struct lightningd *ld, + struct log *log); void wait_for_bitcoind(struct bitcoind *bitcoind); diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index fa5a0b273be1..b153b8afd881 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -551,9 +551,9 @@ static void destroy_outgoing_txs(struct chain_topology *topo) tal_free(otx); } -struct chain_topology *new_topology(const tal_t *ctx, struct log *log) +struct chain_topology *new_topology(struct lightningd *ld, struct log *log) { - struct chain_topology *topo = tal(ctx, struct chain_topology); + struct chain_topology *topo = tal(ld, struct chain_topology); block_map_init(&topo->block_map); list_head_init(&topo->outgoing_txs); @@ -562,7 +562,7 @@ struct chain_topology *new_topology(const tal_t *ctx, struct log *log) topo->log = log; topo->default_fee_rate = 40000; topo->override_fee_rate = 0; - topo->bitcoind = new_bitcoind(topo, log); + topo->bitcoind = new_bitcoind(topo, ld, log); #if DEVELOPER topo->dev_no_broadcast = false; #endif diff --git a/lightningd/chaintopology.h b/lightningd/chaintopology.h index 5e69ce7341a9..f3a8c51392dc 100644 --- a/lightningd/chaintopology.h +++ b/lightningd/chaintopology.h @@ -149,7 +149,7 @@ void broadcast_tx(struct chain_topology *topo, int exitstatus, const char *err)); -struct chain_topology *new_topology(const tal_t *ctx, struct log *log); +struct chain_topology *new_topology(struct lightningd *ld, struct log *log); void setup_topology(struct chain_topology *topology, struct timers *timers, struct timerel poll_time, u32 first_peer_block); diff --git a/lightningd/test/run-cryptomsg.c b/lightningd/test/run-cryptomsg.c index fead63c04748..35126ede92c7 100644 --- a/lightningd/test/run-cryptomsg.c +++ b/lightningd/test/run-cryptomsg.c @@ -47,8 +47,6 @@ void dev_blackhole_fd(int fd UNNEEDED) /* Generated stub for dev_sabotage_fd */ void dev_sabotage_fd(int fd UNNEEDED) { fprintf(stderr, "dev_sabotage_fd called!\n"); abort(); } -/* Could not find declaration for secp256k1_ecdsa_recoverable_signature_parse_compact */ -/* Could not find declaration for secp256k1_ecdsa_recoverable_signature_serialize_compact */ /* AUTOGENERATED MOCKS END */ enum dev_disconnect dev_disconnect(int pkt_type) diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 13b6736188eb..8372232e27d3 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -38,7 +38,7 @@ struct log_book *new_log_book(const tal_t *ctx UNNEEDED, enum log_level printlevel UNNEEDED) { fprintf(stderr, "new_log_book called!\n"); abort(); } /* Generated stub for new_topology */ -struct chain_topology *new_topology(const tal_t *ctx UNNEEDED, struct log *log UNNEEDED) +struct chain_topology *new_topology(struct lightningd *ld UNNEEDED, struct log *log UNNEEDED) { fprintf(stderr, "new_topology called!\n"); abort(); } /* Generated stub for populate_peer */ void populate_peer(struct lightningd *ld UNNEEDED, struct peer *peer UNNEEDED) @@ -46,8 +46,6 @@ void populate_peer(struct lightningd *ld UNNEEDED, struct peer *peer UNNEEDED) /* Generated stub for register_opts */ void register_opts(struct lightningd *ld UNNEEDED) { fprintf(stderr, "register_opts called!\n"); abort(); } -/* Could not find declaration for secp256k1_ecdsa_recoverable_signature_parse_compact */ -/* Could not find declaration for secp256k1_ecdsa_recoverable_signature_serialize_compact */ /* Generated stub for setup_color_and_alias */ void setup_color_and_alias(struct lightningd *ld UNNEEDED) { fprintf(stderr, "setup_color_and_alias called!\n"); abort(); } From c14b159166dfa39ca7d4930327db6b10158350e8 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 2 Nov 2017 10:20:47 +1030 Subject: [PATCH 0081/1428] lightningd: remove --deadline-blocks option. We will derive it from other factors. Signed-off-by: Rusty Russell --- lightningd/lightningd.h | 3 --- lightningd/options.c | 20 -------------------- lightningd/peer_htlcs.c | 21 +++++++++------------ tests/utils.py | 1 - 4 files changed, 9 insertions(+), 36 deletions(-) diff --git a/lightningd/lightningd.h b/lightningd/lightningd.h index 543d3d5f295c..6135a1d9288d 100644 --- a/lightningd/lightningd.h +++ b/lightningd/lightningd.h @@ -55,9 +55,6 @@ struct config { /* Maximum time for an expiring HTLC (blocks). */ u32 max_htlc_expiry; - /* How many blocks before upstream HTLC expiry do we panic and dump? */ - u32 deadline_blocks; - /* Fee rates. */ u32 fee_base; s32 fee_per_satoshi; diff --git a/lightningd/options.c b/lightningd/options.c index 47a6bb212fc1..cdd85be9f3c1 100644 --- a/lightningd/options.c +++ b/lightningd/options.c @@ -238,9 +238,6 @@ static void config_register_opts(struct lightningd *ld) opt_register_arg("--max-htlc-expiry", opt_set_u32, opt_show_u32, &ld->config.max_htlc_expiry, "Maximum number of blocks to accept an HTLC before expiry"); - opt_register_arg("--deadline-blocks", opt_set_u32, opt_show_u32, - &ld->config.deadline_blocks, - "Number of blocks before HTLC timeout before we drop connection"); opt_register_arg("--bitcoind-poll", opt_set_time, opt_show_time, &ld->config.poll_time, "Time between polling for new transactions"); @@ -338,9 +335,6 @@ static const struct config testnet_config = { /* Don't lock up channel for more than 5 days. */ .max_htlc_expiry = 5 * 6 * 24, - /* If we're closing on HTLC expiry, and you're unresponsive, we abort. */ - .deadline_blocks = 4, - /* How often to bother bitcoind. */ .poll_time = TIME_FROM_SEC(10), @@ -406,9 +400,6 @@ static const struct config mainnet_config = { /* Don't lock up channel for more than 5 days. */ .max_htlc_expiry = 5 * 6 * 24, - /* If we're closing on HTLC expiry, and you're unresponsive, we abort. */ - .deadline_blocks = 10, - /* How often to bother bitcoind. */ .poll_time = TIME_FROM_SEC(30), @@ -441,17 +432,6 @@ static void check_config(struct lightningd *ld) if (ld->config.anchor_confirms == 0) fatal("anchor-confirms must be greater than zero"); - - /* FIXME-OLD #2: - * - * a node MUST estimate the deadline for successful redemption - * for each HTLC it offers. A node MUST NOT offer a HTLC - * after this deadline */ - if (ld->config.deadline_blocks >= ld->config.cltv_final - || ld->config.deadline_blocks >= ld->config.cltv_expiry_delta) - fatal("Deadline %u can't be more than final/expiry %u/%u", - ld->config.deadline_blocks, - ld->config.cltv_final, ld->config.cltv_expiry_delta); } static void setup_default_config(struct lightningd *ld) diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 48b5fbcfc54f..c235bc17f9b5 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -527,22 +527,19 @@ static void forward_htlc(struct htlc_in *hin, goto fail; } - /* BOLT #4: + /* BOLT #2: * - * If the `cltv_expiry` is too near, we tell them the the current channel - * setting for the outgoing channel: - * 1. type: UPDATE|14 (`expiry_too_soon`) - * 2. data: - * * [`2`:`len`] - * * [`len`:`channel_update`] + * A node MUST estimate a timeout deadline for each HTLC it offers. A + * node MUST NOT offer an HTLC with a timeout deadline before its + * `cltv_expiry` */ - if (get_block_height(next->ld->topology) - + next->ld->config.deadline_blocks >= outgoing_cltv_value) { + /* In our case, G = 1, so we need to expire it one after it's expiration. + * But never offer an expired HTLC; that's dumb. */ + if (get_block_height(next->ld->topology) >= outgoing_cltv_value) { log_debug(hin->key.peer->log, - "Expiry cltv %u too close to current %u + deadline %u", + "Expiry cltv %u too close to current %u", outgoing_cltv_value, - get_block_height(next->ld->topology), - next->ld->config.deadline_blocks); + get_block_height(next->ld->topology)); failcode = WIRE_EXPIRY_TOO_SOON; goto fail; } diff --git a/tests/utils.py b/tests/utils.py index 1785506551a1..9af80821c763 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -20,7 +20,6 @@ LIGHTNINGD_CONFIG = { "bitcoind-poll": "1s", "log-level": "debug", - "deadline-blocks": 4, "cltv-delta": 6, "cltv-final": 5, "locktime-blocks": 5, From bdabb247346d04aec991181b73686dcb5f0ebf40 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 2 Nov 2017 10:22:39 +1030 Subject: [PATCH 0082/1428] lightningd: note our complience on not forwarding an HTLC post deadline. Signed-off-by: Rusty Russell --- lightningd/peer_htlcs.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index c235bc17f9b5..950b4d0dbbe3 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -647,6 +647,16 @@ static bool peer_accepted_htlc(struct peer *peer, goto out; } + /* BOLT #2: + * + * A node MUST estimate a fulfillment deadline for each HTLC it is + * attempting to fulfill. A node MUST fail (and not forward) an HTLC + * whose fulfillment deadline is already past + */ + /* Our deadline is half the cltv_delta we insist on, so this check is + * a subset of the cltv check done in handle_localpay and + * forward_htlc. */ + /* channeld tests this, so it should have set ss to zeroes. */ op = parse_onionpacket(tmpctx, hin->onion_routing_packet, sizeof(hin->onion_routing_packet)); From 9662589ed81ae9d7a621120b26aaca5f83c4533b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 2 Nov 2017 10:23:19 +1030 Subject: [PATCH 0083/1428] lightningd: move notify_new_block() callback to peer_htlcs. And change prototype to take the lightningd structure. Signed-off-by: Rusty Russell --- lightningd/chaintopology.c | 4 ++-- lightningd/chaintopology.h | 2 +- lightningd/lightningd.c | 6 ------ lightningd/peer_htlcs.c | 6 ++++++ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/lightningd/chaintopology.c b/lightningd/chaintopology.c index b153b8afd881..aa4a9598c276 100644 --- a/lightningd/chaintopology.c +++ b/lightningd/chaintopology.c @@ -92,8 +92,8 @@ static void connect_block(struct chain_topology *topo, } b->full_txs = tal_free(b->full_txs); - /* Tell peers about new block. */ - notify_new_block(topo, b->height); + /* Tell lightningd about new block. */ + notify_new_block(topo->bitcoind->ld, b->height); } static const struct bitcoin_tx *tx_in_block(const struct block *b, diff --git a/lightningd/chaintopology.h b/lightningd/chaintopology.h index f3a8c51392dc..16ebe7f982e6 100644 --- a/lightningd/chaintopology.h +++ b/lightningd/chaintopology.h @@ -156,7 +156,7 @@ void setup_topology(struct chain_topology *topology, struct txlocator *locate_tx(const void *ctx, const struct chain_topology *topo, const struct sha256_double *txid); -void notify_new_block(struct chain_topology *topo, unsigned int height); +void notify_new_block(struct lightningd *ld, unsigned int height); #if DEVELOPER void json_dev_broadcast(struct command *cmd, diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 3a3e1336e777..71268a82640b 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -32,12 +32,6 @@ char *bitcoin_datadir; -void notify_new_block(struct chain_topology *topo, u32 height); -void notify_new_block(struct chain_topology *topo, u32 height) -{ - /* FIXME */ -} - void db_resolve_invoice(struct lightningd *ld, const char *label); void db_resolve_invoice(struct lightningd *ld, diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 950b4d0dbbe3..249614372622 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -1425,3 +1425,9 @@ void peer_htlcs(const tal_t *ctx, fulfilled_htlcs, fulfilled_sides); } } + +void notify_new_block(struct lightningd *ld, u32 height) +{ + /* FIXME */ +} + From 1142c44c2932b93c94d6da51f62f5c9b0b0655f4 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 2 Nov 2017 10:24:00 +1030 Subject: [PATCH 0084/1428] lightningd: fail htlcs we offer if peer unresponsive after deadline. Signed-off-by: Rusty Russell --- lightningd/peer_control.c | 2 +- lightningd/peer_control.h | 2 ++ lightningd/peer_htlcs.c | 54 +++++++++++++++++++++++++++++++++++++-- tests/test_lightningd.py | 40 +++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index c1799ffc3b5c..22a9d307014d 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -176,7 +176,7 @@ void peer_fail_permanent(struct peer *peer, const u8 *msg TAKES) return; } -static void peer_fail_permanent_str(struct peer *peer, const char *str TAKES) +void peer_fail_permanent_str(struct peer *peer, const char *str TAKES) { /* Don't use tal_strdup, since we need tal_len */ u8 *msg = tal_dup_arr(peer, u8, (const u8 *)str, strlen(str) + 1, 0); diff --git a/lightningd/peer_control.h b/lightningd/peer_control.h index f22911657588..4025d06e611c 100644 --- a/lightningd/peer_control.h +++ b/lightningd/peer_control.h @@ -201,6 +201,8 @@ u8 *get_supported_local_features(const tal_t *ctx); PRINTF_FMT(2,3) void peer_fail_transient(struct peer *peer, const char *fmt,...); /* Peer has failed, give up on it. */ void peer_fail_permanent(struct peer *peer, const u8 *msg TAKES); +/* Version where we supply the reason string. */ +void peer_fail_permanent_str(struct peer *peer, const char *str TAKES); /* Permanent error, but due to internal problems, not peer. */ void peer_internal_error(struct peer *peer, const char *fmt, ...); diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 249614372622..00bf77bbb233 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -1426,8 +1426,58 @@ void peer_htlcs(const tal_t *ctx, } } -void notify_new_block(struct lightningd *ld, u32 height) +/* BOLT #2: + * + * For HTLCs we offer: the timeout deadline when we have to fail the channel + * and time it out on-chain. This is `G` blocks after the HTLC + * `cltv_expiry`; 1 block is reasonable. + */ +static u32 htlc_out_deadline(const struct htlc_out *hout) { - /* FIXME */ + return hout->cltv_expiry + 1; } +void notify_new_block(struct lightningd *ld, u32 height) +{ + bool removed; + + /* BOLT #2: + * + * A node ... MUST fail the channel if an HTLC which it offered is in + * either node's current commitment transaction past this timeout + * deadline. + */ + /* FIXME: use db to look this up in one go (earliest deadline per-peer) */ + do { + struct htlc_out *hout; + struct htlc_out_map_iter outi; + + removed = false; + + for (hout = htlc_out_map_first(&ld->htlcs_out, &outi); + hout; + hout = htlc_out_map_next(&ld->htlcs_out, &outi)) { + /* Not timed out yet? */ + if (height < htlc_out_deadline(hout)) + continue; + + /* Peer on chain already? */ + if (peer_on_chain(hout->key.peer)) + continue; + + /* Peer already failed, or we hit it? */ + if (hout->key.peer->error) + continue; + + peer_fail_permanent_str(hout->key.peer, + take(tal_fmt(hout, + "Offered HTLC %"PRIu64 + " %s cltv %u hit deadline", + hout->key.id, + htlc_state_name(hout->hstate), + hout->cltv_expiry))); + removed = true; + } + /* Iteration while removing is safe, but can skip entries! */ + } while (removed); +} diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index bf2ca90c6ca0..6f11c415b767 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -1595,6 +1595,46 @@ def test_forward_pad_fees_and_cltv(self): l1.rpc.sendpay(to_json(route), rhash) assert l3.rpc.listinvoice('test_forward_pad_fees_and_cltv')[0]['complete'] == True + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") + def test_htlc_out_timeout(self): + """Test that we drop onchain if the peer doesn't time out HTLC""" + + # HTLC 1->2, 1 fails after it's irrevocably committed, can't reconnect + disconnects = ['@WIRE_REVOKE_AND_ACK'] + l1 = self.node_factory.get_node(disconnect=disconnects, + options=['--no-reconnect']) + l2 = self.node_factory.get_node() + + l1.rpc.connect(l2.info['id'], 'localhost:{}'.format(l2.info['port'])) + chanid = self.fund_channel(l1, l2, 10**6) + + # Wait for route propagation. + bitcoind.rpc.generate(5) + l1.daemon.wait_for_logs(['Received channel_update for channel {}\(0\)' + .format(chanid), + 'Received channel_update for channel {}\(1\)' + .format(chanid)]) + + amt = 200000000 + inv = l2.rpc.invoice(amt, 'test_htlc_out_timeout', 'desc')['bolt11'] + assert l2.rpc.listinvoice('test_htlc_out_timeout')[0]['complete'] == False + + payfuture = self.executor.submit(l1.rpc.pay, inv); + + # l1 will drop to chain, not reconnect. + l1.daemon.wait_for_log('dev_disconnect: @WIRE_REVOKE_AND_ACK') + + # Takes 6 blocks to timeout (cltv-final + 1), but we also give grace period of 1 block. + bitcoind.rpc.generate(5 + 1) + assert not l1.daemon.is_in_log('hit deadline') + bitcoind.rpc.generate(1) + + l1.daemon.wait_for_log('Offered HTLC 0 SENT_ADD_ACK_REVOCATION cltv {} hit deadline'.format(bitcoind.rpc.getblockcount()-1)) + l1.daemon.wait_for_log('sendrawtx exit 0') + l1.bitcoin.rpc.generate(1) + l1.daemon.wait_for_log('-> ONCHAIND_OUR_UNILATERAL') + l2.daemon.wait_for_log('-> ONCHAIND_THEIR_UNILATERAL') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_disconnect(self): # These should all make us fail, and retry. From 8cef36cbd7f933466f56e697f53c2a3a7245edcd Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 2 Nov 2017 10:26:07 +1030 Subject: [PATCH 0085/1428] lightningd: fail htlcs we fulfill if peer unresponsive after deadline. Closes: #241 Signed-off-by: Rusty Russell --- lightningd/peer_htlcs.c | 63 ++++++++++++++++++++++++++++++++++++++-- tests/test_lightningd.py | 40 +++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 00bf77bbb233..81c596801730 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -289,9 +289,6 @@ static void fulfill_htlc(struct htlc_in *hin, const struct preimage *preimage) if (peer_state_on_chain(hin->key.peer->state)) { msg = towire_onchain_known_preimage(hin, preimage); } else { - /* FIXME: fail the peer if it doesn't tell us that htlc - * fulfill is committed before deadline. - */ msg = towire_channel_fulfill_htlc(hin, hin->key.id, preimage); } subd_send_msg(hin->key.peer->owner, take(msg)); @@ -1437,6 +1434,21 @@ static u32 htlc_out_deadline(const struct htlc_out *hout) return hout->cltv_expiry + 1; } +/* BOLT #2: + * + * For HTLCs we accept and have a preimage: the fulfillment deadline when we + * have to fail the channel and fulfill the HTLC onchain before its + * `cltv_expiry`. This is steps 4-7 above, which means a deadline of `2R+G+S` + * blocks before `cltv_expiry`; 7 blocks is reasonable. + */ +/* We approximate this, by using half the cltv_expiry_delta (3R+2G+2S), + * rounded up. */ +static u32 htlc_in_deadline(const struct lightningd *ld, + const struct htlc_in *hin) +{ + return hin->cltv_expiry - (ld->config.cltv_expiry_delta + 1)/2; +} + void notify_new_block(struct lightningd *ld, u32 height) { bool removed; @@ -1480,4 +1492,49 @@ void notify_new_block(struct lightningd *ld, u32 height) } /* Iteration while removing is safe, but can skip entries! */ } while (removed); + + + /* BOLT #2: + * + * A node MUST estimate a fulfillment deadline for each HTLC it is + * attempting to fulfill. A node ... MUST fail the connection if a + * HTLC it has fulfilled is in either node's current commitment + * transaction past this fulfillment deadline. + */ + do { + struct htlc_in *hin; + struct htlc_in_map_iter ini; + + removed = false; + + for (hin = htlc_in_map_first(&ld->htlcs_in, &ini); + hin; + hin = htlc_in_map_next(&ld->htlcs_in, &ini)) { + /* Not fulfilled? If overdue, that's their problem... */ + if (!hin->preimage) + continue; + + /* Not timed out yet? */ + if (height < htlc_in_deadline(ld, hin)) + continue; + + /* Peer on chain already? */ + if (peer_on_chain(hin->key.peer)) + continue; + + /* Peer already failed, or we hit it? */ + if (hin->key.peer->error) + continue; + + peer_fail_permanent_str(hin->key.peer, + take(tal_fmt(hin, + "Fulfilled HTLC %"PRIu64 + " %s cltv %u hit deadline", + hin->key.id, + htlc_state_name(hin->hstate), + hin->cltv_expiry))); + removed = true; + } + /* Iteration while removing is safe, but can skip entries! */ + } while (removed); } diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 6f11c415b767..6b8d39163ebb 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -1635,6 +1635,46 @@ def test_htlc_out_timeout(self): l1.daemon.wait_for_log('-> ONCHAIND_OUR_UNILATERAL') l2.daemon.wait_for_log('-> ONCHAIND_THEIR_UNILATERAL') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") + def test_htlc_in_timeout(self): + """Test that we drop onchain if the peer doesn't accept fulfilled HTLC""" + + # HTLC 1->2, 1 fails after 2 has sent committed the fulfill + disconnects = ['-WIRE_REVOKE_AND_ACK*2'] + l1 = self.node_factory.get_node(disconnect=disconnects, + options=['--no-reconnect']) + l2 = self.node_factory.get_node() + + l1.rpc.connect(l2.info['id'], 'localhost:{}'.format(l2.info['port'])) + chanid = self.fund_channel(l1, l2, 10**6) + + # Wait for route propagation. + bitcoind.rpc.generate(5) + l1.daemon.wait_for_logs(['Received channel_update for channel {}\(0\)' + .format(chanid), + 'Received channel_update for channel {}\(1\)' + .format(chanid)]) + + amt = 200000000 + inv = l2.rpc.invoice(amt, 'test_htlc_in_timeout', 'desc')['bolt11'] + assert l2.rpc.listinvoice('test_htlc_in_timeout')[0]['complete'] == False + + payfuture = self.executor.submit(l1.rpc.pay, inv); + + # l1 will drop to chain, not reconnect. + l1.daemon.wait_for_log('dev_disconnect: -WIRE_REVOKE_AND_ACK') + + # Deadline HTLC expity minus 1/2 cltv-expiry delta (rounded up) (== cltv - 3). ctlv is 5+1. + bitcoind.rpc.generate(2) + assert not l2.daemon.is_in_log('hit deadline') + bitcoind.rpc.generate(1) + + l2.daemon.wait_for_log('Fulfilled HTLC 0 SENT_REMOVE_COMMIT cltv {} hit deadline'.format(bitcoind.rpc.getblockcount()+3)) + l2.daemon.wait_for_log('sendrawtx exit 0') + l2.bitcoin.rpc.generate(1) + l2.daemon.wait_for_log('-> ONCHAIND_OUR_UNILATERAL') + l1.daemon.wait_for_log('-> ONCHAIND_THEIR_UNILATERAL') + @unittest.skipIf(not DEVELOPER, "needs DEVELOPER=1") def test_disconnect(self): # These should all make us fail, and retry. From 2543bc81b98ec4ca5c568947c5175dfe4d95b5e9 Mon Sep 17 00:00:00 2001 From: Jonas Nick Date: Fri, 3 Nov 2017 23:46:51 +0000 Subject: [PATCH 0086/1428] Fix docker tag in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index eda801b08ca9..c84c57df3545 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ sudo docker run \ -v $HOME/.lightning:/root/.lightning \ -v $HOME/.bitcoin:/root/.bitcoin \ -p 9735:9735 \ - cdecker/lightningd:master + cdecker/lightningd:latest ``` ### Starting `lightningd` @@ -104,7 +104,7 @@ The route contains the path that the payment will take throught the Lightning Ne ``` route=$(cli/lightning-cli getroute 1 | jq --raw-output .route -) -cli/lightning-cli sendpay $route +cli/lightning-cli sendpay "$route" ``` Notice that in the first step we stored the route in a variable and reused it in the second step. From 62ca15d6aa2f73febd1602fc02e64c8a4450a13c Mon Sep 17 00:00:00 2001 From: Konstantin Nick Date: Sun, 5 Nov 2017 13:38:15 +0100 Subject: [PATCH 0087/1428] Fix python-api example --- contrib/pylightning/lightning/lightning.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/contrib/pylightning/lightning/lightning.py b/contrib/pylightning/lightning/lightning.py index 4f245dd37d13..a11d0dccb6d8 100644 --- a/contrib/pylightning/lightning/lightning.py +++ b/contrib/pylightning/lightning/lightning.py @@ -108,16 +108,14 @@ def dev_add_route(self, src, dst, base, var, delay, minblocks): if __name__ == "__main__": l1 = LightningRpc("/tmp/lightning1/lightning-rpc") - l1.connect_rpc() l5 = LightningRpc("/tmp/lightning5/lightning-rpc") - l5.connect_rpc() import random info5 = l5.getinfo() print(info5) - invoice = l5.invoice(100, "lbl{}".format(random.random())) + invoice = l5.invoice(100, "lbl{}".format(random.random()), "testpayment") print(invoice) route = l1.getroute(info5['id'], 100, 1) print(route) - print(l1.sendpay(route, invoice['rhash'])) + print(l1.sendpay(route['route'], invoice['rhash'])) From 77789bb705ae648835da40591d02e0ba318d8c71 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Wed, 1 Nov 2017 11:40:23 +1030 Subject: [PATCH 0088/1428] db: Implemented poor mans nested transactions Nesting is provided by only actually performing the outermost transaction and simulating the nested ones. This still allows us to ensure on lower levels that we are in the context of a transaction without having to resort to keeping explicitly track of it in the calling code. Signed-off-by: Christian Decker --- wallet/db.c | 15 +++++++++------ wallet/db.h | 8 +++++--- wallet/wallet.c | 39 +++++++++++++++++++++++---------------- 3 files changed, 37 insertions(+), 25 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index e6fe5db4cabf..5aec8358dd4f 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -227,12 +227,15 @@ static void close_db(struct db *db) { sqlite3_close(db->sql); } bool db_begin_transaction(struct db *db) { - assert(!db->in_transaction); - /* Clear any errors from previous transactions and - * non-transactional queries */ - db_clear_error(db); - db->in_transaction = db_exec(__func__, db, "BEGIN TRANSACTION;"); - return db->in_transaction; + if (!db->in_transaction) { + /* Clear any errors from previous transactions and + * non-transactional queries */ + db_clear_error(db); + db->in_transaction = db_exec(__func__, db, "BEGIN TRANSACTION;"); + assert(db->in_transaction); + return db->in_transaction; + } + return false; } bool db_commit_transaction(struct db *db) diff --git a/wallet/db.h b/wallet/db.h index 9064bb21d356..3c84f8a15e50 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -44,9 +44,11 @@ bool PRINTF_FMT(3, 4) /** * db_begin_transaction - Begin a transaction * - * We do not support nesting multiple transactions, so make sure that - * we are not in a transaction when calling this. Returns true if we - * succeeded in starting a transaction. + * Begin a new DB transaction if we aren't already in one. Returns + * true if the call started a transaction, i.e., the caller MUST take + * care to either commit or rollback. If false, this is a nested + * transaction and the caller MUST not commit/rollback, since the + * transaction is handled at a higher level in the callstack. */ bool db_begin_transaction(struct db *db); diff --git a/wallet/wallet.c b/wallet/wallet.c index 2df4edfe6d96..eaa668204410 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -139,6 +139,7 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, u64 *fee_estimate, u64 *changesatoshi) { size_t i = 0; + bool should_commit; struct utxo **available; const struct utxo **utxos = tal_arr(ctx, const struct utxo *, 0); *fee_estimate = 0; @@ -147,9 +148,8 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, u64 satoshi_in = 0, weight = (4 + (8 + 22) * 2 + 4) * 4; tal_add_destructor2(utxos, destroy_utxos, w); - if (!db_begin_transaction(w->db)) { - fatal("Unable to begin transaction: %s", w->db->err); - } + should_commit = db_begin_transaction(w->db); + available = wallet_get_utxos(ctx, w, output_state_available); for (i = 0; i < tal_count(available); i++) { @@ -177,10 +177,12 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, if (satoshi_in < *fee_estimate + value) { /* Could not collect enough inputs, cleanup and bail */ utxos = tal_free(utxos); - db_rollback_transaction(w->db); + if (should_commit) + db_rollback_transaction(w->db); } else { /* Commit the db transaction to persist markings */ - db_commit_transaction(w->db); + if (should_commit) + db_commit_transaction(w->db); *changesatoshi = satoshi_in - value - *fee_estimate; } @@ -274,6 +276,7 @@ bool wallet_shachain_add_hash(struct wallet *wallet, const struct sha256 *hash) { sqlite3_stmt *stmt; + bool should_commit; bool ok = true; u32 pos = count_trailing_zeroes(index); assert(index < SQLITE_MAX_UINT); @@ -281,7 +284,7 @@ bool wallet_shachain_add_hash(struct wallet *wallet, return false; } - db_begin_transaction(wallet->db); + should_commit = db_begin_transaction(wallet->db); stmt = db_prepare(wallet->db, "UPDATE shachains SET num_valid=?, min_index=? WHERE id=?"); sqlite3_bind_int(stmt, 1, chain->chain.num_valid); @@ -298,10 +301,12 @@ bool wallet_shachain_add_hash(struct wallet *wallet, sqlite3_bind_blob(stmt, 4, hash, sizeof(*hash), SQLITE_TRANSIENT); ok &= db_exec_prepared(wallet->db, stmt); - if (ok) - ok &= db_commit_transaction(wallet->db); - else - db_rollback_transaction(wallet->db); + if (should_commit) { + if (ok) + ok &= db_commit_transaction(wallet->db); + else + db_rollback_transaction(wallet->db); + } return ok; } @@ -625,12 +630,12 @@ bool wallet_channel_config_load(struct wallet *w, const u64 id, } bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ - bool ok = true; + bool should_commit, ok = true; struct peer *p = chan->peer; tal_t *tmpctx = tal_tmpctx(w); sqlite3_stmt *stmt; - db_begin_transaction(w->db); + should_commit = db_begin_transaction(w->db); if (p->dbid == 0) { /* Need to store the peer first */ @@ -751,10 +756,12 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ ok &= db_exec_prepared(w->db, stmt); } - if (ok) - ok &= db_commit_transaction(w->db); - else - db_rollback_transaction(w->db); + if (should_commit) { + if (ok) + ok &= db_commit_transaction(w->db); + else + db_rollback_transaction(w->db); + } tal_free(tmpctx); return ok; } From 5e46af64fcfbaa9b0daf5583eb603922beca4c98 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0089/1428] db: keep in_transaction as a counter, so we can nest commits. Otherwise we find ourselves outside a commitment. This is a bandaid until we remove nested commitments again at the end of this series. Signed-off-by: Rusty Russell --- wallet/db.c | 5 +++-- wallet/db.h | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index 5aec8358dd4f..8151241a49e4 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -235,6 +235,7 @@ bool db_begin_transaction(struct db *db) assert(db->in_transaction); return db->in_transaction; } + db->in_transaction++; return false; } @@ -242,7 +243,7 @@ bool db_commit_transaction(struct db *db) { assert(db->in_transaction); bool ret = db_exec(__func__, db, "COMMIT;"); - db->in_transaction = false; + db->in_transaction--; return ret; } @@ -250,7 +251,7 @@ bool db_rollback_transaction(struct db *db) { assert(db->in_transaction); bool ret = db_exec(__func__, db, "ROLLBACK;"); - db->in_transaction = false; + db->in_transaction--; return ret; } diff --git a/wallet/db.h b/wallet/db.h index 3c84f8a15e50..d5d7da1df353 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -15,7 +15,7 @@ struct db { char *filename; - bool in_transaction; + unsigned int in_transaction; const char *err; sqlite3 *sql; }; From 360aa15e4d9c6b98f852305845402467ea32014c Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0090/1428] db_query: don't remove transaction or set error if query fails. We return NULL in this case. Signed-off-by: Rusty Russell --- wallet/db.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index 8151241a49e4..88a63de92a0f 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -203,7 +203,6 @@ sqlite3_stmt *PRINTF_FMT(3, 4) va_list ap; char *query; sqlite3_stmt *stmt; - int err; if (db->in_transaction && db->err) return NULL; @@ -214,12 +213,8 @@ sqlite3_stmt *PRINTF_FMT(3, 4) query = tal_vfmt(db, fmt, ap); va_end(ap); - err = sqlite3_prepare_v2(db->sql, query, -1, &stmt, NULL); - if (err != SQLITE_OK) { - db->in_transaction = false; - db->err = tal_fmt(db, "%s:%s:%s:%s", caller, - sqlite3_errstr(err), query, sqlite3_errmsg(db->sql)); - } + /* Sets stmt to NULL if not SQLITE_OK */ + sqlite3_prepare_v2(db->sql, query, -1, &stmt, NULL); return stmt; } From 1f7e370fda3976cd22de5a851dcc6de5560d7156 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0091/1428] db: rollback transaction if we had an error. This is temporary; we'll eventually fail on error. However, since db_exec() is a NOOP if we have an error, we need to do something. --- wallet/db.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/wallet/db.c b/wallet/db.c index 88a63de92a0f..7036ae63928f 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -236,8 +236,24 @@ bool db_begin_transaction(struct db *db) bool db_commit_transaction(struct db *db) { + bool ret; + assert(db->in_transaction); - bool ret = db_exec(__func__, db, "COMMIT;"); + if (db->err) { + char *errmsg; + int err; + + /* Do this manually: db_exec is a NOOP with db->err */ + err = sqlite3_exec(db->sql, "ROLLBACK;", NULL, NULL, &errmsg); + if (err != SQLITE_OK) { + db->err = tal_fmt(db, "%s then ROLLBACK failed:%s:%s", + db->err, sqlite3_errstr(err), errmsg); + sqlite3_free(errmsg); + } + ret = false; + } else { + ret = db_exec(__func__, db, "COMMIT;"); + } db->in_transaction--; return ret; } From b148b89bafc2eb37571e53183c7476624898637e Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0092/1428] db_exec_mayfail: variant of db_exec where we actually expect an error. There's one caller where db_exec can actually fail due to constraints, and we rely on it. Signed-off-by: Rusty Russell --- wallet/db.c | 17 +++++++++++++++++ wallet/db.h | 9 +++++++++ 2 files changed, 26 insertions(+) diff --git a/wallet/db.c b/wallet/db.c index 7036ae63928f..f5cb2a9e6ec9 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -197,6 +197,23 @@ bool PRINTF_FMT(3, 4) return true; } +bool db_exec_prepared_mayfail_(const char *caller, struct db *db, sqlite3_stmt *stmt) +{ + if (db->in_transaction && db->err) { + goto fail; + } + + if (sqlite3_step(stmt) != SQLITE_DONE) { + goto fail; + } + + sqlite3_finalize(stmt); + return true; +fail: + sqlite3_finalize(stmt); + return false; +} + sqlite3_stmt *PRINTF_FMT(3, 4) db_query(const char *caller, struct db *db, const char *fmt, ...) { diff --git a/wallet/db.h b/wallet/db.h index d5d7da1df353..a5c047c9861b 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -112,6 +112,15 @@ sqlite3_stmt *db_prepare_(const char *caller, struct db *db, const char *query); #define db_exec_prepared(db,stmt) db_exec_prepared_(__func__,db,stmt) bool db_exec_prepared_(const char *caller, struct db *db, sqlite3_stmt *stmt); +/** + * db_exec_prepared_mayfail - db_exec_prepared, but don't set db->err if it fails. + */ +#define db_exec_prepared_mayfail(db,stmt) \ + db_exec_prepared_mayfail_(__func__,db,stmt) +bool db_exec_prepared_mayfail_(const char *caller, + struct db *db, + sqlite3_stmt *stmt); + bool sqlite3_bind_short_channel_id(sqlite3_stmt *stmt, int col, const struct short_channel_id *id); bool sqlite3_column_short_channel_id(sqlite3_stmt *stmt, int col, From bbe7a03300fc8b4358fd13a52334d4e6be15b52b Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0093/1428] wallet: use db_exec_mayfail() for wallet_add_utxo. This is the only case where we actually rely on the db to ensure we don't do something twice: don't error out if it fails. Signed-off-by: Rusty Russell --- wallet/wallet.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wallet/wallet.c b/wallet/wallet.c index eaa668204410..4c18035aa59b 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -24,6 +24,7 @@ struct wallet *wallet_new(const tal_t *ctx, struct log *log) return wallet; } +/* We actually use the db constraints to uniquify, so OK if this fails. */ bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, enum wallet_output_type type) { @@ -36,7 +37,7 @@ bool wallet_add_utxo(struct wallet *w, struct utxo *utxo, sqlite3_bind_int(stmt, 4, type); sqlite3_bind_int(stmt, 5, output_state_available); sqlite3_bind_int(stmt, 6, utxo->keyindex); - return db_exec_prepared(w->db, stmt); + return db_exec_prepared_mayfail(w->db, stmt); } /** From 3a596d6dda8ba4d688bef0f750a2f2fe7e20d024 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0094/1428] subd: wrap all message callbacks in a transaction. Including destructors. Signed-off-by: Rusty Russell --- lightningd/subd.c | 59 +++++++++++++++++++++++++++++------------------ lightningd/subd.h | 8 +++---- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/lightningd/subd.c b/lightningd/subd.c index 09c659f0b6b7..95d6703357cf 100644 --- a/lightningd/subd.c +++ b/lightningd/subd.c @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -411,20 +412,26 @@ static struct io_plan *sd_msg_read(struct io_conn *conn, struct subd *sd) int type = fromwire_peektype(sd->msg_in); const tal_t *tmpctx; struct subd_req *sr; + struct db *db = sd->ld->wallet->db; + struct io_plan *plan; - if (type == -1) { - subdaemon_malformed_msg(sd, sd->msg_in); - return io_close(conn); - } + /* Everything we do, we wrap in a database transaction */ + db_begin_transaction(db); + + if (type == -1) + goto malformed; /* First, check for replies. */ sr = get_req(sd, type); if (sr) { - if (sr->num_reply_fds && sd->fds_in == NULL) - return sd_collect_fds(conn, sd, sr->num_reply_fds); + if (sr->num_reply_fds && sd->fds_in == NULL) { + plan = sd_collect_fds(conn, sd, sr->num_reply_fds); + goto out; + } assert(sr->num_reply_fds == tal_count(sd->fds_in)); - return sd_msg_reply(conn, sd, sr); + plan = sd_msg_reply(conn, sd, sr); + goto out; } /* If not stolen, we'll free this below. */ @@ -434,24 +441,18 @@ static struct io_plan *sd_msg_read(struct io_conn *conn, struct subd *sd) if (type == STATUS_TRACE) { int str_len; const char *str = string_from_msg(sd->msg_in, &str_len); - if (!str) { - subdaemon_malformed_msg(sd, sd->msg_in); - return io_close(conn); - } + if (!str) + goto malformed; log_debug(sd->log, "TRACE: %.*s", str_len, str); goto next; } else if (type & STATUS_FAIL) { int str_len; const char *str = string_from_msg(sd->msg_in, &str_len); - if (!str) { - subdaemon_malformed_msg(sd, sd->msg_in); - return io_close(conn); - } + if (!str) + goto malformed; - if (!log_status_fail(sd, type, str, str_len)) { - subdaemon_malformed_msg(sd, sd->msg_in); - return io_close(conn); - } + if (!log_status_fail(sd, type, str, str_len)) + goto malformed; /* If they care, tell them about invalid peer behavior */ if (sd->peer && type == STATUS_FAIL_PEER_BAD) { @@ -464,7 +465,7 @@ static struct io_plan *sd_msg_read(struct io_conn *conn, struct subd *sd) (u8 *)str, str_len, 0))); } - return io_close(conn); + goto close; } log_info(sd->log, "UPDATE %s", sd->msgname(type)); @@ -478,7 +479,7 @@ static struct io_plan *sd_msg_read(struct io_conn *conn, struct subd *sd) i = sd->msgcb(sd, sd->msg_in, sd->fds_in); if (freed) - return io_close(conn); + goto close; tal_del_destructor2(sd, mark_freed, &freed); sd->conn = conn; @@ -489,7 +490,8 @@ static struct io_plan *sd_msg_read(struct io_conn *conn, struct subd *sd) /* Don't free msg_in: we go around again. */ tal_steal(sd, sd->msg_in); tal_free(tmpctx); - return sd_collect_fds(conn, sd, i); + plan = sd_collect_fds(conn, sd, i); + goto out; } } @@ -497,9 +499,20 @@ static struct io_plan *sd_msg_read(struct io_conn *conn, struct subd *sd) sd->msg_in = NULL; sd->fds_in = tal_free(sd->fds_in); tal_free(tmpctx); - return io_read_wire(conn, sd, &sd->msg_in, sd_msg_read, sd); + + plan = io_read_wire(conn, sd, &sd->msg_in, sd_msg_read, sd); + goto out; + +malformed: + subdaemon_malformed_msg(sd, sd->msg_in); +close: + plan = io_close(conn); +out: + db_commit_transaction(db); + return plan; } + static void destroy_subd(struct subd *sd) { int status; diff --git a/lightningd/subd.h b/lightningd/subd.h index ac24353c2800..2fe03c7dee11 100644 --- a/lightningd/subd.h +++ b/lightningd/subd.h @@ -31,7 +31,7 @@ struct subd { /* For logging */ struct log *log; - /* Callback when non-reply message comes in. */ + /* Callback when non-reply message comes in (inside db transaction) */ unsigned (*msgcb)(struct subd *, const u8 *, const int *); const char *(*msgname)(int msgtype); @@ -57,7 +57,7 @@ struct subd { * @ld: global state * @name: basename of daemon * @msgname: function to get name from messages - * @msgcb: function to call when non-fatal message received (or NULL) + * @msgcb: function to call (inside db transaction) when non-fatal message received (or NULL) * @...: NULL-terminated list of pointers to fds to hand as fd 3, 4... * (can be take, if so, set to -1) * @@ -78,7 +78,7 @@ struct subd *new_global_subd(struct lightningd *ld, * @name: basename of daemon * @peer: peer to associate. * @msgname: function to get name from messages - * @msgcb: function to call when non-fatal message received (or NULL) + * @msgcb: function to call (inside db transaction) when non-fatal message received (or NULL) * @...: NULL-terminated list of pointers to fds to hand as fd 3, 4... * (can be take, if so, set to -1) * @@ -122,7 +122,7 @@ void subd_send_fd(struct subd *sd, int fd); * @msg_out: request message (can be take) * @fd_out: if >=0 fd to pass at the end of the message (closed after) * @num_fds_in: how many fds to read in to hand to @replycb if it's a reply. - * @replycb: callback when reply comes in (can free subd) + * @replycb: callback (inside db transaction) when reply comes in (can free subd) * @replycb_data: final arg to hand to @replycb * * @replycb cannot free @sd, so it returns false to remove it. From 01361ab21b5193ab3fa0529ece7a474e8168e710 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0095/1428] jsonrpc: wrap all calls in transaction. --- lightningd/jsonrpc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lightningd/jsonrpc.c b/lightningd/jsonrpc.c index 318814b35fdc..7720fc562905 100644 --- a/lightningd/jsonrpc.c +++ b/lightningd/jsonrpc.c @@ -506,7 +506,9 @@ static void parse_request(struct json_connection *jcon, const jsmntok_t tok[]) return; } + db_begin_transaction(jcon->ld->wallet->db); cmd->dispatch(jcon->current, jcon->buffer, params); + db_commit_transaction(jcon->ld->wallet->db); } static struct io_plan *write_json(struct io_conn *conn, From 7586f3ed54c171cd063de8f2b7439b5500d0e9d8 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0096/1428] timers: wrap all calls in transactions. Signed-off-by: Rusty Russell --- lightningd/lightningd.c | 5 ++++- lightningd/test/run-find_my_path.c | 9 +++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index 71268a82640b..d237b9adfed2 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -309,8 +309,11 @@ int main(int argc, char *argv[]) if (v == ld) break; - if (expired) + if (expired) { + db_begin_transaction(ld->wallet->db); timer_expired(ld, expired); + db_commit_transaction(ld->wallet->db); + } } shutdown_subdaemons(ld); diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 8372232e27d3..8700b06d3adc 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -7,9 +7,18 @@ int unused_main(int argc, char *argv[]); /* Generated stub for crashlog_activate */ void crashlog_activate(const char *argv0 UNNEEDED, struct log *log UNNEEDED) { fprintf(stderr, "crashlog_activate called!\n"); abort(); } +/* Generated stub for db_begin_transaction */ +bool db_begin_transaction(struct db *db UNNEEDED) +{ fprintf(stderr, "db_begin_transaction called!\n"); abort(); } +/* Generated stub for db_commit_transaction */ +bool db_commit_transaction(struct db *db UNNEEDED) +{ fprintf(stderr, "db_commit_transaction called!\n"); abort(); } /* Generated stub for debug_poll */ int debug_poll(struct pollfd *fds UNNEEDED, nfds_t nfds UNNEEDED, int timeout UNNEEDED) { fprintf(stderr, "debug_poll called!\n"); abort(); } +/* Generated stub for fatal */ +void fatal(const char *fmt UNNEEDED, ...) +{ fprintf(stderr, "fatal called!\n"); abort(); } /* Generated stub for gossip_init */ void gossip_init(struct lightningd *ld UNNEEDED) { fprintf(stderr, "gossip_init called!\n"); abort(); } From 822976943868cf52231a96f84984a9d6fe0c38fe Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0097/1428] lightningd: do initial database load within a transaction. Safest, and we can then assert that all db calls are in transactions. Signed-off-by: Rusty Russell --- lightningd/lightningd.c | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index d237b9adfed2..f14224aa11f1 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -255,16 +255,19 @@ int main(int argc, char *argv[]) /* Now we know our ID, we can set our color/alias if not already. */ setup_color_and_alias(ld); - /* Initialize block topology. */ + /* Initialize block topology (does its own transaction) */ setup_topology(ld->topology, &ld->timers, ld->config.poll_time, /* FIXME: Load from peers. */ 0); + /* Everything is within a transaction. */ + db_begin_transaction(ld->wallet->db); + /* Load invoices from the database */ if (!wallet_invoices_load(ld->wallet, ld->invoices)) { - err(1, "Could not load invoices from the database"); + fatal("Could not load invoices from the database"); } /* Set up gossip daemon. */ @@ -282,12 +285,14 @@ int main(int argc, char *argv[]) peer->owner = NULL; if (!wallet_htlcs_load_for_channel(ld->wallet, peer->channel, &ld->htlcs_in, &ld->htlcs_out)) { - err(1, "could not load htlcs for channel: %s", ld->wallet->db->err); + fatal("could not load htlcs for channel"); } } - if (!wallet_htlcs_reconnect(ld->wallet, &ld->htlcs_in, &ld->htlcs_out)) { - errx(1, "could not reconnect htlcs loaded from wallet, wallet may be inconsistent."); - } + if (!wallet_htlcs_reconnect(ld->wallet, &ld->htlcs_in, &ld->htlcs_out)) + fatal("could not reconnect htlcs loaded from wallet, wallet may be inconsistent."); + + db_commit_transaction(ld->wallet->db); + /* Create RPC socket (if any) */ setup_jsonrpc(ld, ld->rpc_filename); From bccd2f8cf47333ac71c2509598df3c225c090170 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0098/1428] bitcoind: wrap callbacks in transaction. Signed-off-by: Rusty Russell --- lightningd/bitcoind.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c index bab699080ff5..4a63bd209d37 100644 --- a/lightningd/bitcoind.c +++ b/lightningd/bitcoind.c @@ -151,7 +151,9 @@ static void bcli_finished(struct io_conn *conn, struct bitcoin_cli *bcli) if (bitcoind->shutdown) return; + db_begin_transaction(bitcoind->ld->wallet->db); bcli->process(bcli); + db_commit_transaction(bitcoind->ld->wallet->db); next_bcli(bitcoind); } From 1eb7e92a30c6495ade5ccda96457bb2b29c0b132 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0099/1428] db_migrate: get version inside transaction. we should never be doing two startups at once, but why take chances? Plus, we can then assert that all db calls are in transactions. Signed-off-by: Rusty Russell --- wallet/db.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index f5cb2a9e6ec9..1f50fc2a0c91 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -366,14 +366,16 @@ static int db_migration_count(void) static bool db_migrate(struct db *db) { /* Attempt to read the version from the database */ - int current = db_get_version(db); - int available = db_migration_count(); + int current, available; if (!db_begin_transaction(db)) { /* No need to rollback, we didn't even start... */ return false; } + current = db_get_version(db); + available = db_migration_count(); + while (++current <= available) { if (!db_exec(__func__, db, "%s", dbmigrations[current])) goto fail; From 32827683021d699952748cd6d925e256a2abeef8 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0100/1428] wallet: don't fail, assume db ops will call fatal. And override fatal() in wallet_tests to be sure. Signed-off-by: Rusty Russell --- wallet/db.c | 29 ++++++----------------------- wallet/db_tests.c | 4 ++-- wallet/wallet.c | 3 --- wallet/wallet_tests.c | 30 ++++++++++++++++++++++++++---- 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index 1f50fc2a0c91..1d054db6f601 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -363,47 +363,30 @@ static int db_migration_count(void) /** * db_migrate - Apply all remaining migrations from the current version */ -static bool db_migrate(struct db *db) +static void db_migrate(struct db *db) { /* Attempt to read the version from the database */ int current, available; - if (!db_begin_transaction(db)) { - /* No need to rollback, we didn't even start... */ - return false; - } + db_begin_transaction(db); current = db_get_version(db); available = db_migration_count(); - while (++current <= available) { - if (!db_exec(__func__, db, "%s", dbmigrations[current])) - goto fail; - } + while (++current <= available) + db_exec(__func__, db, "%s", dbmigrations[current]); /* Finally update the version number in the version table */ db_exec(__func__, db, "UPDATE version SET version=%d;", available); - if (!db_commit_transaction(db)) { - goto fail; - } - - return true; -fail: - db_rollback_transaction(db); - return false; + db_commit_transaction(db); } struct db *db_setup(const tal_t *ctx) { struct db *db = db_open(ctx, DB_FILE); - if (!db) { - return db; - } - if (!db_migrate(db)) { - return tal_free(db); - } + db_migrate(db); return db; } diff --git a/wallet/db_tests.c b/wallet/db_tests.c index ee5d2568f7e5..83feee23c1f9 100644 --- a/wallet/db_tests.c +++ b/wallet/db_tests.c @@ -24,7 +24,7 @@ static bool test_empty_db_migrate(void) struct db *db = create_test_db(__func__); CHECK(db); CHECK(db_get_version(db) == -1); - CHECK(db_migrate(db)); + db_migrate(db); CHECK(db_get_version(db) == db_migration_count()); tal_free(db); @@ -55,7 +55,7 @@ static bool test_vars(void) struct db *db = create_test_db(__func__); char *varname = "testvar"; CHECK(db); - CHECK(db_migrate(db)); + db_migrate(db); /* Check default behavior */ CHECK(db_get_intvar(db, varname, 42) == 42); diff --git a/wallet/wallet.c b/wallet/wallet.c index 4c18035aa59b..ed328394781a 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -18,9 +18,6 @@ struct wallet *wallet_new(const tal_t *ctx, struct log *log) wallet->db = db_setup(wallet); wallet->log = log; wallet->bip32_base = NULL; - if (!wallet->db) { - fatal("Unable to setup the wallet database"); - } return wallet; } diff --git a/wallet/wallet_tests.c b/wallet/wallet_tests.c index bba988e53726..f58a69b302ba 100644 --- a/wallet/wallet_tests.c +++ b/wallet/wallet_tests.c @@ -1,13 +1,32 @@ + #include + +static void wallet_fatal(const char *fmt, ...); +#define fatal wallet_fatal + #include "wallet.c" #include "db.c" #include -#include +#include +#include #include #include #include +static char *wallet_err; +static void wallet_fatal(const char *fmt, ...) +{ + va_list ap; + + /* Fail hard if we're complaining about not being in transaction */ + assert(!strstarts(fmt, "No longer in transaction")); + + va_start(ap, fmt); + wallet_err = tal_vfmt(NULL, fmt, ap); + va_end(ap); +} + void invoice_add(struct invoices *invs, struct invoice *inv){} @@ -37,7 +56,8 @@ static struct wallet *create_test_wallet(const tal_t *ctx) w->db = db_open(w, filename); CHECK_MSG(w->db, "Failed opening the db"); - CHECK_MSG(db_migrate(w->db), "DB migration failed"); + db_migrate(w->db); + CHECK_MSG(!wallet_err, "DB migration failed"); ltmp = tal_tmpctx(ctx); log_book = new_log_book(w, 20*1024*1024, LOG_DBG); @@ -57,7 +77,8 @@ static bool test_wallet_outputs(void) w->db = db_open(w, filename); CHECK_MSG(w->db, "Failed opening the db"); - CHECK_MSG(db_migrate(w->db), "DB migration failed"); + db_migrate(w->db); + CHECK_MSG(!wallet_err, "DB migration failed"); memset(&u, 0, sizeof(u)); @@ -108,7 +129,8 @@ static bool test_shachain_crud(void) w->db = db_open(w, filename); CHECK_MSG(w->db, "Failed opening the db"); - CHECK_MSG(db_migrate(w->db), "DB migration failed"); + db_migrate(w->db); + CHECK_MSG(!wallet_err, "DB migration failed"); CHECK_MSG(fd != -1, "Unable to generate temp filename"); close(fd); From 7133a2f9b3826037cdad5ead36092272f4d54bc9 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0101/1428] wallet: assume db errors will be fatal, don't check. Signed-off-by: Rusty Russell --- lightningd/invoice.c | 7 +-- lightningd/peer_control.c | 30 ++------- lightningd/peer_htlcs.c | 30 +++------ wallet/wallet.c | 124 +++++++++++++------------------------- wallet/wallet.h | 12 ++-- wallet/wallet_tests.c | 43 ++++++++----- 6 files changed, 91 insertions(+), 155 deletions(-) diff --git a/lightningd/invoice.c b/lightningd/invoice.c index 955de51803ab..835dedcd3a7f 100644 --- a/lightningd/invoice.c +++ b/lightningd/invoice.c @@ -168,12 +168,7 @@ static void json_invoice(struct command *cmd, return; } - if (!wallet_invoice_save(cmd->ld->wallet, invoice)) { - printf("Could not save the invoice to the database: %s", - cmd->ld->wallet->db->err); - command_fail(cmd, "database error"); - return; - } + wallet_invoice_save(cmd->ld->wallet, invoice); /* Construct bolt11 string. */ b11 = new_bolt11(cmd, &invoice->msatoshi); diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 22a9d307014d..5bd57a95d039 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -252,10 +252,7 @@ void peer_set_condition(struct peer *peer, enum peer_state old_state, if (peer_persists(peer)) { assert(peer->channel != NULL); /* TODO(cdecker) Selectively save updated fields to DB */ - if (!wallet_channel_save(peer->ld->wallet, peer->channel)) { - fatal("Could not save channel to database: %s", - peer->ld->wallet->db->err); - } + wallet_channel_save(peer->ld->wallet, peer->channel); } } @@ -447,9 +444,7 @@ static struct wallet_channel *peer_channel_new(struct wallet *w, wallet_peer_by_nodeid(w, &peer->id, peer); wc->id = 0; - if (!wallet_channel_save(w, wc)) { - fatal("Unable to save channel to database: %s", w->db->err); - } + wallet_channel_save(w, wc); return wc; } @@ -1668,10 +1663,7 @@ static void peer_got_shutdown(struct peer *peer, const u8 *msg) } /* TODO(cdecker) Selectively save updated fields to DB */ - if (!wallet_channel_save(peer->ld->wallet, peer->channel)) { - fatal("Could not save channel to database: %s", - peer->ld->wallet->db->err); - } + wallet_channel_save(peer->ld->wallet, peer->channel); } void peer_last_tx(struct peer *peer, struct bitcoin_tx *tx, @@ -1733,11 +1725,7 @@ static void peer_received_closing_signature(struct peer *peer, const u8 *msg) /* FIXME: Make sure signature is correct! */ if (better_closing_fee(peer, tx)) { /* TODO(cdecker) Selectively save updated fields to DB */ - if (!wallet_channel_save(peer->ld->wallet, peer->channel)) { - fatal("Could not save channel to database: %s", - peer->ld->wallet->db->err); - } - + wallet_channel_save(peer->ld->wallet, peer->channel); peer_last_tx(peer, tx, &sig); } @@ -2313,10 +2301,7 @@ static void peer_accept_channel(struct lightningd *ld, /* Store the channel in the database in order to get a channel * ID that is unique and which we can base the peer_seed on */ peer->channel = peer_channel_new(ld->wallet, peer); - if (!wallet_channel_save(peer->ld->wallet, peer->channel)) { - fatal("Could not save channel to database: %s", - peer->ld->wallet->db->err); - } + wallet_channel_save(peer->ld->wallet, peer->channel); peer->seed = tal(peer, struct privkey); derive_peer_seed(ld, peer->seed, &peer->id, peer->channel->id); @@ -2377,10 +2362,7 @@ static void peer_offer_channel(struct lightningd *ld, /* Store the channel in the database in order to get a channel * ID that is unique and which we can base the peer_seed on */ fc->peer->channel = peer_channel_new(ld->wallet, fc->peer); - if (!wallet_channel_save(fc->peer->ld->wallet, fc->peer->channel)) { - fatal("Could not save channel to database: %s", - fc->peer->ld->wallet->db->err); - } + wallet_channel_save(fc->peer->ld->wallet, fc->peer->channel); fc->peer->seed = tal(fc->peer, struct privkey); derive_peer_seed(ld, fc->peer->seed, &fc->peer->id, fc->peer->channel->id); diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 81c596801730..691739731190 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -56,8 +56,7 @@ static bool htlc_in_update_state(struct peer *peer, if (!state_update_ok(peer, hin->hstate, newstate, hin->key.id, "in")) return false; - if (!wallet_htlc_update(peer->ld->wallet, hin->dbid, newstate, hin->preimage)) - return false; + wallet_htlc_update(peer->ld->wallet, hin->dbid, newstate, hin->preimage); hin->hstate = newstate; htlc_in_check(hin, __func__); @@ -71,8 +70,7 @@ static bool htlc_out_update_state(struct peer *peer, if (!state_update_ok(peer, hout->hstate, newstate, hout->key.id, "out")) return false; - if (!wallet_htlc_update(peer->ld->wallet, hout->dbid, newstate, NULL)) - return false; + wallet_htlc_update(peer->ld->wallet, hout->dbid, newstate, NULL); hout->hstate = newstate; htlc_out_check(hout, __func__); @@ -929,12 +927,8 @@ static bool update_out_htlc(struct peer *peer, u64 id, enum htlc_state newstate) return false; } - if (!hout->dbid && !wallet_htlc_save_out(peer->ld->wallet, peer->channel, hout)) { - peer_internal_error( - peer, "Unable to save the htlc_out to the database: %s", - peer->ld->wallet->db->err); - return false; - } + if (!hout->dbid) + wallet_htlc_save_out(peer->ld->wallet, peer->channel, hout); if (!htlc_out_update_state(peer, hout, newstate)) return false; @@ -972,10 +966,7 @@ static bool peer_save_commitsig_received(struct peer *peer, u64 commitnum) peer->next_index[LOCAL]++; /* FIXME: Save to database, with sig and HTLCs. */ - if (!wallet_channel_save(peer->ld->wallet, peer->channel)) { - fatal("Could not save channel to database: %s", - peer->ld->wallet->db->err); - } + wallet_channel_save(peer->ld->wallet, peer->channel); return true; } @@ -992,11 +983,7 @@ static bool peer_save_commitsig_sent(struct peer *peer, u64 commitnum) peer->next_index[REMOTE]++; /* FIXME: Save to database, with sig and HTLCs. */ - if (!wallet_channel_save(peer->ld->wallet, peer->channel)) { - fatal("Could not save channel to database: %s", - peer->ld->wallet->db->err); - } - + wallet_channel_save(peer->ld->wallet, peer->channel); return true; } @@ -1293,10 +1280,7 @@ void peer_got_revoke(struct peer *peer, const u8 *msg) hin = find_htlc_in(&peer->ld->htlcs_in, peer, changed[i].id); local_fail_htlc(hin, failcodes[i]); } - if (!wallet_channel_save(peer->ld->wallet, peer->channel)) { - fatal("Could not save channel to database: %s", - peer->ld->wallet->db->err); - } + wallet_channel_save(peer->ld->wallet, peer->channel); } static void *tal_arr_append_(void **p, size_t size) diff --git a/wallet/wallet.c b/wallet/wallet.c index ed328394781a..f3e79f4ccf4d 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -137,7 +137,6 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, u64 *fee_estimate, u64 *changesatoshi) { size_t i = 0; - bool should_commit; struct utxo **available; const struct utxo **utxos = tal_arr(ctx, const struct utxo *, 0); *fee_estimate = 0; @@ -146,8 +145,9 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, u64 satoshi_in = 0, weight = (4 + (8 + 22) * 2 + 4) * 4; tal_add_destructor2(utxos, destroy_utxos, w); - should_commit = db_begin_transaction(w->db); - + if (!db_begin_transaction(w->db)) { + fatal("Unable to begin transaction: %s", w->db->err); + } available = wallet_get_utxos(ctx, w, output_state_available); for (i = 0; i < tal_count(available); i++) { @@ -175,12 +175,10 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, if (satoshi_in < *fee_estimate + value) { /* Could not collect enough inputs, cleanup and bail */ utxos = tal_free(utxos); - if (should_commit) - db_rollback_transaction(w->db); + db_rollback_transaction(w->db); } else { /* Commit the db transaction to persist markings */ - if (should_commit) - db_commit_transaction(w->db); + db_commit_transaction(w->db); *changesatoshi = satoshi_in - value - *fee_estimate; } @@ -274,21 +272,19 @@ bool wallet_shachain_add_hash(struct wallet *wallet, const struct sha256 *hash) { sqlite3_stmt *stmt; - bool should_commit; - bool ok = true; u32 pos = count_trailing_zeroes(index); assert(index < SQLITE_MAX_UINT); if (!shachain_add_hash(&chain->chain, index, hash)) { return false; } - should_commit = db_begin_transaction(wallet->db); + db_begin_transaction(wallet->db); stmt = db_prepare(wallet->db, "UPDATE shachains SET num_valid=?, min_index=? WHERE id=?"); sqlite3_bind_int(stmt, 1, chain->chain.num_valid); sqlite3_bind_int64(stmt, 2, index); sqlite3_bind_int64(stmt, 3, chain->id); - ok &= db_exec_prepared(wallet->db, stmt); + db_exec_prepared(wallet->db, stmt); stmt = db_prepare( wallet->db, @@ -297,15 +293,9 @@ bool wallet_shachain_add_hash(struct wallet *wallet, sqlite3_bind_int(stmt, 2, pos); sqlite3_bind_int64(stmt, 3, index); sqlite3_bind_blob(stmt, 4, hash, sizeof(*hash), SQLITE_TRANSIENT); - ok &= db_exec_prepared(wallet->db, stmt); + db_exec_prepared(wallet->db, stmt); - if (should_commit) { - if (ok) - ok &= db_commit_transaction(wallet->db); - else - db_rollback_transaction(wallet->db); - } - return ok; + return db_commit_transaction(wallet->db); } bool wallet_shachain_load(struct wallet *wallet, u64 id, @@ -569,15 +559,14 @@ bool wallet_channels_load_active(struct wallet *w, struct list_head *peers) return ok; } -bool wallet_channel_config_save(struct wallet *w, struct channel_config *cc) +void wallet_channel_config_save(struct wallet *w, struct channel_config *cc) { - bool ok = true; sqlite3_stmt *stmt; /* Is this an update? If not insert a stub first */ if (!cc->id) { stmt = db_prepare( w->db,"INSERT INTO channel_configs DEFAULT VALUES;"); - ok &= db_exec_prepared(w->db, stmt); + db_exec_prepared(w->db, stmt); cc->id = sqlite3_last_insert_rowid(w->db->sql); } @@ -596,9 +585,7 @@ bool wallet_channel_config_save(struct wallet *w, struct channel_config *cc) sqlite3_bind_int(stmt, 5, cc->to_self_delay); sqlite3_bind_int(stmt, 6, cc->max_accepted_htlcs); sqlite3_bind_int64(stmt, 7, cc->id); - ok &= db_exec_prepared(w->db, stmt); - - return ok; + db_exec_prepared(w->db, stmt); } bool wallet_channel_config_load(struct wallet *w, const u64 id, @@ -627,13 +614,12 @@ bool wallet_channel_config_load(struct wallet *w, const u64 id, return ok; } -bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ - bool should_commit, ok = true; +void wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ struct peer *p = chan->peer; tal_t *tmpctx = tal_tmpctx(w); sqlite3_stmt *stmt; - should_commit = db_begin_transaction(w->db); + db_begin_transaction(w->db); if (p->dbid == 0) { /* Need to store the peer first */ @@ -653,10 +639,10 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ /* Need to initialize the shachain first so we get an id */ if (p->their_shachain.id == 0) { - ok &= wallet_shachain_init(w, &p->their_shachain); + wallet_shachain_init(w, &p->their_shachain); } - ok &= wallet_channel_config_save(w, &p->our_config); + wallet_channel_config_save(w, &p->our_config); /* Now do the real update */ stmt = db_prepare(w->db, tal_fmt(w, "UPDATE channels SET" @@ -718,7 +704,7 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ db_exec_prepared(w->db, stmt); if (chan->peer->channel_info) { - ok &= wallet_channel_config_save(w, &p->channel_info->their_config); + wallet_channel_config_save(w, &p->channel_info->their_config); stmt = db_prepare(w->db, "UPDATE channels SET" " fundingkey_remote=?," " revocation_basepoint_remote=?," @@ -738,7 +724,7 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ sqlite3_bind_int(stmt, 7, p->channel_info->feerate_per_kw); sqlite3_bind_int64(stmt, 8, p->channel_info->their_config.id); sqlite3_bind_int64(stmt, 9, chan->id); - ok &= db_exec_prepared(w->db, stmt); + db_exec_prepared(w->db, stmt); } /* If we have a last_sent_commit, store it */ @@ -751,17 +737,13 @@ bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ sqlite3_bind_int(stmt, 1, p->last_sent_commit->newstate); sqlite3_bind_int64(stmt, 2, p->last_sent_commit->id); sqlite3_bind_int64(stmt, 3, chan->id); - ok &= db_exec_prepared(w->db, stmt); + db_exec_prepared(w->db, stmt); } - if (should_commit) { - if (ok) - ok &= db_commit_transaction(w->db); - else - db_rollback_transaction(w->db); - } + if (!db_commit_transaction(w->db)) + fatal("Could not save channel to database: %s", w->db->err); + tal_free(tmpctx); - return ok; } int wallet_extract_owned_outputs(struct wallet *w, const struct bitcoin_tx *tx, @@ -800,10 +782,9 @@ int wallet_extract_owned_outputs(struct wallet *w, const struct bitcoin_tx *tx, return num_utxos; } -bool wallet_htlc_save_in(struct wallet *wallet, +void wallet_htlc_save_in(struct wallet *wallet, const struct wallet_channel *chan, struct htlc_in *in) { - bool ok = true; tal_t *tmpctx = tal_tmpctx(wallet); sqlite3_stmt *stmt; @@ -839,19 +820,15 @@ bool wallet_htlc_save_in(struct wallet *wallet, sqlite3_bind_blob(stmt, 10, &in->onion_routing_packet, sizeof(in->onion_routing_packet), SQLITE_TRANSIENT); - ok = db_exec_prepared(wallet->db, stmt); + db_exec_prepared(wallet->db, stmt); + in->dbid = sqlite3_last_insert_rowid(wallet->db->sql); tal_free(tmpctx); - if (ok) { - in->dbid = sqlite3_last_insert_rowid(wallet->db->sql); - } - return ok; } -bool wallet_htlc_save_out(struct wallet *wallet, +void wallet_htlc_save_out(struct wallet *wallet, const struct wallet_channel *chan, struct htlc_out *out) { - bool ok = true; tal_t *tmpctx = tal_tmpctx(wallet); sqlite3_stmt *stmt; @@ -890,20 +867,16 @@ bool wallet_htlc_save_out(struct wallet *wallet, sqlite3_bind_blob(stmt, 10, &out->onion_routing_packet, sizeof(out->onion_routing_packet), SQLITE_TRANSIENT); - ok = db_exec_prepared(wallet->db, stmt); + db_exec_prepared(wallet->db, stmt); + out->dbid = sqlite3_last_insert_rowid(wallet->db->sql); tal_free(tmpctx); - if (ok) { - out->dbid = sqlite3_last_insert_rowid(wallet->db->sql); - } - return ok; } -bool wallet_htlc_update(struct wallet *wallet, const u64 htlc_dbid, +void wallet_htlc_update(struct wallet *wallet, const u64 htlc_dbid, const enum htlc_state new_state, const struct preimage *payment_key) { - bool ok = true; sqlite3_stmt *stmt; /* The database ID must be set by a previous call to @@ -919,8 +892,7 @@ bool wallet_htlc_update(struct wallet *wallet, const u64 htlc_dbid, if (payment_key) sqlite3_bind_preimage(stmt, 2, payment_key); - ok = db_exec_prepared(wallet->db, stmt); - return ok; + db_exec_prepared(wallet->db, stmt); } static bool wallet_stmt2htlc_in(const struct wallet_channel *channel, @@ -1090,7 +1062,7 @@ bool wallet_htlcs_reconnect(struct wallet *wallet, return true; } -bool wallet_invoice_save(struct wallet *wallet, struct invoice *inv) +void wallet_invoice_save(struct wallet *wallet, struct invoice *inv) { /* Need to use the lower level API of sqlite3 to bind * label. Otherwise we'd need to implement sanitization of @@ -1099,10 +1071,8 @@ bool wallet_invoice_save(struct wallet *wallet, struct invoice *inv) if (!inv->id) { stmt = db_prepare(wallet->db, "INSERT INTO invoices (payment_hash, payment_key, state, msatoshi, label) VALUES (?, ?, ?, ?, ?);"); - if (!stmt) { - log_broken(wallet->log, "Could not prepare statement: %s", wallet->db->err); - return false; - } + if (!stmt) + fatal("Could not prepare statement: %s", wallet->db->err); sqlite3_bind_blob(stmt, 1, &inv->rhash, sizeof(inv->rhash), SQLITE_TRANSIENT); sqlite3_bind_blob(stmt, 2, &inv->r, sizeof(inv->r), SQLITE_TRANSIENT); @@ -1110,30 +1080,21 @@ bool wallet_invoice_save(struct wallet *wallet, struct invoice *inv) sqlite3_bind_int64(stmt, 4, inv->msatoshi); sqlite3_bind_text(stmt, 5, inv->label, strlen(inv->label), SQLITE_TRANSIENT); - if (!db_exec_prepared(wallet->db, stmt)) { - log_broken(wallet->log, "Could not exec prepared statement: %s", wallet->db->err); - return false; - } + if (!db_exec_prepared(wallet->db, stmt)) + fatal("Could not exec prepared statement: %s", wallet->db->err); inv->id = sqlite3_last_insert_rowid(wallet->db->sql); - return true; } else { stmt = db_prepare(wallet->db, "UPDATE invoices SET state=? WHERE id=?;"); - if (!stmt) { - log_broken(wallet->log, "Could not prepare statement: %s", wallet->db->err); - return false; - } + if (!stmt) + fatal("Could not prepare statement: %s", wallet->db->err); sqlite3_bind_int(stmt, 1, inv->state); sqlite3_bind_int64(stmt, 2, inv->id); - if (!db_exec_prepared(wallet->db, stmt)) { - log_broken(wallet->log, "Could not exec prepared statement: %s", wallet->db->err); - return false; - } else { - return true; - } + if (!db_exec_prepared(wallet->db, stmt)) + fatal("Could not exec prepared statement: %s", wallet->db->err); } } @@ -1195,10 +1156,9 @@ struct htlc_stub *wallet_htlc_stubs(tal_t *ctx, struct wallet *wallet, "SELECT channel_id, direction, cltv_expiry, payment_hash " "FROM channel_htlcs WHERE channel_id = ?;"); - if (!stmt) { - log_broken(wallet->log, "Error preparing select: %s", wallet->db->err); - return NULL; - } + if (!stmt) + fatal("Error preparing select: %s", wallet->db->err); + sqlite3_bind_int64(stmt, 1, chan->id); stubs = tal_arr(ctx, struct htlc_stub, 0); diff --git a/wallet/wallet.h b/wallet/wallet.h index aeac1ad9695a..65924d07d812 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -179,12 +179,12 @@ bool wallet_channel_load(struct wallet *w, const u64 id, * @chan: the instance to store (not const so we can update the unique_id upon * insert) */ -bool wallet_channel_save(struct wallet *w, struct wallet_channel *chan); +void wallet_channel_save(struct wallet *w, struct wallet_channel *chan); /** * wallet_channel_config_save -- Upsert a channel_config into the database */ -bool wallet_channel_config_save(struct wallet *w, struct channel_config *cc); +void wallet_channel_config_save(struct wallet *w, struct channel_config *cc); /** * wallet_channel_config_load -- Load channel_config from database into cc @@ -237,7 +237,7 @@ int wallet_extract_owned_outputs(struct wallet *w, const struct bitcoin_tx *tx, * for state transitions or to set the `payment_key` for completed * HTLCs. */ -bool wallet_htlc_save_in(struct wallet *wallet, +void wallet_htlc_save_in(struct wallet *wallet, const struct wallet_channel *chan, struct htlc_in *in); /** @@ -245,7 +245,7 @@ bool wallet_htlc_save_in(struct wallet *wallet, * * See comment for wallet_htlc_save_in. */ -bool wallet_htlc_save_out(struct wallet *wallet, +void wallet_htlc_save_out(struct wallet *wallet, const struct wallet_channel *chan, struct htlc_out *out); @@ -262,7 +262,7 @@ bool wallet_htlc_save_out(struct wallet *wallet, * `struct htlc_out` and optionally set the `payment_key` should the * HTLC have been settled. */ -bool wallet_htlc_update(struct wallet *wallet, const u64 htlc_dbid, +void wallet_htlc_update(struct wallet *wallet, const u64 htlc_dbid, const enum htlc_state new_state, const struct preimage *payment_key); @@ -312,7 +312,7 @@ bool wallet_htlcs_reconnect(struct wallet *wallet, * @wallet: Wallet to store in * @inv: Invoice to save */ -bool wallet_invoice_save(struct wallet *wallet, struct invoice *inv); +void wallet_invoice_save(struct wallet *wallet, struct invoice *inv); /** * wallet_invoices_load -- Load all invoices into memory diff --git a/wallet/wallet_tests.c b/wallet/wallet_tests.c index f58a69b302ba..b88d6b555aaf 100644 --- a/wallet/wallet_tests.c +++ b/wallet/wallet_tests.c @@ -27,6 +27,9 @@ static void wallet_fatal(const char *fmt, ...) va_end(ap); } +#define transaction_wrap(db, ...) \ + (db_begin_transaction(db), __VA_ARGS__, db_commit_transaction(db)) + void invoice_add(struct invoices *invs, struct invoice *inv){} @@ -259,7 +262,9 @@ static bool test_channel_crud(const tal_t *ctx) ci.old_remote_per_commit = pk; /* Variant 1: insert with null for scid, funding_tx_id, channel_info, last_tx */ - CHECK_MSG(wallet_channel_save(w, &c1), tal_fmt(w, "Insert into DB: %s", w->db->err)); + wallet_channel_save(w, &c1); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Insert into DB: %s", w->db->err)); CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v1)"); @@ -270,7 +275,9 @@ static bool test_channel_crud(const tal_t *ctx) /* Variant 2: update with scid set */ c1.peer->scid = talz(w, struct short_channel_id); - CHECK_MSG(wallet_channel_save(w, &c1), tal_fmt(w, "Insert into DB: %s", w->db->err)); + wallet_channel_save(w, &c1); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Insert into DB: %s", w->db->err)); CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v2)"); @@ -281,32 +288,38 @@ static bool test_channel_crud(const tal_t *ctx) /* Variant 3: update with our_satoshi set */ c1.peer->our_msatoshi = &msat; - CHECK_MSG(wallet_channel_save(w, &c1), tal_fmt(w, "Insert into DB: %s", w->db->err)); + + wallet_channel_save(w, &c1); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v3)"); /* Variant 4: update with funding_tx_id */ c1.peer->funding_txid = hash; - CHECK_MSG(wallet_channel_save(w, &c1), tal_fmt(w, "Insert into DB: %s", w->db->err)); + wallet_channel_save(w, &c1); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v4)"); /* Variant 5: update with channel_info */ p.channel_info = &ci; - CHECK_MSG(wallet_channel_save(w, &c1), tal_fmt(w, "Insert into DB: %s", w->db->err)); + wallet_channel_save(w, &c1); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v5)"); /* Variant 6: update with last_commit_sent */ p.last_sent_commit = &last_commit; - CHECK_MSG(wallet_channel_save(w, &c1), tal_fmt(w, "Insert into DB: %s", w->db->err)); + wallet_channel_save(w, &c1); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v6)"); /* Variant 7: update with last_tx (taken from BOLT #3) */ p.last_tx = bitcoin_tx_from_hex(w, "02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8003a00f0000000000002200208c48d15160397c9731df9bc3b236656efb6665fbfe92b4a6878e88a499f741c4c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de843110ae8f6a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e040047304402206a2679efa3c7aaffd2a447fd0df7aba8792858b589750f6a1203f9259173198a022008d52a0e77a99ab533c36206cb15ad7aeb2aa72b93d4b571e728cb5ec2f6fe260147304402206d6cb93969d39177a09d5d45b583f34966195b77c7e585cf47ac5cce0c90cefb022031d71ae4e33a4e80df7f981d696fbdee517337806a3c7138b7491e2cbb077a0e01475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220", strlen("02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8003a00f0000000000002200208c48d15160397c9731df9bc3b236656efb6665fbfe92b4a6878e88a499f741c4c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de843110ae8f6a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e040047304402206a2679efa3c7aaffd2a447fd0df7aba8792858b589750f6a1203f9259173198a022008d52a0e77a99ab533c36206cb15ad7aeb2aa72b93d4b571e728cb5ec2f6fe260147304402206d6cb93969d39177a09d5d45b583f34966195b77c7e585cf47ac5cce0c90cefb022031d71ae4e33a4e80df7f981d696fbdee517337806a3c7138b7491e2cbb077a0e01475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220")); p.last_sig = sig; - CHECK_MSG(wallet_channel_save(w, &c1), tal_fmt(w, "Insert into DB: %s", w->db->err)); + wallet_channel_save(w, &c1); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v7)"); @@ -328,7 +341,7 @@ static bool test_channel_config_crud(const tal_t *ctx) cc1->to_self_delay = 5; cc1->max_accepted_htlcs = 6; - CHECK(wallet_channel_config_save(w, cc1)); + CHECK(transaction_wrap(w->db, wallet_channel_config_save(w, cc1))); CHECK_MSG( cc1->id == 1, tal_fmt(ctx, "channel_config->id != 1; got %" PRIu64, cc1->id)); @@ -369,23 +382,25 @@ static bool test_htlc_crud(const tal_t *ctx) out.msatoshi = 41; /* Store the htlc_in */ - CHECK_MSG(wallet_htlc_save_in(w, chan, &in), + CHECK_MSG(transaction_wrap(w->db, wallet_htlc_save_in(w, chan, &in)), tal_fmt(ctx, "Save htlc_in failed: %s", w->db->err)); CHECK_MSG(in.dbid != 0, "HTLC DB ID was not set."); /* Saving again should get us a collision */ - CHECK_MSG(!wallet_htlc_save_in(w, chan, &in), + CHECK_MSG(!transaction_wrap(w->db, wallet_htlc_save_in(w, chan, &in)), "Saving two HTLCs with the same data must not succeed."); + /* Update */ - CHECK_MSG(wallet_htlc_update(w, in.dbid, RCVD_ADD_HTLC, NULL), + CHECK_MSG(transaction_wrap(w->db, wallet_htlc_update(w, in.dbid, RCVD_ADD_HTLC, NULL)), "Update HTLC with null payment_key failed"); CHECK_MSG( - wallet_htlc_update(w, in.dbid, SENT_REMOVE_HTLC, &payment_key), + transaction_wrap(w->db, wallet_htlc_update(w, in.dbid, SENT_REMOVE_HTLC, &payment_key)), "Update HTLC with payment_key failed"); - CHECK_MSG(wallet_htlc_save_out(w, chan, &out), + CHECK_MSG(transaction_wrap(w->db, wallet_htlc_save_out(w, chan, &out)), tal_fmt(ctx, "Save htlc_out failed: %s", w->db->err)); CHECK_MSG(out.dbid != 0, "HTLC DB ID was not set."); - CHECK_MSG(!wallet_htlc_save_out(w, chan, &out), + + CHECK_MSG(!transaction_wrap(w->db, wallet_htlc_save_out(w, chan, &out)), "Saving two HTLCs with the same data must not succeed."); /* Attempt to load them from the DB again */ From f4d27eefa185408a63360008ee016a66f3bf3b13 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:40:48 +1030 Subject: [PATCH 0102/1428] wallet: remove internal transactions. We're going to be always in a transaction soon. Note the rollback we used to do was an optimization: the utxo destructors would already clean up the new UTXOs in the database. Signed-off-by: Rusty Russell --- wallet/wallet.c | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/wallet/wallet.c b/wallet/wallet.c index f3e79f4ccf4d..e80d942c25ce 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -145,9 +145,6 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, u64 satoshi_in = 0, weight = (4 + (8 + 22) * 2 + 4) * 4; tal_add_destructor2(utxos, destroy_utxos, w); - if (!db_begin_transaction(w->db)) { - fatal("Unable to begin transaction: %s", w->db->err); - } available = wallet_get_utxos(ctx, w, output_state_available); for (i = 0; i < tal_count(available); i++) { @@ -175,10 +172,7 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, if (satoshi_in < *fee_estimate + value) { /* Could not collect enough inputs, cleanup and bail */ utxos = tal_free(utxos); - db_rollback_transaction(w->db); } else { - /* Commit the db transaction to persist markings */ - db_commit_transaction(w->db); *changesatoshi = satoshi_in - value - *fee_estimate; } @@ -278,8 +272,6 @@ bool wallet_shachain_add_hash(struct wallet *wallet, return false; } - db_begin_transaction(wallet->db); - stmt = db_prepare(wallet->db, "UPDATE shachains SET num_valid=?, min_index=? WHERE id=?"); sqlite3_bind_int(stmt, 1, chain->chain.num_valid); sqlite3_bind_int64(stmt, 2, index); @@ -295,7 +287,7 @@ bool wallet_shachain_add_hash(struct wallet *wallet, sqlite3_bind_blob(stmt, 4, hash, sizeof(*hash), SQLITE_TRANSIENT); db_exec_prepared(wallet->db, stmt); - return db_commit_transaction(wallet->db); + return true; } bool wallet_shachain_load(struct wallet *wallet, u64 id, @@ -619,8 +611,6 @@ void wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ tal_t *tmpctx = tal_tmpctx(w); sqlite3_stmt *stmt; - db_begin_transaction(w->db); - if (p->dbid == 0) { /* Need to store the peer first */ stmt = db_prepare(w->db, "INSERT INTO peers (node_id) VALUES (?);"); @@ -740,9 +730,6 @@ void wallet_channel_save(struct wallet *w, struct wallet_channel *chan){ db_exec_prepared(w->db, stmt); } - if (!db_commit_transaction(w->db)) - fatal("Could not save channel to database: %s", w->db->err); - tal_free(tmpctx); } From 4fb472b7a7fcaa68d9b7463c86ac73bbce8e65f5 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:51:56 +1030 Subject: [PATCH 0103/1428] db: we must always be in a transaction, remove nested, call fatal() We save location where transaction was started, in case we try to nest. There's now no error case; db_exec_mayfail() is the only one. This means the tests need to override fatal() if they want to intercept these errors. Signed-off-by: Rusty Russell --- lightningd/invoice.c | 4 +- lightningd/test/run-find_my_path.c | 8 +- wallet/db.c | 144 +++++++++-------------------- wallet/db.h | 42 ++++----- wallet/db_tests.c | 47 ++++++++-- wallet/wallet.c | 36 +++----- wallet/wallet.h | 2 +- wallet/wallet_tests.c | 81 ++++++++++++---- 8 files changed, 179 insertions(+), 185 deletions(-) diff --git a/lightningd/invoice.c b/lightningd/invoice.c index 835dedcd3a7f..5c21c8b2de5f 100644 --- a/lightningd/invoice.c +++ b/lightningd/invoice.c @@ -287,8 +287,8 @@ static void json_delinvoice(struct command *cmd, } if (!wallet_invoice_remove(cmd->ld->wallet, i)) { - log_broken(cmd->ld->log, "Error attempting to remove invoice %"PRIu64": %s", - i->id, cmd->ld->wallet->db->err); + log_broken(cmd->ld->log, "Error attempting to remove invoice %"PRIu64, + i->id); command_fail(cmd, "Database error"); return; } diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 8700b06d3adc..001dd9f2d338 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -7,11 +7,11 @@ int unused_main(int argc, char *argv[]); /* Generated stub for crashlog_activate */ void crashlog_activate(const char *argv0 UNNEEDED, struct log *log UNNEEDED) { fprintf(stderr, "crashlog_activate called!\n"); abort(); } -/* Generated stub for db_begin_transaction */ -bool db_begin_transaction(struct db *db UNNEEDED) -{ fprintf(stderr, "db_begin_transaction called!\n"); abort(); } +/* Generated stub for db_begin_transaction_ */ +void db_begin_transaction_(struct db *db UNNEEDED, const char *location UNNEEDED) +{ fprintf(stderr, "db_begin_transaction_ called!\n"); abort(); } /* Generated stub for db_commit_transaction */ -bool db_commit_transaction(struct db *db UNNEEDED) +void db_commit_transaction(struct db *db UNNEEDED) { fprintf(stderr, "db_commit_transaction called!\n"); abort(); } /* Generated stub for debug_poll */ int debug_poll(struct pollfd *fds UNNEEDED, nfds_t nfds UNNEEDED, int timeout UNNEEDED) diff --git a/wallet/db.c b/wallet/db.c index 1d054db6f601..86657a1d72ba 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -122,86 +122,64 @@ char *dbmigrations[] = { NULL, }; -/** - * db_clear_error - Clear any errors from previous queries - */ -static void db_clear_error(struct db *db) -{ - db->err = tal_free(db->err); -} - sqlite3_stmt *db_prepare_(const char *caller, struct db *db, const char *query) { int err; sqlite3_stmt *stmt; - if (db->in_transaction && db->err) - return NULL; - db_clear_error(db); + assert(db->in_transaction); + err = sqlite3_prepare_v2(db->sql, query, -1, &stmt, NULL); - if (err != SQLITE_OK) { - db->err = tal_fmt(db, "%s: %s: %s", caller, query, - sqlite3_errmsg(db->sql)); - } + if (err != SQLITE_OK) + fatal("%s: %s: %s", caller, query, sqlite3_errmsg(db->sql)); + return stmt; } -bool db_exec_prepared_(const char *caller, struct db *db, sqlite3_stmt *stmt) +void db_exec_prepared_(const char *caller, struct db *db, sqlite3_stmt *stmt) { - if (db->in_transaction && db->err) { - goto fail; - } - - db_clear_error(db); + assert(db->in_transaction); - if (sqlite3_step(stmt) != SQLITE_DONE) { - db->err = - tal_fmt(db, "%s: %s", caller, sqlite3_errmsg(db->sql)); - goto fail; - } + if (sqlite3_step(stmt) != SQLITE_DONE) + fatal("%s: %s", caller, sqlite3_errmsg(db->sql)); sqlite3_finalize(stmt); - return true; -fail: - sqlite3_finalize(stmt); - return false; } -bool PRINTF_FMT(3, 4) - db_exec(const char *caller, struct db *db, const char *fmt, ...) +/* This one doesn't check if we're in a transaction. */ +static void db_do_exec(const char *caller, struct db *db, const char *cmd) { - va_list ap; - char *cmd, *errmsg; + char *errmsg; int err; - if (db->in_transaction && db->err) - return false; + err = sqlite3_exec(db->sql, cmd, NULL, NULL, &errmsg); + if (err != SQLITE_OK) { + fatal("%s:%s:%s:%s", caller, sqlite3_errstr(err), cmd, errmsg); + /* Only reached in testing */ + sqlite3_free(errmsg); + } +} - db_clear_error(db); +void PRINTF_FMT(3, 4) + db_exec(const char *caller, struct db *db, const char *fmt, ...) +{ + va_list ap; + char *cmd; + + assert(db->in_transaction); va_start(ap, fmt); cmd = tal_vfmt(db, fmt, ap); va_end(ap); - err = sqlite3_exec(db->sql, cmd, NULL, NULL, &errmsg); - if (err != SQLITE_OK) { - tal_free(db->err); - db->err = tal_fmt(db, "%s:%s:%s:%s", caller, - sqlite3_errstr(err), cmd, errmsg); - sqlite3_free(errmsg); - tal_free(cmd); - return false; - } + db_do_exec(caller, db, cmd); tal_free(cmd); - return true; } bool db_exec_prepared_mayfail_(const char *caller, struct db *db, sqlite3_stmt *stmt) { - if (db->in_transaction && db->err) { - goto fail; - } + assert(db->in_transaction); if (sqlite3_step(stmt) != SQLITE_DONE) { goto fail; @@ -221,10 +199,7 @@ sqlite3_stmt *PRINTF_FMT(3, 4) char *query; sqlite3_stmt *stmt; - if (db->in_transaction && db->err) - return NULL; - - db_clear_error(db); + assert(db->in_transaction); va_start(ap, fmt); query = tal_vfmt(db, fmt, ap); @@ -237,50 +212,20 @@ sqlite3_stmt *PRINTF_FMT(3, 4) static void close_db(struct db *db) { sqlite3_close(db->sql); } -bool db_begin_transaction(struct db *db) -{ - if (!db->in_transaction) { - /* Clear any errors from previous transactions and - * non-transactional queries */ - db_clear_error(db); - db->in_transaction = db_exec(__func__, db, "BEGIN TRANSACTION;"); - assert(db->in_transaction); - return db->in_transaction; - } - db->in_transaction++; - return false; -} - -bool db_commit_transaction(struct db *db) +void db_begin_transaction_(struct db *db, const char *location) { - bool ret; + if (db->in_transaction) + fatal("Already in transaction from %s", db->in_transaction); - assert(db->in_transaction); - if (db->err) { - char *errmsg; - int err; - - /* Do this manually: db_exec is a NOOP with db->err */ - err = sqlite3_exec(db->sql, "ROLLBACK;", NULL, NULL, &errmsg); - if (err != SQLITE_OK) { - db->err = tal_fmt(db, "%s then ROLLBACK failed:%s:%s", - db->err, sqlite3_errstr(err), errmsg); - sqlite3_free(errmsg); - } - ret = false; - } else { - ret = db_exec(__func__, db, "COMMIT;"); - } - db->in_transaction--; - return ret; + db_do_exec(location, db, "BEGIN TRANSACTION;"); + db->in_transaction = location; } -bool db_rollback_transaction(struct db *db) +void db_commit_transaction(struct db *db) { assert(db->in_transaction); - bool ret = db_exec(__func__, db, "ROLLBACK;"); - db->in_transaction--; - return ret; + db_exec(__func__, db, "COMMIT;"); + db->in_transaction = NULL; } /** @@ -308,11 +253,8 @@ static struct db *db_open(const tal_t *ctx, char *filename) db->filename = tal_dup_arr(db, char, filename, strlen(filename), 0); db->sql = sql; tal_add_destructor(db, close_db); - db->in_transaction = false; - db->err = NULL; - if (!db_exec(__func__, db, "PRAGMA foreign_keys = ON;")) { - fatal("Could not enable foreignkeys on database: %s", db->err); - } + db->in_transaction = NULL; + db_do_exec(__func__, db, "PRAGMA foreign_keys = ON;"); return db; } @@ -411,16 +353,14 @@ s64 db_get_intvar(struct db *db, char *varname, s64 defval) return res; } -bool db_set_intvar(struct db *db, char *varname, s64 val) +void db_set_intvar(struct db *db, char *varname, s64 val) { /* Attempt to update */ db_exec(__func__, db, "UPDATE vars SET val='%" PRId64 "' WHERE name='%s';", val, varname); - if (sqlite3_changes(db->sql) > 0) - return true; - else - return db_exec( + if (sqlite3_changes(db->sql) == 0) + db_exec( __func__, db, "INSERT INTO vars (name, val) VALUES ('%s', '%" PRId64 "');", diff --git a/wallet/db.h b/wallet/db.h index a5c047c9861b..8b8e3f89f215 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -15,8 +15,7 @@ struct db { char *filename; - unsigned int in_transaction; - const char *err; + const char *in_transaction; sqlite3 *sql; }; @@ -25,6 +24,7 @@ struct db { * * Opens the database, creating it if necessary, and applying * migrations until the schema is updated to the current state. + * Calls fatal() on error. * * Params: * @ctx: the tal_t context to allocate from @@ -33,37 +33,33 @@ struct db { struct db *db_setup(const tal_t *ctx); /** - * db_query - Prepare and execute a query, and return the result + * db_query - Prepare and execute a query, and return the result (or NULL) */ sqlite3_stmt *PRINTF_FMT(3, 4) db_query(const char *caller, struct db *db, const char *fmt, ...); -bool PRINTF_FMT(3, 4) +/** + * db_exec - execute a statement, call fatal() if it fails. + */ +void PRINTF_FMT(3, 4) db_exec(const char *caller, struct db *db, const char *fmt, ...); /** * db_begin_transaction - Begin a transaction * - * Begin a new DB transaction if we aren't already in one. Returns - * true if the call started a transaction, i.e., the caller MUST take - * care to either commit or rollback. If false, this is a nested - * transaction and the caller MUST not commit/rollback, since the - * transaction is handled at a higher level in the callstack. + * Begin a new DB transaction. fatal() on database error. */ -bool db_begin_transaction(struct db *db); +#define db_begin_transaction(db) \ + db_begin_transaction_((db), __FILE__ ":" stringify(__LINE__)) +void db_begin_transaction_(struct db *db, const char *location); /** * db_commit_transaction - Commit a running transaction * - * Requires that we are currently in a transaction. Returns whether - * the commit was successful. - */ -bool db_commit_transaction(struct db *db); - -/** - * db_rollback_transaction - Whoops... undo! undo! + * Requires that we are currently in a transaction. fatal() if we + * fail to commit. */ -bool db_rollback_transaction(struct db *db); +void db_commit_transaction(struct db *db); /** * db_set_intvar - Set an integer variable in the database @@ -71,7 +67,7 @@ bool db_rollback_transaction(struct db *db); * Utility function to store generic integer values in the * database. */ -bool db_set_intvar(struct db *db, char *varname, s64 val); +void db_set_intvar(struct db *db, char *varname, s64 val); /** * db_get_intvar - Retrieve an integer variable from the database @@ -102,18 +98,18 @@ sqlite3_stmt *db_prepare_(const char *caller, struct db *db, const char *query); * After preparing a statement using `db_prepare`, and after binding * all non-null variables using the `sqlite3_bind_*` functions, it can * be executed with this function. It is a small, transaction-aware, - * wrapper around `sqlite3_step`, that also sets `db->err` if the - * execution fails. This will take ownership of `stmt` and will free + * wrapper around `sqlite3_step`, that calls fatal() if the execution + * fails. This will take ownership of `stmt` and will free * it before returning. * * @db: The database to execute on * @stmt: The prepared statement to execute */ #define db_exec_prepared(db,stmt) db_exec_prepared_(__func__,db,stmt) -bool db_exec_prepared_(const char *caller, struct db *db, sqlite3_stmt *stmt); +void db_exec_prepared_(const char *caller, struct db *db, sqlite3_stmt *stmt); /** - * db_exec_prepared_mayfail - db_exec_prepared, but don't set db->err if it fails. + * db_exec_prepared_mayfail - db_exec_prepared, but don't fatal() it fails. */ #define db_exec_prepared_mayfail(db,stmt) \ db_exec_prepared_mayfail_(__func__,db,stmt) diff --git a/wallet/db_tests.c b/wallet/db_tests.c index 83feee23c1f9..2df08caf9f8a 100644 --- a/wallet/db_tests.c +++ b/wallet/db_tests.c @@ -1,3 +1,8 @@ + #include + +static void db_fatal(const char *fmt, ...); +#define fatal db_fatal + #include "db.c" #include "wallet/test_utils.h" @@ -5,6 +10,19 @@ #include #include +static char *db_err; +static void db_fatal(const char *fmt, ...) +{ + va_list ap; + + /* Fail hard if we're complaining about not being in transaction */ + assert(!strstarts(fmt, "No longer in transaction")); + + va_start(ap, fmt); + db_err = tal_vfmt(NULL, fmt, ap); + va_end(ap); +} + static struct db *create_test_db(const char *testname) { struct db *db; @@ -23,9 +41,13 @@ static bool test_empty_db_migrate(void) { struct db *db = create_test_db(__func__); CHECK(db); + db_begin_transaction(db); CHECK(db_get_version(db) == -1); + db_commit_transaction(db); db_migrate(db); + db_begin_transaction(db); CHECK(db_get_version(db) == db_migration_count()); + db_commit_transaction(db); tal_free(db); return true; @@ -34,17 +56,22 @@ static bool test_empty_db_migrate(void) static bool test_primitives(void) { struct db *db = create_test_db(__func__); - CHECK_MSG(db_begin_transaction(db), "Starting a new transaction"); + db_begin_transaction(db); CHECK(db->in_transaction); - CHECK_MSG(db_commit_transaction(db), "Committing a transaction"); + db_commit_transaction(db); CHECK(!db->in_transaction); - CHECK_MSG(db_begin_transaction(db), "Starting a transaction after commit"); - CHECK(db_rollback_transaction(db)); + db_begin_transaction(db); + db_commit_transaction(db); + + db_begin_transaction(db); + db_exec(__func__, db, "SELECT name FROM sqlite_master WHERE type='table';"); + CHECK_MSG(!db_err, "Simple correct SQL command"); - CHECK_MSG(db_exec(__func__, db, "SELECT name FROM sqlite_master WHERE type='table';"), "Simple correct SQL command"); - CHECK_MSG(!db_exec(__func__, db, "not a valid SQL statement"), "Failing SQL command"); + db_exec(__func__, db, "not a valid SQL statement"); + CHECK_MSG(db_err, "Failing SQL command"); + db_err = tal_free(db_err); + db_commit_transaction(db); CHECK(!db->in_transaction); - CHECK_MSG(db_begin_transaction(db), "Starting a transaction after a failed transaction"); tal_free(db); return true; @@ -57,16 +84,18 @@ static bool test_vars(void) CHECK(db); db_migrate(db); + db_begin_transaction(db); /* Check default behavior */ CHECK(db_get_intvar(db, varname, 42) == 42); /* Check setting and getting */ - CHECK(db_set_intvar(db, varname, 1)); + db_set_intvar(db, varname, 1); CHECK(db_get_intvar(db, varname, 42) == 1); /* Check updating */ - CHECK(db_set_intvar(db, varname, 2)); + db_set_intvar(db, varname, 2); CHECK(db_get_intvar(db, varname, 42) == 2); + db_commit_transaction(db); tal_free(db); return true; diff --git a/wallet/wallet.c b/wallet/wallet.c index e80d942c25ce..bbce351d4bd5 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -106,7 +106,7 @@ static void unreserve_utxo(struct wallet *w, const struct utxo *unres) if (!wallet_update_output_status(w, &unres->txid, unres->outnum, output_state_reserved, output_state_available)) { - fatal("Unable to unreserve output: %s", w->db->err); + fatal("Unable to unreserve output"); } } @@ -126,7 +126,7 @@ void wallet_confirm_utxos(struct wallet *w, const struct utxo **utxos) if (!wallet_update_output_status( w, &utxos[i]->txid, utxos[i]->outnum, output_state_reserved, output_state_spent)) { - fatal("Unable to mark output as spent: %s", w->db->err); + fatal("Unable to mark output as spent"); } } } @@ -154,7 +154,7 @@ const struct utxo **wallet_select_coins(const tal_t *ctx, struct wallet *w, if (!wallet_update_output_status( w, &available[i]->txid, available[i]->outnum, output_state_available, output_state_reserved)) - fatal("Unable to reserve output: %s", w->db->err); + fatal("Unable to reserve output"); weight += (32 + 4 + 4) * 4; if (utxos[i]->is_p2sh) @@ -229,19 +229,16 @@ s64 wallet_get_newindex(struct lightningd *ld) return newidx; } -bool wallet_shachain_init(struct wallet *wallet, struct wallet_shachain *chain) +void wallet_shachain_init(struct wallet *wallet, struct wallet_shachain *chain) { sqlite3_stmt *stmt; /* Create shachain */ shachain_init(&chain->chain); stmt = db_prepare(wallet->db, "INSERT INTO shachains (min_index, num_valid) VALUES (?, 0);"); sqlite3_bind_int64(stmt, 1, chain->chain.min_index); - if (!db_exec_prepared(wallet->db, stmt)) { - return false; - } + db_exec_prepared(wallet->db, stmt); chain->id = sqlite3_last_insert_rowid(wallet->db->sql); - return true; } /* TODO(cdecker) Stolen from shachain, move to some appropriate location */ @@ -971,7 +968,7 @@ bool wallet_htlcs_load_for_channel(struct wallet *wallet, DIRECTION_INCOMING, chan->id, SENT_REMOVE_ACK_REVOCATION); if (!stmt) { - log_broken(wallet->log, "Could not select htlc_ins: %s", wallet->db->err); + log_broken(wallet->log, "Could not select htlc_ins"); return false; } @@ -992,7 +989,7 @@ bool wallet_htlcs_load_for_channel(struct wallet *wallet, DIRECTION_OUTGOING, chan->id, RCVD_REMOVE_ACK_REVOCATION); if (!stmt) { - log_broken(wallet->log, "Could not select htlc_outs: %s", wallet->db->err); + log_broken(wallet->log, "Could not select htlc_outs"); return false; } @@ -1058,8 +1055,6 @@ void wallet_invoice_save(struct wallet *wallet, struct invoice *inv) if (!inv->id) { stmt = db_prepare(wallet->db, "INSERT INTO invoices (payment_hash, payment_key, state, msatoshi, label) VALUES (?, ?, ?, ?, ?);"); - if (!stmt) - fatal("Could not prepare statement: %s", wallet->db->err); sqlite3_bind_blob(stmt, 1, &inv->rhash, sizeof(inv->rhash), SQLITE_TRANSIENT); sqlite3_bind_blob(stmt, 2, &inv->r, sizeof(inv->r), SQLITE_TRANSIENT); @@ -1067,21 +1062,16 @@ void wallet_invoice_save(struct wallet *wallet, struct invoice *inv) sqlite3_bind_int64(stmt, 4, inv->msatoshi); sqlite3_bind_text(stmt, 5, inv->label, strlen(inv->label), SQLITE_TRANSIENT); - if (!db_exec_prepared(wallet->db, stmt)) - fatal("Could not exec prepared statement: %s", wallet->db->err); + db_exec_prepared(wallet->db, stmt); inv->id = sqlite3_last_insert_rowid(wallet->db->sql); } else { stmt = db_prepare(wallet->db, "UPDATE invoices SET state=? WHERE id=?;"); - if (!stmt) - fatal("Could not prepare statement: %s", wallet->db->err); - sqlite3_bind_int(stmt, 1, inv->state); sqlite3_bind_int64(stmt, 2, inv->id); - if (!db_exec_prepared(wallet->db, stmt)) - fatal("Could not exec prepared statement: %s", wallet->db->err); + db_exec_prepared(wallet->db, stmt); } } @@ -1109,7 +1099,7 @@ bool wallet_invoices_load(struct wallet *wallet, struct invoices *invs) "SELECT id, state, payment_key, payment_hash, " "label, msatoshi FROM invoices;"); if (!stmt) { - log_broken(wallet->log, "Could not load invoices: %s", wallet->db->err); + log_broken(wallet->log, "Could not load invoices"); return false; } @@ -1131,7 +1121,8 @@ bool wallet_invoice_remove(struct wallet *wallet, struct invoice *inv) { sqlite3_stmt *stmt = db_prepare(wallet->db, "DELETE FROM invoices WHERE id=?"); sqlite3_bind_int64(stmt, 1, inv->id); - return db_exec_prepared(wallet->db, stmt) && sqlite3_changes(wallet->db->sql) == 1; + db_exec_prepared(wallet->db, stmt); + return sqlite3_changes(wallet->db->sql) == 1; } struct htlc_stub *wallet_htlc_stubs(tal_t *ctx, struct wallet *wallet, @@ -1143,9 +1134,6 @@ struct htlc_stub *wallet_htlc_stubs(tal_t *ctx, struct wallet *wallet, "SELECT channel_id, direction, cltv_expiry, payment_hash " "FROM channel_htlcs WHERE channel_id = ?;"); - if (!stmt) - fatal("Error preparing select: %s", wallet->db->err); - sqlite3_bind_int64(stmt, 1, chan->id); stubs = tal_arr(ctx, struct htlc_stub, 0); diff --git a/wallet/wallet.h b/wallet/wallet.h index 65924d07d812..6573f345f886 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -141,7 +141,7 @@ s64 wallet_get_newindex(struct lightningd *ld); /** * wallet_shachain_init -- wallet wrapper around shachain_init */ -bool wallet_shachain_init(struct wallet *wallet, struct wallet_shachain *chain); +void wallet_shachain_init(struct wallet *wallet, struct wallet_shachain *chain); /** * wallet_shachain_add_hash -- wallet wrapper around shachain_add_hash diff --git a/wallet/wallet_tests.c b/wallet/wallet_tests.c index b88d6b555aaf..2165a8cf5792 100644 --- a/wallet/wallet_tests.c +++ b/wallet/wallet_tests.c @@ -22,13 +22,16 @@ static void wallet_fatal(const char *fmt, ...) /* Fail hard if we're complaining about not being in transaction */ assert(!strstarts(fmt, "No longer in transaction")); + /* Fail hard if we're complaining about not being in transaction */ + assert(!strstarts(fmt, "No longer in transaction")); + va_start(ap, fmt); wallet_err = tal_vfmt(NULL, fmt, ap); va_end(ap); } #define transaction_wrap(db, ...) \ - (db_begin_transaction(db), __VA_ARGS__, db_commit_transaction(db)) + (db_begin_transaction(db), __VA_ARGS__, db_commit_transaction(db), wallet_err == NULL) void invoice_add(struct invoices *invs, struct invoice *inv){} @@ -85,6 +88,8 @@ static bool test_wallet_outputs(void) memset(&u, 0, sizeof(u)); + db_begin_transaction(w->db); + /* Should work, it's the first time we add it */ CHECK_MSG(wallet_add_utxo(w, &u, p2sh_wpkh), "wallet_add_utxo failed on first add"); @@ -117,6 +122,7 @@ static bool test_wallet_outputs(void) output_state_spent), "could not change output state ignoring oldstate"); + db_commit_transaction(w->db); tal_free(w); return true; } @@ -143,7 +149,10 @@ static bool test_shachain_crud(void) memset(&b, 0, sizeof(b)); w->db = db_open(w, filename); - CHECK(wallet_shachain_init(w, &a)); + db_begin_transaction(w->db); + CHECK_MSG(!wallet_err, "db_begin_transaction failed"); + wallet_shachain_init(w, &a); + CHECK(!wallet_err); CHECK(a.id == 1); @@ -158,6 +167,9 @@ static bool test_shachain_crud(void) CHECK(wallet_shachain_load(w, a.id, &b)); CHECK_MSG(memcmp(&a, &b, sizeof(a)) == 0, "Loading from database doesn't match"); + + db_commit_transaction(w->db); + CHECK(!wallet_err); tal_free(w); return true; } @@ -261,11 +273,16 @@ static bool test_channel_crud(const tal_t *ctx) ci.remote_per_commit = pk; ci.old_remote_per_commit = pk; + db_begin_transaction(w->db); + CHECK(!wallet_err); + /* Variant 1: insert with null for scid, funding_tx_id, channel_info, last_tx */ wallet_channel_save(w, &c1); CHECK_MSG(!wallet_err, - tal_fmt(w, "Insert into DB: %s", w->db->err)); - CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); + tal_fmt(w, "Insert into DB: %s", wallet_err)); + CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB")); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Load from DB: %s", wallet_err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v1)"); /* We just inserted them into an empty DB so this must be 1 */ @@ -277,8 +294,10 @@ static bool test_channel_crud(const tal_t *ctx) c1.peer->scid = talz(w, struct short_channel_id); wallet_channel_save(w, &c1); CHECK_MSG(!wallet_err, - tal_fmt(w, "Insert into DB: %s", w->db->err)); - CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); + tal_fmt(w, "Insert into DB: %s", wallet_err)); + CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB")); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Insert into DB: %s", wallet_err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v2)"); /* Updates should not result in new ids */ @@ -290,39 +309,51 @@ static bool test_channel_crud(const tal_t *ctx) c1.peer->our_msatoshi = &msat; wallet_channel_save(w, &c1); - CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); - CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", wallet_err)); + CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB")); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Insert into DB: %s", wallet_err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v3)"); /* Variant 4: update with funding_tx_id */ c1.peer->funding_txid = hash; wallet_channel_save(w, &c1); - CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); - CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", wallet_err)); + CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB")); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Insert into DB: %s", wallet_err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v4)"); /* Variant 5: update with channel_info */ p.channel_info = &ci; wallet_channel_save(w, &c1); - CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); - CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", wallet_err)); + CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB")); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Insert into DB: %s", wallet_err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v5)"); /* Variant 6: update with last_commit_sent */ p.last_sent_commit = &last_commit; wallet_channel_save(w, &c1); - CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); - CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", wallet_err)); + CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB")); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Insert into DB: %s", wallet_err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v6)"); /* Variant 7: update with last_tx (taken from BOLT #3) */ p.last_tx = bitcoin_tx_from_hex(w, "02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8003a00f0000000000002200208c48d15160397c9731df9bc3b236656efb6665fbfe92b4a6878e88a499f741c4c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de843110ae8f6a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e040047304402206a2679efa3c7aaffd2a447fd0df7aba8792858b589750f6a1203f9259173198a022008d52a0e77a99ab533c36206cb15ad7aeb2aa72b93d4b571e728cb5ec2f6fe260147304402206d6cb93969d39177a09d5d45b583f34966195b77c7e585cf47ac5cce0c90cefb022031d71ae4e33a4e80df7f981d696fbdee517337806a3c7138b7491e2cbb077a0e01475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220", strlen("02000000000101bef67e4e2fb9ddeeb3461973cd4c62abb35050b1add772995b820b584a488489000000000038b02b8003a00f0000000000002200208c48d15160397c9731df9bc3b236656efb6665fbfe92b4a6878e88a499f741c4c0c62d0000000000160014ccf1af2f2aabee14bb40fa3851ab2301de843110ae8f6a00000000002200204adb4e2f00643db396dd120d4e7dc17625f5f2c11a40d857accc862d6b7dd80e040047304402206a2679efa3c7aaffd2a447fd0df7aba8792858b589750f6a1203f9259173198a022008d52a0e77a99ab533c36206cb15ad7aeb2aa72b93d4b571e728cb5ec2f6fe260147304402206d6cb93969d39177a09d5d45b583f34966195b77c7e585cf47ac5cce0c90cefb022031d71ae4e33a4e80df7f981d696fbdee517337806a3c7138b7491e2cbb077a0e01475221023da092f6980e58d2c037173180e9a465476026ee50f96695963e8efe436f54eb21030e9f7b623d2ccc7c9bd44d66d5ce21ce504c0acf6385a132cec6d3c39fa711c152ae3e195220")); p.last_sig = sig; wallet_channel_save(w, &c1); - CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", w->db->err)); - CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB: %s", w->db->err)); + CHECK_MSG(!wallet_err, tal_fmt(w, "Insert into DB: %s", wallet_err)); + CHECK_MSG(wallet_channel_load(w, c1.id, c2), tal_fmt(w, "Load from DB")); + CHECK_MSG(!wallet_err, + tal_fmt(w, "Insert into DB: %s", wallet_err)); CHECK_MSG(channelseq(&c1, c2), "Compare loaded with saved (v7)"); + db_commit_transaction(w->db); + CHECK(!wallet_err); tal_free(w); return true; } @@ -346,7 +377,7 @@ static bool test_channel_config_crud(const tal_t *ctx) cc1->id == 1, tal_fmt(ctx, "channel_config->id != 1; got %" PRIu64, cc1->id)); - CHECK(wallet_channel_config_load(w, cc1->id, cc2)); + CHECK(transaction_wrap(w->db, wallet_channel_config_load(w, cc1->id, cc2))); CHECK(memeq(cc1, sizeof(*cc1), cc2, sizeof(*cc2))); return true; } @@ -363,7 +394,8 @@ static bool test_htlc_crud(const tal_t *ctx) struct htlc_out_map *htlcs_out = tal(ctx, struct htlc_out_map); /* Make sure we have our references correct */ - db_exec(__func__, w->db, "INSERT INTO channels (id) VALUES (1);"); + CHECK(transaction_wrap(w->db, + db_exec(__func__, w->db, "INSERT INTO channels (id) VALUES (1);"))); chan->id = 1; chan->peer = peer; @@ -383,11 +415,13 @@ static bool test_htlc_crud(const tal_t *ctx) /* Store the htlc_in */ CHECK_MSG(transaction_wrap(w->db, wallet_htlc_save_in(w, chan, &in)), - tal_fmt(ctx, "Save htlc_in failed: %s", w->db->err)); + tal_fmt(ctx, "Save htlc_in failed: %s", wallet_err)); CHECK_MSG(in.dbid != 0, "HTLC DB ID was not set."); /* Saving again should get us a collision */ CHECK_MSG(!transaction_wrap(w->db, wallet_htlc_save_in(w, chan, &in)), "Saving two HTLCs with the same data must not succeed."); + CHECK(wallet_err); + wallet_err = tal_free(wallet_err); /* Update */ CHECK_MSG(transaction_wrap(w->db, wallet_htlc_update(w, in.dbid, RCVD_ADD_HTLC, NULL)), @@ -397,21 +431,28 @@ static bool test_htlc_crud(const tal_t *ctx) "Update HTLC with payment_key failed"); CHECK_MSG(transaction_wrap(w->db, wallet_htlc_save_out(w, chan, &out)), - tal_fmt(ctx, "Save htlc_out failed: %s", w->db->err)); + tal_fmt(ctx, "Save htlc_out failed: %s", wallet_err)); CHECK_MSG(out.dbid != 0, "HTLC DB ID was not set."); CHECK_MSG(!transaction_wrap(w->db, wallet_htlc_save_out(w, chan, &out)), "Saving two HTLCs with the same data must not succeed."); + CHECK(wallet_err); + wallet_err = tal_free(wallet_err); /* Attempt to load them from the DB again */ htlc_in_map_init(htlcs_in); htlc_out_map_init(htlcs_out); + db_begin_transaction(w->db); + CHECK(!wallet_err); + CHECK_MSG(wallet_htlcs_load_for_channel(w, chan, htlcs_in, htlcs_out), "Failed loading HTLCs"); CHECK_MSG(wallet_htlcs_reconnect(w, htlcs_in, htlcs_out), "Unable to reconnect htlcs."); + db_commit_transaction(w->db); + CHECK(!wallet_err); hin = htlc_in_map_get(htlcs_in, &in.key); hout = htlc_out_map_get(htlcs_out, &out.key); From 9fd81ab06daacdeeac859e414d46a6ed8f5159c6 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Wed, 1 Nov 2017 11:51:57 +1030 Subject: [PATCH 0104/1428] db: make db_exec() an internal function. Every caller is using prepared statements now. Signed-off-by: Rusty Russell --- wallet/db.c | 2 +- wallet/db.h | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/wallet/db.c b/wallet/db.c index 86657a1d72ba..bc4792a82863 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -161,7 +161,7 @@ static void db_do_exec(const char *caller, struct db *db, const char *cmd) } } -void PRINTF_FMT(3, 4) +static void PRINTF_FMT(3, 4) db_exec(const char *caller, struct db *db, const char *fmt, ...) { va_list ap; diff --git a/wallet/db.h b/wallet/db.h index 8b8e3f89f215..2e7d186129c8 100644 --- a/wallet/db.h +++ b/wallet/db.h @@ -38,12 +38,6 @@ struct db *db_setup(const tal_t *ctx); sqlite3_stmt *PRINTF_FMT(3, 4) db_query(const char *caller, struct db *db, const char *fmt, ...); -/** - * db_exec - execute a statement, call fatal() if it fails. - */ -void PRINTF_FMT(3, 4) - db_exec(const char *caller, struct db *db, const char *fmt, ...); - /** * db_begin_transaction - Begin a transaction * From 5320cab2de86d12a373d9c06d3e123696ac997d3 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 3 Nov 2017 10:40:57 +1030 Subject: [PATCH 0105/1428] generate-wire.py: create structures for optional fields. If a structure foo has a optional fields opt1 and opt2, this creates towire_foo, towire_foo_opt1 and towire_foo_opt2 (since opt2 implies opt1), similarly for fromwire_*. This requires the callers to be updated to call the correct routines (eg. try fromwire_foo_opt2, then fromwire_foo_opt1, then finally fromwire_foo), but this is a minimal change to the generation code. Signed-off-by: Rusty Russell --- tools/generate-wire.py | 63 ++++++++++++++++++++++++++++++------------ 1 file changed, 46 insertions(+), 17 deletions(-) diff --git a/tools/generate-wire.py b/tools/generate-wire.py index c203f6dee419..9a66768f9ac7 100755 --- a/tools/generate-wire.py +++ b/tools/generate-wire.py @@ -2,6 +2,7 @@ # Read from stdin, spit out C header or body. import argparse +import copy from collections import namedtuple import fileinput import re @@ -387,6 +388,28 @@ def print_towire(self,is_header): subcalls='\n'.join(subcalls), ) +def find_message(messages, name): + for m in messages: + if m.name == name: + return m + + return None + +def find_message_with_option(messages, optional_messages, name, option): + fullname = name + "_" + option; + + base = find_message(messages, name) + if not base: + raise ValueError('Unknown message {}'.format(name)) + + m = find_message(optional_messages, name) + if not m: + # Add a new option. + m = copy.deepcopy(base) + m.name = fullname + optional_messages.append(m) + return m + parser = argparse.ArgumentParser(description='Generate C from from CSV') parser.add_argument('--header', action='store_true', help="Create wire header") parser.add_argument('--bolt', action='store_true', help="Generate wire-format for BOLT") @@ -397,6 +420,7 @@ def print_towire(self,is_header): # Maps message names to messages messages = [] +messages_with_option = [] comments = [] includes = [] prevfield = None @@ -423,22 +447,27 @@ def print_towire(self,is_header): messages.append(Message(parts[0],Enumtype("WIRE_" + parts[0].upper(), parts[1]), comments)) comments=[] prevfield = None - elif len(parts) == 4: - # eg commit_sig,0,channel-id,8 OR - # commit_sig,0,channel-id,u64 - for m in messages: - if m.name == parts[0]: - f = Field(parts[0], parts[2], parts[3], comments, prevfield) - m.addField(f) - # If it used prevfield as lenvar, keep that for next - # time (multiple fields can use the same lenvar). - if not f.lenvar: - prevfield = parts[2] - break - comments=[] else: - raise ValueError('Line {} malformed'.format(line.rstrip())) - + if len(parts) == 4: + # eg commit_sig,0,channel-id,8 OR + # commit_sig,0,channel-id,u64 + m = find_message(messages, parts[0]) + if m is None: + raise ValueError('Unknown message {}'.format(name)) + elif len(parts) == 5: + # eg. + # channel_reestablish,48,your_last_per_commitment_secret,32,option209 + m = find_message_with_option(messages, messages_with_option, parts[0], parts[4]) + else: + raise ValueError('Line {} malformed'.format(line.rstrip())) + + f = Field(m.name, parts[2], parts[3], comments, prevfield) + m.addField(f) + # If it used prevfield as lenvar, keep that for next + # time (multiple fields can use the same lenvar). + if not f.lenvar: + prevfield = parts[2] + comments=[] header_template = """#ifndef LIGHTNING_{idem} #define LIGHTNING_{idem} @@ -485,8 +514,8 @@ def print_towire(self,is_header): includes = '\n'.join(includes) cases = ['case {enum.name}: return "{enum.name}";'.format(enum=m.enum) for m in messages] -fromwire_decls = [m.print_fromwire(options.header) for m in messages] -towire_decls = [m.print_towire(options.header) for m in messages] +fromwire_decls = [m.print_fromwire(options.header) for m in messages + messages_with_option] +towire_decls = [m.print_towire(options.header) for m in messages + messages_with_option] print(template.format( headerfilename=options.headerfilename, From fc176b6cc071c3276a306286848dd38cc8239f72 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 9 Nov 2017 14:36:56 +0100 Subject: [PATCH 0106/1428] pytest: Use the py.test utility if available The py.test unit test runner offers a number of more advanced features than simply running using unittest.main. In particular it allows us to capture a tests output and print it if it fails. This change checks whether we have pytest available and if yes, enables verbose tests and runs using pytest. This'll give the usual experience (with colors!) and show us the stdout if a test fails making travis a lot easier to handle. Signed-off-by: Christian Decker --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 66ae00c555d1..04c3d5628cc0 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,8 @@ ifeq ($(COVERAGE),1) COVFLAGS = --coverage endif +PYTEST := $(shell command -v pytest 2> /dev/null) + # This is where we add new features as bitcoin adds them. FEATURES := @@ -170,7 +172,11 @@ check: $(MAKE) pytest pytest: $(ALL_PROGRAMS) +ifndef PYTEST PYTHONPATH=contrib/pylightning DEVELOPER=$(DEVELOPER) python3 tests/test_lightningd.py -f +else + PYTHONPATH=contrib/pylightning TEST_DEBUG=1 DEVELOPER=$(DEVELOPER) $(PYTEST) -vx tests/test_lightningd.py +endif # Keep includes in alpha order. check-src-include-order/%: % From bab0693fc438b5df7e0f0af90b0220db2f5c7cd6 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 9 Nov 2017 14:42:03 +0100 Subject: [PATCH 0107/1428] contrib: Add py.test to builder Dockerfiles With the previous commit this enables py.test on travis, which should give us some better way of hunting down bugs on travis. Signed-off-by: Christian Decker --- contrib/Dockerfile.builder | 3 ++- contrib/Dockerfile.builder.i386 | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/Dockerfile.builder b/contrib/Dockerfile.builder index 11950f4a59ff..8b844484db43 100644 --- a/contrib/Dockerfile.builder +++ b/contrib/Dockerfile.builder @@ -23,6 +23,7 @@ RUN apt-get -qq update && \ valgrind \ net-tools \ python3-pip \ + python-pkg-resources \ wget && \ rm -rf /var/lib/apt/lists/* @@ -32,4 +33,4 @@ RUN cd /tmp/ && \ mv /tmp/bitcoin-0.15.0/bin/bitcoin* /usr/local/bin/ && \ rm -rf bitcoin.tar.gz /tmp/bitcoin-0.15.0 -RUN pip3 install python-bitcoinlib==0.7.0 +RUN pip3 install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 diff --git a/contrib/Dockerfile.builder.i386 b/contrib/Dockerfile.builder.i386 index b71fc7bbf824..1a91fd9f9ccb 100644 --- a/contrib/Dockerfile.builder.i386 +++ b/contrib/Dockerfile.builder.i386 @@ -23,7 +23,8 @@ RUN apt-get -qq update && \ valgrind \ net-tools \ python3-pip \ - wget && \ + python-pkg-resources \ + wget && \ rm -rf /var/lib/apt/lists/* RUN cd /tmp/ && \ @@ -32,4 +33,4 @@ RUN cd /tmp/ && \ mv /tmp/bitcoin-0.15.0/bin/bitcoin* /usr/local/bin/ && \ rm -rf bitcoin.tar.gz /tmp/bitcoin-0.15.0 -RUN pip3 install python-bitcoinlib==0.7.0 +RUN pip3 install python-bitcoinlib==0.7.0 pytest==3.0.5 setuptools==36.6.0 From d6af14a8693aaf8f09cb258a73e47df6a8feb2d7 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 9 Nov 2017 17:07:18 +0100 Subject: [PATCH 0108/1428] pytest: Valgrind errors trump exit value errors Raising the exception about non-zero exit values into the teardown. This was previously masking the valgrind errors. Now valgrind errors > crash errors > non-zero return value. Still hoping to catch that elusive [7, 0] return value on travis. Signed-off-by: Christian Decker --- tests/test_lightningd.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 6b8d39163ebb..1a8846f2559e 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -140,8 +140,8 @@ def killall(self): except: failed = True rcs.append(n.daemon.proc.returncode) - if failed: - raise Exception("At least one lightning exited with non-zero return code: {}".format(rcs)) + return rcs + class BaseLightningDTests(unittest.TestCase): @@ -190,7 +190,7 @@ def printCrashLog(self, node): return 1 if errors else 0 def tearDown(self): - self.node_factory.killall() + rcs = self.node_factory.killall() self.executor.shutdown(wait=False) err_count = 0 @@ -199,14 +199,18 @@ def tearDown(self): for node in self.node_factory.nodes: err_count += self.printValgrindErrors(node) if err_count: - raise ValueError( - "{} nodes reported valgrind errors".format(err_count)) + raise ValueError("{} nodes reported valgrind errors".format(err_count)) for node in self.node_factory.nodes: err_count += self.printCrashLog(node) if err_count: - raise ValueError( - "{} nodes had crash.log files".format(err_count)) + raise ValueError("{} nodes had crash.log files".format(err_count)) + + # Which nodes may fail? Mask away the ones that we know will fail + failmask = [not n.may_fail for n in self.node_factory.nodes] + unexpected = [(failmask[i] * rcs[i]) for i in range(len(rcs))] + if len([u for u in unexpected if u > 0]) > 0: + raise Exception("At least one lightning exited with unexpected non-zero return code: {}".format(unexpected)) class LightningDTests(BaseLightningDTests): def connect(self): From 07e5a9ef9fa02db9ca2de5793e39f1f9e39216d4 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Wed, 8 Nov 2017 14:34:11 +0100 Subject: [PATCH 0109/1428] htlc: Allow for exactly min_final_cltv_expiry cltv delta We are announcing that we are willing to accept incoming payments with current_height + min_final_cltv_expiry + slack, assuming that the sender adds some slack. In particular we'd reject the payment if slack=0 which is allowed by the spec. Signed-off-by: Christian Decker --- lightningd/peer_htlcs.c | 4 +-- tests/test_lightningd.py | 62 ++++++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c index 691739731190..723d47973a55 100644 --- a/lightningd/peer_htlcs.c +++ b/lightningd/peer_htlcs.c @@ -359,9 +359,9 @@ static void handle_localpay(struct htlc_in *hin, * If the `cltv_expiry` is too low, the final node MUST fail the HTLC: */ if (get_block_height(ld->topology) + ld->config.cltv_final - >= cltv_expiry) { + > cltv_expiry) { log_debug(hin->key.peer->log, - "Expiry cltv %u too close to current %u + %u", + "Expiry cltv too soon %u < %u + %u", cltv_expiry, get_block_height(ld->topology), ld->config.cltv_final); diff --git a/tests/test_lightningd.py b/tests/test_lightningd.py index 1a8846f2559e..803398984541 100644 --- a/tests/test_lightningd.py +++ b/tests/test_lightningd.py @@ -299,7 +299,7 @@ def test_invoice(self): assert b11['description'] == 'description' assert b11['expiry'] == 3600 assert b11['payee'] == l1.info['id'] - + def test_connect(self): l1,l2 = self.connect() @@ -329,9 +329,9 @@ def test_decodepay(self): # BOLT #11: # > ### Please make a donation of any amount using payment_hash 0001020304050607080900010203040506070809000102030405060708090102 to me @03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad # > lnbc1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdpl2pkx2ctnv5sxxmmwwd5kgetjypeh2ursdae8g6twvus8g6rfwvs8qun0dfjkxaq8rkx3yf5tcsyz3d73gafnh3cax9rn449d9p5uxz9ezhhypd0elx87sjle52x86fux2ypatgddc6k63n7erqz25le42c4u4ecky03ylcqca784w - # + # # Breakdown: - # + # # * `lnbc`: prefix, lightning on bitcoin mainnet # * `1`: Bech32 separator # * `pvjluez`: timestamp (1496314658) @@ -350,13 +350,13 @@ def test_decodepay(self): assert b11['description'] == 'Please consider supporting this project' assert b11['expiry'] == 3600 assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' - + # BOLT #11: # > ### Please send $3 for a cup of coffee to the same peer, within 1 minute # > lnbc2500u1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqdq5xysxxatsyp3k7enxv4jsxqzpuaztrnwngzn3kdzw5hydlzf03qdgm2hdq27cqv3agm2awhz5se903vruatfhq77w3ls4evs3ch9zw97j25emudupq63nyw24cg27h2rspfj9srp - # + # # Breakdown: - # + # # * `lnbc`: prefix, lightning on bitcoin mainnet # * `2500u`: amount (2500 micro-bitcoin) # * `1`: Bech32 separator @@ -382,9 +382,9 @@ def test_decodepay(self): # BOLT #11: # > ### Now send $24 for an entire list of things (hashed) # > lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqscc6gd6ql3jrc5yzme8v4ntcewwz5cnw92tz0pc8qcuufvq7khhr8wpald05e92xw006sq94mg8v2ndf4sefvf9sygkshp5zfem29trqq2yxxz7 - # + # # Breakdown: - # + # # * `lnbc`: prefix, lightning on bitcoin mainnet # * `20m`: amount (20 milli-bitcoin) # * `1`: Bech32 separator @@ -405,9 +405,9 @@ def test_decodepay(self): # > ### The same, on testnet, with a fallback address mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP # > lntb20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfpp3x9et2e20v6pu37c5d9vax37wxq72un98kmzzhznpurw9sgl2v0nklu2g4d0keph5t7tj9tcqd8rexnd07ux4uv2cjvcqwaxgj7v4uwn5wmypjd5n69z2xm3xgksg28nwht7f6zspwp3f9t - # + # # Breakdown: - # + # # * `lntb`: prefix, lightning on bitcoin testnet # * `20m`: amount (20 milli-bitcoin) # * `1`: Bech32 separator @@ -428,12 +428,12 @@ def test_decodepay(self): assert b11['payee'] == '03e7156ae33b0a208d0744199163177e909e80176e55d97a2f221ede0f934dd9ad' assert b11['fallback']['type'] == 'P2PKH' assert b11['fallback']['addr'] == 'mk2QpYatsKicvFVuTAQLBryyccRXMUaGHP' - + # > ### On mainnet, with fallback address 1RustyRX2oai4EYYDpQGWvEL62BBGqN9T with extra routing info to go via nodes 029e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255 then 039e03a901b85534ff1e92c43c74431f7ce72046060fcf7a95c37e148f78c77255 # > lnbc20m1pvjluezpp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqsfpp3qjmp7lwpagxun9pygexvgpjdc4jdj85fr9yq20q82gphp2nflc7jtzrcazrra7wwgzxqc8u7754cdlpfrmccae92qgzqvzq2ps8pqqqqqqqqqqqq9qqqvpeuqafqxu92d8lr6fvg0r5gv0heeeqgcrqlnm6jhphu9y00rrhy4grqszsvpcgpy9qqqqqqqqqqqq7qqzqfnlkwydm8rg30gjku7wmxmk06sevjp53fmvrcfegvwy7d5443jvyhxsel0hulkstws7vqv400q4j3wgpk4crg49682hr4scqvmad43cqd5m7tf - # + # # Breakdown: - # + # # * `lnbc`: prefix, lightning on bitcoin mainnet # * `20m`: amount (20 milli-bitcoin) # * `1`: Bech32 separator @@ -470,12 +470,12 @@ def test_decodepay(self): assert b11['routes'][0][1]['short_channel_id'] == '197637:395016:2314' assert b11['routes'][0][1]['fee'] == 30 assert b11['routes'][0][1]['cltv_expiry_delta'] == 4 - + # > ### On mainnet, with fallback (P2SH) address 3EktnHQD7RiAE6uzMj2ZifT9YgRrkSgzQX # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppj3a24vwu6r8ejrss3axul8rxldph2q7z9kmrgvr7xlaqm47apw3d48zm203kzcq357a4ls9al2ea73r8jcceyjtya6fu5wzzpe50zrge6ulk4nvjcpxlekvmxl6qcs9j3tz0469gq5g658y - # + # # Breakdown: - # + # # * `lnbc`: prefix, lightning on bitcoin mainnet # * `20m`: amount (20 milli-bitcoin) # * `1`: Bech32 separator @@ -499,7 +499,7 @@ def test_decodepay(self): # > ### On mainnet, with fallback (P2WPKH) address bc1qw508d6qejxtdg4y5r3zarvary0c5xw7kv8f3t4 # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfppqw508d6qejxtdg4y5r3zarvary0c5xw7kepvrhrm9s57hejg0p662ur5j5cr03890fa7k2pypgttmh4897d3raaq85a293e9jpuqwl0rnfuwzam7yr8e690nd2ypcq9hlkdwdvycqa0qza8 - # + # # * `lnbc`: prefix, lightning on bitcoin mainnet # * `20m`: amount (20 milli-bitcoin) # * `1`: Bech32 separator @@ -507,7 +507,7 @@ def test_decodepay(self): # * `p`: payment hash... # * `f`: tagged field: fallback address. # * `pp`: `data_length` (`p` = 1. 1 * 32 + 1 == 33) - # * `q`: 0, so witness version 0. + # * `q`: 0, so witness version 0. # * `qw508d6qejxtdg4y5r3zarvary0c5xw7k`: 160 bits = P2WPKH. # * `h`: tagged field: hash of description... # * `gw6tk8z0p0qdy9ulggx65lvfsg3nxxhqjxuf2fvmkhl9f4jc74gy44d5ua9us509prqz3e7vjxrftn3jnk7nrglvahxf7arye5llphgq`: signature @@ -524,7 +524,7 @@ def test_decodepay(self): # > ### On mainnet, with fallback (P2WSH) address bc1qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3qccfmv3 # > lnbc20m1pvjluezhp58yjmdan79s6qqdhdzgynm4zwqd5d7xmw5fk98klysy043l2ahrqspp5qqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqqqsyqcyq5rqwzqfqypqfp4qrp33g0q5c5txsp9arysrx4k6zdkfs4nce4xj0gdcccefvpysxf3q28j0v3rwgy9pvjnd48ee2pl8xrpxysd5g44td63g6xcjcu003j3qe8878hluqlvl3km8rm92f5stamd3jw763n3hck0ct7p8wwj463cql26ava - # + # # * `lnbc`: prefix, lightning on bitcoin mainnet # * `20m`: amount (20 milli-bitcoin) # * `1`: Bech32 separator @@ -1426,12 +1426,12 @@ def test_forward_different_fees_and_cltv(self): # B # / \ # / \ - # A C + # A C # \ / # \ / # D # ``` - # + # # Each advertises the following `cltv_expiry_delta` on its end of every # channel: # @@ -1482,7 +1482,7 @@ def test_forward_different_fees_and_cltv(self): 'Received channel_update for channel {}\\(1\\)'.format(c2)]) # BOLT #7: - # + # # If B were to send 4,999,999 millisatoshi directly to C, it wouldn't # charge itself a fee nor add its own `cltv_expiry_delta`, so it would # use C's requested `cltv_expiry` of 9. We also assume it adds a @@ -1510,14 +1510,14 @@ def test_forward_different_fees_and_cltv(self): # If A were to send an 4,999,999 millisatoshi to C via B, it needs to # pay B the fee it specified in the B->C `channel_update`, calculated as # per [HTLC Fees](#htlc_fees): - # + # # 200 + 4999999 * 2000 / 1000000 = 10199 - # + # # Similarly, it would need to add the `cltv_expiry` from B->C's # `channel_update` (20), plus C's requested minimum (9), plus 42 for the # "shadow route". Thus the `update_add_htlc` message from A to B would # be: - # + # # * `amount_msat`: 5010198 # * `cltv_expiry`: current-block-height + 20 + 9 + 42 # * `onion_routing_packet`: @@ -1566,7 +1566,7 @@ def test_forward_pad_fees_and_cltv(self): l2.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') l3.daemon.wait_for_log('WIRE_GOSSIPCTL_HANDLE_PEER') - + c1 = self.fund_channel(l1, l2, 10**6) c2 = self.fund_channel(l2, l3, 10**6) @@ -1631,9 +1631,9 @@ def test_htlc_out_timeout(self): # Takes 6 blocks to timeout (cltv-final + 1), but we also give grace period of 1 block. bitcoind.rpc.generate(5 + 1) assert not l1.daemon.is_in_log('hit deadline') - bitcoind.rpc.generate(1) + bitcoind.rpc.generate(2) - l1.daemon.wait_for_log('Offered HTLC 0 SENT_ADD_ACK_REVOCATION cltv {} hit deadline'.format(bitcoind.rpc.getblockcount()-1)) + l1.daemon.wait_for_log('Offered HTLC 0 SENT_ADD_ACK_REVOCATION cltv .* hit deadline') l1.daemon.wait_for_log('sendrawtx exit 0') l1.bitcoin.rpc.generate(1) l1.daemon.wait_for_log('-> ONCHAIND_OUR_UNILATERAL') @@ -1668,12 +1668,12 @@ def test_htlc_in_timeout(self): # l1 will drop to chain, not reconnect. l1.daemon.wait_for_log('dev_disconnect: -WIRE_REVOKE_AND_ACK') - # Deadline HTLC expity minus 1/2 cltv-expiry delta (rounded up) (== cltv - 3). ctlv is 5+1. + # Deadline HTLC expiry minus 1/2 cltv-expiry delta (rounded up) (== cltv - 3). ctlv is 5+1. bitcoind.rpc.generate(2) assert not l2.daemon.is_in_log('hit deadline') - bitcoind.rpc.generate(1) + bitcoind.rpc.generate(2) - l2.daemon.wait_for_log('Fulfilled HTLC 0 SENT_REMOVE_COMMIT cltv {} hit deadline'.format(bitcoind.rpc.getblockcount()+3)) + l2.daemon.wait_for_log('Fulfilled HTLC 0 SENT_REMOVE_COMMIT cltv .* hit deadline') l2.daemon.wait_for_log('sendrawtx exit 0') l2.bitcoin.rpc.generate(1) l2.daemon.wait_for_log('-> ONCHAIND_OUR_UNILATERAL') From e9337820a0a745fc91fe84090daeb2b9734fdb1e Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 10 Nov 2017 12:17:15 +1030 Subject: [PATCH 0110/1428] onchaind: remove htlcs when peer is irrevocably committed. We don't track them accurately when in onchaind, but we don't want to: onchaind can be restarted at any time. Once it's all settled, we're clear to clean them up. Before this, valgrind could complain about deferncing hout->key.peer: Valgrind error file: valgrind-errors.10876 ==10876== Invalid read of size 4 ==10876== at 0x41F8AF: peer_on_chain (peer_control.h:127) ==10876== by 0x42340D: notify_new_block (peer_htlcs.c:1461) ==10876== by 0x40A08D: connect_block (chaintopology.c:96) ==10876== by 0x40A96B: topology_changed (chaintopology.c:313) ==10876== by 0x40AC85: add_block (chaintopology.c:384) ==10876== by 0x40ABF0: gather_previous_blocks (chaintopology.c:363) ==10876== by 0x4051B3: process_rawblock (bitcoind.c:410) ==10876== by 0x4044DD: bcli_finished (bitcoind.c:155) ==10876== by 0x454665: destroy_conn (poll.c:183) ==10876== by 0x454685: destroy_conn_close_fd (poll.c:189) ==10876== by 0x45DF89: notify (tal.c:240) ==10876== by 0x45E43A: del_tree (tal.c:400) ==10876== Address 0x6929208 is 2,120 bytes inside a block of size 2,416 free'd ==10876== at 0x4C2EDEB: free (in /usr/lib/valgrind/vgpreload_memcheck-amd64-linux.so) ==10876== by 0x45E513: del_tree (tal.c:421) ==10876== by 0x45E849: tal_free (tal.c:509) ==10876== by 0x41A8E9: handle_irrevocably_resolved (peer_control.c:1172) Signed-off-by: Rusty Russell --- lightningd/peer_control.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 5bd57a95d039..035a710f370e 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -1165,6 +1165,36 @@ static void handle_onchain_htlc_timeout(struct peer *peer, const u8 *msg) static void handle_irrevocably_resolved(struct peer *peer, const u8 *msg) { + struct htlc_out_map_iter outi; + struct htlc_out *hout; + struct htlc_in_map_iter ini; + struct htlc_in *hin; + bool deleted; + + /* FIXME: Implement check_htlcs to ensure no dangling hout->in ptrs! */ + + do { + deleted = false; + for (hout = htlc_out_map_first(&peer->ld->htlcs_out, &outi); + hout; + hout = htlc_out_map_next(&peer->ld->htlcs_out, &outi)) { + if (hout->key.peer != peer) + continue; + tal_free(hout); + deleted = true; + } + + for (hin = htlc_in_map_first(&peer->ld->htlcs_in, &ini); + hin; + hin = htlc_in_map_next(&peer->ld->htlcs_in, &ini)) { + if (hin->key.peer != peer) + continue; + tal_free(hin); + deleted = true; + } + /* Can skip over elements due to iterating while deleting. */ + } while (deleted); + /* FIXME: Remove peer from db. */ log_info(peer->log, "onchaind complete, forgetting peer"); From 956350e62e5baacf4b7edb59061b99548059e3c0 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 10 Nov 2017 12:31:10 +1030 Subject: [PATCH 0111/1428] lightningd: check peers don't leave dangling HTLCs when they die. Signed-off-by: Rusty Russell --- lightningd/lightningd.c | 2 ++ lightningd/peer_control.c | 47 +++++++++++++++++++++++++----- lightningd/peer_control.h | 2 ++ lightningd/test/run-find_my_path.c | 3 ++ 4 files changed, 47 insertions(+), 7 deletions(-) diff --git a/lightningd/lightningd.c b/lightningd/lightningd.c index f14224aa11f1..e7562d1f9875 100644 --- a/lightningd/lightningd.c +++ b/lightningd/lightningd.c @@ -198,6 +198,8 @@ static void shutdown_subdaemons(struct lightningd *ld) close(ld->hsm_fd); subd_shutdown(ld->gossip, 10); + free_htlcs(ld, NULL); + while ((p = list_top(&ld->peers, struct peer, list)) != NULL) tal_free(p); } diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 035a710f370e..f8c9f2931d61 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -90,6 +90,32 @@ static void peer_set_owner(struct peer *peer, struct subd *owner) static void destroy_peer(struct peer *peer) { + /* Must not have any HTLCs! */ + struct htlc_out_map_iter outi; + struct htlc_out *hout; + struct htlc_in_map_iter ini; + struct htlc_in *hin; + + for (hout = htlc_out_map_first(&peer->ld->htlcs_out, &outi); + hout; + hout = htlc_out_map_next(&peer->ld->htlcs_out, &outi)) { + if (hout->key.peer != peer) + continue; + fatal("Freeing peer %s has hout %s", + peer_state_name(peer->state), + htlc_state_name(hout->hstate)); + } + + for (hin = htlc_in_map_first(&peer->ld->htlcs_in, &ini); + hin; + hin = htlc_in_map_next(&peer->ld->htlcs_in, &ini)) { + if (hin->key.peer != peer) + continue; + fatal("Freeing peer %s has hin %s", + peer_state_name(peer->state), + htlc_state_name(hin->hstate)); + } + /* Free any old owner still hanging around. */ peer_set_owner(peer, NULL); list_del_from(&peer->ld->peers, &peer->list); @@ -1163,7 +1189,8 @@ static void handle_onchain_htlc_timeout(struct peer *peer, const u8 *msg) onchain_failed_our_htlc(peer, &htlc, "timed out"); } -static void handle_irrevocably_resolved(struct peer *peer, const u8 *msg) +/* If peer is NULL, free them all (for shutdown) */ +void free_htlcs(struct lightningd *ld, const struct peer *peer) { struct htlc_out_map_iter outi; struct htlc_out *hout; @@ -1175,25 +1202,31 @@ static void handle_irrevocably_resolved(struct peer *peer, const u8 *msg) do { deleted = false; - for (hout = htlc_out_map_first(&peer->ld->htlcs_out, &outi); + for (hout = htlc_out_map_first(&ld->htlcs_out, &outi); hout; - hout = htlc_out_map_next(&peer->ld->htlcs_out, &outi)) { - if (hout->key.peer != peer) + hout = htlc_out_map_next(&ld->htlcs_out, &outi)) { + if (peer && hout->key.peer != peer) continue; tal_free(hout); deleted = true; } - for (hin = htlc_in_map_first(&peer->ld->htlcs_in, &ini); + for (hin = htlc_in_map_first(&ld->htlcs_in, &ini); hin; - hin = htlc_in_map_next(&peer->ld->htlcs_in, &ini)) { - if (hin->key.peer != peer) + hin = htlc_in_map_next(&ld->htlcs_in, &ini)) { + if (peer && hin->key.peer != peer) continue; tal_free(hin); deleted = true; } /* Can skip over elements due to iterating while deleting. */ } while (deleted); +} + +static void handle_irrevocably_resolved(struct peer *peer, const u8 *msg) +{ + /* FIXME: Implement check_htlcs to ensure no dangling hout->in ptrs! */ + free_htlcs(peer->ld, peer); /* FIXME: Remove peer from db. */ log_info(peer->log, "onchaind complete, forgetting peer"); diff --git a/lightningd/peer_control.h b/lightningd/peer_control.h index 4025d06e611c..58038172e8e7 100644 --- a/lightningd/peer_control.h +++ b/lightningd/peer_control.h @@ -210,4 +210,6 @@ const char *peer_state_name(enum peer_state state); void peer_set_condition(struct peer *peer, enum peer_state oldstate, enum peer_state state); void setup_listeners(struct lightningd *ld); + +void free_htlcs(struct lightningd *ld, const struct peer *peer); #endif /* LIGHTNING_LIGHTNINGD_PEER_CONTROL_H */ diff --git a/lightningd/test/run-find_my_path.c b/lightningd/test/run-find_my_path.c index 001dd9f2d338..559478eb9e48 100644 --- a/lightningd/test/run-find_my_path.c +++ b/lightningd/test/run-find_my_path.c @@ -19,6 +19,9 @@ int debug_poll(struct pollfd *fds UNNEEDED, nfds_t nfds UNNEEDED, int timeout UN /* Generated stub for fatal */ void fatal(const char *fmt UNNEEDED, ...) { fprintf(stderr, "fatal called!\n"); abort(); } +/* Generated stub for free_htlcs */ +void free_htlcs(struct lightningd *ld UNNEEDED, const struct peer *peer UNNEEDED) +{ fprintf(stderr, "free_htlcs called!\n"); abort(); } /* Generated stub for gossip_init */ void gossip_init(struct lightningd *ld UNNEEDED) { fprintf(stderr, "gossip_init called!\n"); abort(); } From 89f016f524887823d076e6e9a087c4bd2c55df60 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Thu, 9 Nov 2017 14:04:31 +0100 Subject: [PATCH 0112/1428] jsonrpc: Only print netaddr in getpeers when we know it Fixes #285 Signed-off-by: Christian Decker --- lightningd/peer_control.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index f8c9f2931d61..04307cc32fff 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -842,9 +842,10 @@ static void gossipd_getpeers_complete(struct subd *gossip, const u8 *msg, list_for_each(&gpa->cmd->ld->peers, p, list) { json_object_start(response, NULL); json_add_string(response, "state", peer_state_name(p->state)); - json_add_string(response, "netaddr", - type_to_string(response, struct wireaddr, - &p->addr)); + if (p->addr.type != ADDR_TYPE_PADDING) + json_add_string(response, "netaddr", + type_to_string(response, struct wireaddr, + &p->addr)); json_add_pubkey(response, "peerid", &p->id); json_add_bool(response, "connected", p->owner != NULL); if (p->owner) From f95afc55d8f9fbb3766bb70653956b47d4279a12 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sat, 11 Nov 2017 12:09:23 +1030 Subject: [PATCH 0113/1428] rpc: report netaddr as array. Thought we don't handle it at the moment, nodes can certainly have multiple addresses, and we should display them all. Signed-off-by: Rusty Russell --- lightningd/peer_control.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lightningd/peer_control.c b/lightningd/peer_control.c index 04307cc32fff..fa852c4b8a6f 100644 --- a/lightningd/peer_control.c +++ b/lightningd/peer_control.c @@ -842,10 +842,12 @@ static void gossipd_getpeers_complete(struct subd *gossip, const u8 *msg, list_for_each(&gpa->cmd->ld->peers, p, list) { json_object_start(response, NULL); json_add_string(response, "state", peer_state_name(p->state)); + json_array_start(response, "netaddr"); if (p->addr.type != ADDR_TYPE_PADDING) - json_add_string(response, "netaddr", + json_add_string(response, NULL, type_to_string(response, struct wireaddr, &p->addr)); + json_array_end(response); json_add_pubkey(response, "peerid", &p->id); json_add_bool(response, "connected", p->owner != NULL); if (p->owner) @@ -880,9 +882,12 @@ static void gossipd_getpeers_complete(struct subd *gossip, const u8 *msg, /* Fake state. */ json_add_string(response, "state", "GOSSIPING"); json_add_pubkey(response, "peerid", ids+i); - json_add_string(response, "netaddr", - type_to_string(response, struct wireaddr, - addrs + i)); + json_array_start(response, "netaddr"); + if (addrs[i].type != ADDR_TYPE_PADDING) + json_add_string(response, NULL, + type_to_string(response, struct wireaddr, + addrs + i)); + json_array_end(response); json_add_bool(response, "connected", "true"); json_add_string(response, "owner", gossip->name); json_object_end(response); From d07b827050266fe6d41aa0140f54d6ca7932eef2 Mon Sep 17 00:00:00 2001 From: Jonas Nick Date: Mon, 13 Nov 2017 10:03:12 +0000 Subject: [PATCH 0114/1428] Mention in README that 6 confirmations are required to fund a channel --- README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c84c57df3545..494e816f1ff4 100644 --- a/README.md +++ b/README.md @@ -84,16 +84,17 @@ cli/lightning-cli fundchannel ``` This opens a connection and, on top of that connection, then opens a channel. -You can check the status of the channel using `cli/lightning-cli getpeers`. -The funding transaction needs to confirm in order for the channel to be usable, so wait a few minutes, and once that is complete it `getpeers` should say that the status is in _Normal operation_. +The funding transaction needs 6 confirmations in order for the channel to be usable. +You can check the status of the channel using `cli/lightning-cli getpeers`, which after 1 confirmation should say that the status is in _Normal operation_. +After 6 confirmations you can use `cli/lightning-cli getchannels` to verify that the channel shows up in the list of open channels. ### Receiving and receiving payments Payments in Lightning are invoice based. -The recipient creates an invoice with the expected `` in millisatoshi and a `