From a63c44021b78a2a5fda92777bb9a5594e688c845 Mon Sep 17 00:00:00 2001 From: Christian Decker Date: Fri, 28 May 2021 17:10:05 +0200 Subject: [PATCH] pay: Add grouping key to the payments table We add `groupid` as a uniqueness constraint on the table. This allows us to retry an invoice multiple times, without having to partition the partids to differentiate them. --- contrib/pyln-client/pyln/client/lightning.py | 6 +- doc/lightning-listsendpays.7.md | 3 +- doc/lightning-sendpay.7.md | 3 +- doc/lightning-waitsendpay.7.md | 3 +- doc/schemas/listsendpays.schema.json | 7 ++ doc/schemas/sendpay.schema.json | 6 ++ doc/schemas/waitsendpay.schema.json | 5 ++ lightningd/pay.c | 16 ++-- plugins/libplugin-pay.c | 3 + plugins/libplugin-pay.h | 1 + wallet/db.c | 82 ++++++++++++++++++++ wallet/wallet.c | 17 +++- wallet/wallet.h | 3 +- 13 files changed, 141 insertions(+), 14 deletions(-) diff --git a/contrib/pyln-client/pyln/client/lightning.py b/contrib/pyln-client/pyln/client/lightning.py index c128ff4f0cc1..246111408610 100644 --- a/contrib/pyln-client/pyln/client/lightning.py +++ b/contrib/pyln-client/pyln/client/lightning.py @@ -1138,7 +1138,7 @@ def plugin_rescan(self): } return self.call("plugin", payload) - def sendpay(self, route, payment_hash, label=None, msatoshi=None, bolt11=None, payment_secret=None, partid=None): + def sendpay(self, route, payment_hash, label=None, msatoshi=None, bolt11=None, payment_secret=None, partid=None, groupid=None): """ Send along {route} in return for preimage of {payment_hash}. """ @@ -1150,6 +1150,7 @@ def sendpay(self, route, payment_hash, label=None, msatoshi=None, bolt11=None, p "bolt11": bolt11, "payment_secret": payment_secret, "partid": partid, + "groupid": groupid, } return self.call("sendpay", payload) @@ -1206,7 +1207,7 @@ def waitinvoice(self, label): } return self.call("waitinvoice", payload) - def waitsendpay(self, payment_hash, timeout=None, partid=None): + def waitsendpay(self, payment_hash, timeout=None, partid=None, groupid=None): """ Wait for payment for preimage of {payment_hash} to complete. """ @@ -1214,6 +1215,7 @@ def waitsendpay(self, payment_hash, timeout=None, partid=None): "payment_hash": payment_hash, "timeout": timeout, "partid": partid, + "groupid": groupid, } return self.call("waitsendpay", payload) diff --git a/doc/lightning-listsendpays.7.md b/doc/lightning-listsendpays.7.md index 804d55e31c88..67c42e9bdfd0 100644 --- a/doc/lightning-listsendpays.7.md +++ b/doc/lightning-listsendpays.7.md @@ -29,6 +29,7 @@ On success, an object containing **payments** is returned. It is an array of ob - **status** (string): status of the payment (one of "pending", "failed", "complete") - **created_at** (u64): the UNIX timestamp showing when this payment was initiated - **amount_sent_msat** (msat): The amount sent +- **groupid** (u64, optional): Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash - **amount_msat** (msat, optional): The amount delivered to destination (if known) - **destination** (pubkey, optional): the final destination of the payment if known - **label** (string, optional): the label, if given to sendpay @@ -59,4 +60,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:e6a71b4e168fe64b774dc4738a46470edf00b4566ae2927166621ba79f89ca79) +[comment]: # ( SHA256STAMP:2cc3f9f1b830c2ef9f2027c79547af7e2096313b68c045c517870659f3499d38) diff --git a/doc/lightning-sendpay.7.md b/doc/lightning-sendpay.7.md index 6fde3345d8fd..1bcddd7fc180 100644 --- a/doc/lightning-sendpay.7.md +++ b/doc/lightning-sendpay.7.md @@ -61,6 +61,7 @@ On success, an object is returned, containing: - **status** (string): status of the payment (could be complete if already sent previously) (one of "pending", "complete") - **created_at** (u64): the UNIX timestamp showing when this payment was initiated - **amount_sent_msat** (msat): The amount sent +- **groupid** (u64, optional): Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash - **amount_msat** (msat, optional): The amount delivered to destination (if known) - **destination** (pubkey, optional): the final destination of the payment if known - **label** (string, optional): the *label*, if given to sendpay @@ -126,4 +127,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:4dd638126f66c3c4b233b3bb0aaeec5f225a80ec7186edd1901f8ed4cf23380e) +[comment]: # ( SHA256STAMP:f7572da509a442c08f73460c042d8e2aa950747ce175ebb9b89d32b88add6de6) diff --git a/doc/lightning-waitsendpay.7.md b/doc/lightning-waitsendpay.7.md index b062928bd878..42d526d094df 100644 --- a/doc/lightning-waitsendpay.7.md +++ b/doc/lightning-waitsendpay.7.md @@ -39,6 +39,7 @@ On success, an object is returned, containing: - **status** (string): status of the payment (always "complete") - **created_at** (u64): the UNIX timestamp showing when this payment was initiated - **amount_sent_msat** (msat): The amount sent +- **groupid** (u64, optional): Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash - **amount_msat** (msat, optional): The amount delivered to destination (if known) - **destination** (pubkey, optional): the final destination of the payment if known - **label** (string, optional): the label, if given to sendpay @@ -100,4 +101,4 @@ RESOURCES Main web site: -[comment]: # ( SHA256STAMP:7a51daf13b7275c49a6c6f411de005998c674e9d5290ee83f3b41ea711da42f1) +[comment]: # ( SHA256STAMP:c40814f929fb6d741e0724ba75f0833e52fae1f03ed2d1fac9a8ba1186ceabab) diff --git a/doc/schemas/listsendpays.schema.json b/doc/schemas/listsendpays.schema.json index a66ec344b17b..8985a1b79495 100644 --- a/doc/schemas/listsendpays.schema.json +++ b/doc/schemas/listsendpays.schema.json @@ -15,6 +15,10 @@ "type": "u64", "description": "unique ID for this payment attempt" }, + "groupid": { + "type": "u64", + "description": "Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash" + }, "payment_hash": { "type": "hex", "description": "the hash of the *payment_preimage* which will prove payment", @@ -76,6 +80,7 @@ "required": [ "payment_preimage" ], "properties": { "id": { }, + "groupid": { }, "payment_hash": { }, "status": { }, "msatoshi": { }, @@ -110,6 +115,7 @@ "required": [ ], "properties": { "id": { }, + "groupid": { }, "payment_hash": { }, "status": { }, "msatoshi": { }, @@ -142,6 +148,7 @@ "required": [ ], "properties": { "id": { }, + "groupid": { }, "payment_hash": { }, "status": { }, "msatoshi": { }, diff --git a/doc/schemas/sendpay.schema.json b/doc/schemas/sendpay.schema.json index 64802e62852a..5089d45534de 100644 --- a/doc/schemas/sendpay.schema.json +++ b/doc/schemas/sendpay.schema.json @@ -8,6 +8,10 @@ "type": "u64", "description": "unique ID for this payment attempt" }, + "groupid": { + "type": "u64", + "description": "Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash" + }, "payment_hash": { "type": "hex", "description": "the hash of the *payment_preimage* which will prove payment", @@ -73,6 +77,7 @@ "required": [ "payment_preimage" ], "properties": { "id": { }, + "groupid": { }, "payment_hash": { }, "status": { }, "msatoshi": { }, @@ -108,6 +113,7 @@ "required": [ "message" ], "properties": { "id": { }, + "groupid": { }, "payment_hash": { }, "status": { }, "msatoshi": { }, diff --git a/doc/schemas/waitsendpay.schema.json b/doc/schemas/waitsendpay.schema.json index 1ec0ff4d7d9e..5932a9b37640 100644 --- a/doc/schemas/waitsendpay.schema.json +++ b/doc/schemas/waitsendpay.schema.json @@ -8,6 +8,10 @@ "type": "u64", "description": "unique ID for this payment attempt" }, + "groupid": { + "type": "u64", + "description": "Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash" + }, "payment_hash": { "type": "hex", "description": "the hash of the *payment_preimage* which will prove payment", @@ -73,6 +77,7 @@ "required": [ "payment_preimage" ], "properties": { "id": { }, + "groupid": { }, "payment_hash": { }, "status": { }, "msatoshi": { }, diff --git a/lightningd/pay.c b/lightningd/pay.c index b1a68903d5cf..31a81757b14e 100644 --- a/lightningd/pay.c +++ b/lightningd/pay.c @@ -111,6 +111,7 @@ void json_add_payment_fields(struct json_stream *response, { json_add_u64(response, "id", t->id); json_add_sha256(response, "payment_hash", &t->payment_hash); + json_add_u64(response, "groupid", t->groupid); if (t->partid) json_add_u64(response, "partid", t->partid); if (t->destination != NULL) @@ -839,6 +840,7 @@ send_payment_core(struct lightningd *ld, struct command *cmd, const struct sha256 *rhash, u64 partid, + u64 group, const struct route_hop *first_hop, struct amount_msat msat, struct amount_msat total_msat, @@ -1038,6 +1040,7 @@ send_payment_core(struct lightningd *ld, payment->id = 0; payment->payment_hash = *rhash; payment->partid = partid; + payment->groupid = group; if (destination) payment->destination = tal_dup(payment, struct node_id, destination); else @@ -1083,6 +1086,7 @@ send_payment(struct lightningd *ld, struct command *cmd, const struct sha256 *rhash, u64 partid, + u64 group, const struct route_hop *route, struct amount_msat msat, struct amount_msat total_msat, @@ -1169,7 +1173,7 @@ send_payment(struct lightningd *ld, type_to_string(tmpctx, struct amount_msat, &route[0].amount), n_hops, type_to_string(tmpctx, struct amount_msat, &msat)); packet = create_onionpacket(tmpctx, path, ROUTING_INFO_SIZE, &path_secrets); - return send_payment_core(ld, cmd, rhash, partid, &route[0], + return send_payment_core(ld, cmd, rhash, partid, group, &route[0], msat, total_msat, label, invstring, packet, &ids[n_hops - 1], ids, channels, path_secrets, local_offer_id); @@ -1245,7 +1249,7 @@ static struct command_result *json_sendonion(struct command *cmd, struct node_id *destination; struct secret *path_secrets; struct amount_msat *msat; - u64 *partid; + u64 *partid, *group; struct sha256 *local_offer_id = NULL; if (!param(cmd, buffer, params, @@ -1260,6 +1264,7 @@ static struct command_result *json_sendonion(struct command *cmd, p_opt_def("msatoshi", param_msat, &msat, AMOUNT_MSAT(0)), p_opt("destination", param_node_id, &destination), p_opt("localofferid", param_sha256, &local_offer_id), + p_opt_def("groupid", param_u64, &group, 0), NULL)) return command_param_failed(); @@ -1271,7 +1276,7 @@ static struct command_result *json_sendonion(struct command *cmd, "with failcode=%d", failcode); - return send_payment_core(ld, cmd, payment_hash, *partid, + return send_payment_core(ld, cmd, payment_hash, *partid, *group, first_hop, *msat, AMOUNT_MSAT(0), label, invstring, packet, destination, NULL, NULL, path_secrets, local_offer_id); @@ -1397,7 +1402,7 @@ static struct command_result *json_sendpay(struct command *cmd, struct route_hop *route; struct amount_msat *msat; const char *invstring, *label; - u64 *partid; + u64 *partid, *group; struct secret *payment_secret; struct sha256 *local_offer_id = NULL; @@ -1412,6 +1417,7 @@ static struct command_result *json_sendpay(struct command *cmd, p_opt("payment_secret", param_secret, &payment_secret), p_opt_def("partid", param_u64, &partid, 0), p_opt("localofferid", param_sha256, &local_offer_id), + p_opt_def("groupid", param_u64, &group, 0), NULL)) return command_param_failed(); @@ -1451,7 +1457,7 @@ static struct command_result *json_sendpay(struct command *cmd, return command_fail(cmd, JSONRPC2_INVALID_PARAMS, "partid requires payment_secret"); - return send_payment(cmd->ld, cmd, rhash, *partid, + return send_payment(cmd->ld, cmd, rhash, *partid, *group, route, final_amount, msat ? *msat : final_amount, diff --git a/plugins/libplugin-pay.c b/plugins/libplugin-pay.c index d3b2ff04122b..8fad8c58581a 100644 --- a/plugins/libplugin-pay.c +++ b/plugins/libplugin-pay.c @@ -89,6 +89,7 @@ struct payment *payment_new(tal_t *ctx, struct command *cmd, p->id = parent->id; p->local_id = parent->local_id; p->local_offer_id = parent->local_offer_id; + p->groupid = parent->groupid; } else { assert(cmd != NULL); p->partid = 0; @@ -100,6 +101,7 @@ struct payment *payment_new(tal_t *ctx, struct command *cmd, /* Caller must set this. */ p->local_id = NULL; p->local_offer_id = NULL; + p->groupid = 0; } /* Initialize all modifier data so we can point to the fields when @@ -1525,6 +1527,7 @@ static struct command_result *payment_createonion_success(struct command *cmd, json_array_end(req->js); json_add_num(req->js, "partid", p->partid); + json_add_u64(req->js, "groupid", p->groupid); if (p->label) json_add_string(req->js, "label", p->label); diff --git a/plugins/libplugin-pay.h b/plugins/libplugin-pay.h index 1165fd1bf128..81307bded507 100644 --- a/plugins/libplugin-pay.h +++ b/plugins/libplugin-pay.h @@ -186,6 +186,7 @@ struct payment { u32 partid; u32 next_partid; + u64 groupid; /* Destination we should ask `getroute` for. This might differ from * the above destination if we use rendez-vous routing of blinded diff --git a/wallet/db.c b/wallet/db.c index 7184ad7ff864..cd3e9671c995 100644 --- a/wallet/db.c +++ b/wallet/db.c @@ -750,6 +750,88 @@ static struct migration dbmigrations[] = { NULL}, {SQL("CREATE INDEX channel_state_changes_channel_id" " ON channel_state_changes (channel_id);"), NULL}, + /* We need to switch the unique key to cover the groupid as well, + * so we can attempt payments multiple times. */ + {SQL("ALTER TABLE payments RENAME TO temp_payments;"), NULL}, + {SQL("CREATE TABLE payments (" + " id BIGSERIAL" + ", timestamp INTEGER" + ", status INTEGER" + ", payment_hash BLOB" + ", destination BLOB" + ", msatoshi BIGINT" + ", payment_preimage BLOB" + ", path_secrets BLOB" + ", route_nodes BLOB" + ", route_channels BLOB" + ", failonionreply BLOB" + ", faildestperm INTEGER" + ", failindex INTEGER" + ", failcode INTEGER" + ", failnode BLOB" + ", failchannel TEXT" + ", failupdate BLOB" + ", msatoshi_sent BIGINT" + ", faildetail TEXT" + ", description TEXT" + ", faildirection INTEGER" + ", bolt11 TEXT" + ", total_msat BIGINT" + ", partid BIGINT" + ", groupid BIGINT NOT NULL DEFAULT 0" + ", local_offer_id BLOB DEFAULT NULL REFERENCES offers(offer_id)" + ", PRIMARY KEY (id)" + ", UNIQUE (payment_hash, partid, groupid))"), NULL}, + {SQL("INSERT INTO payments (" + "id" + ", timestamp" + ", status" + ", payment_hash" + ", destination" + ", msatoshi" + ", payment_preimage" + ", path_secrets" + ", route_nodes" + ", route_channels" + ", failonionreply" + ", faildestperm" + ", failindex" + ", failcode" + ", failnode" + ", failchannel" + ", failupdate" + ", msatoshi_sent" + ", faildetail" + ", description" + ", faildirection" + ", bolt11" + ", groupid" + ", local_offer_id)" + "SELECT id" + ", timestamp" + ", status" + ", payment_hash" + ", destination" + ", msatoshi" + ", payment_preimage" + ", path_secrets" + ", route_nodes" + ", route_channels" + ", failonionreply" + ", faildestperm" + ", failindex" + ", failcode" + ", failnode" + ", failchannel" + ", failupdate" + ", msatoshi_sent" + ", faildetail" + ", description" + ", faildirection" + ", bolt11" + ", 0" + ", local_offer_id FROM temp_payments;"), NULL}, + {SQL("DROP TABLE temp_payments;"), NULL}, }; /* Leak tracking. */ diff --git a/wallet/wallet.c b/wallet/wallet.c index 61e392ab953c..342f71e4018e 100644 --- a/wallet/wallet.c +++ b/wallet/wallet.c @@ -2931,8 +2931,9 @@ void wallet_payment_store(struct wallet *wallet, " bolt11," " total_msat," " partid," - " local_offer_id" - ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")); + " local_offer_id," + " groupid" + ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);")); db_bind_int(stmt, 0, payment->status); db_bind_sha256(stmt, 1, &payment->payment_hash); @@ -2979,6 +2980,8 @@ void wallet_payment_store(struct wallet *wallet, else db_bind_null(stmt, 13); + db_bind_u64(stmt, 14, payment->groupid); + db_exec_prepared_v2(stmt); payment->id = db_last_insert_id_v2(stmt); assert(payment->id > 0); @@ -3105,6 +3108,11 @@ static struct wallet_payment *wallet_stmt2payment(const tal_t *ctx, } else payment->local_offer_id = NULL; + if (!db_column_is_null(stmt, 17)) + payment->groupid = db_column_u64(stmt, 17); + else + payment->groupid = 0; + return payment; } @@ -3139,6 +3147,7 @@ wallet_payment_by_hash(const tal_t *ctx, struct wallet *wallet, ", total_msat" ", partid" ", local_offer_id" + ", groupid" " FROM payments" " WHERE payment_hash = ?" " AND partid = ?")); @@ -3370,9 +3379,10 @@ wallet_payment_list(const tal_t *ctx, ", total_msat" ", partid" ", local_offer_id" + ", groupid" " FROM payments" " WHERE (payment_hash = ? OR ?) AND (status = ? OR ?)" - " ORDER BY id;")); + " ORDER BY timestamp, id;")); if (payment_hash) db_bind_sha256(stmt, 0, payment_hash); @@ -3433,6 +3443,7 @@ wallet_payments_by_offer(const tal_t *ctx, ", total_msat" ", partid" ", local_offer_id" + ", groupid" " FROM payments" " WHERE local_offer_id = ?;")); db_bind_sha256(stmt, 0, local_offer_id); diff --git a/wallet/wallet.h b/wallet/wallet.h index 4814b2022d42..21bd5dd0039b 100644 --- a/wallet/wallet.h +++ b/wallet/wallet.h @@ -222,9 +222,10 @@ struct wallet_payment { u64 id; u32 timestamp; - /* The combination of these two fields is unique: */ + /* The combination of these three fields is unique: */ struct sha256 payment_hash; u64 partid; + u64 groupid; enum wallet_payment_status status;