pay: Add grouping key to the payments table

We add `groupid` as a uniqueness constraint on the table. This allows
us to retry an invoice multiple times, without having to partition the
partids to differentiate them.

contrib/pyln-client/pyln/client/lightning.py

@@ -1138,7 +1138,7 @@ def plugin_rescan(self):
         }
         return self.call("plugin", payload)
 
-    def sendpay(self, route, payment_hash, label=None, msatoshi=None, bolt11=None, payment_secret=None, partid=None):
+    def sendpay(self, route, payment_hash, label=None, msatoshi=None, bolt11=None, payment_secret=None, partid=None, groupid=None):
         """
         Send along {route} in return for preimage of {payment_hash}.
         """
@@ -1150,6 +1150,7 @@ def sendpay(self, route, payment_hash, label=None, msatoshi=None, bolt11=None, p
             "bolt11": bolt11,
             "payment_secret": payment_secret,
             "partid": partid,
+            "groupid": groupid,
         }
         return self.call("sendpay", payload)
 
@@ -1206,14 +1207,15 @@ def waitinvoice(self, label):
         }
         return self.call("waitinvoice", payload)
 
-    def waitsendpay(self, payment_hash, timeout=None, partid=None):
+    def waitsendpay(self, payment_hash, timeout=None, partid=None, groupid=None):
         """
         Wait for payment for preimage of {payment_hash} to complete.
         """
         payload = {
             "payment_hash": payment_hash,
             "timeout": timeout,
             "partid": partid,
+            "groupid": groupid,
         }
         return self.call("waitsendpay", payload)
 

doc/lightning-listsendpays.7.md

@@ -29,6 +29,7 @@ On success, an object containing **payments** is returned.  It is an array of ob
 - **status** (string): status of the payment (one of "pending", "failed", "complete")
 - **created_at** (u64): the UNIX timestamp showing when this payment was initiated
 - **amount_sent_msat** (msat): The amount sent
+- **groupid** (u64, optional): Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash
 - **amount_msat** (msat, optional): The amount delivered to destination (if known)
 - **destination** (pubkey, optional): the final destination of the payment if known
 - **label** (string, optional): the label, if given to sendpay
@@ -59,4 +60,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:e6a71b4e168fe64b774dc4738a46470edf00b4566ae2927166621ba79f89ca79)
+[comment]: # ( SHA256STAMP:2cc3f9f1b830c2ef9f2027c79547af7e2096313b68c045c517870659f3499d38)

doc/lightning-sendpay.7.md

@@ -61,6 +61,7 @@ On success, an object is returned, containing:
 - **status** (string): status of the payment (could be complete if already sent previously) (one of "pending", "complete")
 - **created_at** (u64): the UNIX timestamp showing when this payment was initiated
 - **amount_sent_msat** (msat): The amount sent
+- **groupid** (u64, optional): Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash
 - **amount_msat** (msat, optional): The amount delivered to destination (if known)
 - **destination** (pubkey, optional): the final destination of the payment if known
 - **label** (string, optional): the *label*, if given to sendpay
@@ -126,4 +127,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:4dd638126f66c3c4b233b3bb0aaeec5f225a80ec7186edd1901f8ed4cf23380e)
+[comment]: # ( SHA256STAMP:f7572da509a442c08f73460c042d8e2aa950747ce175ebb9b89d32b88add6de6)

doc/lightning-waitsendpay.7.md

@@ -39,6 +39,7 @@ On success, an object is returned, containing:
 - **status** (string): status of the payment (always "complete")
 - **created_at** (u64): the UNIX timestamp showing when this payment was initiated
 - **amount_sent_msat** (msat): The amount sent
+- **groupid** (u64, optional): Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash
 - **amount_msat** (msat, optional): The amount delivered to destination (if known)
 - **destination** (pubkey, optional): the final destination of the payment if known
 - **label** (string, optional): the label, if given to sendpay
@@ -100,4 +101,4 @@ RESOURCES
 
 Main web site: <https://github.com/ElementsProject/lightning>
 
-[comment]: # ( SHA256STAMP:7a51daf13b7275c49a6c6f411de005998c674e9d5290ee83f3b41ea711da42f1)
+[comment]: # ( SHA256STAMP:c40814f929fb6d741e0724ba75f0833e52fae1f03ed2d1fac9a8ba1186ceabab)

doc/schemas/listsendpays.schema.json

@@ -15,6 +15,10 @@
 			"type": "u64",
 			"description": "unique ID for this payment attempt"
 		    },
+		    "groupid": {
+			"type": "u64",
+			"description": "Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash"
+		    },
 		    "payment_hash": {
 			"type": "hex",
 			"description": "the hash of the *payment_preimage* which will prove payment",
@@ -76,6 +80,7 @@
 			    "required": [ "payment_preimage" ],
 			    "properties": {
 				"id": { },
+				"groupid": { },
 				"payment_hash": { },
 				"status": { },
 				"msatoshi": { },
@@ -110,6 +115,7 @@
 			    "required": [  ],
 			    "properties": {
 				"id": { },
+				"groupid": { },
 				"payment_hash": { },
 				"status": { },
 				"msatoshi": { },
@@ -142,6 +148,7 @@
 			    "required": [  ],
 			    "properties": {
 				"id": { },
+				"groupid": { },
 				"payment_hash": { },
 				"status": { },
 				"msatoshi": { },

doc/schemas/sendpay.schema.json

@@ -8,6 +8,10 @@
 	    "type": "u64",
 	    "description": "unique ID for this payment attempt"
 	},
+	"groupid": {
+	    "type": "u64",
+	    "description": "Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash"
+	},
 	"payment_hash": {
 	    "type": "hex",
 	    "description": "the hash of the *payment_preimage* which will prove payment",
@@ -73,6 +77,7 @@
 		"required": [ "payment_preimage" ],
 		"properties": {
 		    "id": { },
+		    "groupid": { },
 		    "payment_hash": { },
 		    "status": { },
 		    "msatoshi": { },
@@ -108,6 +113,7 @@
 		"required": [ "message" ],
 		"properties": {
 		    "id": { },
+		    "groupid": { },
 		    "payment_hash": { },
 		    "status": { },
 		    "msatoshi": { },

doc/schemas/waitsendpay.schema.json

@@ -8,6 +8,10 @@
 	    "type": "u64",
 	    "description": "unique ID for this payment attempt"
 	},
+	"groupid": {
+	    "type": "u64",
+	    "description": "Grouping key to disambiguate multiple attempts to pay an invoice or the same payment_hash"
+	},
 	"payment_hash": {
 	    "type": "hex",
 	    "description": "the hash of the *payment_preimage* which will prove payment",
@@ -73,6 +77,7 @@
 		"required": [ "payment_preimage" ],
 		"properties": {
 		    "id": { },
+		    "groupid": { },
 		    "payment_hash": { },
 		    "status": { },
 		    "msatoshi": { },

lightningd/pay.c

@@ -111,6 +111,7 @@ void json_add_payment_fields(struct json_stream *response,
 {
 	json_add_u64(response, "id", t->id);
 	json_add_sha256(response, "payment_hash", &t->payment_hash);
+	json_add_u64(response, "groupid", t->groupid);
 	if (t->partid)
 		json_add_u64(response, "partid", t->partid);
 	if (t->destination != NULL)
@@ -839,6 +840,7 @@ send_payment_core(struct lightningd *ld,
 		  struct command *cmd,
 		  const struct sha256 *rhash,
 		  u64 partid,
+		  u64 group,
 		  const struct route_hop *first_hop,
 		  struct amount_msat msat,
 		  struct amount_msat total_msat,
@@ -1038,6 +1040,7 @@ send_payment_core(struct lightningd *ld,
 	payment->id = 0;
 	payment->payment_hash = *rhash;
 	payment->partid = partid;
+	payment->groupid = group;
 	if (destination)
 		payment->destination = tal_dup(payment, struct node_id, destination);
 	else
@@ -1083,6 +1086,7 @@ send_payment(struct lightningd *ld,
 	     struct command *cmd,
 	     const struct sha256 *rhash,
 	     u64 partid,
+	     u64 group,
 	     const struct route_hop *route,
 	     struct amount_msat msat,
 	     struct amount_msat total_msat,
@@ -1169,7 +1173,7 @@ send_payment(struct lightningd *ld,
 		 type_to_string(tmpctx, struct amount_msat, &route[0].amount),
 		 n_hops, type_to_string(tmpctx, struct amount_msat, &msat));
 	packet = create_onionpacket(tmpctx, path, ROUTING_INFO_SIZE, &path_secrets);
-	return send_payment_core(ld, cmd, rhash, partid, &route[0],
+	return send_payment_core(ld, cmd, rhash, partid, group, &route[0],
 				 msat, total_msat, label, invstring,
 				 packet, &ids[n_hops - 1], ids,
 				 channels, path_secrets, local_offer_id);
@@ -1245,7 +1249,7 @@ static struct command_result *json_sendonion(struct command *cmd,
 	struct node_id *destination;
 	struct secret *path_secrets;
 	struct amount_msat *msat;
-	u64 *partid;
+	u64 *partid, *group;
 	struct sha256 *local_offer_id = NULL;
 
 	if (!param(cmd, buffer, params,
@@ -1260,6 +1264,7 @@ static struct command_result *json_sendonion(struct command *cmd,
 		   p_opt_def("msatoshi", param_msat, &msat, AMOUNT_MSAT(0)),
 		   p_opt("destination", param_node_id, &destination),
 		   p_opt("localofferid", param_sha256, &local_offer_id),
+		   p_opt_def("groupid", param_u64, &group, 0),
 		   NULL))
 		return command_param_failed();
 
@@ -1271,7 +1276,7 @@ static struct command_result *json_sendonion(struct command *cmd,
 				    "with failcode=%d",
 				    failcode);
 
-	return send_payment_core(ld, cmd, payment_hash, *partid,
+	return send_payment_core(ld, cmd, payment_hash, *partid, *group,
 				 first_hop, *msat, AMOUNT_MSAT(0),
 				 label, invstring, packet, destination, NULL, NULL,
 				 path_secrets, local_offer_id);
@@ -1397,7 +1402,7 @@ static struct command_result *json_sendpay(struct command *cmd,
 	struct route_hop *route;
 	struct amount_msat *msat;
 	const char *invstring, *label;
-	u64 *partid;
+	u64 *partid, *group;
 	struct secret *payment_secret;
 	struct sha256 *local_offer_id = NULL;
 
@@ -1412,6 +1417,7 @@ static struct command_result *json_sendpay(struct command *cmd,
 		   p_opt("payment_secret", param_secret, &payment_secret),
 		   p_opt_def("partid", param_u64, &partid, 0),
 		   p_opt("localofferid", param_sha256, &local_offer_id),
+		   p_opt_def("groupid", param_u64, &group, 0),
 		   NULL))
 		return command_param_failed();
 
@@ -1451,7 +1457,7 @@ static struct command_result *json_sendpay(struct command *cmd,
 		return command_fail(cmd, JSONRPC2_INVALID_PARAMS,
 				    "partid requires payment_secret");
 
-	return send_payment(cmd->ld, cmd, rhash, *partid,
+	return send_payment(cmd->ld, cmd, rhash, *partid, *group,
 			    route,
 			    final_amount,
 			    msat ? *msat : final_amount,

plugins/libplugin-pay.c

@@ -89,6 +89,7 @@ struct payment *payment_new(tal_t *ctx, struct command *cmd,
 		p->id = parent->id;
 		p->local_id = parent->local_id;
 		p->local_offer_id = parent->local_offer_id;
+		p->groupid = parent->groupid;
 	} else {
 		assert(cmd != NULL);
 		p->partid = 0;
@@ -100,6 +101,7 @@ struct payment *payment_new(tal_t *ctx, struct command *cmd,
 		/* Caller must set this.  */
 		p->local_id = NULL;
 		p->local_offer_id = NULL;
+		p->groupid = 0;
 	}
 
 	/* Initialize all modifier data so we can point to the fields when
@@ -1525,6 +1527,7 @@ static struct command_result *payment_createonion_success(struct command *cmd,
 	json_array_end(req->js);
 
 	json_add_num(req->js, "partid", p->partid);
+	json_add_u64(req->js, "groupid", p->groupid);
 
 	if (p->label)
 		json_add_string(req->js, "label", p->label);

plugins/libplugin-pay.h

@@ -186,6 +186,7 @@ struct payment {
 
 	u32 partid;
 	u32 next_partid;
+	u64 groupid;
 
 	/* Destination we should ask `getroute` for. This might differ from
 	 * the above destination if we use rendez-vous routing of blinded

wallet/db.c

@@ -750,6 +750,88 @@ static struct migration dbmigrations[] = {
      NULL},
     {SQL("CREATE INDEX channel_state_changes_channel_id"
 	 " ON channel_state_changes (channel_id);"), NULL},
+    /* We need to switch the unique key to cover the groupid as well,
+     * so we can attempt payments multiple times. */
+    {SQL("ALTER TABLE payments RENAME TO temp_payments;"), NULL},
+    {SQL("CREATE TABLE payments ("
+	 " id BIGSERIAL"
+	 ", timestamp INTEGER"
+	 ", status INTEGER"
+	 ", payment_hash BLOB"
+	 ", destination BLOB"
+	 ", msatoshi BIGINT"
+	 ", payment_preimage BLOB"
+	 ", path_secrets BLOB"
+	 ", route_nodes BLOB"
+	 ", route_channels BLOB"
+	 ", failonionreply BLOB"
+	 ", faildestperm INTEGER"
+	 ", failindex INTEGER"
+	 ", failcode INTEGER"
+	 ", failnode BLOB"
+	 ", failchannel TEXT"
+	 ", failupdate BLOB"
+	 ", msatoshi_sent BIGINT"
+	 ", faildetail TEXT"
+	 ", description TEXT"
+	 ", faildirection INTEGER"
+	 ", bolt11 TEXT"
+	 ", total_msat BIGINT"
+	 ", partid BIGINT"
+	 ", groupid BIGINT NOT NULL DEFAULT 0"
+	 ", local_offer_id BLOB DEFAULT NULL REFERENCES offers(offer_id)"
+	 ", PRIMARY KEY (id)"
+	 ", UNIQUE (payment_hash, partid, groupid))"), NULL},
+    {SQL("INSERT INTO payments ("
+	 "id"
+	 ", timestamp"
+	 ", status"
+	 ", payment_hash"
+	 ", destination"
+	 ", msatoshi"
+	 ", payment_preimage"
+	 ", path_secrets"
+	 ", route_nodes"
+	 ", route_channels"
+	 ", failonionreply"
+	 ", faildestperm"
+	 ", failindex"
+	 ", failcode"
+	 ", failnode"
+	 ", failchannel"
+	 ", failupdate"
+	 ", msatoshi_sent"
+	 ", faildetail"
+	 ", description"
+	 ", faildirection"
+	 ", bolt11"
+	 ", groupid"
+	 ", local_offer_id)"
+	 "SELECT id"
+	 ", timestamp"
+	 ", status"
+	 ", payment_hash"
+	 ", destination"
+	 ", msatoshi"
+	 ", payment_preimage"
+	 ", path_secrets"
+	 ", route_nodes"
+	 ", route_channels"
+	 ", failonionreply"
+	 ", faildestperm"
+	 ", failindex"
+	 ", failcode"
+	 ", failnode"
+	 ", failchannel"
+	 ", failupdate"
+	 ", msatoshi_sent"
+	 ", faildetail"
+	 ", description"
+	 ", faildirection"
+	 ", bolt11"
+	 ", 0"
+	 ", local_offer_id FROM temp_payments;"), NULL},
+    {SQL("DROP TABLE temp_payments;"), NULL},
 };
 
 /* Leak tracking. */