From 50e05a9beadaaeac95dac04a221bb862671c2691 Mon Sep 17 00:00:00 2001 From: Steven Danna Date: Fri, 28 Jan 2022 18:46:25 +0000 Subject: [PATCH] sql: add ForcePut option to indexes This PR adds ForcePut mode to indexes in which all writes are handled via Put, even in cases where they would typically used CPut or InitPut. This change is in support of the mvcc-compatible index backfiller. As part of the backfill process, newly added indexes will be added in a BACKFILLING state in which they see no writes or deletes. However, this would allow for the following sequence of events: t0: `CREATE INDEX` creates "new index" in BACKFILLING state t1: `INSERT INTO` with a = 1 (new index does not see this write) t2: Backfill starts at t1 (a=1 included in backfill, ends up in "new index") t3: UPDATE a = 2 where a = 1 (new index still in backfilling does not see these writes) t4: Backfilling completes t5: "new index" to DELETE-ONLY t6: "new index" to DELETE-AND-WRITE-ONLY t7: `INSERT INTO` into with a = 1 At t7, the user would encounter an erroneous duplicate key violation when they attempt their insert because the attempted CPut into "new index" would fail as there is an existing value (a=1). The ForcePut mode solves this case. While use of this option would be unsafe in general because it allows for writes that violate the unique constraint on the index. However, when used as part of the index backfilling process it should be safe because indexes are checked for uniqueness before being made public. The tests added here are nearly identical to the tests added for DeletePreservingEncoding. Release note: None --- pkg/cli/testdata/doctor/test_recreate_zipdir | 12 +- pkg/sql/catalog/descpb/structured.proto | 7 + pkg/sql/catalog/table_elements.go | 18 ++ pkg/sql/catalog/tabledesc/index.go | 7 + pkg/sql/catalog/tabledesc/validate_test.go | 1 + .../exec/execbuilder/testdata/force_put_index | 61 ++++ .../testdata/show_trace_nonmetamorphic | 12 +- .../testdata/update_nonmetamorphic | 300 ++++++++++++++++++ pkg/sql/row/inserter.go | 4 +- pkg/sql/row/updater.go | 20 +- 10 files changed, 416 insertions(+), 26 deletions(-) create mode 100644 pkg/sql/opt/exec/execbuilder/testdata/force_put_index diff --git a/pkg/cli/testdata/doctor/test_recreate_zipdir b/pkg/cli/testdata/doctor/test_recreate_zipdir index 9bf65d0498cf..e52f0f585ec5 100644 --- a/pkg/cli/testdata/doctor/test_recreate_zipdir +++ b/pkg/cli/testdata/doctor/test_recreate_zipdir @@ -12,16 +12,16 @@ SELECT crdb_internal.unsafe_upsert_descriptor(50, decode('12380a0964656661756c74 SELECT crdb_internal.unsafe_upsert_namespace_entry(0, 0, 'defaultdb', 50, true); SELECT crdb_internal.unsafe_upsert_descriptor(51, decode('12370a08706f73746772657310331a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f7418012200280140004a00', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(0, 0, 'postgres', 51, true); -SELECT crdb_internal.unsafe_upsert_descriptor(53, decode('0aeb040a0575736572731835203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d0807100018003007509308600020003000680070007800800100880100980100422a0a046e616d6510031a0d0807100018003007509308600020013000680070007800800100880100980100422d0a076164647265737310041a0d080710001800300750930860002001300068007000780080010088010098010042310a0b6372656469745f6361726410051a0d08071000180030075093086000200130006800700078008001008801009801004806525a0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800101880103980100b2013d0a077072696d61727910001a0269641a04636974791a046e616d651a07616464726573731a0b6372656469745f63617264200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200aa02270836100210041802180120352a11666b5f636974795f7265665f75736572733002380040004800aa02270837100210041802180120352a11666b5f636974795f7265665f75736572733002380040004800aa0227083a100110021802180120352a11666b5f636974795f7265665f75736572733002380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(53, decode('0aee040a0575736572731835203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d0807100018003007509308600020003000680070007800800100880100980100422a0a046e616d6510031a0d0807100018003007509308600020013000680070007800800100880100980100422d0a076164647265737310041a0d080710001800300750930860002001300068007000780080010088010098010042310a0b6372656469745f6361726410051a0d08071000180030075093086000200130006800700078008001008801009801004806525d0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800101880103980100b2013d0a077072696d61727910001a0269641a04636974791a046e616d651a07616464726573731a0b6372656469745f63617264200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200aa02270836100210041802180120352a11666b5f636974795f7265665f75736572733002380040004800aa02270837100210041802180120352a11666b5f636974795f7265665f75736572733002380040004800aa0227083a100110021802180120352a11666b5f636974795f7265665f75736572733002380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'users', 53, true); -SELECT crdb_internal.unsafe_upsert_descriptor(54, decode('0a8e070a0876656869636c65731836203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d0807100018003007509308600020003000680070007800800100880100980100422a0a047479706510031a0d0807100018003007509308600020013000680070007800800100880100980100422e0a086f776e65725f696410041a0d080e10001800300050861760002001300068007000780080010088010098010042330a0d6372656174696f6e5f74696d6510051a0d080510001800300050da08600020013000680070007800800100880100980100422c0a0673746174757310061a0d080710001800300750930860002001300068007000780080010088010098010042360a1063757272656e745f6c6f636174696f6e10071a0d080710001800300750930860002001300068007000780080010088010098010042290a0365787410081a0d081210001800300050da1d6000200130006800700078008001008801009801004809525a0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c001005a80010a2576656869636c65735f6175746f5f696e6465785f666b5f636974795f7265665f75736572731002180022046369747922086f776e65725f6964300230043801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010060036a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b201650a077072696d61727910001a0269641a04636974791a04747970651a086f776e65725f69641a0d6372656174696f6e5f74696d651a067374617475731a1063757272656e745f6c6f636174696f6e1a03657874200120022003200420052006200720082800b80101c20100e80100f2010408001200f801008002009202009a0200a202270836100210041802180120352a11666b5f636974795f7265665f75736572733000380040004800aa02320837100310051802180120362a1c666b5f76656869636c655f636974795f7265665f76656869636c65733002380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(54, decode('0a94070a0876656869636c65731836203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d0807100018003007509308600020003000680070007800800100880100980100422a0a047479706510031a0d0807100018003007509308600020013000680070007800800100880100980100422e0a086f776e65725f696410041a0d080e10001800300050861760002001300068007000780080010088010098010042330a0d6372656174696f6e5f74696d6510051a0d080510001800300050da08600020013000680070007800800100880100980100422c0a0673746174757310061a0d080710001800300750930860002001300068007000780080010088010098010042360a1063757272656e745f6c6f636174696f6e10071a0d080710001800300750930860002001300068007000780080010088010098010042290a0365787410081a0d081210001800300050da1d6000200130006800700078008001008801009801004809525d0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c801005a83010a2576656869636c65735f6175746f5f696e6465785f666b5f636974795f7265665f75736572731002180022046369747922086f776e65725f6964300230043801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010060036a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b201650a077072696d61727910001a0269641a04636974791a04747970651a086f776e65725f69641a0d6372656174696f6e5f74696d651a067374617475731a1063757272656e745f6c6f636174696f6e1a03657874200120022003200420052006200720082800b80101c20100e80100f2010408001200f801008002009202009a0200a202270836100210041802180120352a11666b5f636974795f7265665f75736572733000380040004800aa02320837100310051802180120362a1c666b5f76656869636c655f636974795f7265665f76656869636c65733002380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'vehicles', 54, true); -SELECT crdb_internal.unsafe_upsert_descriptor(55, decode('0a960a0a0572696465731837203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d080710001800300750930860002000300068007000780080010088010098010042320a0c76656869636c655f6369747910031a0d0807100018003007509308600020013000680070007800800100880100980100422e0a0872696465725f696410041a0d080e10001800300050861760002001300068007000780080010088010098010042300a0a76656869636c655f696410051a0d080e10001800300050861760002001300068007000780080010088010098010042330a0d73746172745f6164647265737310061a0d080710001800300750930860002001300068007000780080010088010098010042310a0b656e645f6164647265737310071a0d080710001800300750930860002001300068007000780080010088010098010042300a0a73746172745f74696d6510081a0d080510001800300050da08600020013000680070007800800100880100980100422e0a08656e645f74696d6510091a0d080510001800300050da08600020013000680070007800800100880100980100422d0a07726576656e7565100a1a0d08031002180a300050a40d600020013000680070007800800100880100980100480b525a0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c001005a7d0a2272696465735f6175746f5f696e6465785f666b5f636974795f7265665f757365727310021800220463697479220872696465725f6964300230043801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c001005a94010a2d72696465735f6175746f5f696e6465785f666b5f76656869636c655f636974795f7265665f76656869636c657310031800220c76656869636c655f63697479220a76656869636c655f69643003300538023801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010060046a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800103880103980100a201380a1376656869636c655f63697479203d20636974791217636865636b5f76656869636c655f636974795f6369747918002802280330003800b2018a010a077072696d61727910001a0269641a04636974791a0c76656869636c655f636974791a0872696465725f69641a0a76656869636c655f69641a0d73746172745f616464726573731a0b656e645f616464726573731a0a73746172745f74696d651a08656e645f74696d651a07726576656e7565200120022003200420052006200720082009200a2800b80101c20100e80100f2010408001200f801008002009202009a0200a202270837100210041802180120352a11666b5f636974795f7265665f75736572733000380040004800a202320837100310051802180120362a1c666b5f76656869636c655f636974795f7265665f76656869636c65733000380040004800aa02270838100110021802180120372a11666b5f636974795f7265665f72696465733002380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(55, decode('0aa00a0a0572696465731837203428013a0042280a02696410011a0d080e100018003000508617600020003000680070007800800100880100980100422a0a046369747910021a0d080710001800300750930860002000300068007000780080010088010098010042320a0c76656869636c655f6369747910031a0d0807100018003007509308600020013000680070007800800100880100980100422e0a0872696465725f696410041a0d080e10001800300050861760002001300068007000780080010088010098010042300a0a76656869636c655f696410051a0d080e10001800300050861760002001300068007000780080010088010098010042330a0d73746172745f6164647265737310061a0d080710001800300750930860002001300068007000780080010088010098010042310a0b656e645f6164647265737310071a0d080710001800300750930860002001300068007000780080010088010098010042300a0a73746172745f74696d6510081a0d080510001800300050da08600020013000680070007800800100880100980100422e0a08656e645f74696d6510091a0d080510001800300050da08600020013000680070007800800100880100980100422d0a07726576656e7565100a1a0d08031002180a300050a40d600020013000680070007800800100880100980100480b525d0a077072696d617279100118012204636974792202696430023001400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c801005a80010a2272696465735f6175746f5f696e6465785f666b5f636974795f7265665f757365727310021800220463697479220872696465725f6964300230043801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c801005a97010a2d72696465735f6175746f5f696e6465785f666b5f76656869636c655f636974795f7265665f76656869636c657310031800220c76656869636c655f63697479220a76656869636c655f69643003300538023801400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010060046a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800103880103980100a201380a1376656869636c655f63697479203d20636974791217636865636b5f76656869636c655f636974795f6369747918002802280330003800b2018a010a077072696d61727910001a0269641a04636974791a0c76656869636c655f636974791a0872696465725f69641a0a76656869636c655f69641a0d73746172745f616464726573731a0b656e645f616464726573731a0a73746172745f74696d651a08656e645f74696d651a07726576656e7565200120022003200420052006200720082009200a2800b80101c20100e80100f2010408001200f801008002009202009a0200a202270837100210041802180120352a11666b5f636974795f7265665f75736572733000380040004800a202320837100310051802180120362a1c666b5f76656869636c655f636974795f7265665f76656869636c65733000380040004800aa02270838100110021802180120372a11666b5f636974795f7265665f72696465733002380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'rides', 55, true); -SELECT crdb_internal.unsafe_upsert_descriptor(56, decode('0abe040a1a76656869636c655f6c6f636174696f6e5f686973746f726965731838203428013a00422a0a046369747910011a0d0807100018003007509308600020003000680070007800800100880100980100422d0a07726964655f696410021a0d080e100018003000508617600020003000680070007800800100880100980100422f0a0974696d657374616d7010031a0d080510001800300050da0860002000300068007000780080010088010098010042290a036c617410041a0d080210401800300050bd05600020013000680070007800800100880100980100422a0a046c6f6e6710051a0d080210401800300050bd056000200130006800700078008001008801009801004806526e0a077072696d617279100118012204636974792207726964655f6964220974696d657374616d703001300230034000400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b2013c0a077072696d61727910001a04636974791a07726964655f69641a0974696d657374616d701a036c61741a046c6f6e67200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200a202270838100110021802180120372a11666b5f636974795f7265665f72696465733000380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(56, decode('0ac1040a1a76656869636c655f6c6f636174696f6e5f686973746f726965731838203428013a00422a0a046369747910011a0d0807100018003007509308600020003000680070007800800100880100980100422d0a07726964655f696410021a0d080e100018003000508617600020003000680070007800800100880100980100422f0a0974696d657374616d7010031a0d080510001800300050da0860002000300068007000780080010088010098010042290a036c617410041a0d080210401800300050bd05600020013000680070007800800100880100980100422a0a046c6f6e6710051a0d080210401800300050bd05600020013000680070007800800100880100980100480652710a077072696d617279100118012204636974792207726964655f6964220974696d657374616d703001300230034000400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b2013c0a077072696d61727910001a04636974791a07726964655f69641a0974696d657374616d701a036c61741a046c6f6e67200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200a202270838100110021802180120372a11666b5f636974795f7265665f72696465733000380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'vehicle_location_histories', 56, true); -SELECT crdb_internal.unsafe_upsert_descriptor(57, decode('0a93040a0b70726f6d6f5f636f6465731839203428013a00422a0a04636f646510011a0d080710001800300750930860002000300068007000780080010088010098010042310a0b6465736372697074696f6e10021a0d080710001800300750930860002001300068007000780080010088010098010042330a0d6372656174696f6e5f74696d6510031a0d080510001800300050da0860002001300068007000780080010088010098010042350a0f65787069726174696f6e5f74696d6510041a0d080510001800300050da08600020013000680070007800800100880100980100422b0a0572756c657310051a0d081210001800300050da1d600020013000680070007800800100880100980100480652520a077072696d617279100118012204636f6465300140004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800101880103980100b201510a077072696d61727910001a04636f64651a0b6465736372697074696f6e1a0d6372656174696f6e5f74696d651a0f65787069726174696f6e5f74696d651a0572756c6573200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200f00200', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(57, decode('0a96040a0b70726f6d6f5f636f6465731839203428013a00422a0a04636f646510011a0d080710001800300750930860002000300068007000780080010088010098010042310a0b6465736372697074696f6e10021a0d080710001800300750930860002001300068007000780080010088010098010042330a0d6372656174696f6e5f74696d6510031a0d080510001800300050da0860002001300068007000780080010088010098010042350a0f65787069726174696f6e5f74696d6510041a0d080510001800300050da08600020013000680070007800800100880100980100422b0a0572756c657310051a0d081210001800300050da1d600020013000680070007800800100880100980100480652550a077072696d617279100118012204636f6465300140004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800101880103980100b201510a077072696d61727910001a04636f64651a0b6465736372697074696f6e1a0d6372656174696f6e5f74696d651a0f65787069726174696f6e5f74696d651a0572756c6573200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200b20200b80200c0021dc80200e00200f00200', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'promo_codes', 57, true); -SELECT crdb_internal.unsafe_upsert_descriptor(58, decode('0abe040a10757365725f70726f6d6f5f636f646573183a203428013a00422a0a046369747910011a0d0807100018003007509308600020003000680070007800800100880100980100422d0a07757365725f696410021a0d080e100018003000508617600020003000680070007800800100880100980100422a0a04636f646510031a0d0807100018003007509308600020003000680070007800800100880100980100422f0a0974696d657374616d7010041a0d080510001800300050da0860002001300068007000780080010088010098010042300a0b75736167655f636f756e7410051a0c08011040180030005014600020013000680070007800800100880100980100480652690a077072696d617279100118012204636974792207757365725f69642204636f64653001300230034000400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c0010060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b201440a077072696d61727910001a04636974791a07757365725f69641a04636f64651a0974696d657374616d701a0b75736167655f636f756e74200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200a20227083a100110021802180120352a11666b5f636974795f7265665f75736572733000380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); +SELECT crdb_internal.unsafe_upsert_descriptor(58, decode('0ac1040a10757365725f70726f6d6f5f636f646573183a203428013a00422a0a046369747910011a0d0807100018003007509308600020003000680070007800800100880100980100422d0a07757365725f696410021a0d080e100018003000508617600020003000680070007800800100880100980100422a0a04636f646510031a0d0807100018003007509308600020003000680070007800800100880100980100422f0a0974696d657374616d7010041a0d080510001800300050da0860002001300068007000780080010088010098010042300a0b75736167655f636f756e7410051a0c080110401800300050146000200130006800700078008001008801009801004806526c0a077072696d617279100118012204636974792207757365725f69642204636f64653001300230034000400040004a10080010001a00200028003000380040005a007a0408002000800100880100900101980100a20106080012001800a80100b20100ba0100c00100c8010060026a210a0b0a0561646d696e100218000a0a0a04726f6f74100218001204726f6f741801800102880103980100b201440a077072696d61727910001a04636974791a07757365725f69641a04636f64651a0974696d657374616d701a0b75736167655f636f756e74200120022003200420052800b80101c20100e80100f2010408001200f801008002009202009a0200a20227083a100110021802180120352a11666b5f636974795f7265665f75736572733000380040004800b20200b80200c0021dc80200e00200f00200', 'hex'), true); SELECT crdb_internal.unsafe_upsert_namespace_entry(52, 29, 'user_promo_codes', 58, true); COMMIT; diff --git a/pkg/sql/catalog/descpb/structured.proto b/pkg/sql/catalog/descpb/structured.proto index 7a118e6bff4e..6f3dddf10dae 100644 --- a/pkg/sql/catalog/descpb/structured.proto +++ b/pkg/sql/catalog/descpb/structured.proto @@ -512,6 +512,13 @@ message IndexDescriptor { // index backfiller // docs/RFCS/20211004_incremental_index_backfiller.md#new-index-encoding-for-deletions-vs-mvcc optional bool use_delete_preserving_encoding = 24 [(gogoproto.nullable) = false]; + + // ForcePut, if true, forces all writes to the the index to + // use Put (rather than CPut or InitPut). + // + // This is used by the MVCC-compatible index backfiller to ensure + // that unique indexes do not produce erroneous conflicts. + optional bool force_put = 25 [(gogoproto.nullable) = false]; } // ConstraintToUpdate represents a constraint to be added to the table and diff --git a/pkg/sql/catalog/table_elements.go b/pkg/sql/catalog/table_elements.go index f8ec82b129d7..55b1dd107dcb 100644 --- a/pkg/sql/catalog/table_elements.go +++ b/pkg/sql/catalog/table_elements.go @@ -186,6 +186,24 @@ type Index interface { NumCompositeColumns() int GetCompositeColumnID(compositeColumnOrdinal int) descpb.ColumnID UseDeletePreservingEncoding() bool + // ForcePut, if true, forces all writes to use Put rather than CPut or InitPut. + // + // Users of this options should take great care as it + // effectively mean unique constraints are not respected. + // + // Currently (2022-01-19) this two users: delete preserving + // indexes and backfilling indexes. + // + // Delete preserving encoding indexes are used only as a log of + // index writes during backfill, thus we can blindly put values into + // them. + // + // Indexes in the backfilling state will be checked for uniqueness + // at the end of the backfilling process and may miss updates during + // the backfilling process that would lead to CPut failures until + // the missed updates are merged into the index. Uniqueness for such indexes + // is checked by the schema changer before they are brought back online. + ForcePut() bool } // Column is an interface around the column descriptor types. diff --git a/pkg/sql/catalog/tabledesc/index.go b/pkg/sql/catalog/tabledesc/index.go index 67ff328a275f..9076b40798a1 100644 --- a/pkg/sql/catalog/tabledesc/index.go +++ b/pkg/sql/catalog/tabledesc/index.go @@ -327,6 +327,13 @@ func (w index) UseDeletePreservingEncoding() bool { return w.desc.UseDeletePreservingEncoding } +// ForcePut returns true if writes to the index should only use Put +// (rather than CPut or InitPut). This is used by indexes currently +// being backfilled by the MVCC-compliant index backfiller. +func (w index) ForcePut() bool { + return w.desc.ForcePut || w.desc.UseDeletePreservingEncoding +} + // partitioning is the backing struct for a catalog.Partitioning interface. type partitioning struct { desc *descpb.PartitioningDescriptor diff --git a/pkg/sql/catalog/tabledesc/validate_test.go b/pkg/sql/catalog/tabledesc/validate_test.go index 66d842f23828..ceacce26c79e 100644 --- a/pkg/sql/catalog/tabledesc/validate_test.go +++ b/pkg/sql/catalog/tabledesc/validate_test.go @@ -168,6 +168,7 @@ var validationMap = []struct { "GeoConfig": {status: thisFieldReferencesNoObjects}, "Predicate": {status: iSolemnlySwearThisFieldIsValidated}, "UseDeletePreservingEncoding": {status: thisFieldReferencesNoObjects}, + "ForcePut": {status: thisFieldReferencesNoObjects}, }, }, { diff --git a/pkg/sql/opt/exec/execbuilder/testdata/force_put_index b/pkg/sql/opt/exec/execbuilder/testdata/force_put_index new file mode 100644 index 000000000000..16f2bbd6c986 --- /dev/null +++ b/pkg/sql/opt/exec/execbuilder/testdata/force_put_index @@ -0,0 +1,61 @@ +# LogicTest: local !metamorphic + +# --------------------------------------------------------- +# Ensure ForcePut indexes forces Puts even on unique indexes. +# --------------------------------------------------------- +statement ok +CREATE TABLE ti ( + a INT PRIMARY KEY, + b INT, + FAMILY (a, b), + UNIQUE INDEX (b) +); + +let $t_id +SELECT id FROM system.namespace WHERE name = 'ti' + +let $updated_t_jsonb +WITH + descs + AS ( + SELECT + id, + crdb_internal.pb_to_json( + 'cockroach.sql.sqlbase.Descriptor', + descriptor + ) + AS descriptor + FROM + system.descriptor + ) +SELECT + CAST (json_set(descriptor, ARRAY['table', 'indexes', '0', 'forcePut'], 'true') AS STRING) +FROM + descs WHERE id = $t_id; + +statement ok +SELECT * FROM crdb_internal.unsafe_upsert_descriptor($t_id, crdb_internal.json_to_pb('cockroach.sql.sqlbase.Descriptor',$$ $updated_t_jsonb $$), true) + +statement ok +INSERT INTO ti VALUES (1, 1), (2, 2) + +# Test that it's okay to insert into the unique +# force put index. +query T kvtrace +INSERT INTO ti VALUES (3, 1) +---- +CPut /Table/56/1/3/0 -> /TUPLE/2:2:Int/1 +Put /Table/56/2/1/0 -> /BYTES/0x8b + +# Test that Put's triggered but update that should fail, continue to +# work. +statement ok +INSERT INTO ti VALUES (4, 4) + +query T kvtrace +UPDATE ti SET b = 2 WHERE a = 4 +---- +Scan /Table/56/1/4/0 +Put /Table/56/1/4/0 -> /TUPLE/2:2:Int/2 +Del /Table/56/2/4/0 +Put /Table/56/2/2/0 -> /BYTES/0x8c diff --git a/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic b/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic index f2328dcefaa2..616fd6f3a7ee 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic +++ b/pkg/sql/opt/exec/execbuilder/testdata/show_trace_nonmetamorphic @@ -40,7 +40,7 @@ WHERE message NOT LIKE '%Z/%' AND operation != 'dist sender send' ---- batch flow coordinator CPut /NamespaceTable/30/1/56/57/"kv"/4/1 -> 58 -batch flow coordinator CPut /Table/3/1/58/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<> temporary:false partition_all_by:false > +batch flow coordinator CPut /Table/3/1/58/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false force_put:false > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<> temporary:false partition_all_by:false > exec stmt rows affected: 0 # We avoid using the full trace output, because that would make the @@ -66,7 +66,7 @@ WHERE message NOT LIKE '%Z/%' AND message NOT LIKE 'querying next range at%' AND tag NOT LIKE '%IndexBackfiller%' AND operation != 'dist sender send' ---- -batch flow coordinator Put /Table/3/1/58/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > mutations: interleave:<> partitioning: type:FORWARD created_explicitly:true encoding_type:0 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false > state:DELETE_ONLY direction:ADD mutation_id:1 rollback:false > next_mutation_id:2 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false mutationJobs:<...> new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false > +batch flow coordinator Put /Table/3/1/58/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false force_put:false > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > mutations: interleave:<> partitioning: type:FORWARD created_explicitly:true encoding_type:0 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false force_put:false > state:DELETE_ONLY direction:ADD mutation_id:1 rollback:false > next_mutation_id:2 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false mutationJobs:<...> new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false > exec stmt rows affected: 0 statement ok @@ -121,7 +121,7 @@ WHERE message NOT LIKE '%Z/%' AND operation != 'dist sender send' ---- batch flow coordinator CPut /NamespaceTable/30/1/56/57/"kv2"/4/1 -> 59 -batch flow coordinator CPut /Table/3/1/59/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:false default_expr:"unique_rowid()" hidden:true inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:4 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:ADD offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"TABLE t.public.kv" create_as_of_time:<> temporary:false partition_all_by:false > +batch flow coordinator CPut /Table/3/1/59/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:false default_expr:"unique_rowid()" hidden:true inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:4 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false force_put:false > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:ADD offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"TABLE t.public.kv" create_as_of_time:<> temporary:false partition_all_by:false > exec stmt rows affected: 0 statement ok @@ -170,7 +170,7 @@ WHERE message NOT LIKE '%Z/%' AND message NOT LIKE 'querying next range at%' AND operation != 'dist sender send' ---- batch flow coordinator Del /NamespaceTable/30/1/56/57/"kv2"/4/1 -batch flow coordinator Put /Table/3/1/59/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:false default_expr:"unique_rowid()" hidden:true inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:4 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:DROP offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:... replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"TABLE t.public.kv" create_as_of_time:<...> temporary:false partition_all_by:false > +batch flow coordinator Put /Table/3/1/59/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:false default_expr:"unique_rowid()" hidden:true inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:4 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false force_put:false > next_index_id:2 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:1 format_version:3 state:DROP offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:... replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"TABLE t.public.kv" create_as_of_time:<...> temporary:false partition_all_by:false > exec stmt rows affected: 0 statement ok @@ -204,7 +204,7 @@ WHERE message NOT LIKE '%Z/%' AND message NOT LIKE 'querying next range at%' AND tag NOT LIKE '%IndexBackfiller%' AND operation != 'dist sender send' ---- -batch flow coordinator Put /Table/3/1/58/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > mutations: interleave:<> partitioning: type:FORWARD created_explicitly:true encoding_type:0 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false > state:DELETE_AND_WRITE_ONLY direction:DROP mutation_id:2 rollback:false > next_mutation_id:3 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false mutationJobs:<...> new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false > +batch flow coordinator Put /Table/3/1/58/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false force_put:false > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > mutations: interleave:<> partitioning: type:FORWARD created_explicitly:true encoding_type:0 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false force_put:false > state:DELETE_AND_WRITE_ONLY direction:DROP mutation_id:2 rollback:false > next_mutation_id:3 format_version:3 state:PUBLIC offline_reason:"" view_query:"" is_materialized_view:false mutationJobs:<...> new_schema_change_job_id:0 drop_time:0 replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false > exec stmt rows affected: 0 statement ok @@ -222,7 +222,7 @@ WHERE message NOT LIKE '%Z/%' AND message NOT LIKE 'querying next range at%' AND operation != 'dist sender send' ---- batch flow coordinator Del /NamespaceTable/30/1/56/57/"kv"/4/1 -batch flow coordinator Put /Table/3/1/58/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:3 format_version:3 state:DROP offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:... replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false > +batch flow coordinator Put /Table/3/1/58/2/1 -> table: parent_id:56 unexposed_parent_schema_id:57 columns: nullable:false hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > columns: nullable:true hidden:false inaccessible:false generated_as_identity_type:NOT_IDENTITY_COLUMN virtual:false pg_attribute_num:0 alter_column_type_in_progress:false system_column_kind:NONE > next_column_id:3 families: next_family_id:1 primary_index: interleave:<> partitioning: type:FORWARD created_explicitly:false encoding_type:1 sharded: disabled:false geo_config:<> predicate:"" use_delete_preserving_encoding:false force_put:false > next_index_id:3 privileges: users: users: owner_proto:"root" version:2 > next_mutation_id:3 format_version:3 state:DROP offline_reason:"" view_query:"" is_materialized_view:false new_schema_change_job_id:0 drop_time:... replacement_of: > audit_mode:DISABLED drop_job_id:0 create_query:"" create_as_of_time:<...> temporary:false partition_all_by:false > exec stmt rows affected: 0 # Check that session tracing does not inhibit the fast path for inserts & diff --git a/pkg/sql/opt/exec/execbuilder/testdata/update_nonmetamorphic b/pkg/sql/opt/exec/execbuilder/testdata/update_nonmetamorphic index 584190f38219..84829c053c1c 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/update_nonmetamorphic +++ b/pkg/sql/opt/exec/execbuilder/testdata/update_nonmetamorphic @@ -397,3 +397,303 @@ Put (delete) /Table/61/2/2/"a"/"foo"/1/0 Put (delete) /Table/61/2/2/"b"/"bar"/1/0 Put /Table/61/2/3/"a"/"foobar"/1/0 -> /BYTES/0x0a0103 Put /Table/61/2/3/"c"/"baz"/1/0 -> /BYTES/0x0a0103 + +# --------------------------------------------------------- +# Index With ForcePut +# --------------------------------------------------------- +statement ok +CREATE TABLE tfp ( + a INT PRIMARY KEY, + b INT, + c INT, + FAMILY (a, b, c), + INDEX (b, c) +); + +let $t_id +SELECT id FROM system.namespace WHERE name = 'tfp' + +let $updated_t_jsonb +WITH + descs + AS ( + SELECT + id, + crdb_internal.pb_to_json( + 'cockroach.sql.sqlbase.Descriptor', + descriptor + ) + AS descriptor + FROM + system.descriptor + ) +SELECT + CAST (json_set(descriptor, ARRAY['table', 'indexes', '0', 'forcePut'], 'true') AS STRING) +FROM + descs WHERE id = $t_id; + +statement ok +SELECT * FROM crdb_internal.unsafe_upsert_descriptor($t_id, crdb_internal.json_to_pb('cockroach.sql.sqlbase.Descriptor',$$ $updated_t_jsonb $$), true) + +statement ok +INSERT INTO tfp VALUES (1, 2, 100) + +query T kvtrace +UPDATE tfp SET b = b + 1, c = c + 1 +---- +Scan /Table/62/{1-2} +Put /Table/62/1/1/0 -> /TUPLE/2:2:Int/3/1:3:Int/101 +Del /Table/62/2/2/100/1/0 +Put /Table/62/2/3/101/1/0 -> /BYTES/ + +# --------------------------------------------------------- +# Partial Index With ForcePut +# --------------------------------------------------------- +statement ok +CREATE TABLE tpfp ( + a INT PRIMARY KEY, + b INT, + c STRING, + FAMILY (a, b, c), + INDEX partial (c) WHERE a > b AND c IN ('foo', 'foobar') +); + +let $t_id +SELECT id FROM system.namespace WHERE name = 'tpfp' + +let $updated_t_jsonb +WITH + descs + AS ( + SELECT + id, + crdb_internal.pb_to_json( + 'cockroach.sql.sqlbase.Descriptor', + descriptor + ) + AS descriptor + FROM + system.descriptor + ) +SELECT + CAST (json_set(descriptor, ARRAY['table', 'indexes', '0', 'forcePut'], 'true') AS STRING) +FROM + descs WHERE id = $t_id; + +statement ok +SELECT * FROM crdb_internal.unsafe_upsert_descriptor($t_id, crdb_internal.json_to_pb('cockroach.sql.sqlbase.Descriptor',$$ $updated_t_jsonb $$), true) + +statement ok +INSERT INTO tpfp VALUES (3, 4, 'bar') + +# Update a row that doesn't match the partial index. +query T kvtrace +UPDATE tpfp SET b = b + 1 +---- +Scan /Table/63/{1-2} +Put /Table/63/1/3/0 -> /TUPLE/2:2:Int/5/1:3:Bytes/bar + +# Update a row that didn't match the partial index before but matches after. +query T kvtrace +UPDATE tpfp SET b = b - 3, c = 'foo' +---- +Scan /Table/63/{1-2} +Put /Table/63/1/3/0 -> /TUPLE/2:2:Int/2/1:3:Bytes/foo +Put /Table/63/2/"foo"/3/0 -> /BYTES/ + +# Update a row that matches the partial index before and after, but the index +# entry doesn't change. +query T kvtrace +UPDATE tpfp SET b = b - 1 +---- +Scan /Table/63/{1-2} +Put /Table/63/1/3/0 -> /TUPLE/2:2:Int/1/1:3:Bytes/foo + +# Update a row that matches the partial index before and after, and the index +# entry changes. +query T kvtrace +UPDATE tpfp SET b = b + 1, c = 'foobar' +---- +Scan /Table/63/{1-2} +Put /Table/63/1/3/0 -> /TUPLE/2:2:Int/2/1:3:Bytes/foobar +Del /Table/63/2/"foo"/3/0 +Put /Table/63/2/"foobar"/3/0 -> /BYTES/ + +# Update a row that matches the partial index before but not after. +query T kvtrace +UPDATE tpfp SET c = 'baz' +---- +Scan /Table/63/{1-2} +Put /Table/63/1/3/0 -> /TUPLE/2:2:Int/2/1:3:Bytes/baz +Del /Table/63/2/"foobar"/3/0 + +# --------------------------------------------------------- +# Expression Index With ForcePut +# --------------------------------------------------------- +statement ok +CREATE TABLE tefp ( + k INT PRIMARY KEY, + a INT, + b INT, + FAMILY (k, a, b), + INDEX t_a_plus_b_idx ((a + b)) +) + +let $t_id +SELECT id FROM system.namespace WHERE name = 'tefp' + +let $updated_t_jsonb +WITH + descs + AS ( + SELECT + id, + crdb_internal.pb_to_json( + 'cockroach.sql.sqlbase.Descriptor', + descriptor + ) + AS descriptor + FROM + system.descriptor + ) +SELECT + CAST ( + json_set( + json_set(descriptor, ARRAY['table', 'indexes', '0', 'forcePut'], 'true'), + ARRAY['table', 'modificationTime'], json_build_object('wallTime', cluster_logical_timestamp()::INT8::STRING) + ) AS STRING + ) +FROM + descs WHERE id = $t_id; + +statement ok +SELECT * FROM crdb_internal.unsafe_upsert_descriptor($t_id, crdb_internal.json_to_pb('cockroach.sql.sqlbase.Descriptor',$$ $updated_t_jsonb $$), true) + +statement ok +INSERT INTO tefp VALUES (1, 2, 100) + +# Update a row which changes the index entry. +query T kvtrace +UPDATE tefp SET a = a + 1, b = b + 100 +---- +Scan /Table/64/{1-2} +Put /Table/64/1/1/0 -> /TUPLE/2:2:Int/3/1:3:Int/200 +Del /Table/64/2/102/1/0 +Put /Table/64/2/203/1/0 -> /BYTES/ + +# Update a row with different values without changing the index entry. +query T kvtrace +UPDATE tefp SET a = a + 1, b = b - 1 +---- +Scan /Table/64/{1-2} +Put /Table/64/1/1/0 -> /TUPLE/2:2:Int/4/1:3:Int/199 + +# --------------------------------------------------------- +# Inverted Index With ForcePut +# --------------------------------------------------------- + +statement ok +CREATE TABLE tifp ( + a INT PRIMARY KEY, + b INT[], + FAMILY (a,b), + INVERTED INDEX(b) +) + +let $t_id +SELECT id FROM system.namespace WHERE name = 'tifp' + +let $updated_t_jsonb +WITH + descs + AS ( + SELECT + id, + crdb_internal.pb_to_json( + 'cockroach.sql.sqlbase.Descriptor', + descriptor + ) + AS descriptor + FROM + system.descriptor + ) +SELECT + CAST ( + json_set( + json_set(descriptor, ARRAY['table', 'indexes', '0', 'forcePut'], 'true'), + ARRAY['table', 'modificationTime'], json_build_object('wallTime', cluster_logical_timestamp()::INT8::STRING) + ) AS STRING + ) +FROM + descs WHERE id = $t_id; + +statement ok +SELECT * FROM crdb_internal.unsafe_upsert_descriptor($t_id, crdb_internal.json_to_pb('cockroach.sql.sqlbase.Descriptor',$$ $updated_t_jsonb $$), true) + +statement ok +INSERT INTO tifp VALUES (1, ARRAY[1, 2, 3, 2, 2, NULL, 3]) + +# Update a row that has 1 new entry and 1 removed entry in the index. +query T kvtrace +UPDATE tifp SET b = ARRAY[1, 2, 2, NULL, 4, 4] +---- +Scan /Table/65/{1-2} +Put /Table/65/1/1/0 -> /TUPLE/ +Del /Table/65/2/3/1/0 +Put /Table/65/2/4/1/0 -> /BYTES/ + +# --------------------------------------------------------- +# Multicolumn Inverted Index With ForcePut +# --------------------------------------------------------- + +statement ok +CREATE TABLE tmfp ( + a INT PRIMARY KEY, + b INT, + c JSON, + FAMILY (a, b, c), + INVERTED INDEX(b, c) +) + +let $t_id +SELECT id FROM system.namespace WHERE name = 'tmfp' + +let $updated_t_jsonb +WITH + descs + AS ( + SELECT + id, + crdb_internal.pb_to_json( + 'cockroach.sql.sqlbase.Descriptor', + descriptor + ) + AS descriptor + FROM + system.descriptor + ) +SELECT + CAST ( + json_set( + json_set(descriptor, ARRAY['table', 'indexes', '0', 'forcePut'], 'true'), + ARRAY['table', 'modificationTime'], json_build_object('wallTime', cluster_logical_timestamp()::INT8::STRING) + ) AS STRING + ) +FROM + descs WHERE id = $t_id; + +statement ok +SELECT * FROM crdb_internal.unsafe_upsert_descriptor($t_id, crdb_internal.json_to_pb('cockroach.sql.sqlbase.Descriptor',$$ $updated_t_jsonb $$), true) + +statement ok +INSERT INTO tmfp VALUES (1, 2, '{"a": "foo", "b": "bar"}'::json) + +query T kvtrace +UPDATE tmfp SET b = 3, c = '{"a": "foobar", "c": "baz"}'::json +---- +Scan /Table/66/{1-2} +Put /Table/66/1/1/0 -> /TUPLE/2:2:Int/3/ +Del /Table/66/2/2/"a"/"foo"/1/0 +Del /Table/66/2/2/"b"/"bar"/1/0 +Put /Table/66/2/3/"a"/"foobar"/1/0 -> /BYTES/ +Put /Table/66/2/3/"c"/"baz"/1/0 -> /BYTES/ diff --git a/pkg/sql/row/inserter.go b/pkg/sql/row/inserter.go index 8cda4572c676..407970067441 100644 --- a/pkg/sql/row/inserter.go +++ b/pkg/sql/row/inserter.go @@ -192,8 +192,8 @@ func (ri *Inserter) InsertRow( for i := range entries { e := &entries[i] - // We don't want to check any conflicts when trying to preserve deletes. - if ri.Helper.Indexes[idx].UseDeletePreservingEncoding() { + if ri.Helper.Indexes[idx].ForcePut() { + // See the comemnt on (catalog.Index).ForcePut() for more details. insertPutFn(ctx, b, &e.Key, &e.Value, traceKV) } else { putFn(ctx, b, &e.Key, &e.Value, traceKV) diff --git a/pkg/sql/row/updater.go b/pkg/sql/row/updater.go index 4aa25addaeae..c5d35b3f4e56 100644 --- a/pkg/sql/row/updater.go +++ b/pkg/sql/row/updater.go @@ -421,10 +421,8 @@ func (ru *Updater) UpdateRow( continue } - if index.UseDeletePreservingEncoding() { - // Delete preserving encoding indexes are used only as a log of - // index writes during backfill, thus we can blindly put values into - // them. + if index.ForcePut() { + // See the comemnt on (catalog.Index).ForcePut() for more details. insertPutFn(ctx, batch, &newEntry.Key, &newEntry.Value, traceKV) } else { if traceKV { @@ -459,10 +457,8 @@ func (ru *Updater) UpdateRow( ) } - if index.UseDeletePreservingEncoding() { - // Delete preserving encoding indexes are used only as a log of - // index writes during backfill, thus we can blindly put values into - // them. + if index.ForcePut() { + // See the comemnt on (catalog.Index).ForcePut() for more details. insertPutFn(ctx, batch, &newEntry.Key, &newEntry.Value, traceKV) } else { // In this case, the index now has a k/v that did not exist in the @@ -497,7 +493,8 @@ func (ru *Updater) UpdateRow( // and the old row values do not match the partial index // predicate. newEntry := &newEntries[newIdx] - if index.UseDeletePreservingEncoding() { + if index.ForcePut() { + // See the comemnt on (catalog.Index).ForcePut() for more details. insertPutFn(ctx, batch, &newEntry.Key, &newEntry.Value, traceKV) } else { if traceKV { @@ -518,9 +515,8 @@ func (ru *Updater) UpdateRow( } // We're adding all of the inverted index entries from the row being updated. for j := range ru.newIndexEntries[i] { - if index.UseDeletePreservingEncoding() { - // Delete preserving encoding indexes are used only as a log of index - // writes during backfill, thus we can blindly put values into them. + if index.ForcePut() { + // See the comemnt on (catalog.Index).ForcePut() for more details. insertPutFn(ctx, batch, &ru.newIndexEntries[i][j].Key, &ru.newIndexEntries[i][j].Value, traceKV) } else { insertInvertedPutFn(ctx, batch, &ru.newIndexEntries[i][j].Key, &ru.newIndexEntries[i][j].Value, traceKV)