diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array b/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array new file mode 100644 index 000000000000..2b2697039074 --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array @@ -0,0 +1,270 @@ +# Test cases for using invertedFilterer on an inverted json or array index. + +statement ok +CREATE TABLE json_tab ( + a INT PRIMARY KEY, + b JSONB +) + +statement ok +CREATE INVERTED INDEX foo_inv ON json_tab(b) + +statement ok +CREATE TABLE array_tab ( + a INT PRIMARY KEY, + b INT[] +) + +statement ok +CREATE INVERTED INDEX foo_inv ON array_tab(b) + +statement ok +INSERT INTO json_tab VALUES + (1, '{"a": "b"}'), + (2, '[1,2,3,4, "foo"]'), + (3, '{"a": {"b": "c"}}'), + (4, '{"a": {"b": [1]}}'), + (5, '{"a": {"b": [1, [2]]}}'), + (6, '{"a": {"b": [[2]]}}'), + (7, '{"a": "b", "c": "d"}'), + (8, '{"a": {"b":true}}'), + (9, '{"a": {"b":false}}'), + (10, '"a"'), + (11, 'null'), + (12, 'true'), + (13, 'false'), + (14, '1'), + (15, '1.23'), + (16, '[{"a": {"b": [1, [2]]}}, "d"]'), + (17, '{}'), + (18, '[]'), + (19, '["a", "a"]'), + (20, '[{"a": "a"}, {"a": "a"}]'), + (21, '[[[["a"]]], [[["a"]]]]'), + (22, '[1,2,3,1]'), + (23, '{"a": 123.123}'), + (24, '{"a": 123.123000}'), + (25, '{"a": [{}]}'), + (26, '[[], {}]'), + (27, '[true, false, null, 1.23, "a"]'), + (28, '{"a": {}}'), + (29, NULL), + (30, '{"a": []}'), + (31, '{"a": {"b": "c", "d": "e"}, "f": "g"}'), + (32, '{"a": [1]}'), + (33, '[1, "bar"]'), + (34, '{"a": 1}'), + (35, '[1]'), + (36, '[2]'), + (37, '[[1]]'), + (38, '[[2]]'), + (39, '["a"]'), + (40, '{"a": [[]]}'), + (41, '[[1, 2]]'), + (42, '[[1], [2]]'), + (43, '[{"a": "b", "c": "d"}]'), + (44, '[{"a": "b"}, {"c": "d"}]') + +# Filter with a scalar. +query IT +SELECT * FROM json_tab WHERE b @> '1' ORDER BY a +---- +2 [1, 2, 3, 4, "foo"] +14 1 +22 [1, 2, 3, 1] +33 [1, "bar"] +35 [1] + +query I +SELECT a FROM json_tab WHERE b @> '1' ORDER BY a +---- +2 +14 +22 +33 +35 + +# Filter with a fully-specified array. This should use a zigzag join. +query IT +SELECT * FROM json_tab WHERE b @> '[1, 2]' ORDER BY a +---- +2 [1, 2, 3, 4, "foo"] +22 [1, 2, 3, 1] + +# Filter with fully-specified arrays. This should use an inverted filter. +query I +SELECT a FROM json_tab WHERE b @> '[1, 2]' OR b @> '[3, 4]' ORDER BY a +---- +2 +22 + +# Filter with a path ending in an empty object. +query IT +SELECT * FROM json_tab WHERE b @> '{"a": {}}' ORDER BY a +---- +3 {"a": {"b": "c"}} +4 {"a": {"b": [1]}} +5 {"a": {"b": [1, [2]]}} +6 {"a": {"b": [[2]]}} +8 {"a": {"b": true}} +9 {"a": {"b": false}} +28 {"a": {}} +31 {"a": {"b": "c", "d": "e"}, "f": "g"} + +query I +SELECT a FROM json_tab WHERE b @> '{"a": {}}' ORDER BY a +---- +3 +4 +5 +6 +8 +9 +28 +31 + +# Filter with a path ending in an empty array. +query IT +SELECT * FROM json_tab WHERE b @> '{"a": []}' ORDER BY a +---- +25 {"a": [{}]} +30 {"a": []} +32 {"a": [1]} +40 {"a": [[]]} + +query I +SELECT a FROM json_tab WHERE b @> '{"a": []}' ORDER BY a +---- +25 +30 +32 +40 + +# Filter with a nested array. This index expression is not tight. +# This should use a zigzag join. +query IT +SELECT * FROM json_tab WHERE b @> '[[1, 2]]' ORDER BY a +---- +41 [[1, 2]] + +# Filter with a nested array. This index expression is not tight. +# This should use an inverted filter +query I +SELECT a FROM json_tab WHERE b @> '[[1, 2]]' OR b @> '[[3, 4]]' ORDER BY a +---- +41 + +# Combine predicates with AND. Should have the same output as b @> '[1, 2]'. +# This should use a zigzag join. +query I +SELECT a FROM json_tab WHERE b @> '[1]' AND b @> '[2]' ORDER BY a +---- +2 +22 + +# Combine predicates with OR. +query IT +SELECT * FROM json_tab@foo_inv WHERE b @> '[1]' OR b @> '[2]' ORDER BY a +---- +2 [1, 2, 3, 4, "foo"] +22 [1, 2, 3, 1] +33 [1, "bar"] +35 [1] +36 [2] + +query I +SELECT a FROM json_tab@foo_inv WHERE b @> '[1]' OR b @> '[2]' ORDER BY a +---- +2 +22 +33 +35 +36 + +# More complex combination. +query IT +SELECT * FROM json_tab +WHERE (b @> '[1]'::json OR b @> '[2]'::json) AND (b @> '3'::json OR b @> '"bar"'::json) +ORDER BY a +---- +2 [1, 2, 3, 4, "foo"] +22 [1, 2, 3, 1] +33 [1, "bar"] + +# Combined with non-JSON predicates. +query IT +SELECT * FROM json_tab WHERE b @> '[1]' AND a % 2 = 0 ORDER BY a +---- +2 [1, 2, 3, 4, "foo"] +22 [1, 2, 3, 1] + +# The split disjunction rule allows us to use the index for this query. +query IT +SELECT * FROM json_tab WHERE b @> '[1]' OR a = 44 ORDER BY a +---- +2 [1, 2, 3, 4, "foo"] +22 [1, 2, 3, 1] +33 [1, "bar"] +35 [1] +44 [{"a": "b"}, {"c": "d"}] + +# We cannot use the index for this query. +query IT +SELECT * FROM json_tab WHERE b @> '[1]' OR sqrt(a::decimal) = 2 ORDER BY a +---- +2 [1, 2, 3, 4, "foo"] +4 {"a": {"b": [1]}} +22 [1, 2, 3, 1] +33 [1, "bar"] +35 [1] + +statement ok +INSERT INTO array_tab VALUES + (1, '{}'), + (2, '{1}'), + (3, '{2}'), + (4, '{1, 2}'), + (5, '{1, 3}'), + (6, '{1, 2, 3, 4}') + +# Array operations. +query I +SELECT a FROM array_tab@foo_inv WHERE b @> '{}' ORDER BY a +---- +1 +2 +3 +4 +5 +6 + +# This should use a zigzag join. +query IT +SELECT * FROM array_tab@foo_inv WHERE b @> '{1, 2}' ORDER BY a +---- +4 {1,2} +6 {1,2,3,4} + +# Combined with non-Array predicates. +query IT +SELECT * FROM array_tab WHERE b @> '{1}' AND a % 2 = 0 ORDER BY a +---- +2 {1} +4 {1,2} +6 {1,2,3,4} + +# The split disjunction rule allows us to use the index for this query. +query IT +SELECT * FROM array_tab WHERE b @> '{1}' OR a = 1 ORDER BY a +---- +1 {} +2 {1} +4 {1,2} +5 {1,3} +6 {1,2,3,4} + +# We cannot use the index for this query. +query IT +SELECT * FROM array_tab WHERE (b @> '{2}' AND a = 3) OR b[0] = a ORDER BY a +---- +3 {2} diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_dist b/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_dist new file mode 100644 index 000000000000..3a43d17980f5 --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_dist @@ -0,0 +1,222 @@ +# LogicTest: 5node-default-configs !5node-metadata + +statement ok +CREATE TABLE json_tab ( + a INT PRIMARY KEY, + b JSONB +) + +statement ok +CREATE INVERTED INDEX foo_inv ON json_tab(b) + +statement ok +CREATE TABLE array_tab ( + a INT PRIMARY KEY, + b INT[] +) + +statement ok +CREATE INVERTED INDEX foo_inv ON array_tab(b) + +statement ok +INSERT INTO json_tab VALUES + (1, '{"a": "b"}'), + (2, '[1,2,3,4, "foo"]'), + (3, '{"a": {"b": "c"}}'), + (4, '{"a": {"b": [1]}}'), + (5, '{"a": {"b": [1, [2]]}}'), + (6, '{"a": {"b": [[2]]}}'), + (7, '{"a": "b", "c": "d"}'), + (8, '{"a": {"b":true}}'), + (9, '{"a": {"b":false}}'), + (10, '"a"'), + (11, 'null'), + (12, 'true'), + (13, 'false'), + (14, '1'), + (15, '1.23'), + (16, '[{"a": {"b": [1, [2]]}}, "d"]'), + (17, '{}'), + (18, '[]'), + (19, '["a", "a"]'), + (20, '[{"a": "a"}, {"a": "a"}]'), + (21, '[[[["a"]]], [[["a"]]]]'), + (22, '[1,2,3,1]'), + (23, '{"a": 123.123}'), + (24, '{"a": 123.123000}'), + (25, '{"a": [{}]}'), + (26, '[[], {}]'), + (27, '[true, false, null, 1.23, "a"]'), + (28, '{"a": {}}'), + (29, NULL), + (30, '{"a": []}'), + (31, '{"a": {"b": "c", "d": "e"}, "f": "g"}'), + (32, '{"a": [1]}'), + (33, '[1, "bar"]') + +statement ok +ALTER TABLE json_tab SPLIT AT VALUES (10), (20) + +statement ok +ALTER TABLE json_tab EXPERIMENTAL_RELOCATE VALUES (ARRAY[1], 1), (ARRAY[2], 10), (ARRAY[3], 20) + +query TTTI colnames +SELECT start_key, end_key, replicas, lease_holder +FROM [SHOW EXPERIMENTAL_RANGES FROM TABLE json_tab] ORDER BY lease_holder +---- +start_key end_key replicas lease_holder +NULL /10 {1} 1 +/10 /20 {2} 2 +/20 NULL {3} 3 + +# Filter with a scalar. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '1' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUkEFr20AQhe_9FcNc3NIt0rq3PamtVSJwLEcyJMERZq0dOwrKrrK7Cgaj_x4sJSQOOCTHefPeN4_Zo3uoUWAeT-N_C2htDf-z9ByW8dV8-ieZwfdJki_yi-kPeLbIwXDnjF55uYbLsziLYQ3RTRuGvwlGfARpNokz-HsNskCG2iiayXtyKJbIsWDYWFOSc8YepH1vSNQORciw0k3rD3LBsDSWUOzRV74mFLiQ65oykopsECJDRV5WdY99qRNtjFlV-hEZ5o3UTkDAfwU8mFvaVLtYK5BaAQfjb8kiw7T1AiKORcfQtP71tvNySyh4xz7fLzfWkw34cbWI_zyJH38Fn5FrjHZ0hD9FDruCIaktDS92prUlza0p-zPDmPa5XlDk_LDlw5DoYXUo-DbMPwyP34WL7ttTAAAA__9zosNt + +# Filter with a fully-specified array. This should use a zigzag join. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '[1, 2]' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUkk9v1DAQxe98itFcCsLSxtmbT6E0iK2WTUkq8WeJKjcZIpfUE2wHAat8d5QEFRbaVfc48-a995PlHfqvLSos0nX68hJ618KrPHsD2_T9xfrFagNPz1bFZfF2_Qx-n-j54MazvQr6Gt69TvMUriH51EfRkuBkKwXE5Qlk-Vmaw-kH0CUKtFzTRt-SR7VFiaXAznFF3rMbV7vpYFV_RxUJNLbrw7guBVbsCNUOgwktocKPpvmpm3M2ltwiQoE1BW3aKbcwNUGk7uCSz8xXxn5DMUvyXinrg4JEYjkI5D78qfZBN4RKDuLxeCNYTromt5D7cHfNnTO32v1AgWvmL30HN2wssJ0gBGYbSJb_PadS6rzINqeP4I2P4S3YBXKLeJ81kc8fjF8eE5-T79h62ot_KDkaSoFUNzT_CM-9q-jCcTXVzGM2-aZFTT7MqpyHlZ2lEfBvszxojg-b44Pm5T_mcnjyKwAA___NrxBc + +# Filter with fully-specified arrays. This should use an inverted filter. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '[1, 2]' OR b @> '[3, 4]' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUUmFv0zAU_M6veHpftoFR4gQJyZ86WCYilbUklQA10eTWbyMo2MF2pkpV_ztKzIAgtbAvTt69u_Od5D267y0KLLN59nYFvW3huli8h3X2aTm_zG_g_CovV-WH-QX8pMhA-OqMvvVyAx_fZUUG5xuYVX0cpwRna84gqc8uYFFM8ZTBq4BfZQW8-QyyRobaKLqR38ihWCPHmmFnzZacM3aA9iMhVzsUMcNGd70f4Jrh1lhCsUff-JZQ4EpuWipIKrJRjAwVedm0o-1j2tmdMbeNfkCGZSe1ExBdWhvxl-ETLS3dNbtMK5BaQQrGfyHrsD4wNL3_fbXz8p5Q8AP7_3i5fiDrSV03rSdLNuLTjI_7bNdZMBpmXIAbQoLz0npR4euqiuN4ONJff_z5cCZhRiCt_k3kFcK0IMNF7wXM-NGqyVOqlsZ6slEyLTjjL47ap0-xL8h1Rjua2B9zjg81Q1L3FB6TM73d0tKa7XhNGBejbgQUOR-2PAy5Dqsh4J9iflKcnBYnJ8XpX-L68OxHAAAA__-W8SfB + +# Filter with a path ending in an empty object. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '{"a": {}}' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUktFr1EAQxt_9K4Z5qeJKsncP4j5FbYqBs3cmByrdUPay0xqJu3F3Uw5C_ndJYrVXuKN9CZlv5vvmN7A9-t8NCizSVfpxC51r4CJff4ar9Ntm9T67hJfnWbEtvqxewd8RNQ_89NZcB7WDr5_SPIUdJLKL4yXBWS9RSRTQD8MZrPPzNIcP30GVyNBYTZfqF3kUV8ixZNg6W5H31o1SPw1keo8iZlibtgujXDKsrCMUPYY6NIQCt2rXUE5Kk4tiZKgpqLqZYu_Bkhtrr2tzhwyLVhkvIJrAon548-8v2ji6qfep0aCMBg42_CCH5cDQduH_fh_ULaHgA3s6Y2buyAXSF3UTyJGL-CHofT_dtw6sgYQL8CMp-KBcEBLfKinjOB4__J1EIKMfqUIiPCT3yHDdBQEJP3rE4jlHFNYFctHiED3hr4_GL58Tn5NvrfF0EH8sOR5KhqRvaX4r3nauoo2z1bRmLteTbxI0-TB3-VxkZm6NgA_N_KR5cdq8OGlePjKXw4s_AQAA__9oexw3 + +# Filter with a path ending in an empty array. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '{"a": []}' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUktFr1EAQxt_9K4Z5qeJKsncP6j5FbYqBs1eTA5XLUvayY43E3bi7KYWQ_12StNgr3NF7CZlv5vvmN7A9-r8NCizSVfppA51r4CJff4Ft-v1q9SG7hJfnWbEpvq5ewf2Imgd-e2uug9rBt89pnsIOkrKL4yXBWV-iKlHAVg5nsM7P0xw-_gAlkaGxmi7VH_IotshRMmydrch760apnwYyfYciZlibtgujLBlW1hGKHkMdGkKBG7VrKCelyUUxMtQUVN1MsQ9gyU9rr2tziwyLVhkvIJrAoq18c__XD6CMBg42_CKHcmBou_B_qw_qhlDwgT2fLDO35ALpi7oJ5MhFfB_voZ_etQ6sgYQL8CMf-KBcECW-VWUZx_H44e9KBDL6ifq-RHhM7pHhugsCEn7wiMUpRxTWBXLRYh894a8Pxi9Pic_Jt9Z42os_lBwPkiHpG5pfiLedq-jK2WpaM5fryTcJmnyYu3wuMjO3RsDHZn7UvDhuXhw1L5-Y5fDiXwAAAP__7rAYAQ== + +# Filter with a nested array. This index expression is not tight. +# This should use a zigzag join. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '[[1, 2]]' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUkl9v0zAUxd_5FFf3ZSAsNU7f_BTGguhUmpFM4k-xJq-5VB6Zb7AdBFT57igJGnTaqvXx3nPPOT9Z3mH43qDCKl_mry-h8w28KYt3sM4_XixfLVbw_GxRXVbvly_g74mZDm4Cu6toruHD27zM4RqyL12SzAlO1mspINX6BIryLC_h9BMYjQId17QytxRQrVGiFth63lAI7IfVbjxY1D9RJQKta7s4rLXADXtCtcNoY0Oo8LPd_jbbc7aO_CxBgTVFY5sxt7I1QaLu8LKvzFfW_UAxSfJBqeiigkyi7gVyF_9Vh2i2hEr24ul4A1hJpiY_k_twd82tt7fG_0KBS-ZvXQs3bB2wGyEEFivI5g88qFLqvCpWp08gTo8hrthH8rN0nzaTLx-Nnx8TX1Jo2QXai38sOem1QKq3NP2JwJ3f0IXnzVgzjcXoGxc1hTipchoWbpIGwP_N8qA5PWxOD5rn98y6f_YnAAD__2R8Ecw= + +# Filter with a nested array. This index expression is not tight. +# This should use an inverted filter +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '[[1, 2]]' OR b @> '[[3, 4]]' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUU1Fv2jAQft-vON1L6eaJOEGa5Ke0a6pRMWAJ0jZBVBly7TKldmY7FRPiv09J6ChVicoDlu_u-_x9x102aP8UKDCJRtHnGVSmgOt48hXm0Y_p6GI4ht7VMJkl30bnsIPIFvDbanXr5BK-f4niCHpLCBeV5wUEZ_M5Z-Cn6dk5TOIXlYDBYFe5imK4_AkyRYZKZzSWD2RRzJFjyrA0ekXWalOnNg1gmK1ReAxzVVauTqcMV9oQig263BWEAmdyWVBMMiPT95BhRk7mRfPsk-PwTuvbXD0iw6SUygroXxjT_PjH_bU_NXSXryOVgVQZBKDdLzIW0y1DXbm9BevkPaHgW_Z2m0P1SMZRdp0XjgyZPj_0-lSP1qUBrSDkAmxtFqyTxokFflosPM-rj-CVG39fn34bI5DKTqHwBcJh0wwnlRMQ8qPt-6e0f6NztRuSf2RIpckfpPm7V2ahf1Q8OEU80caR6QeHwiH_gAzbeQjohf4r-yyEuEkm48t2rw8hu8X-D3nDXzY4xXVMttTK0oHrYy9725QhZffUfjxWV2ZFU6NXjUwbThpek8jIurbK22Co2lJt8DmZd5L9brLfSQ66yUEnefCCnG7f_QsAAP__2umEPg== + +# Combine predicates with AND. Should have the same output as b @> '[1, 2]'. +# This should use a zigzag join. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '[1]' AND b @> '[2]' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUUl1v00AQfOdXrPalqTgpPuftntwSI1IFu9iV-AhWdY2X6Ep6a-7OCIjy35HtqtSojZrHm9mZnVndDv2PLSos02X65gpat4W3Rf4eVumny-XZIoPJfFFelR-Wp3A_ooeBW8_2Ougb-PguLVKY3EDytY2iGcHJSlYnp3CWzcdo3KF5MU8LOP8MukKBlmvK9B15VCuUWAlsHK_Je3YdtOsHFvUvVJFAY5s2dHAlcM2OUO0wmLAlVPjFbP7ozQUbS24aocCagjbb3rc0NUGkHhIn35ivjf2JYqDkk1TeBgWJxGovkNvwb7UPekOo5F68PF4XrCBdk5vKcbiHzY0zd9r9RoFL5u9tA7dsLLDtQwjMM5gks_GNlVIXZZ6d3996RMeP6Be0iY9pU7IL5KbxuEkiXz9rPzvGviDfsPU0sn_OOdpXAqne0PBfPLduTZeO1_2a4Zn3uh6oyYeBlcNjYQeqC_hYLA-K48Pi-KB49p-42r_6GwAA__8MxBzL + +# Combine predicates with OR. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab@foo_inv WHERE b @> '[1]' OR b @> '[2]' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUUl1r2zAUfd-vuNyXtpuGLedhoKd0q8sMWZPZgW3EpijRbefhSd6VXAIh_33Y3kdTSLa-yL7nnnN0DmiH_keDCot0lr5bQscNXOfzD7BKPy9ml9kNnF9lxbL4OLuAXxQ9Er55Z2-DXk_vnLut7QN8ep_mKZyvYVp2cTwhOFvJ6uwC5vkhmIzgVZrD2y-gKxRonaEb_Z08qhVKrAS27DbkveMe2g2EzGxRxQJr23ahhyuBG8eEaoehDg2hwqVeN5STNsRRjAINBV03g-3TvCiwaLX1CqJL5ki-Hj_Rgumu3qbWgLYGJLjwlRirvUDXhb83-6DvCZXci_9Pl9kH4kDmum4CMXEkDyP-3qfblsFZmEoFvs8IPmgOqsQ3ZRnHcX9M_vzJl_2ZjDMCWfNvoiwRHvfzKHDeBQVTebRq8pyqheNAHCWHBafy1VH7yXPsc_Kts54O7I85x_tKIJl7Gt-Sdx1vaMFuM1wzjvNBNwCGfBi3chwyO676gI_F8qQ4OS1OToonT8TV_sXPAAAA__-cIyl7 + +# More complex combination. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab +WHERE (b @> '[1]'::json OR b @> '[2]'::json) AND (b @> '3'::json OR b @> '"bar"'::json) +ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUklFr2zAUhd_3Ky73pQnTiOUwBnpKu7gsI4szO7CN2BQluu08PMmT5BII-e_DdkPqQbL2xVj33O_oHNAe3Z8SBabRPPq4gtqWcJvEX2AdfV_Or2cLGExn6Sr9Oh_C04rsFn45o--83MC3T1ESwWCwgUlWB8GY4GrN8yshPqfx4mYIcQI9LTxpQ7heTPvo-CyY4UbaDJ_BcTKNErj5ATJHhtooWsjf5FCskWPOsLJmS84Z24z27cJM7VAEDAtd1b4Z5wy3xhKKPfrCl4QCV3JTUkJSkR0FyFCRl0XZ2h5LT-6NuSv0IzJMK6mdgNFTvHfHn9HS0n2xi7QCqRW8B-N_knWYHxia2p-ud14-EAp-YC-PONOPZD2p26L0ZMmOeD_nUY92lQWjYcIFuCYoOC-tFxl-yLIgCJoPz7IgDJvMpwkCafWfrTBD6FdjGNdewISfLRm-pmRqrCc7CvvVJvztWfvxa-wTcpXRjnr255yDQ86Q1AN1T8mZ2m5pac22vaY7xi3XDhQ536m8O8x0JzUBn8P8IhxehsOL8PgfOD-8-RsAAP__85I68g== + +# Combined with non-JSON predicates. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '[1]' AND a % 2 = 0 ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUkeHL0zAQxr_7VxwH8rYYaTO_SEDodH2xMNvZFlRmecnae0elJjVJZTD6v0vbiU6Y6Kfk7p7nye_IGe23DgUW8TZ-U8JgOrjPs3ewjz_utuskBW-TFGXxfuvDRSIXwRer1YOTB_jwNs5j8A4QfR7C8AXB3Z5Xdz6s0w14noSnsPLhFYQ-ZPkmzuH1J5AVMlS6oVR-JYtijxwrhr3RNVmrzdQ6z4KkOaEIGbaqH9zUrhjW2hCKM7rWdYQCS3noKCfZkAlCZNiQk203x_6EjB61fmjVd2RY9FJZAcHamIA_X45gZ-ixPcWqQYb3befICPAiPrELIZK0fDmvcLkjw2xwAiKO1chQD-4XnHXySCj4yP59gUIbRybg1-wRf3YzfvU_8TnZXitLV_G3ksOxYkjNkZY_sHowNe2MrudnljKbfXOjIeuWKV-KRC2jCfB3M_-refWHuRqf_AgAAP__hfXPag== + +# The split disjunction rule allows us to use the index for this query. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '[1]' OR a = 44 ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUktFv2jAQxt_3V1i3h1LNKDgxfYg0Kd3INCYGLEHaJhZVJrlWmVI7s52pE-J_n5K0oqAmhSewfd_9vi93WzB_CvAhDmfhxxWpdEE-RYuvZB3-WM6up3MymEzjVfxtdkkeS0Rb8NsoeWPFhnz_HEYhGWxI8KsajTwkF2uWXFySRUQGgrwnnNf_J2FEPvwkIgEKUmU4F_dowF8DAwoeJBRKrVI0Run6etsUTbMH8EcUcllWtr5OKKRKI_hbsLktEHxYiU2BEYoMtVP3ytCKvGhaP1kMbpW6yeVfoBCXQhqfONdaO2zY_jhLjbf5QygzoLCorE8CBsmOgqrsnmusuEPw3R093dsXlctHa26HtVLn90L_25Np4HbCvU74nllJpTPUmB0Ak90L9uZqqErHOyp8Gc3PyR0rbVE7_DBzwN51Jhuf036SG5vL1DrjY0D9Gdv0zRBfH-fVAZedvmqjV-f5tGqcDx3OnbedHtg52SM0pZIGTxraqJ46ZnfYrohRlU5xqVXaYNrjotE1Fxka276y9jCV7VNt8LmY9YrdAzE7Fru9Yq-f7PWKeb-Y94qv-sXjszInuzf_AwAA__-3jLp4 + +# We cannot use the index for this query. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM json_tab WHERE b @> '[1]' OR sqrt(a::decimal) = 2 ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJzMktGL00AQxt_9K4Z5uRS3JLv1aUHI3TWHkV5zJgWVGmTbDEckl83tbkAp_d8lidKLXCXigz7OzvftzG_4DmgfK5SYRavoegOtqeAmTW5hG324W13Ga_CWcbbJ3q1m8EOiBsEXq-vPTu3g_ZsojcDbQfipDYIFwcWW5xczSFLw7KNxnpJyGV3Ht5erGbwG0XWWUQpXH0HlyLDWBa3VA1mUW-TIUCDDBeYMG6P3ZK02XevQC-PiK8qAYVk3reuec4Z7bQjlAV3pKkKJG7WrKCVVkPEDZFiQU2XVf_9z6bAx5YMy35Bh1qjaSpj7vNPelJUjI8ELxZhHSvk2S9ZXT7hCPgaTpxIZJq2TEHLMjwx1607rWqfuCSU_sjNIJxJtCjJUjBlC_hLz4zPcaz3Xjb8Yqc9NF6PpfPpB-dSD-jyY--Jf3lRMpxKTqUQw_09i8gxSSrbRtaVJEQi6DFFxT0PmrG7Nnu6M3vdjhjLpff1DQdYNXT4Ucd23-gWfmvlvza9G5uBXs_ibyYs_MufHF98DAAD__5J4ij8= + +statement ok +INSERT INTO array_tab VALUES + (1, '{}'), + (2, '{1}'), + (3, '{1, 2}'), + (4, '{1, 3}'), + (5, '{1, 2, 3, 4}') + +statement ok +ALTER TABLE array_tab SPLIT AT VALUES (3), (3) + +statement ok +ALTER TABLE array_tab EXPERIMENTAL_RELOCATE VALUES (ARRAY[1], 1), (ARRAY[2], 3), (ARRAY[3], 5) + +query TTTI colnames +SELECT start_key, end_key, replicas, lease_holder +FROM [SHOW EXPERIMENTAL_RANGES FROM TABLE array_tab] ORDER BY lease_holder +---- +start_key end_key replicas lease_holder +NULL /3 {1} 1 +/3 NULL {3} 3 + +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM array_tab@foo_inv WHERE b @> '{}' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUkd-K1DAUh-99isO5WcXINO3FQq6qbhcLdbq2IyrbsmSa41KoSU3SZaX03aWtoqPMsHMTOH--3_kgI7rvHQoskyx5u4PBdnBd5O_hNvl8k71Ot_D8Ki135YfsBfxakeuCtFb-uPNyH3815q7VD_DpXVIksIe4GoIgIrgYpwvIi6ukgDdfQNbIUBtFW_mNHIpb5Fgz7K1pyDlj59a4LKTqEUXAsNX94Od2zbAxllCM6FvfEQrcyX1HBUlFdhMgQ0Vett0S-58YMix7qZ2AV5vtxyxbHqwnhmbwf044L-8JBZ_Y0zVS_UDWk7puO0-W7IYfuvyeJ4-9BaMh5gLc7AKklaiwqqLLy_VBQIb54AXE_KhceI5caawnuwkPlWL-8mh8dE58Qa432tFB_LHkYKoZkrqn9ZudGWxDN9Y0y5m1zBduaShyfp3ytUj1OpoF_4b5STg8DYcn4egfuJ6e_QwAAP__txEHZg== + +# This should use a zigzag join. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM array_tab@foo_inv WHERE b @> '{1, 2}' ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUkkFv00AQhe_8itFcCmKleJ0L2pMDNcIoxGUdCUqwqm08RAvpjtldI0rk_45iV5RAG7U3z7x5732ydofh-xYVVvk8f7WEzm_htS7fwSr_eDafFQt4elpUy-r9_BncnJjxwHhvri-iucy-MF9Y9wM-vMl1DpeQfe6SZEpwspMC0v4ESn2aa3h5DqZGgY4bWpgrCqhWKLEW2HpeUwjs96vdcFA0P1ElAq1ru7hf1wLX7AnVDqONW0KFn-zml9m8ZevITxIU2FA0djvkVrYhSNT_lChGTd6tlV1UkEmse4HcxdvyEM2GUMlePBxwj6bJNOQn8hDvtrr19sr4axQ4Z_7WtfCVrQN2A4XAcgHZ9M8fnWk9O19JpVSxWL4Q6c1H_QDw9DHgFftIfpIeQmfy-b3x08fEawotu0AH8fclJ30tkJoNjY8jcOfXdOZ5PdSMYzn4hkVDIY6qHIfCjdIe8G-zPGpOj5vTo-bpP-a6f_I7AAD__5XSF-k= + +# Combined with non-Array predicates. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM array_tab WHERE b @> '{1}' AND a % 2 = 0 ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJyUkFHrlEAUxd_7FJcL8Vea0LGXGAjcWpeETTcVKjZZZvW2CObYzBiF-N1DXaoNNupt7rnnnPlxRzRfWhSYR_voVQGDbmGXpW_gGL0_7DdxAs42zov87d6Fq0WuBqm1_H6y8gzvXkdZBM4Zwo-D7z8jeBj59ODCJtmC40h4DIELL8B3Ic22UQYvP4AskWGnakrkZzIojsixZNhrVZExSs_SuBji-hsKn2HT9YOd5ZJhpTShGNE2tiUUWMhzSxnJmrTnI8OarGzapfYnZfhJqVPTfUWGeS87I8DjT70AGe6a1pIW4IR8RhVCxEnxfCG-vpFhOlgBIcdyYqgG-4vFWHkhFHxi_86bK21Je_wWNeRP7tYH_1OfkelVZ-im_l6zP5UMqb7QenKjBl3RQatq-WYd0yW3CDUZu275OsTdupoBfw_zv4aDP8Ll9OhHAAAA__-mCsoZ + +# The split disjunction rule allows us to use the index for this query. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM array_tab WHERE b @> '{1}' OR a = 1 ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJykklGPk0AQx9_9FJPx4XpxL7DQ-kBiglqMNbU9oYmak1y2MF5IOBZ3F-Ol4bubLo0njVCrb-zM_Of3nxl2qL-VGGASLaPXG2hUCW_i9Xu4iT5dL18uVjCZL5JN8mF5CYcS0RUIpcTDrRFb-Pg2iiOYbCH80riuT3Cx4-3FJaxjmAh4AXz_OY9iePUZRIoMK5nTStyTxuAGOaYMayUz0lqqfWhnCxb5DwxchkVVN2YfThlmUhEGOzSFKQkD3IhtSTGJnJTjIsOcjChK2_aXvbBWxb1QD8gwqUWlA3D4lcOdp5i2DGVjHttrI-4IA96yf7PAhyx8lfK2qL73LXjIcN2YAEI-aMU7x8o7WVQHJ97pZRzQLPQG6f4g_REqVU6K8j4P0_YP_lbyStaO3ysdQrvnDJ5IZUg5076JkD8bnGx6Tvt5oU1RZcaZHQP2a-zmt1c8fc_ZOdyYdC0rTX-5sJQh5XfUnUfLRmV0rWRmMd1zbXU2kJM2XdbvHovKpuy__7uYj4q9ntg9Fnv_Q_ZHxdNx8nRUPBsXz0bFz4_EafvkZwAAAP__EAe0CA== + +# We cannot use the index for this query. +query T +SELECT url FROM [EXPLAIN (DISTSQL) +SELECT a FROM array_tab WHERE (b @> '{2}' AND a = 3) OR b[0] = a ORDER BY a] +---- +https://cockroachdb.github.io/distsqlplan/decode.html#eJzMkVGLm0AUhd_7Ky73ZQ2dEEdfysCCaeNSITVbE2gXK2USL4vgOnZmhC7B_14cQ7aWTUnf-njPnOO93_GI5keNArfxOv6wg07XcJdtPkEef71fL5MUvFWy3W0_r2dwssjRILWWz9-t3MOXj3EWg-ftIfrW-X5IcHMM-psZLNMVeBJuIZzNYJOBt8_9Am5BDtMqzuD9A8gCGTaqpFQ-kUGRI0eGIRYMW60OZIzSg3x0pqT8icJnWDVtZwe5YHhQmlAc0Va2JhS4k_uaMpIl6YWPDEuysqrdp883R62unqR-RobbVjZGwHwRIsO7qrakBXheFJxpllm2fMgDIUSS7t4VJ66ID2An8cTntCjI_bMXGW46KyDiWPQMVWdfDjdWPhIK3rMLcC9MSpekqZzSRPwtFv0rDaRqrtpFMHFf2h5MtvPrq-VXV7sI5_9dta_AZWRa1Ri6qjZ_6J3KRxr_k1GdPtC9Vge3Zhw3LueEkowdX_k4JI17cgf-HuZ_DYeTsP9nOPinzUX_5lcAAAD__zvmOHw= diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_explain_local b/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_explain_local new file mode 100644 index 000000000000..d695e4b29a55 --- /dev/null +++ b/pkg/sql/logictest/testdata/logic_test/inverted_filter_json_array_explain_local @@ -0,0 +1,404 @@ +# LogicTest: local + +statement ok +CREATE TABLE json_tab ( + a INT PRIMARY KEY, + b JSONB +) + +statement ok +CREATE INVERTED INDEX foo_inv ON json_tab(b) + +statement ok +CREATE TABLE array_tab ( + a INT PRIMARY KEY, + b INT[] +) + +statement ok +CREATE INVERTED INDEX foo_inv ON array_tab(b) + +# Filter with a scalar. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '1' ORDER BY a +---- +distribution: local +vectorized: true +· +• sort +│ order: +a +│ +└── • scan + missing stats + table: json_tab@foo_inv + spans: 2 spans + +# Filter with a fully-specified array. This should use a zigzag join. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '[1, 2]' ORDER BY a +---- +distribution: local +vectorized: true +· +• sort +│ order: +a +│ +└── • lookup join + │ table: json_tab@primary + │ equality: (a) = (a) + │ equality cols are key + │ pred: b @> '[1, 2]' + │ + └── • zigzag join + left table: json_tab@foo_inv + left columns: (a) + left fixed values: 1 column + right table: json_tab@foo_inv + right columns: () + right fixed values: 1 column + +# Filter with fully-specified arrays. This should use an inverted filter. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '[1, 2]' OR b @> '[3, 4]' ORDER BY a +---- +distribution: local +vectorized: false +· +• sort +│ order: +a +│ +└── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 4 + │ + └── • scan + missing stats + table: json_tab@foo_inv + spans: 4 spans + +# Filter with a path ending in an empty object. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '{"a": {}}' ORDER BY a +---- +distribution: local +vectorized: false +· +• sort +│ order: +a +│ +└── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 2 + │ + └── • scan + missing stats + table: json_tab@foo_inv + spans: 2 spans + +# Filter with a path ending in an empty array. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '{"a": []}' ORDER BY a +---- +distribution: local +vectorized: false +· +• sort +│ order: +a +│ +└── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 2 + │ + └── • scan + missing stats + table: json_tab@foo_inv + spans: 2 spans + +# Filter with a nested array. This index expression is not tight. +# This should use a zigzag join. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '[[1, 2]]' ORDER BY a +---- +distribution: local +vectorized: true +· +• sort +│ order: +a +│ +└── • lookup join + │ table: json_tab@primary + │ equality: (a) = (a) + │ equality cols are key + │ pred: b @> '[[1, 2]]' + │ + └── • zigzag join + left table: json_tab@foo_inv + left columns: (a) + left fixed values: 1 column + right table: json_tab@foo_inv + right columns: () + right fixed values: 1 column + +# Filter with a nested array. This index expression is not tight. +# This should use an inverted filter +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '[[1, 2]]' OR b @> '[[3, 4]]' ORDER BY a +---- +distribution: local +vectorized: false +· +• filter +│ filter: (b @> '[[1, 2]]') OR (b @> '[[3, 4]]') +│ +└── • sort + │ order: +a + │ + └── • index join + │ table: json_tab@primary + │ + └── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 4 + │ + └── • scan + missing stats + table: json_tab@foo_inv + spans: 4 spans + +# Combine predicates with AND. Should have the same output as b @> '[1, 2]'. +# This should use a zigzag join. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '[1]' AND b @> '[2]' ORDER BY a +---- +distribution: local +vectorized: true +· +• sort +│ order: +a +│ +└── • lookup join + │ table: json_tab@primary + │ equality: (a) = (a) + │ equality cols are key + │ pred: (b @> '[1]') AND (b @> '[2]') + │ + └── • zigzag join + left table: json_tab@foo_inv + left columns: (a) + left fixed values: 1 column + right table: json_tab@foo_inv + right columns: () + right fixed values: 1 column + +# Combine predicates with OR. +query T +EXPLAIN SELECT a FROM json_tab@foo_inv WHERE b @> '[1]' OR b @> '[2]' ORDER BY a +---- +distribution: local +vectorized: false +· +• sort +│ order: +a +│ +└── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 2 + │ + └── • scan + missing stats + table: json_tab@foo_inv + spans: 2 spans + +# More complex combination. +query T +EXPLAIN SELECT a FROM json_tab +WHERE (b @> '[1]'::json OR b @> '[2]'::json) AND (b @> '3'::json OR b @> '"bar"'::json) +ORDER BY a +---- +distribution: local +vectorized: false +· +• sort +│ order: +a +│ +└── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 6 + │ + └── • scan + missing stats + table: json_tab@foo_inv + spans: 6 spans + +# Combined with non-JSON predicates. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '[1]' AND a % 2 = 0 ORDER BY a +---- +distribution: local +vectorized: true +· +• sort +│ order: +a +│ +└── • filter + │ filter: (a % 2) = 0 + │ + └── • scan + missing stats + table: json_tab@foo_inv + spans: 1 span + +# The split disjunction rule allows us to use the index for this query. +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '[1]' OR a = 44 ORDER BY a +---- +distribution: local +vectorized: true +· +• distinct +│ distinct on: a +│ order key: a +│ +└── • sort + │ order: +a + │ + └── • union all + │ + ├── • index join + │ │ table: json_tab@primary + │ │ + │ └── • scan + │ missing stats + │ table: json_tab@foo_inv + │ spans: 1 span + │ + └── • scan + missing stats + table: json_tab@primary + spans: [/44 - /44] + +# We cannot use the index for this query. +query error pq: index "foo_inv" is inverted and cannot be used for this query +EXPLAIN SELECT a FROM json_tab@foo_inv WHERE b @> '[1]' OR sqrt(a::decimal) = 2 ORDER BY a + +query T +EXPLAIN SELECT a FROM json_tab WHERE b @> '[1]' OR sqrt(a::decimal) = 2 ORDER BY a +---- +distribution: local +vectorized: true +· +• filter +│ filter: (b @> '[1]') OR (sqrt(a::DECIMAL) = 2) +│ +└── • scan + missing stats + table: json_tab@primary + spans: FULL SCAN + +# Array operations. +query T +EXPLAIN SELECT a FROM array_tab@foo_inv WHERE b @> '{}' ORDER BY a +---- +distribution: local +vectorized: false +· +• sort +│ order: +a +│ +└── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 1 + │ + └── • scan + missing stats + table: array_tab@foo_inv + spans: 1 span + +# This should use a zigzag join. +query T +EXPLAIN SELECT a FROM array_tab@foo_inv WHERE b @> '{1, 2}' ORDER BY a +---- +distribution: local +vectorized: true +· +• sort +│ order: +a +│ +└── • lookup join + │ table: array_tab@primary + │ equality: (a) = (a) + │ equality cols are key + │ pred: b @> ARRAY[1,2] + │ + └── • zigzag join + left table: array_tab@foo_inv + left columns: (a) + left fixed values: 1 column + right table: array_tab@foo_inv + right columns: () + right fixed values: 1 column + +# Combined with non-Array predicates. +query T +EXPLAIN SELECT a FROM array_tab WHERE b @> '{1}' AND a % 2 = 0 ORDER BY a +---- +distribution: local +vectorized: true +· +• sort +│ order: +a +│ +└── • filter + │ filter: (a % 2) = 0 + │ + └── • scan + missing stats + table: array_tab@foo_inv + spans: 1 span + +# The split disjunction rule allows us to use the index for this query. +query T +EXPLAIN SELECT a FROM array_tab WHERE b @> '{1}' OR a = 1 ORDER BY a +---- +distribution: local +vectorized: true +· +• distinct +│ distinct on: a +│ order key: a +│ +└── • sort + │ order: +a + │ + └── • union all + │ + ├── • index join + │ │ table: array_tab@primary + │ │ + │ └── • scan + │ missing stats + │ table: array_tab@foo_inv + │ spans: 1 span + │ + └── • scan + missing stats + table: array_tab@primary + spans: [/1 - /1] + +# We cannot use the index for this query. +query error pq: index "foo_inv" is inverted and cannot be used for this query +EXPLAIN SELECT a FROM array_tab@foo_inv WHERE (b @> '{2}' AND a = 3) OR b[0] = a ORDER BY a + +query T +EXPLAIN SELECT a FROM array_tab WHERE (b @> '{2}' AND a = 3) OR b[0] = a ORDER BY a +---- +distribution: local +vectorized: true +· +• filter +│ filter: ((b @> ARRAY[2]) AND (a = 3)) OR (a = b[0]) +│ +└── • scan + missing stats + table: array_tab@primary + spans: FULL SCAN diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array b/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array index ad3e262f6e8d..cd51e166f97c 100644 --- a/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array +++ b/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array @@ -328,8 +328,8 @@ WHERE inv_join.a1 IS NULL OR cross_join.a1 IS NULL # This query performs an inverted join with an additional filter. query ITIT -SELECT * FROM json_tab@foo_inv AS j1, json_tab AS j2 -WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 +SELECT j1.*, j2.* FROM json_tab AS j2 INNER INVERTED JOIN json_tab AS j1 +ON j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 ORDER BY j1.a, j2.a ---- 3 {"a": {"b": "c"}} 3 {"a": {"b": "c"}} @@ -379,8 +379,8 @@ ORDER BY j1.a, j2.a query IIII SELECT * FROM ( - SELECT j1.a, j2.a FROM json_tab@foo_inv AS j1, json_tab AS j2 - WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 + SELECT j1.a, j2.a FROM json_tab AS j2 INNER INVERTED JOIN json_tab AS j1 + ON j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 ) AS inv_join(a1, a2) FULL OUTER JOIN ( @@ -456,29 +456,19 @@ NULL NULL 44 [{"a": "b"}, {"c": "d"}] query IT SELECT * FROM json_tab AS j2 WHERE EXISTS ( SELECT * FROM json_tab@foo_inv AS j1 - WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 + WHERE j1.b @> j2.b AND j2.a < 20 ) ORDER BY j2.a ---- +1 {"a": "b"} +2 [1, 2, 3, 4, "foo"] 3 {"a": {"b": "c"}} 4 {"a": {"b": [1]}} 5 {"a": {"b": [1, [2]]}} 6 {"a": {"b": [[2]]}} +7 {"a": "b", "c": "d"} 8 {"a": {"b": true}} 9 {"a": {"b": false}} -17 {} - -# This query performs an anti inverted join with an additional filter. -query IT -SELECT * FROM json_tab AS j2 WHERE NOT EXISTS ( - SELECT * FROM json_tab@foo_inv AS j1 - WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 -) -ORDER BY j2.a ----- -1 {"a": "b"} -2 [1, 2, 3, 4, "foo"] -7 {"a": "b", "c": "d"} 10 "a" 11 null 12 true @@ -486,8 +476,18 @@ ORDER BY j2.a 14 1 15 1.23 16 [{"a": {"b": [1, [2]]}}, "d"] +17 {} 18 [] 19 ["a", "a"] + +# This query performs an anti inverted join with an additional filter. +query IT +SELECT * FROM json_tab AS j2 WHERE NOT EXISTS ( + SELECT * FROM json_tab@foo_inv AS j1 + WHERE j1.b @> j2.b AND j2.a < 20 +) +ORDER BY j2.a +---- 20 [{"a": "a"}, {"a": "a"}] 21 [[[["a"]]], [[["a"]]]] 22 [1, 2, 3, 1] diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array_dist b/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array_dist index 2c46b3a2d030..e8010dfa9307 100644 --- a/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array_dist +++ b/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array_dist @@ -86,11 +86,11 @@ https://cockroachdb.github.io/distsqlplan/decode.html#eJysk9Fv0z4Qx99_f4V1Tz-Yu8 # This query performs an inverted join with an additional filter. query T SELECT url FROM [EXPLAIN (DISTSQL) -SELECT * FROM json_tab@foo_inv AS j1, json_tab AS j2 -WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 +SELECT * FROM json_tab AS j2 INNER INVERTED JOIN json_tab AS j1 +ON j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 ORDER BY j1.a, j2.a] ---- -https://cockroachdb.github.io/distsqlplan/decode.html#eJzUk19P2zwUxu_fT3F03gvaYZrYafnjq8DItKKuZQ3SNkGF3MZD6UqcOQliqvrdpySlNIi44WLSdunj8_NznpM8S0x-LpCj7w2891eQ6QV8GI8-wbX39XJw2h9C67zvX_mfB21Yt7wrG-aJim5TMXW_K3UbRg9w6sOckk29ODP48tEbe9BqzWlnCu5NZtuOhDnrTNtwOjyHan1veYPiBjksV6u99lMH6wgoGmbA7DaMxufeGM6-wZx2BMnfEhMkGKlADsW9TJBfI0WCDCcEY61mMkmUzsvLoqkfPCK3CYZRnKV5eUJwprREvsQ0TBcSOV6J6UKOpQiktmwkGMhUhIvi6Y3vWIf3Qv9Cgn4sooTDgUVtnKwIqix9fjpJxZ1ETlekuXw_epA6lcGFCiOpLVYzwXrzSDaA9xhraLndzUZdtt7iVq2yZc75hT8anrWR4ChLObgOcSlxWa0T9hYnuYP1Hrs79zhQ6kcWw1yFEaiIg5t_xtEQWm7v2ZDzZKjX1FDuhuS2erWenLd48pVOpbYOq35cuk9ctr-t2i2Fa1W7tarPYkoHUsvgFS1nHyerV8YbqgMVW8cVom6CXmUC2jwUtGkoLGofWPTE-r9hMnbM8CIZzl-cjB1OtpLR-2eSscPTOhlHfzAZr6iOZRKrKJGN_ng7j4wM7mQZs0RleiYvtZoVMuVxVHBFIZBJWt7S8tCPyqt8wG2YGmFmhpkRdsywY4S7FZi-hLtG-MSs3DPCh2b40AgfmeEjI3xsho_ftLDJ6r_fAQAA__-NdN5s +https://cockroachdb.github.io/distsqlplan/decode.html#eJzUk09vm0AQxe_9FKPpIXazMSzY-bMnkppKRC6kYFWtEitam22E67B0gSiV5e9eYTt_iMzalnpoj8zOb-a9EW-O-a8ZMozcgftxCKWawacw-AzX7rerwbnnQ6vvRcPoy6AN65YPq4ZpLtPbgo_hPIKpBZ7vuyF4_lc3HLp9uAw8v95CIfCh1ZrSzhicm9I0bQFTqzNuw7nfh3r9YH6D_AYZzBeLg_ZTh9XhsGyYgGW2IQj7bggX32FKO5xUs_gICaYyFj6_Fzmya6RI0MIRwUzJichzqaryfNnkxY_ITIJJmpVFVR4RnEglkM2xSIqZQIZDPp6JUPBYKMNEgrEoeDJbjn4y52QquefqNxKMMp7mDI4MauJoQVCWxcvovOB3AhldkN3Xe-mDUIWIL2WSCmVYDQp-SHmbpA9IngH3MVPQcrrPF3Ws9RVf1WpXZoxdRoF_0UaCQVkwcChxLOLYjU6sfZxUDtZ37G6940DKn2UGU5mkIFMGjl2J8qHl9DYY6u1piDi9Rk_2Pp4iqQqhjOO6H8c-JA49bFzRbVzxMlmqWCgRNw3eoMWXRzIzTmtEk4JeTQHdPQF01wQY1Dwy6JnxfscYbNHwJgb2PxyDLU5exaD338Rgi6d1DE7-Vgw2rAhFnsk0Fzv93maVDxHfiVWmclmqibhScrJcs_oMltyyEIu8WL3S1YeXrp4qga9hqoUtPWxpYVsP21q4W4PpW7irhc_0m3ta-FgPH2vhEz18ooVP9fDpXgcbLd79CQAA__9K-9cD # This query performs a cross join followed by a filter. query T @@ -111,28 +111,26 @@ ORDER BY j1.a, j2.a] https://cockroachdb.github.io/distsqlplan/decode.html#eJzklVFv4jgQx9_vU4zmpXA1ECdAqZ_SHqmUikt6kFvtqkVVIG4VlsZZJ1StEN99FUILQcVJ1X1qH23Pf2b88_zlJSa_5shwZA2sfzxYyDlcDN1_4dr6fjU4sx2o9e2RN_pvUIdNyN95wCwR0W3qT-BsBDMdBtaFB7bzzRp6Vh8uXdspRlBwHajVZrQ5AfNmoWkGh5nenNThzOlDcf9oeYP-DTJYrlZH9ZcIvenDOmAKulYHd9i3hnD-A2a06ZMslz9GgpEIuOM_8ATZNVIkqCNBA8cEYymmPEmEzI6W60A7eEKmEQyjeJFm22OCUyE5siWmYTrnyNDzJ3M-5H7AZUtDggFP_XC-Tv9yQTOW4YMvn5HgKPajhEGjRTUcrwiKRbpNnaT-PUdGV6R6eTt65DLlwaUIIy5bRrED7znmLGfv_u9ZwzV4JNvO7oS4DaNHJK-JrKdYQs1sv9I29Q3hnb3CCzDGLkeuc15Hgq4DJt0-A2PMdrweErwLZZLCTIQRhBHEfih50MiWmWiRMjApMXViGsTsHCSjv4dMRmTzLt13Utm-10CIn4s471tEDEwjv2TN7L4BqFsOKOFTEQXVQHQPgjDeA2IkZMpl67QIwTSOiUmPD5ZoHyyxzSxkwCUPDiV-oxdHNETconpBcqiFTqEFWt2HtKoPW1RrtPSqVizpYM-K7S9kxRIyO1Y8-dxWLAGxsSLVPuJFvboR9MpG0LVGRReUlN9zQecLuaCEzI4Lep_bBSUgXlxA_9SP9EaNIU9iESW80kejZV8VD-55_r0lYiGn_EqK6bpMvnTXuvVGwJM0P6X5wo7yo6zBXTFVinW1WFeKDbXYUIrbBTHdF7fVdy4p3VGqu2pxVyk-UYtPlOKeWtz7CLFTNTGtZExKhqxsytRjRkvmjKoHbf_m49VfvwMAAP__HTRo-Q== # This query performs a semi inverted join with an additional filter. -# TODO(rytaft): A better plan would use an inner inverted join followed by a -# semi lookup join. Adjust the costing so that plan is chosen instead. query T SELECT url FROM [EXPLAIN (DISTSQL) SELECT * FROM json_tab AS j2 WHERE EXISTS ( SELECT * FROM json_tab@foo_inv AS j1 - WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 + WHERE j1.b @> j2.b AND j2.a < 20 ) ORDER BY j2.a] ---- -https://cockroachdb.github.io/distsqlplan/decode.html#eJzUk99u2kwQxe-_pxjNdxFoN7HXhiTslZPiqkQUUojUVAmKDN4mpsTr7q6jRIh3r2xT_jUsoKKqvfR4fjPnjH3GqL6PkGHXb_rvriCVI3jfaX-EG__6snnWaEGp3uhedT81yzBteVM0DJWI73TQh7MuDB34_MHv-OBfZ81Qer3V-yrEXRQ_5QidIqXSkB71wbtNbdvlMHSO-mU4a9VhuX4wvsXgFhmMJ5OD8s8O5yiAvGEAjl0uQ7tT9ztw_iUbE_SQYCxC3goeuUJ2gxQJOtgjmEgx4EoJmZXHeVMjfEZmE4ziJNVZuUdwICRHNkYd6RFHhldBf8Q7PAi5tGwkGHIdRKN89MxiIqPHQL4gwW4SxIrBoUVt7E0IilTPRysd3HNkdEK2X9-In7jUPLwQUcyl5axRMD0ykhngPycSSl5ldkzPmR5wobZ0YMbYRbfdOi8jwXaqGXiUeA7x3LVOnF2cZA6md6xsvGNTiG9pAkMRxSBiBp6biWpByau-Yqi6i6G1btxd3NQjpaN4oK3jZS9e9r-1ZcglD7OFK9vmA_ov8BCoh1_o3mSuqLJW0XyOKHatznlbDFor-3Qn2XNJ1b1LaolDkWR5WWxft_54aT3dPrp02-ha1D60aM36f8v8btCwkl_3L87vBicL-a3-A_nd4GYWhJM_ll-6z_zW9p7fV7Z1uEpErPhW0bQzvTy854U_JVI54JdSDPI1xWM75_JCyJUu3tLioREXrzKBizA1wo4Zdoywa4bdVZguwpUlmO4GU9tMV4y6q2a4aj73hnsfG-kTM3xihE_N8KkRrpnh2u98LDO86WNR8_-9-rV6k_9-BAAA__9MCKhZ +https://cockroachdb.github.io/distsqlplan/decode.html#eJzUkkFvm0AQhe_9FaPpxW7XgQXbVTmRNkQlcuzUWGqqFEUYJhHU2aW7ECWK_N8rwEpCZBPn1h53eB_z3ug9oP6zQgcDb-J9XUCpVnA8n53ChXd-Njn0p9A78oNF8H3Sh43kQyPItBSXRbSEwwAyC3588-YeeOeVGHrbpe6VlJepuK0RvkF6GT9YgvurNE2bILMOln04nB5BL7MOIqjHMVhmvw-z-ZE3hy8_K1EUIkMhE5pGN6TRuUCODC0MGeZKxqS1VNX4oRb5yR06JsNU5GVRjUOGsVSEzgMWabEidHARLVc0pyghZZjIMKEiSlf1rx_95yq9idQ9MgzySGgHBgY3MVwzlGXx9GtdRNeEDl-z_df74pZUQcmJTAUpw9rhYHNBZI-Ad5crcIePF3Qr9CpVuoBMpgJSAXmUKkoG1RMZzsrCAZcz12KuzdzRTv_WW_xXvjfXG7a9L-5zcmDiHS8g8E59OJn5U2TbjjqR8neZN7alcMC1K7tTcMftdJpiKZJX4-0MZu8M9pRHqoQUJe0oLv-I4XpL-qkcyNwYt9S7tg9b2_n-reT7ttLg5sDgn433e1bzFQ8vqmn_c9V8xf-zao7-32puCTYnnUuhaa_amVVvKbmmpudaliqmMyXjek3znNVcPUhIF81X3jx80XyqDD6HeSdsdcNWJ2y3YP4StjvhT92bh53wqBsedcLjbnj8pszh-t3fAAAA__8qVXLL # This query performs an anti inverted join with an additional filter. query T SELECT url FROM [EXPLAIN (DISTSQL) SELECT * FROM json_tab AS j2 WHERE NOT EXISTS ( SELECT * FROM json_tab@foo_inv AS j1 - WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 + WHERE j1.b @> j2.b AND j2.a < 20 ) ORDER BY j2.a] ---- -https://cockroachdb.github.io/distsqlplan/decode.html#eJzkld9T2kAQx9_7V-zsi9AekrsA4j0Fa5zGoYmFdGpHGSeQ0wnFXHoJjg7D_94JoUKoJHH6po93t5_98d3dZIHx7xlyHJp987MLczWDs4HzFa7My4t-z7KhdmoN3eG3fh3WJh8zg2ksw5vEG0NvCFMGP76YAxNsxwXzMgWg9rK5cSvlTRA-rDC6xmq1KT0cg3E91zRdwJQdjuvQs08hf3-wuEbvGjkslsuD-l8LdujBymACTKvXwRmcmgM4-Zm68UZIMJS-sL17ESO_QooEGRLUcUQwUnIi4liq9GmxMrT8R-QawSCM5kl6PSI4kUogX2ASJDOBHF1vPBMD4flCNTUk6IvEC2Yr989lRiq499QTEhxGXhhzaDSphqMlQTlPNq7jxLsTyOmSVA9vhQ9CJcI_l0EoVFPPZ-A-RYJD3zxzwfnumgM4dywbCe42AMmzI_MxUlAzWs9CG2wt7tZdTnzO-fnQsU_qSNCxwaCbDnDOLdvtIsHbQMUJTGUQQhBC5AVK-I30mELzhINBicGIoROjvVcZ9hplUkXWfensVaVnu9Y_omza1Zfy1zzK0pYhB0PPaqwZnRf06ZTrE4uJDP1SHfYqoO9VYFO4VL5Qws_XbNBPOFq-IJMtGzJqHues90Vv5aLT6otBqy5Gk2qNJqu6GyUZ7OxG6x3tRokyW7tx9EZ3g1WfTlZ5OpnWqDiaJeF3RrP9jkazRJmt0ey-0dEs-XENRBzJMBaVPsla-k0X_p3I_gGxnKuJuFBysgqTHZ0Vt7rwRZxkrzQ7WGH2lCa4DdNCmBXDrBDWczDdhfXitLXi0K1Cul0MtwvhTjHc-Z-ijwrhbnHkbiF8XAwfvyrt0fLDnwAAAP__TMTMFw== +https://cockroachdb.github.io/distsqlplan/decode.html#eJzUlFFv2joUx9_vpzg6T3CvKXEClOYp3NtUNxVLupBpnTpUBeJWYdTO7KRqVfHdpySoFFZMqj3xGPv8fM7P-csvqH4u0caJO3b_i6CQS7gIg09w415fjUeeD61zbxJNPo_bsC75uy5YKMFv83gGowksTPj6vxu64AcRuNclAK33y507IW5T_lhhdI21FvRkBs73wjAsBgvzZNaGkX8OrYV5EkO1PAfTaLchCM_dEP79VhbFUyTIRcL8-IEptG-QIkETCVo4JZhJMWdKCVluvVSFXvKEtkEw5VmRl8tTgnMhGdovmKf5kqGNUTxbspDFCZNdAwkmLI_TZXX8q0Mm04dYPiPBSRZzZUOnSw2crgiKIt8crfL4nqFNV6R5e48_Mpmz5FKknMmutT1B9JwxG8buRQTBl8gN4TLwfCS4e7tIXg9ynzIJTu_1dp3yggIfHLq5WNu2PT8aIsG7VKocFiLlkHLI4lSypFN-llCR2-BQ4pjEsYjT3ytsfkS4FF1f92Cv7MiPvN9cN39hLMSPIqvHFtwGx1o7Dra1FZsLnhzU2ytm7RXb-AiZMMmSbRWH_oPT1Tv2vuiIrHu2Vb2ve2-rO20eY9o0xl1qdLpm0yQfmGAnyb3jT_IB4TdJPj2uJJvNs2Q2zpJpdBoG6UD7nSD1jz9IB4TfBGl4XEE68NaHTGWCK9bouTPK95Il96x-X5Uo5JxdSTGv2tSfQcVVCwlTeb1L6w-P11vlgG9hqoVNPWxqYWsLpruwpR_b0Lfuaem-Hu5r4YEeHvyJ9KkWHuo7D7XwmR4--9DY09VfvwIAAP__8vKDjQ== statement ok INSERT INTO array_tab VALUES diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array_explain b/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array_explain index f75b116c5db8..f89fc799d00b 100644 --- a/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array_explain +++ b/pkg/sql/logictest/testdata/logic_test/inverted_join_json_array_explain @@ -67,8 +67,8 @@ vectorized: true # This query performs an inverted join with an additional filter. query T -EXPLAIN SELECT * FROM json_tab@foo_inv AS j1, json_tab AS j2 -WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 +EXPLAIN SELECT * FROM json_tab AS j2 INNER INVERTED JOIN json_tab AS j1 +ON j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 ORDER BY j1.a, j2.a ---- distribution: local @@ -147,41 +147,35 @@ vectorized: true spans: FULL SCAN # This query performs a semi inverted join with an additional filter. -# TODO(rytaft): A better plan would use an inner inverted join followed by a -# semi lookup join. Adjust the costing so that plan is chosen instead. query T EXPLAIN SELECT * FROM json_tab AS j2 WHERE EXISTS ( SELECT * FROM json_tab@foo_inv AS j1 - WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 + WHERE j1.b @> j2.b AND j2.a < 20 ) ORDER BY j2.a ---- distribution: local vectorized: true · -• distinct -│ distinct on: a -│ order key: a +• lookup join (semi) +│ table: json_tab@primary +│ equality: (a) = (a) +│ equality cols are key +│ pred: b @> b │ -└── • lookup join - │ table: json_tab@primary - │ equality: (a) = (a) - │ equality cols are key - │ pred: (b @> b) AND (b @> '{"a": {}}') +└── • inverted join + │ table: json_tab@foo_inv │ - └── • inverted join - │ table: json_tab@foo_inv - │ - └── • scan - missing stats - table: json_tab@primary - spans: [ - /19] + └── • scan + missing stats + table: json_tab@primary + spans: [ - /19] # This query performs an anti inverted join with an additional filter. query T EXPLAIN SELECT * FROM json_tab AS j2 WHERE NOT EXISTS ( SELECT * FROM json_tab@foo_inv AS j1 - WHERE j1.b @> j2.b AND j1.b @> '{"a": {}}' AND j2.a < 20 + WHERE j1.b @> j2.b AND j2.a < 20 ) ORDER BY j2.a ---- @@ -192,7 +186,7 @@ vectorized: true │ table: json_tab@primary │ equality: (a) = (a) │ equality cols are key -│ pred: (b @> b) AND (b @> '{"a": {}}') +│ pred: b @> b │ └── • inverted join (left outer) │ table: json_tab@foo_inv diff --git a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index index c3d4950500dc..3af17e463a95 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/inverted_index +++ b/pkg/sql/opt/exec/execbuilder/testdata/inverted_index @@ -179,7 +179,7 @@ vectorized: true │ └── • scan columns: (a) - estimated row count: 110 (missing stats) + estimated row count: 111 (missing stats) table: d@foo_inv spans: /"a"/"b"-/"a"/"b"/PrefixEnd @@ -197,7 +197,7 @@ vectorized: true │ └── • scan columns: (a) - estimated row count: 110 (missing stats) + estimated row count: 111 (missing stats) table: d@foo_inv spans: /"a"/"b"/Arr/1-/"a"/"b"/Arr/1/PrefixEnd @@ -215,7 +215,7 @@ vectorized: true │ └── • scan columns: (a) - estimated row count: 110 (missing stats) + estimated row count: 111 (missing stats) table: d@foo_inv spans: /"a"/"b"/Arr/Arr/2-/"a"/"b"/Arr/Arr/2/PrefixEnd @@ -233,7 +233,7 @@ vectorized: true │ └── • scan columns: (a) - estimated row count: 110 (missing stats) + estimated row count: 111 (missing stats) table: d@foo_inv spans: /"a"/"b"/True-/"a"/"b"/False @@ -251,7 +251,7 @@ vectorized: true │ └── • scan columns: (a) - estimated row count: 110 (missing stats) + estimated row count: 111 (missing stats) table: d@foo_inv spans: /Arr/1-/Arr/1/PrefixEnd @@ -269,44 +269,66 @@ vectorized: true │ └── • scan columns: (a) - estimated row count: 110 (missing stats) + estimated row count: 111 (missing stats) table: d@foo_inv spans: /Arr/"a"/"b"/Arr/1-/Arr/"a"/"b"/Arr/1/PrefixEnd +# TODO(rytaft): Fix the stats so we don't choose an inverted index scan here. query T EXPLAIN (VERBOSE) SELECT * from d where b @> '[]'; ---- distribution: local -vectorized: true +vectorized: false · -• filter +• index join │ columns: (a, b) │ estimated row count: 110 (missing stats) -│ filter: b @> '[]' +│ table: d@primary +│ key columns: a │ -└── • scan - columns: (a, b) - estimated row count: 1000 (missing stats) - table: d@primary - spans: FULL SCAN +└── • project + │ columns: (a) + │ estimated row count: 111 (missing stats) + │ + └── • inverted filter + │ columns: (a, b_inverted_key) + │ inverted column: b_inverted_key + │ num spans: 2 + │ + └── • scan + columns: (a, b_inverted_key) + estimated row count: 111 (missing stats) + table: d@foo_inv + spans: /[]-/{} /???-/??? +# TODO(rytaft): Fix the stats so we don't choose an inverted index scan here. query T EXPLAIN (VERBOSE) SELECT * from d where b @> '{}'; ---- distribution: local -vectorized: true +vectorized: false · -• filter +• index join │ columns: (a, b) │ estimated row count: 110 (missing stats) -│ filter: b @> '{}' +│ table: d@primary +│ key columns: a │ -└── • scan - columns: (a, b) - estimated row count: 1000 (missing stats) - table: d@primary - spans: FULL SCAN +└── • project + │ columns: (a) + │ estimated row count: 111 (missing stats) + │ + └── • inverted filter + │ columns: (a, b_inverted_key) + │ inverted column: b_inverted_key + │ num spans: 2 + │ + └── • scan + columns: (a, b_inverted_key) + estimated row count: 111 (missing stats) + table: d@foo_inv + spans: /{}-/{}/PrefixEnd /???-/[] # TODO(mgartner): It should not be required to force the index scan. It is @@ -401,29 +423,43 @@ query T EXPLAIN SELECT * from d where b @> '{"a": []}' ORDER BY a; ---- distribution: local -vectorized: true +vectorized: false · -• filter -│ filter: b @> '{"a": []}' +• sort +│ order: +a │ -└── • scan - missing stats - table: d@primary - spans: FULL SCAN +└── • index join + │ table: d@primary + │ + └── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 2 + │ + └── • scan + missing stats + table: d@foo_inv + spans: 2 spans query T EXPLAIN SELECT * from d where b @> '{"a": {}}' ORDER BY a; ---- distribution: local -vectorized: true +vectorized: false · -• filter -│ filter: b @> '{"a": {}}' +• sort +│ order: +a │ -└── • scan - missing stats - table: d@primary - spans: FULL SCAN +└── • index join + │ table: d@primary + │ + └── • inverted filter + │ inverted column: b_inverted_key + │ num spans: 2 + │ + └── • scan + missing stats + table: d@foo_inv + spans: 2 spans # Multi-path contains queries. Should create zigzag joins. @@ -578,41 +614,55 @@ query T EXPLAIN (VERBOSE) SELECT * from d where b @> '{"a": {}, "b": 2}' ---- distribution: local -vectorized: true +vectorized: false · -• filter +• index join │ columns: (a, b) │ estimated row count: 12 (missing stats) -│ filter: b @> '{"a": {}, "b": 2}' +│ table: d@primary +│ key columns: a │ -└── • index join - │ columns: (a, b) - │ estimated row count: 110 (missing stats) - │ table: d@primary - │ key columns: a +└── • project + │ columns: (a) + │ estimated row count: 111 (missing stats) │ - └── • scan - columns: (a) - estimated row count: 110 (missing stats) - table: d@foo_inv - spans: /"b"/2-/"b"/2/PrefixEnd + └── • inverted filter + │ columns: (a, b_inverted_key) + │ inverted column: b_inverted_key + │ num spans: 3 + │ + └── • scan + columns: (a, b_inverted_key) + estimated row count: 111 (missing stats) + table: d@foo_inv + spans: /"a"/{}-/"a"/{}/PrefixEnd /???-/??? /"b"/2-/"b"/2/PrefixEnd query T EXPLAIN (VERBOSE) SELECT * from d where b @> '{"a": {}, "b": {}}' ---- distribution: local -vectorized: true +vectorized: false · -• filter +• index join │ columns: (a, b) │ estimated row count: 12 (missing stats) -│ filter: b @> '{"a": {}, "b": {}}' +│ table: d@primary +│ key columns: a │ -└── • scan - columns: (a, b) - estimated row count: 1000 (missing stats) - table: d@primary - spans: FULL SCAN +└── • project + │ columns: (a) + │ estimated row count: 111 (missing stats) + │ + └── • inverted filter + │ columns: (a, b_inverted_key) + │ inverted column: b_inverted_key + │ num spans: 4 + │ + └── • scan + columns: (a, b_inverted_key) + estimated row count: 111 (missing stats) + table: d@foo_inv + spans: /"a"/{}-/"a"/{}/PrefixEnd /???-/??? /"b"/{}-/"b"/{}/PrefixEnd /???-/??? subtest array @@ -631,7 +681,7 @@ vectorized: true │ └── • scan columns: (a) - estimated row count: 110 (missing stats) + estimated row count: 111 (missing stats) table: e@e_b_idx spans: /1-/2 @@ -653,20 +703,23 @@ vectorized: true spans: FULL SCAN # Test that searching for a NULL element using the inverted index. +# TODO(rytaft): This filter should be normalized to false (See #56799). query T EXPLAIN (VERBOSE) SELECT * from e where b @> ARRAY[NULL]::INT[] ---- distribution: local vectorized: true · -• index join +• filter │ columns: (a, b) │ estimated row count: 110 (missing stats) -│ table: e@primary -│ key columns: a +│ filter: b @> ARRAY[NULL] │ -└── • norows - columns: (a) +└── • scan + columns: (a, b) + estimated row count: 1000 (missing stats) + table: e@primary + spans: FULL SCAN query T EXPLAIN (VERBOSE) SELECT * from e where b @> NULL diff --git a/pkg/sql/opt/exec/execbuilder/testdata/partial_index b/pkg/sql/opt/exec/execbuilder/testdata/partial_index index 9d5d54cc9b6b..d7c8f56cf479 100644 --- a/pkg/sql/opt/exec/execbuilder/testdata/partial_index +++ b/pkg/sql/opt/exec/execbuilder/testdata/partial_index @@ -907,7 +907,7 @@ vectorized: true • scan missing stats table: inv@i (partial index) - spans: [/'{"x": "y"}' - /'{"x": "y"}'] + spans: 1 span query T EXPLAIN SELECT a FROM inv@i WHERE b @> '{"x": "y"}' AND c = 'foo' @@ -924,7 +924,7 @@ vectorized: true └── • scan missing stats table: inv@i (partial index) - spans: [/'{"x": "y"}' - /'{"x": "y"}'] + spans: 1 span query T EXPLAIN SELECT * FROM inv@i WHERE b @> '{"x": "y"}' AND c IN ('foo', 'bar') @@ -938,4 +938,4 @@ vectorized: true └── • scan missing stats table: inv@i (partial index) - spans: [/'{"x": "y"}' - /'{"x": "y"}'] + spans: 1 span diff --git a/pkg/sql/opt/invertedexpr/expression.go b/pkg/sql/opt/invertedexpr/expression.go index 58edc7def372..ce6b44e1204c 100644 --- a/pkg/sql/opt/invertedexpr/expression.go +++ b/pkg/sql/opt/invertedexpr/expression.go @@ -360,6 +360,12 @@ type SpanExpression struct { // Tight mirrors the definition of IsTight(). Tight bool + // Unique is true if the spans are guaranteed not to produce duplicate + // primary keys. Otherwise, Unique is false. Unique may be true for certain + // JSON or Array SpanExpressions, but it does not hold when these + // SpanExpressions are combined with And or Or. + Unique bool + // SpansToRead are the spans to read from the inverted index // to evaluate this SpanExpression. These are non-overlapping // and sorted. If left or right contains a non-SpanExpression, @@ -420,6 +426,7 @@ func (s *SpanExpression) SetNotTight() { func (s *SpanExpression) Copy() InvertedExpression { res := &SpanExpression{ Tight: s.Tight, + Unique: s.Unique, SpansToRead: s.SpansToRead, FactoredUnionSpans: s.FactoredUnionSpans, Operator: s.Operator, @@ -442,7 +449,7 @@ func (s *SpanExpression) String() string { // Format pretty-prints the SpanExpression. func (s *SpanExpression) Format(tp treeprinter.Node, includeSpansToRead bool) { - tp.Childf("tight: %t", s.Tight) + tp.Childf("tight: %t, unique: %t", s.Tight, s.Unique) if includeSpansToRead { s.SpansToRead.Format(tp, "to read") } diff --git a/pkg/sql/opt/invertedexpr/json_array_expression.go b/pkg/sql/opt/invertedexpr/json_array_expression.go index 47d2bffee22d..01b6efcbba43 100644 --- a/pkg/sql/opt/invertedexpr/json_array_expression.go +++ b/pkg/sql/opt/invertedexpr/json_array_expression.go @@ -25,7 +25,7 @@ func JSONOrArrayToContainingSpanExpr( evalCtx *tree.EvalContext, d tree.Datum, ) (*SpanExpression, error) { var b []byte - spansSlice, tight, _, err := rowenc.EncodeContainingInvertedIndexSpans( + spansSlice, tight, unique, err := rowenc.EncodeContainingInvertedIndexSpans( evalCtx, d, b, descpb.EmptyArraysInInvertedIndexesVersion, ) if err != nil { @@ -60,6 +60,7 @@ func JSONOrArrayToContainingSpanExpr( } if spanExpr, ok := invExpr.(*SpanExpression); ok { + spanExpr.Unique = unique return spanExpr, nil } return nil, nil diff --git a/pkg/sql/opt/invertedexpr/testdata/expression b/pkg/sql/opt/invertedexpr/testdata/expression index 03ed544485e8..03b9e53b88ad 100644 --- a/pkg/sql/opt/invertedexpr/testdata/expression +++ b/pkg/sql/opt/invertedexpr/testdata/expression @@ -1,7 +1,7 @@ new-span-leaf name=b tight=true span=b ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "b"] └── union spans: ["b", "b"] @@ -22,12 +22,12 @@ unknown expression: tight=false or result=bt left=b right=u-tight ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "b"] ├── union spans: ["b", "b"] └── UNION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ └── union spans: empty └── unknown expression: tight=true @@ -37,12 +37,12 @@ span expression or result=bt left=u-tight right=b ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "b"] ├── union spans: ["b", "b"] └── UNION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ └── union spans: empty └── unknown expression: tight=true @@ -52,12 +52,12 @@ span expression or result=bnt left=b right=u-not-tight ---- span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] ├── union spans: ["b", "b"] └── UNION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ └── union spans: empty └── unknown expression: tight=false @@ -67,12 +67,12 @@ span expression and result=b-and-unknown left=b right=u-tight ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "b"] ├── union spans: empty └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ └── union spans: ["b", "b"] └── unknown expression: tight=true @@ -82,12 +82,12 @@ span expression and result=b-and-unknown left=b right=u-not-tight ---- span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] ├── union spans: empty └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ └── union spans: ["b", "b"] └── unknown expression: tight=false @@ -97,27 +97,27 @@ span expression and result=bt-and-bnt left=bt right=bnt ---- span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] ├── union spans: ["b", "b"] └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ ├── union spans: empty │ └── UNION │ ├── span expression - │ │ ├── tight: true + │ │ ├── tight: true, unique: false │ │ ├── to read: ["b", "b"] │ │ └── union spans: empty │ └── unknown expression: tight=true └── span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] ├── union spans: empty └── UNION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ └── union spans: empty └── unknown expression: tight=false @@ -126,27 +126,27 @@ span expression or result=bt-or-bnt left=bnt right=bt ---- span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] ├── union spans: ["b", "b"] └── UNION ├── span expression - │ ├── tight: false + │ ├── tight: false, unique: false │ ├── to read: ["b", "b"] │ ├── union spans: empty │ └── UNION │ ├── span expression - │ │ ├── tight: true + │ │ ├── tight: true, unique: false │ │ ├── to read: ["b", "b"] │ │ └── union spans: empty │ └── unknown expression: tight=false └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "b"] ├── union spans: empty └── UNION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ └── union spans: empty └── unknown expression: tight=true @@ -163,12 +163,12 @@ new-non-inverted-leaf name=niexpr and result=bt-and-niexpr left=bt right=niexpr ---- span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] ├── union spans: ["b", "b"] └── UNION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "b"] │ └── union spans: empty └── unknown expression: tight=true @@ -187,7 +187,7 @@ or result=bt-or-niexpr left=niexpr right=bt or result=b-or-b left=b right=b ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "b"] └── union spans: ["b", "b"] @@ -195,14 +195,14 @@ span expression and result=b-and-b left=b right=b ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "b"] └── union spans: ["b", "b"] new-span-leaf name=b-not-tight tight=false span=b ---- span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] └── union spans: ["b", "b"] @@ -210,7 +210,7 @@ span expression or result=_ left=b right=b-not-tight ---- span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] └── union spans: ["b", "b"] @@ -218,14 +218,14 @@ span expression and result=_ left=b-not-tight right=b ---- span expression - ├── tight: false + ├── tight: false, unique: false ├── to read: ["b", "b"] └── union spans: ["b", "b"] new-span-leaf name=ac tight=true span=a,c ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "c") └── union spans: ["a", "c") @@ -233,7 +233,7 @@ span expression or result=_ left=b right=ac ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "c") └── union spans: ["a", "c") @@ -241,14 +241,14 @@ span expression and result=_ left=b right=ac ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "c") └── union spans: ["b", "b"] new-span-leaf name=bj tight=true span=b,j ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "j") └── union spans: ["b", "j") @@ -256,7 +256,7 @@ span expression or result=_ left=bj right=b ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "j") └── union spans: ["b", "j") @@ -264,7 +264,7 @@ span expression and result=_ left=b right=bj ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["b", "j") └── union spans: ["b", "b"] @@ -272,7 +272,7 @@ span expression or result=aj left=bj right=ac ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") └── union spans: ["a", "j") @@ -280,16 +280,16 @@ span expression and result=bj-and-ac left=bj right=ac ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans: ["b", "b"] └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "j") │ └── union spans: ["c", "j") └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "c") └── union spans: ["a", "a"] @@ -297,27 +297,27 @@ span expression and result=foo left=aj right=bj-and-ac ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans: ["b", "b"] └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["a", "j") │ └── union spans │ ├── ["a", "a"] │ └── ["c", "j") └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans: empty └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "j") │ └── union spans: ["c", "j") └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "c") └── union spans: ["a", "a"] @@ -325,25 +325,25 @@ span expression and result=foo left=bj-and-ac right=aj ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans: ["b", "b"] └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["a", "j") │ ├── union spans: empty │ └── INTERSECTION │ ├── span expression - │ │ ├── tight: true + │ │ ├── tight: true, unique: false │ │ ├── to read: ["b", "j") │ │ └── union spans: ["c", "j") │ └── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["a", "c") │ └── union spans: ["a", "a"] └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") └── union spans ├── ["a", "a"] @@ -357,108 +357,108 @@ span expression or result=bar left=aj right=bj-and-ac ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans: ["a", "j") └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "j") │ └── union spans: ["c", "j") └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "c") └── union spans: ["a", "a"] and result=foo-and-bar left=foo right=bar ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans: ["b", "b"] └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["a", "j") │ ├── union spans: empty │ └── INTERSECTION │ ├── span expression - │ │ ├── tight: true + │ │ ├── tight: true, unique: false │ │ ├── to read: ["a", "j") │ │ ├── union spans: empty │ │ └── INTERSECTION │ │ ├── span expression - │ │ │ ├── tight: true + │ │ │ ├── tight: true, unique: false │ │ │ ├── to read: ["b", "j") │ │ │ └── union spans: ["c", "j") │ │ └── span expression - │ │ ├── tight: true + │ │ ├── tight: true, unique: false │ │ ├── to read: ["a", "c") │ │ └── union spans: ["a", "a"] │ └── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["a", "j") │ └── union spans │ ├── ["a", "a"] │ └── ["c", "j") └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans │ ├── ["a", "a"] │ └── ["c", "j") └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "j") │ └── union spans: ["c", "j") └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "c") └── union spans: ["a", "a"] or result=foo-or-bar left=foo right=bar ---- span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans: ["a", "j") └── UNION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["a", "j") │ ├── union spans: empty │ └── INTERSECTION │ ├── span expression - │ │ ├── tight: true + │ │ ├── tight: true, unique: false │ │ ├── to read: ["a", "j") │ │ ├── union spans: empty │ │ └── INTERSECTION │ │ ├── span expression - │ │ │ ├── tight: true + │ │ │ ├── tight: true, unique: false │ │ │ ├── to read: ["b", "j") │ │ │ └── union spans: ["c", "j") │ │ └── span expression - │ │ ├── tight: true + │ │ ├── tight: true, unique: false │ │ ├── to read: ["a", "c") │ │ └── union spans: ["a", "a"] │ └── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["a", "j") │ └── union spans │ ├── ["a", "a"] │ └── ["c", "j") └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "j") ├── union spans: empty └── INTERSECTION ├── span expression - │ ├── tight: true + │ ├── tight: true, unique: false │ ├── to read: ["b", "j") │ └── union spans: ["c", "j") └── span expression - ├── tight: true + ├── tight: true, unique: false ├── to read: ["a", "c") └── union spans: ["a", "a"] diff --git a/pkg/sql/opt/invertedidx/geo.go b/pkg/sql/opt/invertedidx/geo.go index aa813a92c870..027d9bf1d6c4 100644 --- a/pkg/sql/opt/invertedidx/geo.go +++ b/pkg/sql/opt/invertedidx/geo.go @@ -21,7 +21,6 @@ import ( "github.com/cockroachdb/cockroach/pkg/geo/geoprojbase" "github.com/cockroachdb/cockroach/pkg/sql/opt" "github.com/cockroachdb/cockroach/pkg/sql/opt/cat" - "github.com/cockroachdb/cockroach/pkg/sql/opt/constraint" "github.com/cockroachdb/cockroach/pkg/sql/opt/invertedexpr" "github.com/cockroachdb/cockroach/pkg/sql/opt/memo" "github.com/cockroachdb/cockroach/pkg/sql/opt/norm" @@ -60,82 +59,11 @@ func GetGeoIndexRelationship(expr opt.ScalarExpr) (_ geoindex.RelationshipType, // getSpanExprForGeoIndexFn is a function that returns a SpanExpression that // constrains the given geo index according to the given constant and // geospatial relationship. It is implemented by getSpanExprForGeographyIndex -// and getSpanExprForGeometryIndex and used in constrainGeoIndex. +// and getSpanExprForGeometryIndex and used in extractInvertedFilterCondition. type getSpanExprForGeoIndexFn func( context.Context, tree.Datum, []tree.Datum, geoindex.RelationshipType, *geoindex.Config, ) *invertedexpr.SpanExpression -// TryConstrainGeoIndex tries to derive an inverted index constraint for the -// given geospatial index from the specified filters. If a constraint is -// derived, it is returned with ok=true. If no constraint can be derived, -// then TryConstrainGeoIndex returns ok=false. -func TryConstrainGeoIndex( - evalCtx *tree.EvalContext, - factory *norm.Factory, - filters memo.FiltersExpr, - optionalFilters memo.FiltersExpr, - tabID opt.TableID, - index cat.Index, -) ( - invertedConstraint *invertedexpr.SpanExpression, - constraint *constraint.Constraint, - remainingFilters memo.FiltersExpr, - preFiltererState *invertedexpr.PreFiltererStateForInvertedFilterer, - ok bool, -) { - // Attempt to constrain the prefix columns, if there are any. If they cannot - // be constrained to single values, the index cannot be used. - constraint, filters, ok = constrainPrefixColumns(evalCtx, factory, filters, optionalFilters, tabID, index) - if !ok { - return nil, nil, nil, nil, false - } - - config := index.GeoConfig() - var getSpanExpr getSpanExprForGeoIndexFn - var typ *types.T - if geoindex.IsGeographyConfig(config) { - getSpanExpr = getSpanExprForGeographyIndex - typ = types.Geography - } else if geoindex.IsGeometryConfig(config) { - getSpanExpr = getSpanExprForGeometryIndex - typ = types.Geometry - } else { - return nil, nil, nil, nil, false - } - - var invertedExpr invertedexpr.InvertedExpression - var pfState *invertedexpr.PreFiltererStateForInvertedFilterer - for i := range filters { - invertedExprLocal, pfStateLocal := constrainGeoIndex( - evalCtx.Context, factory, filters[i].Condition, tabID, index, getSpanExpr, - ) - if invertedExpr == nil { - invertedExpr = invertedExprLocal - pfState = pfStateLocal - } else { - invertedExpr = invertedexpr.And(invertedExpr, invertedExprLocal) - // Do pre-filtering using the first of the conjuncts that provided - // non-nil pre-filtering state. - if pfState == nil { - pfState = pfStateLocal - } - } - } - - if invertedExpr == nil { - return nil, nil, nil, nil, false - } - - spanExpr, ok := invertedExpr.(*invertedexpr.SpanExpression) - if !ok { - return nil, nil, nil, nil, false - } - if pfState != nil { - pfState.Typ = typ - } - return spanExpr, constraint, filters, pfState, true -} - // getSpanExprForGeographyIndex gets a SpanExpression that constrains the given // geography index according to the given constant and geospatial relationship. func getSpanExprForGeographyIndex( @@ -394,28 +322,26 @@ func constructFunction( }) } -// constrainGeoIndex returns an InvertedExpression representing a constraint -// of the given geospatial index. -func constrainGeoIndex( - ctx context.Context, - factory *norm.Factory, - expr opt.ScalarExpr, - tabID opt.TableID, - index cat.Index, - getSpanExpr getSpanExprForGeoIndexFn, -) (invertedexpr.InvertedExpression, *invertedexpr.PreFiltererStateForInvertedFilterer) { - var args memo.ScalarListExpr - switch t := expr.(type) { - case *memo.AndExpr: - l, _ := constrainGeoIndex(ctx, factory, t.Left, tabID, index, getSpanExpr) - r, _ := constrainGeoIndex(ctx, factory, t.Right, tabID, index, getSpanExpr) - return invertedexpr.And(l, r), nil +type geoFilterPlanner struct { + factory *norm.Factory + tabID opt.TableID + index cat.Index + getSpanExpr getSpanExprForGeoIndexFn +} - case *memo.OrExpr: - l, _ := constrainGeoIndex(ctx, factory, t.Left, tabID, index, getSpanExpr) - r, _ := constrainGeoIndex(ctx, factory, t.Right, tabID, index, getSpanExpr) - return invertedexpr.Or(l, r), nil +var _ invertedFilterPlanner = &geoFilterPlanner{} +// extractInvertedFilterConditionFromLeaf is part of the invertedFilterPlanner +// interface. +func (g *geoFilterPlanner) extractInvertedFilterConditionFromLeaf( + evalCtx *tree.EvalContext, expr opt.ScalarExpr, +) ( + invertedExpr invertedexpr.InvertedExpression, + remainingFilters opt.ScalarExpr, + _ *invertedexpr.PreFiltererStateForInvertedFilterer, +) { + var args memo.ScalarListExpr + switch t := expr.(type) { case *memo.FunctionExpr: args = t.Args @@ -426,18 +352,18 @@ func constrainGeoIndex( // Cast the arguments to type Geometry if they are type Box2d. for i := 0; i < len(args); i++ { if args[i].DataType().Family() == types.Box2DFamily { - args[i] = factory.ConstructCast(args[i], types.Geometry) + args[i] = g.factory.ConstructCast(args[i], types.Geometry) } } default: - return invertedexpr.NonInvertedColExpression{}, nil + return invertedexpr.NonInvertedColExpression{}, expr, nil } - // Try to constrain the index with the given expression. If the resulting - // inverted expression is not a SpanExpression, try constraining the index - // with an equivalent function in which the arguments are commuted. For - // example: + // Try to extract an inverted filter condition from the given expression. + // If the resulting inverted expression is not a SpanExpression, try + // extracting the condition with an equivalent function in which the + // arguments are commuted. For example: // // ST_Intersects(g1, g2) <-> ST_Intersects(g2, g1) // ST_Covers(g1, g2) <-> ST_CoveredBy(g2, g1) @@ -445,23 +371,26 @@ func constrainGeoIndex( // g1 ~ g2 -> ST_CoveredBy(g2, g1) // // See geoindex.CommuteRelationshipMap for the full list of mappings. - invertedExpr, pfState := constrainGeoIndexFromExpr( - ctx, factory, expr, args, false /* commuteArgs */, tabID, index, getSpanExpr, + invertedExpr, pfState := extractGeoFilterCondition( + evalCtx.Context, g.factory, expr, args, false /* commuteArgs */, g.tabID, g.index, g.getSpanExpr, ) if _, ok := invertedExpr.(invertedexpr.NonInvertedColExpression); ok { - invertedExpr, pfState = constrainGeoIndexFromExpr( - ctx, factory, expr, args, true /* commuteArgs */, tabID, index, getSpanExpr, + invertedExpr, pfState = extractGeoFilterCondition( + evalCtx.Context, g.factory, expr, args, true /* commuteArgs */, g.tabID, g.index, g.getSpanExpr, ) } - return invertedExpr, pfState + if !invertedExpr.IsTight() { + remainingFilters = expr + } + return invertedExpr, remainingFilters, pfState } -// constrainGeoIndexFromExpr returns an InvertedExpression representing a -// constraint of the given geospatial index, based on the given expression. -// If commuteArgs is true, constrainGeoIndexFromExpr constrains the index -// based on an equivalent version of the given expression in which the first -// two arguments are swapped. -func constrainGeoIndexFromExpr( +// extractGeoFilterCondition extracts an InvertedExpression representing an +// inverted filter condition over the given geospatial index, based on the +// given expression. If commuteArgs is true, extractGeoFilterCondition extracts +// the InvertedExpression based on an equivalent version of the given +// expression in which the first two arguments are swapped. +func extractGeoFilterCondition( ctx context.Context, factory *norm.Factory, expr opt.ScalarExpr, diff --git a/pkg/sql/opt/invertedidx/geo_test.go b/pkg/sql/opt/invertedidx/geo_test.go index 08caefea5cf9..884efa4af733 100644 --- a/pkg/sql/opt/invertedidx/geo_test.go +++ b/pkg/sql/opt/invertedidx/geo_test.go @@ -310,7 +310,7 @@ func TestTryJoinGeoIndex(t *testing.T) { } } -func TestTryConstrainGeoIndex(t *testing.T) { +func TestTryFilterGeoIndex(t *testing.T) { semaCtx := tree.MakeSemaContext() evalCtx := tree.NewTestingEvalContext(nil /* st */) @@ -490,12 +490,15 @@ func TestTryConstrainGeoIndex(t *testing.T) { // We're not testing that the correct SpanExpression is returned here; // that is tested elsewhere. This is just testing that we are constraining // the index when we expect to. - _, _, _, pfState, ok := invertedidx.TryConstrainGeoIndex( + spanExpr, _, _, pfState, ok := invertedidx.TryFilterInvertedIndex( evalCtx, &f, filters, nil /* optionalFilters */, tab, md.Table(tab).Index(tc.indexOrd), ) if tc.ok != ok { t.Fatalf("expected %v, got %v", tc.ok, ok) } + if spanExpr != nil && spanExpr.Unique { + t.Fatalf("geospatial indexes should never have Unique=true") + } if ok { if len(tc.preFilterExpr) == 0 { require.Nil(t, pfState) diff --git a/pkg/sql/opt/invertedidx/inverted_index_expr.go b/pkg/sql/opt/invertedidx/inverted_index_expr.go index 3402841afe49..292c1a48ff5f 100644 --- a/pkg/sql/opt/invertedidx/inverted_index_expr.go +++ b/pkg/sql/opt/invertedidx/inverted_index_expr.go @@ -251,6 +251,106 @@ func evalInvertedExpr( } } +// TryFilterInvertedIndex tries to derive an inverted filter condition for +// the given inverted index from the specified filters. If an inverted filter +// condition is derived, it is returned with ok=true. If no condition can be +// derived, then TryFilterInvertedIndex returns ok=false. +// +// In addition to the inverted filter condition (spanExpr), returns: +// - a constraint of the prefix columns if there are any, +// - remaining filters that must be applied if the span expression is not tight, +// - pre-filterer state that can be used by the invertedFilterer operator to +// reduce the number of false positives returned by the span expression, +// - ok=true if the spanExpr is a valid inverted filter condition. Otherwise, +// returns ok=false. +func TryFilterInvertedIndex( + evalCtx *tree.EvalContext, + factory *norm.Factory, + filters memo.FiltersExpr, + optionalFilters memo.FiltersExpr, + tabID opt.TableID, + index cat.Index, +) ( + spanExpr *invertedexpr.SpanExpression, + constraint *constraint.Constraint, + remainingFilters memo.FiltersExpr, + preFiltererState *invertedexpr.PreFiltererStateForInvertedFilterer, + ok bool, +) { + // Attempt to constrain the prefix columns, if there are any. If they cannot + // be constrained to single values, the index cannot be used. + constraint, filters, ok = constrainPrefixColumns( + evalCtx, factory, filters, optionalFilters, tabID, index, + ) + if !ok { + return nil, nil, nil, nil, false + } + + config := index.GeoConfig() + var typ *types.T + var filterPlanner invertedFilterPlanner + if geoindex.IsGeographyConfig(config) { + filterPlanner = &geoFilterPlanner{ + factory: factory, + tabID: tabID, + index: index, + getSpanExpr: getSpanExprForGeographyIndex, + } + typ = types.Geography + } else if geoindex.IsGeometryConfig(config) { + filterPlanner = &geoFilterPlanner{ + factory: factory, + tabID: tabID, + index: index, + getSpanExpr: getSpanExprForGeometryIndex, + } + typ = types.Geometry + } else { + filterPlanner = &jsonOrArrayFilterPlanner{ + tabID: tabID, + index: index, + } + col := index.VirtualInvertedColumn().InvertedSourceColumnOrdinal() + typ = factory.Metadata().Table(tabID).Column(col).DatumType() + } + + var invertedExpr invertedexpr.InvertedExpression + var pfState *invertedexpr.PreFiltererStateForInvertedFilterer + for i := range filters { + invertedExprLocal, remFiltersLocal, pfStateLocal := extractInvertedFilterCondition( + evalCtx, factory, filters[i].Condition, filterPlanner, + ) + if invertedExpr == nil { + invertedExpr = invertedExprLocal + pfState = pfStateLocal + } else { + invertedExpr = invertedexpr.And(invertedExpr, invertedExprLocal) + // Do pre-filtering using the first of the conjuncts that provided + // non-nil pre-filtering state. + if pfState == nil { + pfState = pfStateLocal + } + } + if remFiltersLocal != nil { + remainingFilters = append(remainingFilters, factory.ConstructFiltersItem(remFiltersLocal)) + } + } + + if invertedExpr == nil { + return nil, nil, nil, nil, false + } + + spanExpr, ok = invertedExpr.(*invertedexpr.SpanExpression) + if !ok { + return nil, nil, nil, nil, false + } + if pfState != nil { + pfState.Typ = typ + } + + return spanExpr, constraint, remainingFilters, pfState, true +} + // constrainPrefixColumns attempts to build a constraint for the non-inverted // prefix columns of the given index. If a constraint is successfully built, it // is returned along with remaining filters and ok=true. The function is only @@ -305,3 +405,74 @@ func constrainPrefixColumns( remainingFilters = ic.RemainingFilters() return ©, remainingFilters, true } + +type invertedFilterPlanner interface { + // extractInvertedFilterConditionFromLeaf extracts an inverted filter + // condition from the given expression, which represents a leaf of an + // expression tree in which the internal nodes are And and/or Or expressions. + // Returns an empty InvertedExpression if no inverted filter condition could + // be extracted. + // + // Additionally, returns: + // - remaining filters that must be applied if the inverted expression is not + // tight, + // - pre-filterer state that can be used to reduce false positives, and + extractInvertedFilterConditionFromLeaf(evalCtx *tree.EvalContext, expr opt.ScalarExpr) ( + invertedExpr invertedexpr.InvertedExpression, + remainingFilters opt.ScalarExpr, + _ *invertedexpr.PreFiltererStateForInvertedFilterer, + ) +} + +// extractInvertedFilterCondition extracts an InvertedExpression from the given +// filter condition, where the InvertedExpression represents an inverted filter +// over the given inverted index. Returns an empty InvertedExpression if no +// inverted filter condition could be extracted. +// +// The filter condition should be an expression tree of And, Or, and leaf +// expressions. Extraction of the InvertedExpression from the leaves is +// delegated to the given invertedFilterPlanner. +// +// In addition to the InvertedExpression, returns: +// - remaining filters that must be applied if the inverted expression is not +// tight, +// - pre-filterer state that can be used to reduce false positives, and +func extractInvertedFilterCondition( + evalCtx *tree.EvalContext, + factory *norm.Factory, + filterCond opt.ScalarExpr, + filterPlanner invertedFilterPlanner, +) ( + invertedExpr invertedexpr.InvertedExpression, + remainingFilters opt.ScalarExpr, + _ *invertedexpr.PreFiltererStateForInvertedFilterer, +) { + switch t := filterCond.(type) { + case *memo.AndExpr: + l, remLeft, _ := extractInvertedFilterCondition(evalCtx, factory, t.Left, filterPlanner) + r, remRight, _ := extractInvertedFilterCondition(evalCtx, factory, t.Right, filterPlanner) + if remLeft == nil { + remainingFilters = remRight + } else if remRight == nil { + remainingFilters = remLeft + } else { + remainingFilters = factory.ConstructAnd(remLeft, remRight) + } + return invertedexpr.And(l, r), remainingFilters, nil + + case *memo.OrExpr: + l, remLeft, _ := extractInvertedFilterCondition(evalCtx, factory, t.Left, filterPlanner) + r, remRight, _ := extractInvertedFilterCondition(evalCtx, factory, t.Right, filterPlanner) + if remLeft == nil { + remainingFilters = remRight + } else if remRight == nil { + remainingFilters = remLeft + } else { + remainingFilters = factory.ConstructOr(remLeft, remRight) + } + return invertedexpr.Or(l, r), remainingFilters, nil + + default: + return filterPlanner.extractInvertedFilterConditionFromLeaf(evalCtx, filterCond) + } +} diff --git a/pkg/sql/opt/invertedidx/json_array.go b/pkg/sql/opt/invertedidx/json_array.go index 74d2936d2595..a3ee13194fa3 100644 --- a/pkg/sql/opt/invertedidx/json_array.go +++ b/pkg/sql/opt/invertedidx/json_array.go @@ -224,7 +224,8 @@ func (g *jsonOrArrayDatumsToInvertedExpr) Convert( if d == tree.DNull { return nil, nil } - return getSpanExprForJSONOrArrayIndex(g.evalCtx, d), nil + spanExpr := getSpanExprForJSONOrArrayIndex(g.evalCtx, d) + return spanExpr, nil default: return nil, fmt.Errorf("unsupported expression %v", t) @@ -257,3 +258,73 @@ func (g *jsonOrArrayDatumsToInvertedExpr) PreFilter( ) (bool, error) { return false, errors.AssertionFailedf("PreFilter called on jsonOrArrayDatumsToInvertedExpr") } + +type jsonOrArrayFilterPlanner struct { + tabID opt.TableID + index cat.Index +} + +var _ invertedFilterPlanner = &jsonOrArrayFilterPlanner{} + +// extractInvertedFilterConditionFromLeaf is part of the invertedFilterPlanner +// interface. +func (j *jsonOrArrayFilterPlanner) extractInvertedFilterConditionFromLeaf( + evalCtx *tree.EvalContext, expr opt.ScalarExpr, +) ( + invertedExpr invertedexpr.InvertedExpression, + remainingFilters opt.ScalarExpr, + _ *invertedexpr.PreFiltererStateForInvertedFilterer, +) { + switch t := expr.(type) { + case *memo.ContainsExpr: + invertedExpr := j.extractJSONOrArrayFilterCondition(evalCtx, t.Left, t.Right) + if !invertedExpr.IsTight() { + remainingFilters = expr + } + + // We do not currently support pre-filtering for JSON and Array indexes, so + // the returned pre-filter state is nil. + return invertedExpr, remainingFilters, nil + + default: + return invertedexpr.NonInvertedColExpression{}, expr, nil + } +} + +// extractJSONOrArrayFilterCondition extracts an InvertedExpression +// representing an inverted filter over the given inverted index, based +// on the given left and right expression arguments. Returns an empty +// InvertedExpression if no inverted filter could be extracted. +func (j *jsonOrArrayFilterPlanner) extractJSONOrArrayFilterCondition( + evalCtx *tree.EvalContext, left, right opt.ScalarExpr, +) invertedexpr.InvertedExpression { + // The first argument should be a variable corresponding to the index + // column. + variable, ok := left.(*memo.VariableExpr) + if !ok { + return invertedexpr.NonInvertedColExpression{} + } + if variable.Col != j.tabID.ColumnID( + j.index.VirtualInvertedColumn().InvertedSourceColumnOrdinal(), + ) { + // The column does not match the index column. + return invertedexpr.NonInvertedColExpression{} + } + if variable.Typ.Family() == types.ArrayFamily && + j.index.Version() < descpb.EmptyArraysInInvertedIndexesVersion { + // We cannot constrain array indexes that do not include + // keys for empty arrays. + // TODO(rytaft): If the spans to read do not include the key for the empty + // array, we should still be able to use this index. + return invertedexpr.NonInvertedColExpression{} + } + + // The second argument should be a constant. + if !memo.CanExtractConstDatum(right) { + return invertedexpr.NonInvertedColExpression{} + } + d := memo.ExtractConstDatum(right) + + spanExpr := getSpanExprForJSONOrArrayIndex(evalCtx, d) + return spanExpr +} diff --git a/pkg/sql/opt/invertedidx/json_array_test.go b/pkg/sql/opt/invertedidx/json_array_test.go index 9df6ba662617..62b342ebb867 100644 --- a/pkg/sql/opt/invertedidx/json_array_test.go +++ b/pkg/sql/opt/invertedidx/json_array_test.go @@ -212,3 +212,121 @@ func TestTryJoinJsonOrArrayIndex(t *testing.T) { } } } + +func TestTryFilterJsonOrArrayIndex(t *testing.T) { + semaCtx := tree.MakeSemaContext() + evalCtx := tree.NewTestingEvalContext(nil /* st */) + + tc := testcat.New() + if _, err := tc.ExecuteDDL( + "CREATE TABLE t (j JSON, a INT[], INVERTED INDEX (j), INVERTED INDEX (a))", + ); err != nil { + t.Fatal(err) + } + var f norm.Factory + f.Init(evalCtx, tc) + md := f.Metadata() + tn := tree.NewUnqualifiedTableName("t") + tab := md.AddTable(tc.Table(tn), tn) + jsonOrd, arrayOrd := 1, 2 + + testCases := []struct { + filters string + indexOrd int + ok bool + unique bool + }{ + // If we can create an inverted filter with the given filter expression and + // index, ok=true. If the spans in the resulting inverted index constraint + // do not have duplicate primary keys, unique=true. + { + filters: "j @> '1'", + indexOrd: jsonOrd, + ok: true, + unique: true, + }, + { + // Contained by is not yet supported. + filters: "j <@ '1'", + indexOrd: jsonOrd, + ok: false, + }, + { + filters: "a @> '{1}'", + indexOrd: arrayOrd, + ok: true, + unique: true, + }, + { + // Contained by is not yet supported. + filters: "a <@ '{1}'", + indexOrd: arrayOrd, + ok: false, + }, + { + // Wrong index ordinal. + filters: "a @> '{1}'", + indexOrd: jsonOrd, + ok: false, + }, + { + // Wrong index ordinal. + filters: "j @> '1'", + indexOrd: arrayOrd, + ok: false, + }, + { + // When operations affecting two different variables are OR-ed, we cannot + // constrain either index. + filters: "j @> '1' OR a @> '{1}'", + indexOrd: jsonOrd, + ok: false, + }, + { + // We can constrain either index when the functions are AND-ed. + filters: "j @> '1' AND a @> '{1}'", + indexOrd: jsonOrd, + ok: true, + unique: true, + }, + { + // We can constrain either index when the functions are AND-ed. + filters: "j @> '1' AND a @> '{1}'", + indexOrd: arrayOrd, + ok: true, + unique: true, + }, + { + // We cannot guarantee unique primary keys when there are multiple paths. + filters: "j @> '[1, 2]'", + indexOrd: jsonOrd, + ok: true, + unique: false, + }, + { + // We cannot guarantee unique primary keys when there are multiple paths. + filters: "a @> '{1, 2}'", + indexOrd: arrayOrd, + ok: true, + unique: false, + }, + } + + for _, tc := range testCases { + t.Logf("test case: %v", tc) + filters := testutils.BuildFilters(t, &f, &semaCtx, evalCtx, tc.filters) + + // We're not testing that the correct SpanExpression is returned here; + // that is tested elsewhere. This is just testing that we are constraining + // the index when we expect to. + spanExpr, _, _, _, ok := invertedidx.TryFilterInvertedIndex( + evalCtx, &f, filters, nil /* optionalFilters */, tab, md.Table(tab).Index(tc.indexOrd), + ) + if tc.ok != ok { + t.Fatalf("expected %v, got %v", tc.ok, ok) + } + if spanExpr != nil && tc.unique != spanExpr.Unique { + t.Fatalf("expected unique=%v, but got %v", tc.unique, spanExpr.Unique) + } + } +} diff --git a/pkg/sql/opt/memo/testdata/stats/inverted-geo b/pkg/sql/opt/memo/testdata/stats/inverted-geo index b1d733872f91..834765e4c4fb 100644 --- a/pkg/sql/opt/memo/testdata/stats/inverted-geo +++ b/pkg/sql/opt/memo/testdata/stats/inverted-geo @@ -173,7 +173,7 @@ project │ │ └── inverted-filter │ │ ├── columns: rowid:3(int!null) │ │ ├── inverted expression: /5 - │ │ │ ├── tight: false + │ │ │ ├── tight: false, unique: false │ │ │ └── union spans: ["B\xfd\xff\xff\xff\xff\xff\xff\xff\xff", "B\xfd\xff\xff\xff\xff\xff\xff\xff\xff"] │ │ ├── pre-filterer expression │ │ │ └── st_intersects('010200000002000000000000000000594000000000000059400000000000C062400000000000C06240', g:2) [type=bool] @@ -392,7 +392,7 @@ project │ │ └── inverted-filter │ │ ├── columns: rowid:3(int!null) │ │ ├── inverted expression: /5 - │ │ │ ├── tight: false + │ │ │ ├── tight: false, unique: false │ │ │ └── union spans: ["B\xfd\xff\xff\xff\xff\xff\xff\xff\xff", "B\xfd\xff\xff\xff\xff\xff\xff\xff\xff"] │ │ ├── pre-filterer expression │ │ │ └── st_intersects('010200000002000000000000000000594000000000000059400000000000C062400000000000C06240', g:2) [type=bool] @@ -468,7 +468,7 @@ select │ └── inverted-filter │ ├── columns: rowid:3(int!null) │ ├── inverted expression: /5 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") @@ -510,7 +510,7 @@ project │ └── inverted-filter │ ├── columns: rowid:3(int!null) │ ├── inverted expression: /5 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") diff --git a/pkg/sql/opt/memo/testdata/stats/partial-index-scan b/pkg/sql/opt/memo/testdata/stats/partial-index-scan index 60fa0251c9be..a1a4fdc1bd04 100644 --- a/pkg/sql/opt/memo/testdata/stats/partial-index-scan +++ b/pkg/sql/opt/memo/testdata/stats/partial-index-scan @@ -825,7 +825,8 @@ project ├── key: (1) └── scan inv@partial,partial ├── columns: k:1(int!null) - ├── constraint: /3/1: [/'{"x": "y"}' - /'{"x": "y"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7x\x00\x01\x12y\x00\x01", "7x\x00\x01\x12y\x00\x01"] ├── flags: force-index=partial ├── stats: [rows=2.20440882, distinct(4)=2, null(4)=0] └── key: (1) @@ -841,7 +842,8 @@ index-join inv ├── fd: (1)-->(2-4) └── scan inv@partial,partial ├── columns: k:1(int!null) - ├── constraint: /3/1: [/'{"x": "y"}' - /'{"x": "y"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7x\x00\x01\x12y\x00\x01", "7x\x00\x01\x12y\x00\x01"] ├── flags: force-index=partial ├── stats: [rows=2.20440882, distinct(4)=2, null(4)=0] └── key: (1) @@ -867,7 +869,8 @@ project │ ├── fd: (1)-->(3,4) │ └── scan inv@partial,partial │ ├── columns: k:1(int!null) - │ ├── constraint: /3/1: [/'{"x": "y"}' - /'{"x": "y"}'] + │ ├── inverted constraint: /6/1 + │ │ └── spans: ["7x\x00\x01\x12y\x00\x01", "7x\x00\x01\x12y\x00\x01"] │ ├── flags: force-index=partial │ ├── stats: [rows=2.20440882, distinct(4)=2, null(4)=0] │ └── key: (1) @@ -890,7 +893,8 @@ select │ ├── fd: (1)-->(2-4) │ └── scan inv@partial,partial │ ├── columns: k:1(int!null) - │ ├── constraint: /3/1: [/'{"x": "y"}' - /'{"x": "y"}'] + │ ├── inverted constraint: /6/1 + │ │ └── spans: ["7x\x00\x01\x12y\x00\x01", "7x\x00\x01\x12y\x00\x01"] │ ├── flags: force-index=partial │ ├── stats: [rows=2.20440882, distinct(4)=2, null(4)=0] │ └── key: (1) @@ -913,7 +917,8 @@ select │ ├── fd: (1)-->(2-4) │ └── scan inv@partial,partial │ ├── columns: k:1(int!null) - │ ├── constraint: /3/1: [/'{"x": "y"}' - /'{"x": "y"}'] + │ ├── inverted constraint: /6/1 + │ │ └── spans: ["7x\x00\x01\x12y\x00\x01", "7x\x00\x01\x12y\x00\x01"] │ ├── flags: force-index=partial │ ├── stats: [rows=2.20440882, distinct(4)=2, null(4)=0] │ └── key: (1) @@ -994,10 +999,11 @@ project ├── key: (1) └── scan inv_hist@partial,partial ├── columns: k:1(int!null) - ├── constraint: /3/1: [/'{"g": 7}' - /'{"g": 7}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7g\x00\x01*\x0e\x00", "7g\x00\x01*\x0e\x00"] ├── flags: force-index=partial - ├── stats: [rows=17.7777778, distinct(4)=2, null(4)=0] - │ histogram(4)= 0 8.8889 0 8.8889 + ├── stats: [rows=22.2222222, distinct(4)=2, null(4)=0] + │ histogram(4)= 0 11.111 0 11.111 │ <--- 'banana' --- 'cherry' └── key: (1) @@ -1014,10 +1020,11 @@ index-join inv_hist ├── fd: (1)-->(2-4) └── scan inv_hist@partial,partial ├── columns: k:1(int!null) - ├── constraint: /3/1: [/'{"g": 7}' - /'{"g": 7}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7g\x00\x01*\x0e\x00", "7g\x00\x01*\x0e\x00"] ├── flags: force-index=partial - ├── stats: [rows=17.7777778, distinct(4)=2, null(4)=0] - │ histogram(4)= 0 8.8889 0 8.8889 + ├── stats: [rows=22.2222222, distinct(4)=2, null(4)=0] + │ histogram(4)= 0 11.111 0 11.111 │ <--- 'banana' --- 'cherry' └── key: (1) @@ -1039,15 +1046,16 @@ project ├── fd: ()-->(4), (1)-->(3) ├── index-join inv_hist │ ├── columns: k:1(int!null) j:3(jsonb) s:4(string) - │ ├── stats: [rows=17.7777778] + │ ├── stats: [rows=22.2222222] │ ├── key: (1) │ ├── fd: (1)-->(3,4) │ └── scan inv_hist@partial,partial │ ├── columns: k:1(int!null) - │ ├── constraint: /3/1: [/'{"g": 7}' - /'{"g": 7}'] + │ ├── inverted constraint: /6/1 + │ │ └── spans: ["7g\x00\x01*\x0e\x00", "7g\x00\x01*\x0e\x00"] │ ├── flags: force-index=partial - │ ├── stats: [rows=17.7777778, distinct(4)=2, null(4)=0] - │ │ histogram(4)= 0 8.8889 0 8.8889 + │ ├── stats: [rows=22.2222222, distinct(4)=2, null(4)=0] + │ │ histogram(4)= 0 11.111 0 11.111 │ │ <--- 'banana' --- 'cherry' │ └── key: (1) └── filters @@ -1066,15 +1074,16 @@ select ├── fd: ()-->(4), (1)-->(2,3) ├── index-join inv_hist │ ├── columns: k:1(int!null) i:2(int) j:3(jsonb) s:4(string) - │ ├── stats: [rows=17.7777778] + │ ├── stats: [rows=22.2222222] │ ├── key: (1) │ ├── fd: (1)-->(2-4) │ └── scan inv_hist@partial,partial │ ├── columns: k:1(int!null) - │ ├── constraint: /3/1: [/'{"g": 7}' - /'{"g": 7}'] + │ ├── inverted constraint: /6/1 + │ │ └── spans: ["7g\x00\x01*\x0e\x00", "7g\x00\x01*\x0e\x00"] │ ├── flags: force-index=partial - │ ├── stats: [rows=17.7777778, distinct(4)=2, null(4)=0] - │ │ histogram(4)= 0 8.8889 0 8.8889 + │ ├── stats: [rows=22.2222222, distinct(4)=2, null(4)=0] + │ │ histogram(4)= 0 11.111 0 11.111 │ │ <--- 'banana' --- 'cherry' │ └── key: (1) └── filters @@ -1095,15 +1104,16 @@ select ├── fd: (1)-->(2-4) ├── index-join inv_hist │ ├── columns: k:1(int!null) i:2(int) j:3(jsonb) s:4(string) - │ ├── stats: [rows=17.7777778] + │ ├── stats: [rows=22.2222222] │ ├── key: (1) │ ├── fd: (1)-->(2-4) │ └── scan inv_hist@partial,partial │ ├── columns: k:1(int!null) - │ ├── constraint: /3/1: [/'{"g": 7}' - /'{"g": 7}'] + │ ├── inverted constraint: /6/1 + │ │ └── spans: ["7g\x00\x01*\x0e\x00", "7g\x00\x01*\x0e\x00"] │ ├── flags: force-index=partial - │ ├── stats: [rows=17.7777778, distinct(4)=2, null(4)=0] - │ │ histogram(4)= 0 8.8889 0 8.8889 + │ ├── stats: [rows=22.2222222, distinct(4)=2, null(4)=0] + │ │ histogram(4)= 0 11.111 0 11.111 │ │ <--- 'banana' --- 'cherry' │ └── key: (1) └── filters @@ -1166,7 +1176,7 @@ project │ └── inverted-filter │ ├── columns: k:1(int!null) │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") @@ -1212,7 +1222,7 @@ project │ └── inverted-filter │ ├── columns: k:1(int!null) │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") @@ -1277,7 +1287,7 @@ project │ └── inverted-filter │ ├── columns: k:1(int!null) │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") @@ -1321,7 +1331,7 @@ project │ └── inverted-filter │ ├── columns: k:1(int!null) │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") @@ -1409,7 +1419,7 @@ project │ └── inverted-filter │ ├── columns: k:1(int!null) │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") @@ -1461,7 +1471,7 @@ project │ └── inverted-filter │ ├── columns: k:1(int!null) │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") @@ -1547,7 +1557,7 @@ project │ └── inverted-filter │ ├── columns: k:1(int!null) │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") @@ -1597,7 +1607,7 @@ project │ └── inverted-filter │ ├── columns: k:1(int!null) │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x12\x00\x00\x00\x00\x00\x00\x00") diff --git a/pkg/sql/opt/memo/testdata/stats/select b/pkg/sql/opt/memo/testdata/stats/select index 4a7d7cf27906..e7a52f34fced 100644 --- a/pkg/sql/opt/memo/testdata/stats/select +++ b/pkg/sql/opt/memo/testdata/stats/select @@ -860,7 +860,8 @@ index-join t_json_arr ├── fd: (1)-->(2-4) └── scan t_json_arr@b_idx ├── columns: a:1(int!null) - ├── constraint: /2/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] ├── stats: [rows=555.555556] └── key: (1) diff --git a/pkg/sql/opt/xform/select_funcs.go b/pkg/sql/opt/xform/select_funcs.go index 0bc5ab224eca..4e252b912248 100644 --- a/pkg/sql/opt/xform/select_funcs.go +++ b/pkg/sql/opt/xform/select_funcs.go @@ -809,14 +809,14 @@ func (c *CustomFuncs) GenerateInvertedIndexScans( var pfState *invertedexpr.PreFiltererStateForInvertedFilterer var spansToRead invertedexpr.InvertedSpans var constraint *constraint.Constraint - var geoOk, nonGeoOk bool + var filterOk, constraintOk bool // Check whether the filter can constrain the index. // TODO(rytaft): Unify these two cases so both return a spanExpr. - spanExpr, constraint, remainingFilters, pfState, geoOk := invertedidx.TryConstrainGeoIndex( + spanExpr, constraint, remainingFilters, pfState, filterOk := invertedidx.TryFilterInvertedIndex( c.e.evalCtx, c.e.f, filters, optionalFilters, scanPrivate.Table, index, ) - if geoOk { + if filterOk { spansToRead = spanExpr.SpansToRead // Override the filters with remainingFilters. If the index is a // multi-column inverted index, the non-inverted prefix columns are @@ -838,18 +838,24 @@ func (c *CustomFuncs) GenerateInvertedIndexScans( // (a = 1 AND ST_Intersects(.., g)). filters = remainingFilters } else { - constraint, filters, nonGeoOk = c.tryConstrainIndex( + constraint, filters, constraintOk = c.tryConstrainIndex( filters, nil, /* optionalFilters */ scanPrivate.Table, index.Ordinal(), true, /* isInverted */ ) - if !nonGeoOk { + if !constraintOk { return } } + // We will need an inverted filter above the scan if the spanExpr might + // produce duplicate primary keys or requires at least one UNION or + // INTERSECTION. + needInvertedFilter := spanExpr != nil && + (!spanExpr.Unique || spanExpr.Operator != invertedexpr.None) + // Construct new ScanOpDef with the new index and constraint. newScanPrivate := *scanPrivate newScanPrivate.Index = index.Ordinal() @@ -860,8 +866,9 @@ func (c *CustomFuncs) GenerateInvertedIndexScans( // inverted filter. pkCols := sb.primaryKeyCols() newScanPrivate.Cols = pkCols.Copy() - invertedCol := scanPrivate.Table.ColumnID(index.VirtualInvertedColumn().Ordinal()) - if spanExpr != nil { + var invertedCol opt.ColumnID + if needInvertedFilter { + invertedCol = scanPrivate.Table.ColumnID(index.VirtualInvertedColumn().Ordinal()) newScanPrivate.Cols.Add(invertedCol) } @@ -874,8 +881,10 @@ func (c *CustomFuncs) GenerateInvertedIndexScans( // place. sb.setScan(&newScanPrivate) - // Add an inverted filter if it exists. - sb.addInvertedFilter(spanExpr, pfState, invertedCol) + // Add an inverted filter if needed. + if needInvertedFilter { + sb.addInvertedFilter(spanExpr, pfState, invertedCol) + } // If remaining filter exists, split it into one part that can be pushed // below the IndexJoin, and one part that needs to stay above. diff --git a/pkg/sql/opt/xform/testdata/external/postgis-tutorial-idx b/pkg/sql/opt/xform/testdata/external/postgis-tutorial-idx index 35efdcd715ed..1e6f7a61ff00 100644 --- a/pkg/sql/opt/xform/testdata/external/postgis-tutorial-idx +++ b/pkg/sql/opt/xform/testdata/external/postgis-tutorial-idx @@ -37,7 +37,7 @@ sort │ └── inverted-filter │ ├── columns: gid:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\xfd\xff\xff\xff\xff\xff\xff\xff\xff", "B\xfd\xff\xff\xff\xff\xff\xff\xff\xff"] │ ├── pre-filterer expression │ │ └── st_intersects('0101000020266900000000000026CF21410000008016315141', geom:4) @@ -81,7 +81,7 @@ sort │ └── inverted-filter │ ├── columns: gid:1!null │ ├── inverted expression: /8 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\xfd\xff\xff\xff\xff\xff\xff\xff\xff", "B\xfd\xff\xff\xff\xff\xff\xff\xff\xff"] │ ├── pre-filterer expression │ │ └── st_dwithin('0101000020266900000000000026CF21410000008016315141', geom:6, 10.0) @@ -127,7 +127,7 @@ sort │ └── inverted-filter │ ├── columns: gid:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\xfd\xff\xff\xff\xff\xff\xff\xff\xff", "B\xfd\xff\xff\xff\xff\xff\xff\xff\xff"] │ ├── pre-filterer expression │ │ └── st_intersects('01020000202669000002000000000000003CE8214100000080A22E514100000000E0E8214100000000A62E5141', geom:4) @@ -174,7 +174,7 @@ sort │ └── inverted-filter │ ├── columns: gid:1!null │ ├── inverted expression: /8 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\xfd\xff\xff\xff\xff\xff\xff\xff\xff", "B\xfd\xff\xff\xff\xff\xff\xff\xff\xff"] │ ├── pre-filterer expression │ │ └── st_dwithin('01020000202669000002000000000000003CE8214100000080A22E514100000000E0E8214100000000A62E5141', geom:6, 0.1) @@ -218,7 +218,7 @@ scalar-group-by │ │ └── inverted-filter │ │ ├── columns: gid:1!null │ │ ├── inverted expression: /12 - │ │ │ ├── tight: false + │ │ │ ├── tight: false, unique: false │ │ │ └── union spans: ["B\xfd\xff\xff\xff\xff\xff\xff\xff\xff", "B\xfd\xff\xff\xff\xff\xff\xff\xff\xff"] │ │ ├── pre-filterer expression │ │ │ └── st_dwithin('01020000202669000002000000000000003CE8214100000080A22E514100000000E0E8214100000000A62E5141', geom:10, 50.0) diff --git a/pkg/sql/opt/xform/testdata/rules/join b/pkg/sql/opt/xform/testdata/rules/join index ab896dc042a4..5710b1a3dd2d 100644 --- a/pkg/sql/opt/xform/testdata/rules/join +++ b/pkg/sql/opt/xform/testdata/rules/join @@ -3979,7 +3979,7 @@ project │ │ │ └── inverted-filter │ │ │ ├── columns: n.gid:13!null │ │ │ ├── inverted expression: /18 - │ │ │ │ ├── tight: false + │ │ │ │ ├── tight: false, unique: false │ │ │ │ └── union spans: ["B\x89", "B\xfd \x00\x00\x00\x00\x00\x00\x00") │ │ │ ├── pre-filterer expression │ │ │ │ └── st_covers('0102000000020000000000000000000000000000000000000000000000000000000000000000000040', n.geom:16) diff --git a/pkg/sql/opt/xform/testdata/rules/limit b/pkg/sql/opt/xform/testdata/rules/limit index 81e555cb53e8..cdeaa90a6611 100644 --- a/pkg/sql/opt/xform/testdata/rules/limit +++ b/pkg/sql/opt/xform/testdata/rules/limit @@ -1211,7 +1211,7 @@ select │ └── inverted-filter │ ├── columns: id:1!null │ ├── inverted expression: /10 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x14\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x14\x00\x00\x00\x00\x00\x00\x00"] diff --git a/pkg/sql/opt/xform/testdata/rules/project b/pkg/sql/opt/xform/testdata/rules/project index c83c684c5ae8..c5bef4b4615c 100644 --- a/pkg/sql/opt/xform/testdata/rules/project +++ b/pkg/sql/opt/xform/testdata/rules/project @@ -91,7 +91,8 @@ project ├── key: (1) └── scan a@j ├── columns: k:1!null - ├── constraint: /5/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /7/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) # Eliminate the IndexJoin for an inverted index scan when only the primary key @@ -104,7 +105,8 @@ project ├── immutable ├── scan a@j │ ├── columns: k:1!null - │ ├── constraint: /5/1: [/'{"a": "b"}' - /'{"a": "b"}'] + │ ├── inverted constraint: /7/1 + │ │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] │ └── key: (1) └── projections └── k:1 + 1 [as="?column?":8, outer=(1), immutable] @@ -120,5 +122,6 @@ index-join a ├── immutable └── scan a@j ├── columns: k:1!null - ├── constraint: /5/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /7/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) diff --git a/pkg/sql/opt/xform/testdata/rules/select b/pkg/sql/opt/xform/testdata/rules/select index 6627fa2144db..89b9e9e34f90 100644 --- a/pkg/sql/opt/xform/testdata/rules/select +++ b/pkg/sql/opt/xform/testdata/rules/select @@ -1672,7 +1672,8 @@ project ├── key: (1) └── scan b@inv_idx ├── columns: k:1!null - ├── constraint: /4/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) memo @@ -1682,16 +1683,16 @@ memo (optimized, ~7KB, required=[presentation: k:1]) ├── G1: (project G2 G3 k) (project G4 G3 k) │ └── [presentation: k:1] │ ├── best: (project G4 G3 k) - │ └── cost: 118.42 + │ └── cost: 119.56 ├── G2: (select G5 G6) (index-join G4 b,cols=(1,4)) │ └── [] │ ├── best: (index-join G4 b,cols=(1,4)) - │ └── cost: 783.92 + │ └── cost: 789.51 ├── G3: (projections) - ├── G4: (scan b@inv_idx,cols=(1),constrained) + ├── G4: (scan b@inv_idx,cols=(1),constrained inverted) │ └── [] - │ ├── best: (scan b@inv_idx,cols=(1),constrained) - │ └── cost: 117.31 + │ ├── best: (scan b@inv_idx,cols=(1),constrained inverted) + │ └── cost: 118.45 ├── G5: (scan b,cols=(1,4)) │ └── [] │ ├── best: (scan b,cols=(1,4)) @@ -1717,7 +1718,8 @@ project ├── fd: (1)-->(2,4) └── scan b@inv_idx ├── columns: k:1!null - ├── constraint: /4/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) opt @@ -1730,7 +1732,8 @@ index-join b ├── fd: (1)-->(4) └── scan b@inv_idx ├── columns: k:1!null - ├── constraint: /4/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) opt @@ -1743,38 +1746,61 @@ index-join b ├── fd: (1)-->(2-4), (3)~~>(1,2,4) └── scan b@inv_idx ├── columns: k:1!null - ├── constraint: /4/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) +# TODO(rytaft): ensure that the stats prevent us from using the inverted index. opt SELECT * FROM b WHERE j @> '{}' ---- -select +index-join b ├── columns: k:1!null u:2 v:3 j:4!null ├── immutable ├── key: (1) ├── fd: (1)-->(2-4), (3)~~>(1,2,4) - ├── scan b - │ ├── columns: k:1!null u:2 v:3 j:4 - │ ├── key: (1) - │ └── fd: (1)-->(2-4), (3)~~>(1,2,4) - └── filters - └── j:4 @> '{}' [outer=(4), immutable, constraints=(/4: (/NULL - ])] + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /6 + │ ├── tight: true, unique: false + │ └── union spans + │ ├── ["7\x00\x019", "7\x00\x019"] + │ └── ["7\x00\xff", "8") + ├── key: (1) + └── scan b@inv_idx + ├── columns: k:1!null j_inverted_key:6!null + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7\x00\x019", "7\x00\x019"] + │ └── ["7\x00\xff", "8") + ├── key: (1) + └── fd: (1)-->(6) +# TODO(rytaft): ensure that the stats prevent us from using the inverted index. opt SELECT * FROM b WHERE j @> '[]' ---- -select +index-join b ├── columns: k:1!null u:2 v:3 j:4!null ├── immutable ├── key: (1) ├── fd: (1)-->(2-4), (3)~~>(1,2,4) - ├── scan b - │ ├── columns: k:1!null u:2 v:3 j:4 - │ ├── key: (1) - │ └── fd: (1)-->(2-4), (3)~~>(1,2,4) - └── filters - └── j:4 @> '[]' [outer=(4), immutable, constraints=(/4: (/NULL - ])] + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /6 + │ ├── tight: true, unique: false + │ └── union spans + │ ├── ["7\x00\x018", "7\x00\x018"] + │ └── ["7\x00\x03", "7\x00\x03"] + ├── key: (1) + └── scan b@inv_idx + ├── columns: k:1!null j_inverted_key:6!null + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7\x00\x018", "7\x00\x018"] + │ └── ["7\x00\x03", "7\x00\x03"] + ├── key: (1) + └── fd: (1)-->(6) opt SELECT * FROM b WHERE j @> '2' @@ -1786,55 +1812,168 @@ index-join b ├── fd: (1)-->(2-4), (3)~~>(1,2,4) └── scan b@inv_idx ├── columns: k:1!null - ├── constraint: /4/1 - │ ├── [/'2' - /'2'] - │ └── [/'[2]' - /'[2]'] + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7\x00\x01*\x04\x00", "7\x00\x01*\x04\x00"] + │ └── ["7\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x01*\x04\x00"] └── key: (1) +# Disjunction. opt -SELECT * FROM b WHERE j @> '[{}]' +SELECT k FROM b WHERE j @> '2' OR j @> '1' +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /6 + │ ├── tight: true, unique: false + │ └── union spans + │ ├── ["7\x00\x01*\x02\x00", "7\x00\x01*\x02\x00"] + │ ├── ["7\x00\x01*\x04\x00", "7\x00\x01*\x04\x00"] + │ ├── ["7\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x01*\x02\x00"] + │ └── ["7\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x01*\x04\x00"] + ├── key: (1) + └── scan b@inv_idx + ├── columns: k:1!null j_inverted_key:6!null + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7\x00\x01*\x02\x00", "7\x00\x01*\x02\x00"] + │ ├── ["7\x00\x01*\x04\x00", "7\x00\x01*\x04\x00"] + │ ├── ["7\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x01*\x02\x00"] + │ └── ["7\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x01*\x04\x00"] + ├── key: (1) + └── fd: (1)-->(6) + +# Disjunction with non-tight predicate. +opt +SELECT * FROM b WHERE j @> '[[1, 2]]' OR j @> '[[3, 4]]' ---- select ├── columns: k:1!null u:2 v:3 j:4!null ├── immutable ├── key: (1) ├── fd: (1)-->(2-4), (3)~~>(1,2,4) - ├── scan b + ├── index-join b │ ├── columns: k:1!null u:2 v:3 j:4 │ ├── key: (1) - │ └── fd: (1)-->(2-4), (3)~~>(1,2,4) + │ ├── fd: (1)-->(2-4), (3)~~>(1,2,4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /6 + │ │ ├── tight: false, unique: false + │ │ ├── union spans: empty + │ │ └── UNION + │ │ ├── span expression + │ │ │ ├── tight: false, unique: false + │ │ │ ├── union spans: empty + │ │ │ └── INTERSECTION + │ │ │ ├── span expression + │ │ │ │ ├── tight: false, unique: false + │ │ │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x03\x00\x01*\x02\x00"] + │ │ │ └── span expression + │ │ │ ├── tight: false, unique: false + │ │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x03\x00\x01*\x04\x00"] + │ │ └── span expression + │ │ ├── tight: false, unique: false + │ │ ├── union spans: empty + │ │ └── INTERSECTION + │ │ ├── span expression + │ │ │ ├── tight: false, unique: false + │ │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\x06\x00", "7\x00\x03\x00\x03\x00\x01*\x06\x00"] + │ │ └── span expression + │ │ ├── tight: false, unique: false + │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\b\x00", "7\x00\x03\x00\x03\x00\x01*\b\x00"] + │ ├── key: (1) + │ └── scan b@inv_idx + │ ├── columns: k:1!null j_inverted_key:6!null + │ ├── inverted constraint: /6/1 + │ │ └── spans + │ │ ├── ["7\x00\x03\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x03\x00\x01*\x02\x00"] + │ │ ├── ["7\x00\x03\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x03\x00\x01*\x04\x00"] + │ │ ├── ["7\x00\x03\x00\x03\x00\x01*\x06\x00", "7\x00\x03\x00\x03\x00\x01*\x06\x00"] + │ │ └── ["7\x00\x03\x00\x03\x00\x01*\b\x00", "7\x00\x03\x00\x03\x00\x01*\b\x00"] + │ ├── key: (1) + │ └── fd: (1)-->(6) └── filters - └── j:4 @> '[{}]' [outer=(4), immutable, constraints=(/4: (/NULL - ])] + └── (j:4 @> '[[1, 2]]') OR (j:4 @> '[[3, 4]]') [outer=(4), immutable, constraints=(/4: (/NULL - ])] + +opt +SELECT * FROM b WHERE j @> '[{}]' +---- +index-join b + ├── columns: k:1!null u:2 v:3 j:4!null + ├── immutable + ├── key: (1) + ├── fd: (1)-->(2-4), (3)~~>(1,2,4) + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /6 + │ ├── tight: true, unique: false + │ └── union spans + │ ├── ["7\x00\x03\x00\x019", "7\x00\x03\x00\x019"] + │ └── ["7\x00\x03\x00\xff", "7\x00\x04") + ├── key: (1) + └── scan b@inv_idx + ├── columns: k:1!null j_inverted_key:6!null + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7\x00\x03\x00\x019", "7\x00\x03\x00\x019"] + │ └── ["7\x00\x03\x00\xff", "7\x00\x04") + ├── key: (1) + └── fd: (1)-->(6) opt SELECT * FROM b WHERE j @> '{"a": {}}' ---- -select +index-join b ├── columns: k:1!null u:2 v:3 j:4!null ├── immutable ├── key: (1) ├── fd: (1)-->(2-4), (3)~~>(1,2,4) - ├── scan b - │ ├── columns: k:1!null u:2 v:3 j:4 - │ ├── key: (1) - │ └── fd: (1)-->(2-4), (3)~~>(1,2,4) - └── filters - └── j:4 @> '{"a": {}}' [outer=(4), immutable, constraints=(/4: (/NULL - ])] + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /6 + │ ├── tight: true, unique: false + │ └── union spans + │ ├── ["7a\x00\x019", "7a\x00\x019"] + │ └── ["7a\x00\x02\x00\xff", "7a\x00\x03") + ├── key: (1) + └── scan b@inv_idx + ├── columns: k:1!null j_inverted_key:6!null + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7a\x00\x019", "7a\x00\x019"] + │ └── ["7a\x00\x02\x00\xff", "7a\x00\x03") + ├── key: (1) + └── fd: (1)-->(6) opt SELECT * FROM b WHERE j @> '{"a": []}' ---- -select +index-join b ├── columns: k:1!null u:2 v:3 j:4!null ├── immutable ├── key: (1) ├── fd: (1)-->(2-4), (3)~~>(1,2,4) - ├── scan b - │ ├── columns: k:1!null u:2 v:3 j:4 - │ ├── key: (1) - │ └── fd: (1)-->(2-4), (3)~~>(1,2,4) - └── filters - └── j:4 @> '{"a": []}' [outer=(4), immutable, constraints=(/4: (/NULL - ])] + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /6 + │ ├── tight: true, unique: false + │ └── union spans + │ ├── ["7a\x00\x018", "7a\x00\x018"] + │ └── ["7a\x00\x02\x00\x03", "7a\x00\x02\x00\x03"] + ├── key: (1) + └── scan b@inv_idx + ├── columns: k:1!null j_inverted_key:6!null + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7a\x00\x018", "7a\x00\x018"] + │ └── ["7a\x00\x02\x00\x03", "7a\x00\x02\x00\x03"] + ├── key: (1) + └── fd: (1)-->(6) opt SELECT * FROM b WHERE j @> '{"a":[[{"b":{"c":[{"d":"e"}]}}]]}' @@ -1846,7 +1985,8 @@ index-join b ├── fd: (1)-->(2-4), (3)~~>(1,2,4) └── scan b@inv_idx ├── columns: k:1!null - ├── constraint: /4/1: [/'{"a": [[{"b": {"c": [{"d": "e"}]}}]]}' - /'{"a": [[{"b": {"c": [{"d": "e"}]}}]]}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7a\x00\x02\x00\x03\x00\x03b\x00\x02c\x00\x02\x00\x03d\x00\x01\x12e\x00\x01", "7a\x00\x02\x00\x03\x00\x03b\x00\x02c\x00\x02\x00\x03d\x00\x01\x12e\x00\x01"] └── key: (1) # GenerateInvertedIndexScans propagates row-level locking information. @@ -1859,7 +1999,8 @@ project ├── key: (1) └── scan b@inv_idx ├── columns: k:1!null - ├── constraint: /4/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] ├── locking: for-update ├── volatile └── key: (1) @@ -1874,9 +2015,31 @@ project ├── key: (1) └── scan c@inv_idx ├── columns: k:1!null - ├── constraint: /2/1: [/ARRAY[1] - /ARRAY[1]] + ├── inverted constraint: /5/1 + │ └── spans: ["\x89", "\x89"] └── key: (1) +# Disjunction. +opt +SELECT k FROM c WHERE a @> ARRAY[1] OR a @> ARRAY[2] +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /5 + │ ├── tight: true, unique: false + │ └── union spans: ["\x89", "\x8b") + ├── key: (1) + └── scan c@inv_idx + ├── columns: k:1!null a_inverted_key:5!null + ├── inverted constraint: /5/1 + │ └── spans: ["\x89", "\x8b") + ├── key: (1) + └── fd: (1)-->(5) + opt SELECT k FROM c WHERE a @> ARRAY[]::INT[] ---- @@ -1884,17 +2047,18 @@ project ├── columns: k:1!null ├── immutable ├── key: (1) - └── select - ├── columns: k:1!null a:2!null - ├── immutable + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /5 + │ ├── tight: true, unique: false + │ └── union spans: ["", ""] ├── key: (1) - ├── fd: (1)-->(2) - ├── scan c - │ ├── columns: k:1!null a:2 - │ ├── key: (1) - │ └── fd: (1)-->(2) - └── filters - └── a:2 @> ARRAY[] [outer=(2), immutable, constraints=(/2: (/NULL - ])] + └── scan c@inv_idx + ├── columns: k:1!null a_inverted_key:5!null + ├── inverted constraint: /5/1 + │ └── spans: ["", ""] + ├── key: (1) + └── fd: (1)-->(5) opt SELECT k FROM c WHERE a IS NULL @@ -1933,7 +2097,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdL\x00\x00\x00\x00\x00\x00\x00", "B\xfdL\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfdN\x00\x00\x00\x00\x00\x00\x01", "B\xfdP\x00\x00\x00\x00\x00\x00\x00") @@ -1973,7 +2137,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdL\x00\x00\x00\x00\x00\x00\x00", "B\xfdL\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfdN\x00\x00\x00\x00\x00\x00\x01", "B\xfdP\x00\x00\x00\x00\x00\x00\x00") @@ -2012,7 +2176,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] @@ -2052,7 +2216,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\x89", "B\xfd \x00\x00\x00\x00\x00\x00\x00") │ ├── pre-filterer expression │ │ └── st_covers('0103000000010000000500000000000000000000000000000000000000000000000000F03F0000000000000000000000000000F03F000000000000F03F0000000000000000000000000000F03F00000000000000000000000000000000', geom:3) @@ -2085,7 +2249,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\x89", "B\xfd \x00\x00\x00\x00\x00\x00\x00") │ ├── pre-filterer expression │ │ └── st_dwithin('0103000000010000000500000000000000000000000000000000000000000000000000F03F0000000000000000000000000000F03F000000000000F03F0000000000000000000000000000F03F00000000000000000000000000000000', geom:3, 2.0) @@ -2118,7 +2282,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\x89", "B\xfd \x00\x00\x00\x00\x00\x00\x00") │ ├── pre-filterer expression │ │ └── st_dfullywithin('0103000000010000000500000000000000000000000000000000000000000000000000F03F0000000000000000000000000000F03F000000000000F03F0000000000000000000000000000F03F00000000000000000000000000000000', geom:3, 2.0) @@ -2151,7 +2315,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdD\x00\x00\x00\x00\x00\x00\x00", "B\xfdD\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfdD\x00\x00\x00\x00\x00\x00\x01", "B\xfdF\x00\x00\x00\x00\x00\x00\x00") @@ -2204,7 +2368,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdD\x00\x00\x00\x00\x00\x00\x00", "B\xfdD\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfdD\x00\x00\x00\x00\x00\x00\x01", "B\xfdF\x00\x00\x00\x00\x00\x00\x00") @@ -2258,7 +2422,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\x89", "B\xfd \x00\x00\x00\x00\x00\x00\x00") │ ├── pre-filterer expression │ │ └── st_dwithinexclusive('0103000000010000000500000000000000000000000000000000000000000000000000F03F0000000000000000000000000000F03F000000000000F03F0000000000000000000000000000F03F00000000000000000000000000000000', geom:3, 2.0) @@ -2292,7 +2456,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\x89", "B\xfd \x00\x00\x00\x00\x00\x00\x00") │ ├── pre-filterer expression │ │ └── st_dwithin('0103000000010000000500000000000000000000000000000000000000000000000000F03F0000000000000000000000000000F03F000000000000F03F0000000000000000000000000000F03F00000000000000000000000000000000', geom:3, 2.0) @@ -2325,7 +2489,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\x89", "B\xfd \x00\x00\x00\x00\x00\x00\x00") │ ├── pre-filterer expression │ │ └── st_dfullywithinexclusive('0103000000010000000500000000000000000000000000000000000000000000000000F03F0000000000000000000000000000F03F000000000000F03F0000000000000000000000000000F03F00000000000000000000000000000000', geom:3, 2.0) @@ -2363,7 +2527,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdL\x00\x00\x00\x00\x00\x00\x00", "B\xfdL\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfdO\x00\x00\x00\x00\x00\x00\x00", "B\xfdO\x00\x00\x00\x00\x00\x00\x00"] @@ -2404,7 +2568,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdL\x00\x00\x00\x00\x00\x00\x00", "B\xfdL\x00\x00\x00\x00\x00\x00\x00"] │ │ └── ["B\xfdN\x00\x00\x00\x00\x00\x00\x01", "B\xfdP\x00\x00\x00\x00\x00\x00\x01") @@ -2442,30 +2606,30 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ ├── union spans │ │ │ ├── ["B\xfdL\x00\x00\x00\x00\x00\x00\x00", "B\xfdL\x00\x00\x00\x00\x00\x00\x00"] │ │ │ ├── ["B\xfdN\x00\x00\x00\x00\x00\x00\x01", "B\xfdP\x00\x00\x00\x00\x00\x00\x00") │ │ │ └── ["B\xfdP\x00\x00\x00\x00\x00\x00\x00", "B\xfdP\x00\x00\x00\x00\x00\x00\x00"] │ │ └── INTERSECTION │ │ ├── span expression - │ │ │ ├── tight: false + │ │ │ ├── tight: false, unique: false │ │ │ ├── union spans: empty │ │ │ └── INTERSECTION │ │ │ ├── span expression - │ │ │ │ ├── tight: false + │ │ │ │ ├── tight: false, unique: false │ │ │ │ └── union spans │ │ │ │ ├── ["B\xfd\x81\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x81\x00\x00\x00\x00\x00\x00\x00"] │ │ │ │ ├── ["B\xfd\x84\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x84\x00\x00\x00\x00\x00\x00\x00"] │ │ │ │ └── ["B\xfd\x90\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x90\x00\x00\x00\x00\x00\x00\x00"] │ │ │ └── span expression - │ │ │ ├── tight: false + │ │ │ ├── tight: false, unique: false │ │ │ └── union spans │ │ │ ├── ["B\xfdD\x00\x00\x00\x00\x00\x00\x00", "B\xfdD\x00\x00\x00\x00\x00\x00\x00"] │ │ │ ├── ["B\xfdG\x00\x00\x00\x00\x00\x00\x00", "B\xfdG\x00\x00\x00\x00\x00\x00\x00"] │ │ │ └── ["B\xfdP\x00\x00\x00\x00\x00\x00\x00", "B\xfdP\x00\x00\x00\x00\x00\x00\x00"] │ │ └── span expression - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdL\x00\x00\x00\x00\x00\x00\x00", "B\xfdL\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfdO\x00\x00\x00\x00\x00\x00\x00", "B\xfdO\x00\x00\x00\x00\x00\x00\x00"] @@ -2509,7 +2673,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\x89", "B\xfd \x00\x00\x00\x00\x00\x00\x00") │ ├── pre-filterer expression │ │ └── st_overlaps('0102000000020000000000000000000000000000000000000000000000000000000000000000000040', geom:3) @@ -2568,7 +2732,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdL\x00\x00\x00\x00\x00\x00\x00", "B\xfdL\x00\x00\x00\x00\x00\x00\x00"] │ │ └── ["B\xfdN\x00\x00\x00\x00\x00\x00\x01", "B\xfdP\x00\x00\x00\x00\x00\x00\x01") @@ -2606,7 +2770,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfdL\x00\x00\x00\x00\x00\x00\x00", "B\xfdL\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfdN\x00\x00\x00\x00\x00\x00\x01", "B\xfdP\x00\x00\x00\x00\x00\x00\x00") @@ -2653,7 +2817,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ └── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x01", "B\xfd\x18\x00\x00\x00\x00\x00\x00\x00") @@ -2690,7 +2854,7 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ └── ["B\xfd\x14\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x14\x00\x00\x00\x00\x00\x00\x00"] @@ -2727,16 +2891,16 @@ project │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ ├── union spans │ │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ │ └── ["B\xfd\x14\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x14\x00\x00\x00\x00\x00\x00\x00"] │ │ └── INTERSECTION │ │ ├── span expression - │ │ │ ├── tight: false + │ │ │ ├── tight: false, unique: false │ │ │ └── union spans: ["B\xfd\x15\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x15\x00\x00\x00\x00\x00\x00\x00"] │ │ └── span expression - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans: ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] │ ├── pre-filterer expression │ │ └── st_coveredby('01040000000200000001010000009A999999999901409A99999999990140010100000000000000000008400000000000000840', geom:3) @@ -2779,7 +2943,8 @@ project ├── key: (1) └── scan pi@idx,partial ├── columns: k:1!null - ├── constraint: /3/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /5/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) exec-ddl @@ -2801,7 +2966,8 @@ project ├── key: (1) └── scan pi@idx,partial ├── columns: k:1!null - ├── constraint: /3/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /6/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) exec-ddl @@ -2824,7 +2990,8 @@ index-join pi ├── fd: ()-->(2), (1)-->(3) └── scan pi@idx,partial ├── columns: k:1!null - ├── constraint: /3/1: [/'{"a": "b"}' - /'{"a": "b"}'] + ├── inverted constraint: /7/1 + │ └── spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] └── key: (1) exec-ddl @@ -2849,12 +3016,28 @@ select │ ├── columns: k:1!null s:2 j:3 │ ├── key: (1) │ ├── fd: (1)-->(2,3) - │ └── scan pi@idx,partial + │ └── inverted-filter │ ├── columns: k:1!null - │ ├── constraint: /3/1: [/'{"a": "b"}' - /'{"a": "b"}'] - │ └── key: (1) + │ ├── inverted expression: /8 + │ │ ├── tight: false, unique: false + │ │ ├── union spans: empty + │ │ └── INTERSECTION + │ │ ├── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] + │ │ └── span expression + │ │ ├── tight: true, unique: true + │ │ └── union spans: ["7group\x00\x01*\x02\x00", "7group\x00\x01*\x02\x00"] + │ ├── key: (1) + │ └── scan pi@idx,partial + │ ├── columns: k:1!null j_inverted_key:8!null + │ ├── inverted constraint: /8/1 + │ │ └── spans + │ │ ├── ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] + │ │ └── ["7group\x00\x01*\x02\x00", "7group\x00\x01*\x02\x00"] + │ ├── key: (1) + │ └── fd: (1)-->(8) └── filters - ├── j:3 @> '{"group": 1}' [outer=(3), immutable, constraints=(/3: (/NULL - ])] └── s:2 = 'foo' [outer=(2), constraints=(/2: [/'foo' - /'foo']; tight), fd=()-->(2)] exec-ddl @@ -2879,7 +3062,7 @@ memo (optimized, ~10KB, required=[presentation: k:1,s:2,j:3]) ├── G1: (select G2 G3) (select G4 G5) (index-join G6 pi,cols=(1-3)) │ └── [presentation: k:1,s:2,j:3] │ ├── best: (index-join G6 pi,cols=(1-3)) - │ └── cost: 11.82 + │ └── cost: 11.87 ├── G2: (scan pi,cols=(1-3)) │ └── [] │ ├── best: (scan pi,cols=(1-3)) @@ -2888,18 +3071,18 @@ memo (optimized, ~10KB, required=[presentation: k:1,s:2,j:3]) ├── G4: (index-join G9 pi,cols=(1-3)) │ └── [] │ ├── best: (index-join G9 pi,cols=(1-3)) - │ └── cost: 19.62 + │ └── cost: 19.78 ├── G5: (filters G8) - ├── G6: (scan pi@idx2,partial,cols=(1),constrained) + ├── G6: (scan pi@idx2,partial,cols=(1),constrained inverted) │ └── [] - │ ├── best: (scan pi@idx2,partial,cols=(1),constrained) - │ └── cost: 5.14 + │ ├── best: (scan pi@idx2,partial,cols=(1),constrained inverted) + │ └── cost: 5.15 ├── G7: (contains G10 G11) ├── G8: (eq G12 G13) - ├── G9: (scan pi@idx,partial,cols=(1),constrained) + ├── G9: (scan pi@idx,partial,cols=(1),constrained inverted) │ └── [] - │ ├── best: (scan pi@idx,partial,cols=(1),constrained) - │ └── cost: 6.28 + │ ├── best: (scan pi@idx,partial,cols=(1),constrained inverted) + │ └── cost: 6.30 ├── G10: (variable j) ├── G11: (const '{"a": "b"}') ├── G12: (variable s) @@ -2986,7 +3169,7 @@ select │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] @@ -3023,7 +3206,7 @@ select │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /6 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] @@ -3105,7 +3288,7 @@ select │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] @@ -3142,7 +3325,7 @@ select │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] @@ -3288,7 +3471,7 @@ select │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /7 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] @@ -3336,7 +3519,7 @@ select │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /8 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] @@ -3385,7 +3568,7 @@ select │ └── inverted-filter │ ├── columns: k:1!null │ ├── inverted expression: /9 - │ │ ├── tight: false + │ │ ├── tight: false, unique: false │ │ └── union spans │ │ ├── ["B\xfd\x10\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x10\x00\x00\x00\x00\x00\x00\x00"] │ │ ├── ["B\xfd\x11\x00\x00\x00\x00\x00\x00\x00", "B\xfd\x11\x00\x00\x00\x00\x00\x00\x00"] @@ -4377,24 +4560,35 @@ project opt expect-not=GenerateInvertedIndexZigzagJoins SELECT * FROM b WHERE j @> '{"a":1, "c":2}' FOR UPDATE ---- -select +index-join b ├── columns: k:1!null u:2 v:3 j:4!null ├── volatile ├── key: (1) ├── fd: (1)-->(2-4), (3)~~>(1,2,4) - ├── index-join b - │ ├── columns: k:1!null u:2 v:3 j:4 - │ ├── volatile - │ ├── key: (1) - │ ├── fd: (1)-->(2-4), (3)~~>(1,2,4) - │ └── scan b@inv_idx - │ ├── columns: k:1!null - │ ├── constraint: /4/1: [/'{"a": 1}' - /'{"a": 1}'] - │ ├── locking: for-update - │ ├── volatile - │ └── key: (1) - └── filters - └── j:4 @> '{"a": 1, "c": 2}' [outer=(4), immutable, constraints=(/4: (/NULL - ])] + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /6 + │ ├── tight: true, unique: false + │ ├── union spans: empty + │ └── INTERSECTION + │ ├── span expression + │ │ ├── tight: true, unique: false + │ │ └── union spans: ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ └── span expression + │ ├── tight: true, unique: false + │ └── union spans: ["7c\x00\x01*\x04\x00", "7c\x00\x01*\x04\x00"] + ├── volatile + ├── key: (1) + └── scan b@inv_idx + ├── columns: k:1!null j_inverted_key:6!null + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ └── ["7c\x00\x01*\x04\x00", "7c\x00\x01*\x04\x00"] + ├── locking: for-update + ├── volatile + ├── key: (1) + └── fd: (1)-->(6) exec-ddl CREATE TABLE inv_zz_partial ( @@ -4417,21 +4611,27 @@ project ├── columns: k:1!null ├── immutable ├── key: (1) - └── inner-join (lookup inv_zz_partial) - ├── columns: k:1!null j:2!null b:3!null - ├── key columns: [1] = [1] - ├── lookup columns are key - ├── immutable + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /6 + │ ├── tight: true, unique: false + │ ├── union spans: empty + │ └── INTERSECTION + │ ├── span expression + │ │ ├── tight: true, unique: false + │ │ └── union spans: ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ └── span expression + │ ├── tight: true, unique: false + │ └── union spans: ["7b\x00\x01*\x04\x00", "7b\x00\x01*\x04\x00"] ├── key: (1) - ├── fd: ()-->(3), (1)-->(2) - ├── inner-join (zigzag inv_zz_partial@zz_idx,partial inv_zz_partial@zz_idx,partial) - │ ├── columns: k:1!null - │ ├── eq columns: [1] = [1] - │ ├── left fixed columns: [2] = ['{"a": 1}'] - │ ├── right fixed columns: [2] = ['{"b": 2}'] - │ └── filters (true) - └── filters - └── j:2 @> '{"a": 1, "b": 2}' [outer=(2), immutable, constraints=(/2: (/NULL - ])] + └── scan inv_zz_partial@zz_idx,partial + ├── columns: k:1!null j_inverted_key:6!null + ├── inverted constraint: /6/1 + │ └── spans + │ ├── ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ └── ["7b\x00\x01*\x04\x00", "7b\x00\x01*\x04\x00"] + ├── key: (1) + └── fd: (1)-->(6) # Generate a zigzag join on a single index with a remaining filter. opt expect=GenerateInvertedIndexZigzagJoins @@ -4782,7 +4982,8 @@ project │ ├── fd: (7)-->(10) │ └── scan b@inv_idx │ ├── columns: k:7!null - │ ├── constraint: /10/7: [/'{"foo": "bar"}' - /'{"foo": "bar"}'] + │ ├── inverted constraint: /12/7 + │ │ └── spans: ["7foo\x00\x01\x12bar\x00\x01", "7foo\x00\x01\x12bar\x00\x01"] │ └── key: (7) └── aggregations └── const-agg [as=j:4, outer=(4)] @@ -4820,7 +5021,8 @@ project │ ├── fd: (6)-->(7) │ └── scan c@inv_idx │ ├── columns: k:6!null - │ ├── constraint: /7/6: [/ARRAY[2] - /ARRAY[2]] + │ ├── inverted constraint: /10/6 + │ │ └── spans: ["\x8a", "\x8a"] │ └── key: (6) └── aggregations └── const-agg [as=a:2, outer=(2)] @@ -5813,7 +6015,8 @@ project │ ├── fd: (7)-->(8,10) │ └── scan b@inv_idx │ ├── columns: k:7!null - │ ├── constraint: /10/7: [/'{"foo": "bar"}' - /'{"foo": "bar"}'] + │ ├── inverted constraint: /12/7 + │ │ └── spans: ["7foo\x00\x01\x12bar\x00\x01", "7foo\x00\x01\x12bar\x00\x01"] │ └── key: (7) └── aggregations ├── const-agg [as=u:2, outer=(2)] @@ -5855,7 +6058,8 @@ project │ ├── fd: (6)-->(7,8) │ └── scan c@inv_idx │ ├── columns: k:6!null - │ ├── constraint: /7/6: [/ARRAY[2] - /ARRAY[2]] + │ ├── inverted constraint: /10/6 + │ │ └── spans: ["\x8a", "\x8a"] │ └── key: (6) └── aggregations ├── const-agg [as=a:2, outer=(2)]