From dc441a6aed7abbec4fe39dae5966041a68df459f Mon Sep 17 00:00:00 2001 From: Shivam Saraf Date: Thu, 2 Feb 2023 16:38:45 -0500 Subject: [PATCH] opt: inverted-index accelerate queries with filters in the form json->'field' IN (...) Previously, the optimizer did not plan inverted index scans for queries consisting of the IN operator. To address this, we now build JSON arrays from fetch value expressions consisting of both integers and strings. Each entity present inside of the tuple following the IN operator has it's own seperate JSON array being created. From these JSON arrays, inverted spans are built. Each inverted span generated is OR'ed together to replicate the functionality of the IN operator. In other words, a filter of the form: j->0 IN ('1', '2') is thought of as (j->0 = '1') OR (j->0 = '2') and the generated inverted span replicates this behaviour. Epic: CRDB-3301 Fixes: #87856 Release note (performance improvement): The optimizer now plans inverted index scans for queries that filter by JSON fetch value operators (->), consisting of both string and integer index values, alongside the IN operator, e.g, json_col->0 IN ('1', '2') and json_col->'a' IN ('1', '2') --- .../testdata/logic_test/inverted_index | 81 ++- pkg/sql/opt/invertedidx/json_array.go | 37 ++ pkg/sql/opt/invertedidx/json_array_test.go | 82 +++ pkg/sql/opt/xform/testdata/rules/select | 581 ++++++++++++++++++ 4 files changed, 780 insertions(+), 1 deletion(-) diff --git a/pkg/sql/logictest/testdata/logic_test/inverted_index b/pkg/sql/logictest/testdata/logic_test/inverted_index index f18dc6119a53..75a3aedf9449 100644 --- a/pkg/sql/logictest/testdata/logic_test/inverted_index +++ b/pkg/sql/logictest/testdata/logic_test/inverted_index @@ -806,7 +806,8 @@ INSERT INTO f VALUES (36, '[[]]'), (37, '[{"a": [0, "b"]}, null, 1]'), (38, '[[0, 1, 2], {"b": "c"}]'), - (39, '[[0, [1, 2]]]') + (39, '[[0, [1, 2]]]'), + (40, '{"a": "a"}') query T SELECT j FROM f@i WHERE j->0 = '[]' ORDER BY k @@ -1197,6 +1198,84 @@ SELECT j FROM f@i WHERE j->'a'->'b' <@ '{"c": [1, 2], "d": 2}' OR j->'a'->'b' <@ {"a": {"b": "c"}, "d": "e"} {"a": {"b": []}} +#Testing the IN operator +query T +SELECT j FROM f@i WHERE j->'a' IN ('1', '2', '3') ORDER BY k +---- +{"a": 1} +{"a": 1, "b": 2} +{"a": 1, "c": 3} + +query T +SELECT j FROM f@i WHERE j->'a' IN ('"a"') ORDER BY k +---- +{"a": "a"} + +query T +SELECT j FROM f@i WHERE j->'a' IN ('"a"', 'null') ORDER BY k +---- +{"a": null} +{"a": "a"} + +query T +SELECT j FROM f@i WHERE j->'a' IN ('{"b": "c"}', '{"c": "d"}') ORDER BY k +---- +{"a": {"b": "c"}} +{"a": {"b": "c"}, "d": "e"} + +query T +SELECT j FROM f@i WHERE j->'a' IN ('{"b": "c"}', '"a"') ORDER BY k +---- +{"a": {"b": "c"}} +{"a": {"b": "c"}, "d": "e"} +{"a": "a"} + +query T +SELECT j FROM f@i WHERE j->'a' IN ('[1, 2]', '"a"') ORDER BY k +---- +{"a": [1, 2]} +{"a": "a"} + +query T +SELECT j FROM f@i WHERE j->0 IN ('1', '2', '3') ORDER BY k +---- +[1, 2, {"b": "c"}] + +query T +SELECT j FROM f@i WHERE j->1 IN ('1', '2', '3') ORDER BY k +---- +[1, 2, {"b": "c"}] + +query T +SELECT j FROM f@i WHERE j->0 IN ('1', '2', '3', '"d"') AND j->1 IN + ('"e"', '"d"') ORDER BY k +---- + +query T +SELECT j FROM f@i WHERE j->0 IN ('1', '2', '3', '"d"') OR j->1 IN + ('"e"', '"d"') ORDER BY k +---- +[1, 2, {"b": "c"}] +[{"a": {"b": "c"}}, "d", "e"] + +query T +SELECT j FROM f@i WHERE j->0 IN ('{"a": {"b": "c"}}', '{"a": [0, "b"]}') ORDER BY k +---- +[{"a": {"b": "c"}}, "d", "e"] +[{"a": [0, "b"]}, null, 1] + +query T +SELECT j FROM f@i WHERE j->0 IN ('{"a": {"b": "c"}}', '[]') ORDER BY k +---- +[{"a": {"b": "c"}}, "d", "e"] +[[]] + +query T +SELECT j FROM f@i WHERE j->0 IN ('[0, [1, 2]]', '[0, 1, 2]') ORDER BY k +---- +[[0, 1, 2], {"b": "c"}] +[[0, [1, 2]]] + # Regression test for #63180. This query should not cause an index out of range # error in the inverted filterer. query T diff --git a/pkg/sql/opt/invertedidx/json_array.go b/pkg/sql/opt/invertedidx/json_array.go index ff91d93ad594..139d5d4fbc6c 100644 --- a/pkg/sql/opt/invertedidx/json_array.go +++ b/pkg/sql/opt/invertedidx/json_array.go @@ -392,6 +392,10 @@ func (j *jsonOrArrayFilterPlanner) extractInvertedFilterConditionFromLeaf( if fetch, ok := t.Left.(*memo.FetchValExpr); ok { invertedExpr = j.extractJSONFetchValEqCondition(ctx, evalCtx, fetch, t.Right) } + case *memo.InExpr: + if fetch, ok := t.Left.(*memo.FetchValExpr); ok { + invertedExpr = j.extractJSONInCondition(ctx, evalCtx, fetch, t.Right) + } case *memo.OverlapsExpr: invertedExpr = j.extractArrayOverlapsCondition(ctx, evalCtx, t.Left, t.Right) } @@ -412,6 +416,39 @@ func (j *jsonOrArrayFilterPlanner) extractInvertedFilterConditionFromLeaf( return invertedExpr, remainingFilters, nil } +// extractJSONInCondition extracts an InvertedExpression representing +// an inverted filter over the planner's inverted index, based on the +// give left and right expression arguments. If an +// InvertedExpression cannot be generated from the expression, an +// inverted.NonInvertedColExpression is returned. +func (j *jsonOrArrayFilterPlanner) extractJSONInCondition( + ctx context.Context, evalCtx *eval.Context, left *memo.FetchValExpr, right opt.ScalarExpr, +) inverted.Expression { + // The right side of the expression should be a constant JSON value. + if !memo.CanExtractConstDatum(right) { + return inverted.NonInvertedColExpression{} + } + var invertedExpr inverted.Expression + tuple := right.Child(0) + tupleLen := tuple.ChildCount() + for index := 0; index < tupleLen; index++ { + child := tuple.Child(index).(opt.ScalarExpr) + expr := j.extractJSONFetchValEqCondition(ctx, evalCtx, left, child) + if invertedExpr == nil { + invertedExpr = expr + } + invertedExpr = inverted.Or(invertedExpr, expr) + } + + if invertedExpr == nil { + // An inverted expression could not be extracted. + return inverted.NonInvertedColExpression{} + } + + return invertedExpr + +} + // extractArrayOverlapsCondition extracts an InvertedExpression // representing an inverted filter over the planner's inverted index, based // on the given left and right expression arguments. Returns an empty diff --git a/pkg/sql/opt/invertedidx/json_array_test.go b/pkg/sql/opt/invertedidx/json_array_test.go index 542b3d869ec3..763e45d04e4f 100644 --- a/pkg/sql/opt/invertedidx/json_array_test.go +++ b/pkg/sql/opt/invertedidx/json_array_test.go @@ -885,6 +885,88 @@ func TestTryFilterJsonOrArrayIndex(t *testing.T) { unique: true, remainingFilters: "str <@ '{hello}' AND str @> '{hello}'", }, + { + // Testing the IN operator with the fetch value as a string + filters: "j->'a' IN ('1', '2', '3')", + indexOrd: jsonOrd, + ok: true, + tight: true, + unique: false, + remainingFilters: "", + }, + { + filters: `j->'a' IN ('"a"', '"b"')`, + indexOrd: jsonOrd, + ok: true, + tight: true, + unique: false, + remainingFilters: "", + }, + { + filters: `j->'a' IN ('{"a": "b"}', '"a"')`, + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: false, + remainingFilters: `j->'a' IN ('{"a": "b"}', '"a"')`, + }, + { + filters: `j->'a' IN ('[1,2,3]', '[1]')`, + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: false, + remainingFilters: `j->'a' IN ('[1,2,3]', '[1]')`, + }, + { + filters: `j->'a' IN ('[1,2,3]', '{"a": "b"}', '"a"', 'null')`, + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: false, + remainingFilters: `j->'a' IN ('[1,2,3]', '{"a": "b"}', '"a"', 'null')`, + }, + { + // Testing the IN operator with the fetch value as an integer + filters: "j->0->1 IN ('1', '2', '3')", + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: false, + remainingFilters: "j->0->1 IN ('1', '2', '3')", + }, + { + filters: `j->0 IN ('"a"', '"b"')`, + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: false, + remainingFilters: `j->0 IN ('"a"', '"b"')`, + }, + { + filters: `j->0->'a' IN ('{"a": "b"}', '"a"')`, + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: false, + remainingFilters: `j->0->'a' IN ('{"a": "b"}', '"a"')`, + }, + { + filters: `j->'a'->0 IN ('[1,2,3]', '[1]')`, + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: false, + remainingFilters: `j->'a'->0 IN ('[1,2,3]', '[1]')`, + }, + { + filters: `j->0 IN ('[1,2,3]', '{"a": "b"}', '"a"', 'null')`, + indexOrd: jsonOrd, + ok: true, + tight: false, + unique: false, + remainingFilters: `j->0 IN ('[1,2,3]', '{"a": "b"}', '"a"', 'null')`, + }, } for _, tc := range testCases { diff --git a/pkg/sql/opt/xform/testdata/rules/select b/pkg/sql/opt/xform/testdata/rules/select index 06b9de2b5479..a90cf95779d9 100644 --- a/pkg/sql/opt/xform/testdata/rules/select +++ b/pkg/sql/opt/xform/testdata/rules/select @@ -2777,6 +2777,26 @@ select ├── j:4 <@ '{"a": [3]}' [outer=(4), immutable] └── j:4 <@ '[1, 2, 3]' [outer=(4), immutable] +# Query not using the fetch val but using the equality operator. +opt expect-not=GenerateInvertedIndexScans +SELECT k FROM b WHERE j = '"b"' +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4!null + ├── immutable + ├── key: (1) + ├── fd: ()-->(4) + ├── scan b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ └── fd: (1)-->(4) + └── filters + └── j:4 = '"b"' [outer=(4), immutable, constraints=(/4: [/'"b"' - /'"b"']; tight), fd=()-->(4)] + # Query using the fetch val and equality operators. opt expect=GenerateInvertedIndexScans SELECT k FROM b WHERE j->'a' = '"b"' @@ -3072,6 +3092,567 @@ project ├── key: (1) └── fd: (1)-->(7) +# Generate an inverted scan when the index of the fetch val operator is +# a string along with the IN operator consisting of JSON strings in +# the tuple. Since the inverted expression is tight, +# the original filter should not be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->'a' IN ('1', '2', '3') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /7 + │ ├── tight: true, unique: false + │ └── union spans + │ ├── ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ ├── ["7a\x00\x01*\x04\x00", "7a\x00\x01*\x04\x00"] + │ └── ["7a\x00\x01*\x06\x00", "7a\x00\x01*\x06\x00"] + ├── key: (1) + └── scan b@j_inv_idx + ├── columns: k:1!null j_inverted_key:7!null + ├── inverted constraint: /7/1 + │ └── spans + │ ├── ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ ├── ["7a\x00\x01*\x04\x00", "7a\x00\x01*\x04\x00"] + │ └── ["7a\x00\x01*\x06\x00", "7a\x00\x01*\x06\x00"] + ├── key: (1) + └── fd: (1)-->(7) + + +# Query NOT using the fetch val and the IN operator. +# This should result in a full table scan (at the moment) +opt expect-not=GenerateInvertedIndexScans +SELECT k FROM b WHERE j IN ('1', '2', '3') +---- +project + ├── columns: k:1!null + ├── key: (1) + └── select + ├── columns: k:1!null j:4!null + ├── key: (1) + ├── fd: (1)-->(4) + ├── scan b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ └── fd: (1)-->(4) + └── filters + └── j:4 IN ('1', '2', '3') [outer=(4), constraints=(/4: [/'1' - /'1'] [/'2' - /'2'] [/'3' - /'3']; tight)] + +# Generate an inverted scan when the index of the fetch val operator is +# a string along with the IN operator consisting of JSON strings in +# the tuple. Since the inverted expression is tight, +# the original filter should not be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->'a' IN ('"a"', '"b"') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── inverted-filter + ├── columns: k:1!null + ├── inverted expression: /7 + │ ├── tight: true, unique: false + │ └── union spans + │ ├── ["7a\x00\x01\x12a\x00\x01", "7a\x00\x01\x12a\x00\x01"] + │ └── ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] + ├── key: (1) + └── scan b@j_inv_idx + ├── columns: k:1!null j_inverted_key:7!null + ├── inverted constraint: /7/1 + │ └── spans + │ ├── ["7a\x00\x01\x12a\x00\x01", "7a\x00\x01\x12a\x00\x01"] + │ └── ["7a\x00\x01\x12b\x00\x01", "7a\x00\x01\x12b\x00\x01"] + ├── key: (1) + └── fd: (1)-->(7) + + +# Generate an inverted scan when the index of the fetch val operator is +# a string along with the IN operator consisting of JSON objects in +# the tuple. Since the inverted expression is not tight, +# the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->'a' IN ('{"a": "b"}', '{"c": "d"}') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7a\x00\x02a\x00\x01\x12b\x00\x01", "7a\x00\x02a\x00\x01\x12b\x00\x01"] + │ │ └── ["7a\x00\x02c\x00\x01\x12d\x00\x01", "7a\x00\x02c\x00\x01\x12d\x00\x01"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7a\x00\x02a\x00\x01\x12b\x00\x01", "7a\x00\x02a\x00\x01\x12b\x00\x01"] + │ │ └── ["7a\x00\x02c\x00\x01\x12d\x00\x01", "7a\x00\x02c\x00\x01\x12d\x00\x01"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->'a') IN ('{"a": "b"}', '{"c": "d"}') [outer=(4), immutable] + +# Generate an inverted scan when the index of the fetch val operator is +# a string along with the IN operator consisting of a combination of JSON objects, +# JSON strings and JSON integers inside the tuple. Since the inverted expression generated +# is not tight, the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->'a' IN ('{"a": "b"}', '1', '"a"', 'null') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7a\x00\x01\x00", "7a\x00\x01\x00"] + │ │ ├── ["7a\x00\x01\x12a\x00\x01", "7a\x00\x01\x12a\x00\x01"] + │ │ ├── ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ │ └── ["7a\x00\x02a\x00\x01\x12b\x00\x01", "7a\x00\x02a\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7a\x00\x01\x00", "7a\x00\x01\x00"] + │ │ ├── ["7a\x00\x01\x12a\x00\x01", "7a\x00\x01\x12a\x00\x01"] + │ │ ├── ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ │ └── ["7a\x00\x02a\x00\x01\x12b\x00\x01", "7a\x00\x02a\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->'a') IN ('null', '"a"', '1', '{"a": "b"}') [outer=(4), immutable] + +# Generate an inverted scan when the index of the fetch val operator is +# a string along with the IN operator consisting of a combination of JSON objects, +# JSON strings,JSON integers and JSON arrays inside the tuple. Since the inverted expression +# generated is not tight, the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->'a' IN ('{"a": "b"}', '1', '"a"', 'null', '[1,2,3]') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ ├── union spans + │ │ │ ├── ["7a\x00\x01\x00", "7a\x00\x01\x00"] + │ │ │ ├── ["7a\x00\x01\x12a\x00\x01", "7a\x00\x01\x12a\x00\x01"] + │ │ │ ├── ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ │ │ └── ["7a\x00\x02a\x00\x01\x12b\x00\x01", "7a\x00\x02a\x00\x01\x12b\x00\x01"] + │ │ └── INTERSECTION + │ │ ├── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ ├── union spans: empty + │ │ │ └── INTERSECTION + │ │ │ ├── span expression + │ │ │ │ ├── tight: true, unique: true + │ │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x02\x00", "7a\x00\x02\x00\x03\x00\x01*\x02\x00"] + │ │ │ └── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x04\x00", "7a\x00\x02\x00\x03\x00\x01*\x04\x00"] + │ │ └── span expression + │ │ ├── tight: true, unique: true + │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x06\x00", "7a\x00\x02\x00\x03\x00\x01*\x06\x00"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7a\x00\x01\x00", "7a\x00\x01\x00"] + │ │ ├── ["7a\x00\x01\x12a\x00\x01", "7a\x00\x01\x12a\x00\x01"] + │ │ ├── ["7a\x00\x01*\x02\x00", "7a\x00\x01*\x02\x00"] + │ │ ├── ["7a\x00\x02\x00\x03\x00\x01*\x02\x00", "7a\x00\x02\x00\x03\x00\x01*\x02\x00"] + │ │ ├── ["7a\x00\x02\x00\x03\x00\x01*\x04\x00", "7a\x00\x02\x00\x03\x00\x01*\x04\x00"] + │ │ ├── ["7a\x00\x02\x00\x03\x00\x01*\x06\x00", "7a\x00\x02\x00\x03\x00\x01*\x06\x00"] + │ │ └── ["7a\x00\x02a\x00\x01\x12b\x00\x01", "7a\x00\x02a\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->'a') IN ('null', '"a"', '1', '[1, 2, 3]', '{"a": "b"}') [outer=(4), immutable] + + +# Generate an inverted scan when the index of the fetch val operator is +# a string along with the IN operator consisting of JSON arrays in +# the tuple. Since the inverted expression is not tight, +# the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->'a' IN ('[1,2,3]', '[1, 2]', '[1]') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ ├── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x02\x00", "7a\x00\x02\x00\x03\x00\x01*\x02\x00"] + │ │ └── UNION + │ │ ├── span expression + │ │ │ ├── tight: false, unique: false + │ │ │ ├── union spans: empty + │ │ │ └── INTERSECTION + │ │ │ ├── span expression + │ │ │ │ ├── tight: true, unique: true + │ │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x02\x00", "7a\x00\x02\x00\x03\x00\x01*\x02\x00"] + │ │ │ └── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x04\x00", "7a\x00\x02\x00\x03\x00\x01*\x04\x00"] + │ │ └── span expression + │ │ ├── tight: false, unique: true + │ │ ├── union spans: empty + │ │ └── INTERSECTION + │ │ ├── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ ├── union spans: empty + │ │ │ └── INTERSECTION + │ │ │ ├── span expression + │ │ │ │ ├── tight: true, unique: true + │ │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x02\x00", "7a\x00\x02\x00\x03\x00\x01*\x02\x00"] + │ │ │ └── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x04\x00", "7a\x00\x02\x00\x03\x00\x01*\x04\x00"] + │ │ └── span expression + │ │ ├── tight: true, unique: true + │ │ └── union spans: ["7a\x00\x02\x00\x03\x00\x01*\x06\x00", "7a\x00\x02\x00\x03\x00\x01*\x06\x00"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7a\x00\x02\x00\x03\x00\x01*\x02\x00", "7a\x00\x02\x00\x03\x00\x01*\x02\x00"] + │ │ ├── ["7a\x00\x02\x00\x03\x00\x01*\x04\x00", "7a\x00\x02\x00\x03\x00\x01*\x04\x00"] + │ │ └── ["7a\x00\x02\x00\x03\x00\x01*\x06\x00", "7a\x00\x02\x00\x03\x00\x01*\x06\x00"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->'a') IN ('[1]', '[1, 2]', '[1, 2, 3]') [outer=(4), immutable] + + +# Generate an inverted scan when the index of the fetch val operator is +# an integer along with the IN operator consisting of JSON integers in +# the tuple. Since the inverted expression is not tight, +# the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->0 in ('1', '2') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x01*\x02\x00"] + │ │ └── ["7\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x01*\x04\x00"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x01*\x02\x00"] + │ │ └── ["7\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x01*\x04\x00"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->0) IN ('1', '2') [outer=(4), immutable] + +# Generate an inverted scan when the index of the fetch val operator is +# an integer along with the IN operator consisting of JSON strings in +# the tuple. Since the generated inverted expression is not tight, +# the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->0 IN ('"a"', '"b"') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7\x00\x03\x00\x01\x12a\x00\x01", "7\x00\x03\x00\x01\x12a\x00\x01"] + │ │ └── ["7\x00\x03\x00\x01\x12b\x00\x01", "7\x00\x03\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7\x00\x03\x00\x01\x12a\x00\x01", "7\x00\x03\x00\x01\x12a\x00\x01"] + │ │ └── ["7\x00\x03\x00\x01\x12b\x00\x01", "7\x00\x03\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->0) IN ('"a"', '"b"') [outer=(4), immutable] + +# Generate an inverted scan when the index of the fetch val operator is +# an integer along with the IN operator consisting of JSON objects in +# the tuple. Since the generated inverted expression is not tight, +# the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->0 IN ('{"a": "b"}', '{"c": "d"}') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7\x00\x03a\x00\x01\x12b\x00\x01", "7\x00\x03a\x00\x01\x12b\x00\x01"] + │ │ └── ["7\x00\x03c\x00\x01\x12d\x00\x01", "7\x00\x03c\x00\x01\x12d\x00\x01"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7\x00\x03a\x00\x01\x12b\x00\x01", "7\x00\x03a\x00\x01\x12b\x00\x01"] + │ │ └── ["7\x00\x03c\x00\x01\x12d\x00\x01", "7\x00\x03c\x00\x01\x12d\x00\x01"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->0) IN ('{"a": "b"}', '{"c": "d"}') [outer=(4), immutable] + +# Generate an inverted scan when the index of the fetch val operator is +# an integer along with the IN operator consisting of a combination of JSON objects, +# JSON strings and JSON integers inside the tuple. Since the inverted expression generated +# is not tight, the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->0 IN ('{"a": "b"}', '1', '"a"', 'null') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7\x00\x03\x00\x01\x00", "7\x00\x03\x00\x01\x00"] + │ │ ├── ["7\x00\x03\x00\x01\x12a\x00\x01", "7\x00\x03\x00\x01\x12a\x00\x01"] + │ │ ├── ["7\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x01*\x02\x00"] + │ │ └── ["7\x00\x03a\x00\x01\x12b\x00\x01", "7\x00\x03a\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7\x00\x03\x00\x01\x00", "7\x00\x03\x00\x01\x00"] + │ │ ├── ["7\x00\x03\x00\x01\x12a\x00\x01", "7\x00\x03\x00\x01\x12a\x00\x01"] + │ │ ├── ["7\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x01*\x02\x00"] + │ │ └── ["7\x00\x03a\x00\x01\x12b\x00\x01", "7\x00\x03a\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->0) IN ('null', '"a"', '1', '{"a": "b"}') [outer=(4), immutable] + + +# Generate an inverted scan when the index of the fetch val operator is +# an integer along with the IN operator consisting of JSON arrays. +# Since the inverted expression generated is not tight, +# the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->0 IN ('[1,2,3]', '[1]', '[1,2]') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ ├── union spans: ["7\x00\x03\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x03\x00\x01*\x02\x00"] + │ │ └── UNION + │ │ ├── span expression + │ │ │ ├── tight: false, unique: false + │ │ │ ├── union spans: empty + │ │ │ └── INTERSECTION + │ │ │ ├── span expression + │ │ │ │ ├── tight: true, unique: true + │ │ │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x03\x00\x01*\x02\x00"] + │ │ │ └── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x03\x00\x01*\x04\x00"] + │ │ └── span expression + │ │ ├── tight: false, unique: true + │ │ ├── union spans: empty + │ │ └── INTERSECTION + │ │ ├── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ ├── union spans: empty + │ │ │ └── INTERSECTION + │ │ │ ├── span expression + │ │ │ │ ├── tight: true, unique: true + │ │ │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x03\x00\x01*\x02\x00"] + │ │ │ └── span expression + │ │ │ ├── tight: true, unique: true + │ │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x03\x00\x01*\x04\x00"] + │ │ └── span expression + │ │ ├── tight: true, unique: true + │ │ └── union spans: ["7\x00\x03\x00\x03\x00\x01*\x06\x00", "7\x00\x03\x00\x03\x00\x01*\x06\x00"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7\x00\x03\x00\x03\x00\x01*\x02\x00", "7\x00\x03\x00\x03\x00\x01*\x02\x00"] + │ │ ├── ["7\x00\x03\x00\x03\x00\x01*\x04\x00", "7\x00\x03\x00\x03\x00\x01*\x04\x00"] + │ │ └── ["7\x00\x03\x00\x03\x00\x01*\x06\x00", "7\x00\x03\x00\x03\x00\x01*\x06\x00"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── (j:4->0) IN ('[1]', '[1, 2]', '[1, 2, 3]') [outer=(4), immutable] + +# Generate an inverted scan when the index of the fetch val operator is +# a combination of an integer and a string along with the IN operator consisting of +# a combination of JSON arrays, JSON objects, JSON integers, JSON strings and null. +# Since the inverted expression generated is not tight, +# the original filter should be applied on top. +opt expect=GenerateInvertedIndexScans +SELECT k FROM b WHERE j->0->'a' IN ('{"a": "b"}', '1', '"a"', 'null') +---- +project + ├── columns: k:1!null + ├── immutable + ├── key: (1) + └── select + ├── columns: k:1!null j:4 + ├── immutable + ├── key: (1) + ├── fd: (1)-->(4) + ├── index-join b + │ ├── columns: k:1!null j:4 + │ ├── key: (1) + │ ├── fd: (1)-->(4) + │ └── inverted-filter + │ ├── columns: k:1!null + │ ├── inverted expression: /7 + │ │ ├── tight: false, unique: false + │ │ └── union spans + │ │ ├── ["7\x00\x03a\x00\x01\x00", "7\x00\x03a\x00\x01\x00"] + │ │ ├── ["7\x00\x03a\x00\x01\x12a\x00\x01", "7\x00\x03a\x00\x01\x12a\x00\x01"] + │ │ ├── ["7\x00\x03a\x00\x01*\x02\x00", "7\x00\x03a\x00\x01*\x02\x00"] + │ │ └── ["7\x00\x03a\x00\x02a\x00\x01\x12b\x00\x01", "7\x00\x03a\x00\x02a\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── scan b@j_inv_idx + │ ├── columns: k:1!null j_inverted_key:7!null + │ ├── inverted constraint: /7/1 + │ │ └── spans + │ │ ├── ["7\x00\x03a\x00\x01\x00", "7\x00\x03a\x00\x01\x00"] + │ │ ├── ["7\x00\x03a\x00\x01\x12a\x00\x01", "7\x00\x03a\x00\x01\x12a\x00\x01"] + │ │ ├── ["7\x00\x03a\x00\x01*\x02\x00", "7\x00\x03a\x00\x01*\x02\x00"] + │ │ └── ["7\x00\x03a\x00\x02a\x00\x01\x12b\x00\x01", "7\x00\x03a\x00\x02a\x00\x01\x12b\x00\x01"] + │ ├── key: (1) + │ └── fd: (1)-->(7) + └── filters + └── ((j:4->0)->'a') IN ('null', '"a"', '1', '{"a": "b"}') [outer=(4), immutable] + + # Query using the fetch val and equality operators in a disjunction with a # contains operator. opt expect=GenerateInvertedIndexScans