Skip to content

Commit

Permalink
Merge #130957
Browse files Browse the repository at this point in the history 
130957: storage,roachpb: allow reading MVCCValueHeader data via SQL r=michae2 a=stevendanna

Logical Data Replication (LDR) and Import now both write values into
the MVCCValueHeader of keys. We would like to be able to
read these values from SQL via new system columns. In the case of LDR this will allow the
SQL-write-path to continue to function even after the OriginTimetamp
column is moved out of the user's table schema and into the
MVCCValueHeader.

We make that possible by:

- Adding a ReturnRawMVCCValue option to GetRequest, ScanRequest, and
ReverseScanRequest that returns values with the extended encoding rather than
stripping off the MVCCValueHeader.

- Modifying the functions on roachpb.Value to understand the extended
  encoding may be in place and return the correct version.

- Modifying the column and rowfetchers to make requests with this new option
  when the newly added system columns are added.


Release note: None
Epic: none

Co-authored-by: Steven Danna <[email protected]>
  • Loading branch information
@stevendanna
craig[bot] and stevendanna committed Oct 10, 2024
2 parents fd4b146 + 6f51713 commit be52993
Show file tree
Hide file tree
Showing 128 changed files with 7,886 additions and 1,250 deletions.
5 changes: 4 additions & 1 deletion pkg/ccl/changefeedccl/cdcevent/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -622,7 +622,10 @@ func (d *eventDecoder) initForKey(
// In particular, when decoding previous row, we strip table OID column
// since it makes little sense to include it in the previous row value.
var systemColumns = []descpb.ColumnDescriptor{
colinfo.MVCCTimestampColumnDesc, colinfo.TableOIDColumnDesc,
colinfo.MVCCTimestampColumnDesc,
colinfo.TableOIDColumnDesc,
colinfo.OriginIDColumnDesc,
colinfo.OriginTimestampColumnDesc,
}

type fetcher struct {
Expand Down
4 changes: 2 additions & 2 deletions pkg/ccl/logictestccl/testdata/logic_test/as_of
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,9 +189,9 @@ SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('1ms') WHERE j = 2
query T
EXPLAIN (OPT, MEMO) SELECT * FROM t AS OF SYSTEM TIME with_max_staleness('1ms') WHERE j = 2 AND i = 1
----
memo (optimized, ~9KB, required=[presentation: info:6] [distribution: test])
memo (optimized, ~9KB, required=[presentation: info:8] [distribution: test])
├── G1: (explain G2 [presentation: i:1,j:2,k:3] [distribution: test])
│ └── [presentation: info:6] [distribution: test]
│ └── [presentation: info:8] [distribution: test]
│ ├── best: (explain G2="[presentation: i:1,j:2,k:3] [distribution: test]" [presentation: i:1,j:2,k:3] [distribution: test])
│ └── cost: 9.18
├── G2: (select G3 G4) (select G5 G6)
Expand Down
42 changes: 23 additions & 19 deletions pkg/ccl/logictestccl/testdata/logic_test/explain_redact
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ TABLE p
├── q int
├── crdb_internal_mvcc_timestamp decimal [hidden] [system]
├── tableoid oid [hidden] [system]
├── crdb_internal_origin_id int4 [hidden] [system]
├── crdb_internal_origin_timestamp decimal [hidden] [system]
├── FAMILY fam_0_p_q (p, q)
└── PRIMARY INDEX p_pkey
├── p int not null
Expand Down Expand Up @@ -50,6 +52,8 @@ TABLE q
├── r int
├── crdb_internal_mvcc_timestamp decimal [hidden] [system]
├── tableoid oid [hidden] [system]
├── crdb_internal_origin_id int4 [hidden] [system]
├── crdb_internal_origin_timestamp decimal [hidden] [system]
├── FAMILY fam_0_q_r (q, r)
└── PRIMARY INDEX q_pkey
└── q int not null
Expand Down Expand Up @@ -94,20 +98,20 @@ EXPLAIN (OPT, VERBOSE, REDACT) INSERT INTO p VALUES (1, 1)
insert p
├── columns: <none>
├── insert-mapping:
│ ├── column1:5 => p:1
│ └── column2:6 => q:2
│ ├── column1:7 => p:1
│ └── column2:8 => q:2
├── cardinality: [0 - 0]
├── volatile, mutations
├── stats: [rows=0]
├── cost: 0.03
├── distribution: test
└── values
├── columns: column1:5 column2:6
├── columns: column1:7 column2:8
├── cardinality: [1 - 1]
├── stats: [rows=1]
├── cost: 0.02
├── key: ()
├── fd: ()-->(5,6)
├── fd: ()-->(7,8)
├── distribution: test
└── ‹(‹×›, ‹×›)›

Expand All @@ -117,20 +121,20 @@ EXPLAIN (OPT, TYPES, REDACT) INSERT INTO p VALUES (1, 1)
insert p
├── columns: <none>
├── insert-mapping:
│ ├── column1:5 => p:1
│ └── column2:6 => q:2
│ ├── column1:7 => p:1
│ └── column2:8 => q:2
├── cardinality: [0 - 0]
├── volatile, mutations
├── stats: [rows=0]
├── cost: 0.03
├── distribution: test
└── values
├── columns: column1:5(int!null) column2:6(int!null)
├── columns: column1:7(int!null) column2:8(int!null)
├── cardinality: [1 - 1]
├── stats: [rows=1]
├── cost: 0.02
├── key: ()
├── fd: ()-->(5,6)
├── fd: ()-->(7,8)
├── distribution: test
└── tuple [type=tuple{int, int}]
├── const: ‹×› [type=int]
Expand All @@ -139,9 +143,9 @@ insert p
query T
EXPLAIN (OPT, MEMO, REDACT) INSERT INTO p VALUES (1, 1)
----
memo (optimized, ~4KB, required=[presentation: info:7] [distribution: test])
memo (optimized, ~4KB, required=[presentation: info:9] [distribution: test])
├── G1: (explain G2 [distribution: test])
│ └── [presentation: info:7] [distribution: test]
│ └── [presentation: info:9] [distribution: test]
│ ├── best: (explain G2="[distribution: test]" [distribution: test])
│ └── cost: 0.05
├── G2: (insert G3 G4 G5 G6 p)
Expand Down Expand Up @@ -229,9 +233,9 @@ scan p
query T
EXPLAIN (OPT, MEMO, REDACT) SELECT * FROM p WHERE p = 11
----
memo (optimized, ~7KB, required=[presentation: info:5] [distribution: test])
memo (optimized, ~7KB, required=[presentation: info:7] [distribution: test])
├── G1: (explain G2 [presentation: p:1,q:2] [distribution: test])
│ └── [presentation: info:5] [distribution: test]
│ └── [presentation: info:7] [distribution: test]
│ ├── best: (explain G2="[presentation: p:1,q:2] [distribution: test]" [presentation: p:1,q:2] [distribution: test])
│ └── cost: 9.11
├── G2: (select G3 G4) (scan p,cols=(1,2),constrained)
Expand Down Expand Up @@ -313,9 +317,9 @@ scan q
query T
EXPLAIN (OPT, MEMO, REDACT) SELECT * FROM q WHERE q > 2
----
memo (optimized, ~7KB, required=[presentation: info:5] [distribution: test])
memo (optimized, ~7KB, required=[presentation: info:7] [distribution: test])
├── G1: (explain G2 [presentation: q:1,r:2] [distribution: test])
│ └── [presentation: info:5] [distribution: test]
│ └── [presentation: info:7] [distribution: test]
│ ├── best: (explain G2="[presentation: q:1,r:2] [distribution: test]" [presentation: q:1,r:2] [distribution: test])
│ └── cost: 378.04
├── G2: (select G3 G4) (scan q,cols=(1,2),constrained)
Expand Down Expand Up @@ -375,26 +379,26 @@ create-function
query T
EXPLAIN (OPT, MEMO, REDACT) CREATE FUNCTION f() RETURNS STRING LANGUAGE SQL AS $$ SELECT * FROM t128282 WHERE col = 'secret' $$
----
memo (optimized, ~8KB, required=[presentation: info:5] [distribution: test])
memo (optimized, ~8KB, required=[presentation: info:7] [distribution: test])
├── G1: (explain G2 [distribution: test])
│ └── [presentation: info:5] [distribution: test]
│ └── [presentation: info:7] [distribution: test]
│ ├── best: (explain G2="[distribution: test]" [distribution: test])
│ └── cost: 0.03
└── G2: (create-function &{‹×› ‹×›
‹×›
‹×›
‹×› [{‹×› ‹×› map[‹×›:0 ‹×›:1 ‹×›:2 ‹×›:3] false 0}] ‹×› ‹×›})
‹×› [{‹×› ‹×› map[‹×›:0 ‹×›:1 ‹×›:2 ‹×›:3 ‹×›:4 ‹×›:5] false 0}] ‹×› ‹×›})
├── [distribution: test]
│ ├── best: (create-function &{‹×› ‹×›
│ │ ‹×›
│ │ ‹×›
│ │ ‹×› [{‹×› ‹×› map[‹×›:0 ‹×›:1 ‹×›:2 ‹×›:3] false 0}] ‹×› ‹×›})
│ │ ‹×› [{‹×› ‹×› map[‹×›:0 ‹×›:1 ‹×›:2 ‹×›:3 ‹×›:4 ‹×›:5] false 0}] ‹×› ‹×›})
│ └── cost: 0.01
└── []
├── best: (create-function &{‹×› ‹×›
│ ‹×›
│ ‹×›
│ ‹×› [{‹×› ‹×› map[‹×›:0 ‹×›:1 ‹×›:2 ‹×›:3] false 0}] ‹×› ‹×›})
│ ‹×› [{‹×› ‹×› map[‹×›:0 ‹×›:1 ‹×›:2 ‹×›:3 ‹×›:4 ‹×›:5] false 0}] ‹×› ‹×›})
└── cost: 0.01
create-function
├── CREATE FUNCTION f()
Expand Down
4 changes: 2 additions & 2 deletions pkg/ccl/logictestccl/testdata/logic_test/generic
View file
Original file line number Diff line number Diff line change
Expand Up @@ -543,7 +543,7 @@ quality of service: regular
│ KV gRPC calls: 0
│ estimated max memory allocated: 0 B
│ table: t@t_t_idx
│ equality: (column9) = (t)
│ equality: (column11) = (t)
│
└── • values
sql nodes: <hidden>
Expand Down Expand Up @@ -593,7 +593,7 @@ quality of service: regular
│ KV gRPC calls: 0
│ estimated max memory allocated: 0 B
│ table: t@t_t_idx
│ equality: (column9) = (t)
│ equality: (column11) = (t)
│
└── • values
sql nodes: <hidden>
Expand Down
2 changes: 1 addition & 1 deletion pkg/ccl/logictestccl/testdata/logic_test/multi_region_foreign_key_lookup_join
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ vectorized: true
│ table: parent_rbr@parent_rbr_pkey
│ equality: (crdb_region, c_p_id) = (crdb_region, p_id)
│ equality cols are key
│ pred: column14 = p_int
│ pred: column18 = p_int
│
└── • render
│
Expand Down
52 changes: 26 additions & 26 deletions pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error
View file
Original file line number Diff line number Diff line change
Expand Up @@ -640,10 +640,10 @@ EXPLAIN (OPT) SELECT * FROM parent LIMIT 1
----
locality-optimized-search
├── scan parent
│ ├── constraint: /7/6: [/'ap-southeast-2' - /'ap-southeast-2']
│ ├── constraint: /9/8: [/'ap-southeast-2' - /'ap-southeast-2']
│ └── limit: 1
└── scan parent
├── constraint: /11/10: [/'ca-central-1' - /'us-east-1']
├── constraint: /15/14: [/'ca-central-1' - /'us-east-1']
└── limit: 1

# Static checking should not error this query out, because we now generate a
Expand All @@ -658,9 +658,9 @@ project
├── inner-join (lookup orders@orders_cust_id_idx [as=o])
│ ├── locality-optimized-search
│ │ ├── scan customers [as=c]
│ │ │ └── constraint: /15/13: [/'ap-southeast-2'/'69a1c2c2-5b18-459e-94d2-079dc53a4dd0' - /'ap-southeast-2'/'69a1c2c2-5b18-459e-94d2-079dc53a4dd0']
│ │ │ └── constraint: /19/17: [/'ap-southeast-2'/'69a1c2c2-5b18-459e-94d2-079dc53a4dd0' - /'ap-southeast-2'/'69a1c2c2-5b18-459e-94d2-079dc53a4dd0']
│ │ └── scan customers [as=c]
│ │ └── constraint: /20/18
│ │ └── constraint: /26/24
│ │ ├── [/'ca-central-1'/'69a1c2c2-5b18-459e-94d2-079dc53a4dd0' - /'ca-central-1'/'69a1c2c2-5b18-459e-94d2-079dc53a4dd0']
│ │ └── [/'us-east-1'/'69a1c2c2-5b18-459e-94d2-079dc53a4dd0' - /'us-east-1'/'69a1c2c2-5b18-459e-94d2-079dc53a4dd0']
│ └── filters
Expand Down Expand Up @@ -839,10 +839,10 @@ inner-join (lookup messages_global_2 [as=g3])
│ ├── lookup columns are key
│ ├── locality-optimized-search
│ │ ├── scan messages_rbr
│ │ │ ├── constraint: /33/30: [/'ap-southeast-2' - /'ap-southeast-2']
│ │ │ ├── constraint: /43/40: [/'ap-southeast-2' - /'ap-southeast-2']
│ │ │ └── limit: 1
│ │ └── scan messages_rbr
│ │ ├── constraint: /39/36: [/'ca-central-1' - /'us-east-1']
│ │ ├── constraint: /51/48: [/'ca-central-1' - /'us-east-1']
│ │ └── limit: 1
│ └── filters (true)
└── filters (true)
Expand Down Expand Up @@ -1056,65 +1056,65 @@ EXPLAIN (OPT,VERBOSE) SELECT t1.k FROM json_arr2_rbt AS t2 INNER INVERTED JOIN j
AND t1.crdb_region = 'ap-southeast-2' LIMIT 1
----
project
├── columns: k:7
├── columns: k:9
├── cardinality: [0 - 1]
├── immutable
├── stats: [rows=1]
├── cost: 4309.15778
├── key: ()
├── fd: ()-->(7)
├── fd: ()-->(9)
├── distribution: ap-southeast-2
├── prune: (7)
├── prune: (9)
└── limit
├── columns: t2.j:3 t1.k:7 t1.j:9 crdb_region:11
├── columns: t2.j:3 t1.k:9 t1.j:11 crdb_region:13
├── cardinality: [0 - 1]
├── immutable
├── stats: [rows=1]
├── cost: 4309.13778
├── key: ()
├── fd: ()-->(3,7,9,11)
├── fd: ()-->(3,9,11,13)
├── distribution: ap-southeast-2
├── inner-join (lookup json_arr1_rbr [as=t1])
│ ├── columns: t2.j:3 t1.k:7 t1.j:9 crdb_region:11
│ ├── key columns: [22 18] = [11 7]
│ ├── columns: t2.j:3 t1.k:9 t1.j:11 crdb_region:13
│ ├── key columns: [26 22] = [13 9]
│ ├── lookup columns are key
│ ├── immutable
│ ├── stats: [rows=3333.333]
│ ├── cost: 4309.11778
│ ├── fd: ()-->(11), (7)-->(9)
│ ├── fd: ()-->(13), (9)-->(11)
│ ├── limit hint: 1.00
│ ├── distribution: ap-southeast-2
│ ├── lookup table distribution: ap-southeast-2
│ ├── prune: (7)
│ ├── prune: (9)
│ ├── inner-join (inverted json_arr1_rbr@j_idx,inverted [as=t1])
│ │ ├── columns: t2.j:3 t1.k:18 crdb_region:22
│ │ ├── columns: t2.j:3 t1.k:22 crdb_region:26
│ │ ├── flags: force inverted join (into right side)
│ │ ├── prefix key columns: [17] = [22]
│ │ ├── prefix key columns: [21] = [26]
│ │ ├── inverted-expr
│ │ │ └── t1.j:20 @> t2.j:3
│ │ ├── stats: [rows=3333.333, distinct(17)=1, null(17)=0, distinct(18)=964.524, null(18)=0, distinct(22)=1, null(22)=0]
│ │ │ └── t1.j:24 @> t2.j:3
│ │ ├── stats: [rows=3333.333, distinct(21)=1, null(21)=0, distinct(22)=964.524, null(22)=0, distinct(26)=1, null(26)=0]
│ │ ├── cost: 3837.19889
│ │ ├── fd: ()-->(22)
│ │ ├── fd: ()-->(26)
│ │ ├── limit hint: 100.00
│ │ ├── distribution: ap-southeast-2
│ │ ├── project
│ │ │ ├── columns: "inverted_join_const_col_@11":17 t2.j:3
│ │ │ ├── stats: [rows=1000, distinct(17)=1, null(17)=0]
│ │ │ ├── columns: "inverted_join_const_col_@13":21 t2.j:3
│ │ │ ├── stats: [rows=1000, distinct(21)=1, null(21)=0]
│ │ │ ├── cost: 1136.62333
│ │ │ ├── fd: ()-->(17)
│ │ │ ├── fd: ()-->(21)
│ │ │ ├── distribution: ap-southeast-2
│ │ │ ├── scan json_arr2_rbt [as=t2]
│ │ │ │ ├── columns: t2.j:3
│ │ │ │ ├── stats: [rows=1000]
│ │ │ │ ├── cost: 1116.60333
│ │ │ │ ├── distribution: ap-southeast-2
│ │ │ │ ├── prune: (3)
│ │ │ │ └── unfiltered-cols: (1-6)
│ │ │ │ └── unfiltered-cols: (1-8)
│ │ │ └── projections
│ │ │ └── 'ap-southeast-2' [as="inverted_join_const_col_@11":17]
│ │ │ └── 'ap-southeast-2' [as="inverted_join_const_col_@13":21]
│ │ └── filters (true)
│ └── filters
│ └── t1.j:9 @> t2.j:3 [outer=(3,9), immutable]
│ └── t1.j:11 @> t2.j:3 [outer=(3,11), immutable]
└── 1

statement ok
Expand Down
Loading

0 comments on commit be52993

Please sign in to comment.