Skip to content

Commit

Permalink
Merge #59304
Browse files Browse the repository at this point in the history 
59304: config/sql: add global_reads field to ZoneConfig, use for GLOBAL tables r=ajstorm a=nvanbenschoten

Closes #57689.

This PR adds a new `global_reads` field to the `ZoneConfig` struct. `global_reads` specifies whether transactions operating over the range(s) should be configured to provide non-blocking behavior, meaning that reads can be served consistently from all replicas and do not block on writes. In exchange, writes get pushed into the future and must wait on commit to ensure linearizability. For more, see: https://github.com/cockroachdb/cockroach/blob/master/docs/RFCS/20200811_non_blocking_txns.md

The PR then enables `global_reads` for tables with the `GLOBAL` locality config. `global_reads` are not yet hooked up in KV because we don't yet have per-range closed timestamp tracking, but this completes the SQL-level work for GLOBAL tables, with the exception of #57663, which will add non-voting replicas (at the database zone config level) for all database regions that don't already have a voting replica.

Release note (sql change): A new, unused field called "global_reads" was added to zone configurations. The field does not yet have any effect.

Co-authored-by: Nathan VanBenschoten <[email protected]>
  • Loading branch information
@nvanbenschoten
craig[bot] and nvanbenschoten committed Jan 27, 2021
2 parents 6cbbc45 + 91fbbca commit 9a93daf
Show file tree
Hide file tree
Showing 14 changed files with 214 additions and 103 deletions.
5 changes: 5 additions & 0 deletions pkg/ccl/logictestccl/testdata/logic_test/alter_table_locality
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ TABLE global ALTER TABLE global CONFIGURE ZONE USING
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down Expand Up @@ -219,6 +220,7 @@ TABLE global ALTER TABLE global CONFIGURE ZONE USING
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down Expand Up @@ -337,6 +339,7 @@ TABLE regional_by_table_in_primary_region ALTER TABLE regional_by_table_in_prim
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down Expand Up @@ -450,6 +453,7 @@ TABLE regional_by_table_no_region ALTER TABLE regional_by_table_no_region CONFI
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down Expand Up @@ -555,6 +559,7 @@ TABLE regional_by_table_in_us_east ALTER TABLE regional_by_table_in_us_east CON
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down
6 changes: 6 additions & 0 deletions pkg/config/zonepb/zone.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -418,6 +418,12 @@ func (z *ZoneConfig) CopyFromZone(other ZoneConfig, fieldList []tree.Name) {
z.RangeMaxBytes = proto.Int64(*other.RangeMaxBytes)
}
}
if fieldName == "global_reads" {
z.GlobalReads = nil
if other.GlobalReads != nil {
z.GlobalReads = proto.Bool(*other.GlobalReads)
}
}
if fieldName == "gc.ttlseconds" {
z.GC = nil
if other.GC != nil {
Expand Down
220 changes: 138 additions & 82 deletions pkg/config/zonepb/zone.pb.go
View file

Large diffs are not rendered by default.

10 changes: 9 additions & 1 deletion pkg/config/zonepb/zone.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,15 @@ message ZoneConfig {
// in the zone config hierarchy, up to the default policy if necessary.
optional GCPolicy gc = 4 [(gogoproto.customname) = "GC"];

// NumReplicas specifies the desired number of replicas
// GlobalReads specifies whether transactions operating over the range(s)
// should be configured to provide non-blocking behavior, meaning that reads
// can be served consistently from all replicas and do not block on writes. In
// exchange, writes get pushed into the future and must wait on commit to
// ensure linearizability. For more, see:
// https://github.com/cockroachdb/cockroach/blob/master/docs/RFCS/20200811_non_blocking_txns.md
optional bool global_reads = 12 [(gogoproto.moretags) = "yaml:\"global_reads\""];

// NumReplicas specifies the desired number of replicas.
optional int32 num_replicas = 5 [(gogoproto.moretags) = "yaml:\"num_replicas\""];

// Constraints constrains which stores the replicas can be stored on. The
Expand Down
10 changes: 10 additions & 0 deletions pkg/config/zonepb/zone_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -394,6 +394,7 @@ func TestZoneConfigMarshalYAML(t *testing.T) {
GC: &GCPolicy{
TTLSeconds: 1,
},
GlobalReads: proto.Bool(true),
NumReplicas: proto.Int32(1),
}

Expand All @@ -407,6 +408,7 @@ func TestZoneConfigMarshalYAML(t *testing.T) {
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: []
lease_preferences: []
Expand All @@ -428,6 +430,7 @@ lease_preferences: []
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: [+duck=foo]
lease_preferences: []
Expand Down Expand Up @@ -458,6 +461,7 @@ lease_preferences: []
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: [foo, +duck=foo, -duck=foo]
lease_preferences: []
Expand All @@ -480,6 +484,7 @@ lease_preferences: []
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: {+duck=foo: 3}
lease_preferences: []
Expand Down Expand Up @@ -511,6 +516,7 @@ lease_preferences: []
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: {'foo,+duck=foo,-duck=foo': 3}
lease_preferences: []
Expand Down Expand Up @@ -548,6 +554,7 @@ lease_preferences: []
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: {'+duck=bar1,+duck=bar2': 1, +duck=foo: 2}
lease_preferences: []
Expand All @@ -559,6 +566,7 @@ lease_preferences: []
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: []
lease_preferences: []
Expand All @@ -580,6 +588,7 @@ lease_preferences: []
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: []
lease_preferences: [[+duck=foo]]
Expand Down Expand Up @@ -626,6 +635,7 @@ lease_preferences: [[+duck=foo]]
range_max_bytes: 1
gc:
ttlseconds: 1
global_reads: true
num_replicas: 1
constraints: [+duck=foo]
lease_preferences: [[+duck=bar1, +duck=bar2], [-duck=foo]]
Expand Down
7 changes: 7 additions & 0 deletions pkg/config/zonepb/zone_yaml.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ type marshalableZoneConfig struct {
RangeMinBytes *int64 `json:"range_min_bytes" yaml:"range_min_bytes"`
RangeMaxBytes *int64 `json:"range_max_bytes" yaml:"range_max_bytes"`
GC *GCPolicy `json:"gc"`
GlobalReads *bool `json:"global_reads" yaml:"global_reads"`
NumReplicas *int32 `json:"num_replicas" yaml:"num_replicas"`
Constraints ConstraintsList `json:"constraints" yaml:"constraints,flow"`
LeasePreferences []LeasePreference `json:"lease_preferences" yaml:"lease_preferences,flow"`
Expand All @@ -222,6 +223,9 @@ func zoneConfigToMarshalable(c ZoneConfig) marshalableZoneConfig {
tempGC := *c.GC
m.GC = &tempGC
}
if c.GlobalReads != nil {
m.GlobalReads = proto.Bool(*c.GlobalReads)
}
if c.NumReplicas != nil && *c.NumReplicas != 0 {
m.NumReplicas = proto.Int32(*c.NumReplicas)
}
Expand Down Expand Up @@ -250,6 +254,9 @@ func zoneConfigFromMarshalable(m marshalableZoneConfig, c ZoneConfig) ZoneConfig
tempGC := *m.GC
c.GC = &tempGC
}
if m.GlobalReads != nil {
c.GlobalReads = proto.Bool(*m.GlobalReads)
}
if m.NumReplicas != nil {
c.NumReplicas = proto.Int32(*m.NumReplicas)
}
Expand Down
4 changes: 4 additions & 0 deletions pkg/kv/kvserver/replica.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -678,6 +678,10 @@ func (r *Replica) SetZoneConfig(zone *zonepb.ZoneConfig) {

r.mu.largestPreviousMaxRangeSizeBytes = 0
}

// TODO(nvanbenschoten): use the new zone.GlobalReads field to configure
// closed timestamp lead/lag, once we have per-range closed timestamp
// trackers.
}
r.mu.zone = zone
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/sql/logictest/testdata/logic_test/crdb_internal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ SELECT zone_id, target FROM crdb_internal.zones ORDER BY 1
query T
SELECT quote_literal(raw_config_yaml) FROM crdb_internal.zones WHERE zone_id = 0
----
e'range_min_bytes: 134217728\nrange_max_bytes: 536870912\ngc:\n ttlseconds: 90000\nnum_replicas: 3\nconstraints: []\nlease_preferences: []\n'
e'range_min_bytes: 134217728\nrange_max_bytes: 536870912\ngc:\n ttlseconds: 90000\nglobal_reads: null\nnum_replicas: 3\nconstraints: []\nlease_preferences: []\n'

query T
SELECT raw_config_sql FROM crdb_internal.zones WHERE zone_id = 0
Expand Down
4 changes: 4 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/multiregion
View file
Original file line number Diff line number Diff line change
Expand Up @@ -307,6 +307,7 @@ TABLE global_table ALTER TABLE global_table CONFIGURE ZONE USING
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down Expand Up @@ -582,6 +583,7 @@ TABLE t_global ALTER TABLE t_global CONFIGURE ZONE USING
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down Expand Up @@ -663,6 +665,7 @@ TABLE t_global ALTER TABLE t_global CONFIGURE ZONE USING
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down Expand Up @@ -716,6 +719,7 @@ TABLE t_global ALTER TABLE t_global CONFIGURE ZONE USING
range_min_bytes = 134217728,
range_max_bytes = 536870912,
gc.ttlseconds = 90000,
global_reads = true,
num_replicas = 3,
constraints = '{+region=ap-southeast-2: 1, +region=ca-central-1: 1, +region=us-east-1: 1}',
lease_preferences = '[[+region=ca-central-1]]'
Expand Down
6 changes: 6 additions & 0 deletions pkg/sql/logictest/testdata/logic_test/zone_config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ ALTER TABLE a CONFIGURE ZONE USING
range_min_bytes = 200000 + 1,
range_max_bytes = 300000 + 1,
gc.ttlseconds = 3000 + 600,
global_reads = true,
num_replicas = floor(1.2)::int,
constraints = '[+region=test]',
lease_preferences = '[[+region=test]]'
Expand All @@ -86,12 +87,14 @@ WHERE feature_name IN (
'sql.schema.zone_config.table.range_min_bytes',
'sql.schema.zone_config.table.range_max_bytes',
'sql.schema.zone_config.table.gc.ttlseconds',
'sql.schema.zone_config.table.global_reads',
'sql.schema.zone_config.table.num_replicas',
'sql.schema.zone_config.table.constraints'
) AND usage_count > 0 ORDER BY feature_name
----
sql.schema.zone_config.table.constraints
sql.schema.zone_config.table.gc.ttlseconds
sql.schema.zone_config.table.global_reads
sql.schema.zone_config.table.num_replicas
sql.schema.zone_config.table.range_max_bytes
sql.schema.zone_config.table.range_min_bytes
Expand All @@ -103,6 +106,7 @@ SELECT zone_id, raw_config_sql FROM [SHOW ZONE CONFIGURATION FOR TABLE a]
range_min_bytes = 200001,
range_max_bytes = 300001,
gc.ttlseconds = 3600,
global_reads = true,
num_replicas = 1,
constraints = '[+region=test]',
lease_preferences = '[[+region=test]]'
Expand All @@ -118,6 +122,7 @@ SELECT zone_id, raw_config_sql FROM [SHOW ZONE CONFIGURATION FOR TABLE a]
range_min_bytes = 200001,
range_max_bytes = 400000,
gc.ttlseconds = 3600,
global_reads = true,
num_replicas = 1,
constraints = '[+region=test]',
lease_preferences = '[[+region=test]]'
Expand Down Expand Up @@ -154,6 +159,7 @@ SELECT zone_id, raw_config_sql FROM [SHOW ZONE CONFIGURATION FOR TABLE a]
range_min_bytes = 200001,
range_max_bytes = 400000,
gc.ttlseconds = 3600,
global_reads = true,
num_replicas = 1,
constraints = '[+region=test]',
lease_preferences = '[[+region=test]]'
Expand Down
4 changes: 3 additions & 1 deletion pkg/sql/region_util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,9 @@ func zoneConfigFromTableLocalityConfig(

switch l := localityConfig.Locality.(type) {
case *descpb.TableDescriptor_LocalityConfig_Global_:
// Inherit everything from the database.
// Enable non-blocking transactions.
ret.GlobalReads = proto.Bool(true)
// Inherit constraints and leaseholders from the database.
ret.InheritedConstraints = true
ret.InheritedLeasePreferences = true
case *descpb.TableDescriptor_LocalityConfig_RegionalByTable_:
Expand Down
4 changes: 3 additions & 1 deletion pkg/sql/region_util_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,7 @@ func TestZoneConfigFromTableLocalityConfig(t *testing.T) {
SurvivalGoal: descpb.SurvivalGoal_ZONE_FAILURE,
},
expected: &zonepb.ZoneConfig{
GlobalReads: proto.Bool(true),
NumReplicas: proto.Int32(4),
InheritedConstraints: true,
InheritedLeasePreferences: true,
Expand All @@ -228,6 +229,7 @@ func TestZoneConfigFromTableLocalityConfig(t *testing.T) {
SurvivalGoal: descpb.SurvivalGoal_REGION_FAILURE,
},
expected: &zonepb.ZoneConfig{
GlobalReads: proto.Bool(true),
NumReplicas: proto.Int32(4),
InheritedConstraints: true,
InheritedLeasePreferences: true,
Expand Down Expand Up @@ -434,7 +436,7 @@ func TestZoneConfigFromRegionConfigForPartition(t *testing.T) {
},
},
{
desc: "4-region global table with region survivability",
desc: "4-region table with region survivability",
region: descpb.DatabaseDescriptor_RegionConfig_Region{
Name: "region_a",
},
Expand Down
1 change: 1 addition & 0 deletions pkg/sql/set_zone_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ var supportedZoneConfigOptions = map[tree.Name]struct {
"range_min_bytes": {types.Int, func(c *zonepb.ZoneConfig, d tree.Datum) { c.RangeMinBytes = proto.Int64(int64(tree.MustBeDInt(d))) }},
"range_max_bytes": {types.Int, func(c *zonepb.ZoneConfig, d tree.Datum) { c.RangeMaxBytes = proto.Int64(int64(tree.MustBeDInt(d))) }},
"num_replicas": {types.Int, func(c *zonepb.ZoneConfig, d tree.Datum) { c.NumReplicas = proto.Int32(int32(tree.MustBeDInt(d))) }},
"global_reads": {types.Bool, func(c *zonepb.ZoneConfig, d tree.Datum) { c.GlobalReads = proto.Bool(bool(tree.MustBeDBool(d))) }},
"gc.ttlseconds": {types.Int, func(c *zonepb.ZoneConfig, d tree.Datum) {
c.GC = &zonepb.GCPolicy{TTLSeconds: int32(tree.MustBeDInt(d))}
}},
Expand Down
34 changes: 17 additions & 17 deletions pkg/sql/show_zone_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,36 +184,43 @@ func zoneConfigToSQL(zs *tree.ZoneSpecifier, zone *zonepb.ZoneConfig) (string, e
prefs = strings.TrimSpace(prefs)

useComma := false
maybeWriteComma := func(f *tree.FmtCtx) {
if useComma {
f.Printf(",\n")
}
useComma = true
}

f := tree.NewFmtCtx(tree.FmtParsable)
f.WriteString("ALTER ")
f.FormatNode(zs)
f.WriteString(" CONFIGURE ZONE USING\n")
if zone.RangeMinBytes != nil {
maybeWriteComma(f)
f.Printf("\trange_min_bytes = %d", *zone.RangeMinBytes)
useComma = true
}
if zone.RangeMaxBytes != nil {
writeComma(f, useComma)
maybeWriteComma(f)
f.Printf("\trange_max_bytes = %d", *zone.RangeMaxBytes)
useComma = true
}
if zone.GC != nil {
writeComma(f, useComma)
maybeWriteComma(f)
f.Printf("\tgc.ttlseconds = %d", zone.GC.TTLSeconds)
useComma = true
}
if zone.GlobalReads != nil {
maybeWriteComma(f)
f.Printf("\tglobal_reads = %t", *zone.GlobalReads)
}
if zone.NumReplicas != nil {
writeComma(f, useComma)
maybeWriteComma(f)
f.Printf("\tnum_replicas = %d", *zone.NumReplicas)
useComma = true
}
if !zone.InheritedConstraints {
writeComma(f, useComma)
maybeWriteComma(f)
f.Printf("\tconstraints = %s", lex.EscapeSQLString(constraints))
useComma = true
}
if !zone.InheritedLeasePreferences {
writeComma(f, useComma)
maybeWriteComma(f)
f.Printf("\tlease_preferences = %s", lex.EscapeSQLString(prefs))
}
return f.String(), nil
Expand Down Expand Up @@ -317,13 +324,6 @@ func generateZoneConfigIntrospectionValues(
return nil
}

// Writes a comma followed by a newline if useComma is true.
func writeComma(f *tree.FmtCtx, useComma bool) {
if useComma {
f.Printf(",\n")
}
}

func yamlMarshalFlow(v interface{}) (string, error) {
var buf bytes.Buffer
e := yaml.NewEncoder(&buf)
Expand Down

0 comments on commit 9a93daf

Please sign in to comment.