From 59f5cb92dd7681bc9a044c392610117afe8a9ea2 Mon Sep 17 00:00:00 2001 From: Rafi Shamim Date: Thu, 11 Jul 2024 17:15:10 -0400 Subject: [PATCH] bootstrap: create an explicit zoneconfig for timeseries data This will make it more clear that the timeseries zone config can be changed independently from all the other zone configs. Release note: None --- .../testdata/logic_test/crdb_internal | 8 +++ pkg/ccl/logictestccl/testdata/logic_test/zone | 55 ++++++++++--------- .../full_translate_named_zones_deleted | 1 + .../testdata/named_zones | 7 +-- pkg/cmd/roachtest/tests/gossip.go | 1 + pkg/kv/kvserver/replicate_queue_test.go | 1 + pkg/kv/kvserver/reports/reporter_test.go | 6 +- pkg/sql/catalog/bootstrap/metadata.go | 5 ++ pkg/sql/catalog/bootstrap/testdata/testdata | 3 +- .../internal/catkv/testdata/testdata_system | 2 + .../testdata/logic_test/crdb_internal | 1 - .../logic_test/zone_config_system_tenant | 1 + pkg/sql/tests/system_table_test.go | 2 +- pkg/sql/tests/testdata/initial_keys | 3 +- 14 files changed, 60 insertions(+), 36 deletions(-) diff --git a/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal b/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal index 02074c148e72..c764ce53581f 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal +++ b/pkg/ccl/logictestccl/testdata/logic_test/crdb_internal @@ -72,6 +72,7 @@ SELECT zone_id, target FROM crdb_internal.zones ORDER BY 1 0 RANGE default 16 RANGE meta 17 RANGE system +18 RANGE timeseries 22 RANGE liveness 111 TABLE test.public.t4 111 INDEX test.public.t4@myt4index @@ -109,6 +110,13 @@ RANGE system ALTER RANGE system CONFIGURE ZONE USING num_replicas = 5, constraints = '[]', lease_preferences = '[]' +RANGE timeseries ALTER RANGE timeseries CONFIGURE ZONE USING + range_min_bytes = 134217728, + range_max_bytes = 536870912, + gc.ttlseconds = 14400, + num_replicas = 3, + constraints = '[]', + lease_preferences = '[]' TABLE test.public.t4 ALTER TABLE test.public.t4 CONFIGURE ZONE USING num_replicas = 7 diff --git a/pkg/ccl/logictestccl/testdata/logic_test/zone b/pkg/ccl/logictestccl/testdata/logic_test/zone index 5ebf9e9520f9..e1ed1bbfa546 100644 --- a/pkg/ccl/logictestccl/testdata/logic_test/zone +++ b/pkg/ccl/logictestccl/testdata/logic_test/zone @@ -616,33 +616,34 @@ SELECT target, range_name, database_name, table_name, index_name, partition_name FROM crdb_internal.zones ORDER BY target ---- -DATABASE "my database" NULL my database NULL NULL NULL -DATABASE system NULL system NULL NULL NULL -INDEX "my database".public."my table"@"my index" NULL my database my table my index NULL -INDEX test.public.t36642@secondary NULL test t36642 secondary NULL -INDEX test.public.t36642@tertiary NULL test t36642 tertiary NULL -INDEX test.public.t36644@secondary NULL test t36644 secondary NULL -INDEX test.public.t36644@tertiary NULL test t36644 tertiary NULL -INDEX test.public.t@secondary NULL test t secondary NULL -INDEX test.public.t@tertiary NULL test t tertiary NULL -PARTITION "my partition" OF INDEX "my database".public."my table"@"my index" NULL my database my table my index my partition -PARTITION "my partition" OF INDEX "my database".public."my table"@"my table_pkey" NULL my database my table my table_pkey my partition -PARTITION x1_idx OF INDEX test.public.t38391@foo NULL test t38391 foo x1_idx -RANGE default default NULL NULL NULL NULL -RANGE liveness liveness NULL NULL NULL NULL -RANGE meta meta NULL NULL NULL NULL -RANGE system system NULL NULL NULL NULL -TABLE "my database".public."my table" NULL my database my table NULL NULL -TABLE system.public.lease NULL system lease NULL NULL -TABLE system.public.replication_constraint_stats NULL system replication_constraint_stats NULL NULL -TABLE system.public.replication_stats NULL system replication_stats NULL NULL -TABLE system.public.span_stats_tenant_boundaries NULL system span_stats_tenant_boundaries NULL NULL -TABLE system.public.statement_activity NULL system statement_activity NULL NULL -TABLE system.public.statement_statistics NULL system statement_statistics NULL NULL -TABLE system.public.tenant_usage NULL system tenant_usage NULL NULL -TABLE system.public.transaction_activity NULL system transaction_activity NULL NULL -TABLE system.public.transaction_statistics NULL system transaction_statistics NULL NULL -TABLE test.public.t NULL test t NULL NULL +DATABASE "my database" NULL my database NULL NULL NULL +DATABASE system NULL system NULL NULL NULL +INDEX "my database".public."my table"@"my index" NULL my database my table my index NULL +INDEX test.public.t36642@secondary NULL test t36642 secondary NULL +INDEX test.public.t36642@tertiary NULL test t36642 tertiary NULL +INDEX test.public.t36644@secondary NULL test t36644 secondary NULL +INDEX test.public.t36644@tertiary NULL test t36644 tertiary NULL +INDEX test.public.t@secondary NULL test t secondary NULL +INDEX test.public.t@tertiary NULL test t tertiary NULL +PARTITION "my partition" OF INDEX "my database".public."my table"@"my index" NULL my database my table my index my partition +PARTITION "my partition" OF INDEX "my database".public."my table"@"my table_pkey" NULL my database my table my table_pkey my partition +PARTITION x1_idx OF INDEX test.public.t38391@foo NULL test t38391 foo x1_idx +RANGE default default NULL NULL NULL NULL +RANGE liveness liveness NULL NULL NULL NULL +RANGE meta meta NULL NULL NULL NULL +RANGE system system NULL NULL NULL NULL +RANGE timeseries timeseries NULL NULL NULL NULL +TABLE "my database".public."my table" NULL my database my table NULL NULL +TABLE system.public.lease NULL system lease NULL NULL +TABLE system.public.replication_constraint_stats NULL system replication_constraint_stats NULL NULL +TABLE system.public.replication_stats NULL system replication_stats NULL NULL +TABLE system.public.span_stats_tenant_boundaries NULL system span_stats_tenant_boundaries NULL NULL +TABLE system.public.statement_activity NULL system statement_activity NULL NULL +TABLE system.public.statement_statistics NULL system statement_statistics NULL NULL +TABLE system.public.tenant_usage NULL system tenant_usage NULL NULL +TABLE system.public.transaction_activity NULL system transaction_activity NULL NULL +TABLE system.public.transaction_statistics NULL system transaction_statistics NULL NULL +TABLE test.public.t NULL test t NULL NULL # Test the zone information being displayed in SHOW CREATE statement ok diff --git a/pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/full_translate_named_zones_deleted b/pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/full_translate_named_zones_deleted index f6fccf3b144f..dc1446d9e896 100644 --- a/pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/full_translate_named_zones_deleted +++ b/pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/full_translate_named_zones_deleted @@ -12,6 +12,7 @@ SELECT id FROM system.zones 11 16 17 +18 22 25 27 diff --git a/pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/named_zones b/pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/named_zones index cd3ab2fb49f4..a971d9c64294 100644 --- a/pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/named_zones +++ b/pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/testdata/named_zones @@ -3,13 +3,12 @@ # RANGE DEFAULT is akin to a full reconciliation. -# ID 18 is for the time series range which is the only named zone that doesn't -# have an entry in `system.zones` at bootstrap. It should inherit from RANGE -# DEFAULT. +# ID 18 is for the time series range. At bootstrap, it has an explicit zone +# config which should inherit from RANGE DEFAULT. query-sql SELECT count(*) FROM system.zones WHERE id=18 ---- -0 +1 translate named-zone=timeseries ---- diff --git a/pkg/cmd/roachtest/tests/gossip.go b/pkg/cmd/roachtest/tests/gossip.go index d90f688af99d..0bb17abb156a 100644 --- a/pkg/cmd/roachtest/tests/gossip.go +++ b/pkg/cmd/roachtest/tests/gossip.go @@ -400,6 +400,7 @@ func runGossipRestartNodeOne(ctx context.Context, t test.Test, c cluster.Cluster run(`ALTER DATABASE system %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`) run(`ALTER RANGE meta %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`) run(`ALTER RANGE liveness %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`) + run(`ALTER RANGE timeseries %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`) if t.IsBuildVersion("v19.2.0") { run(`ALTER TABLE system.replication_stats %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`) run(`ALTER TABLE system.replication_constraint_stats %[1]s CONFIGURE ZONE %[2]s 'constraints: {"-rack=0"}'`) diff --git a/pkg/kv/kvserver/replicate_queue_test.go b/pkg/kv/kvserver/replicate_queue_test.go index 1f1734754a0f..64eb4ce555aa 100644 --- a/pkg/kv/kvserver/replicate_queue_test.go +++ b/pkg/kv/kvserver/replicate_queue_test.go @@ -2167,6 +2167,7 @@ func TestPromoteNonVoterInAddVoter(t *testing.T) { setConstraintFn("RANGE system", 7, 7, "") setConstraintFn("RANGE liveness", 7, 7, "") setConstraintFn("RANGE meta", 7, 7, "") + setConstraintFn("RANGE timeseries", 7, 7, "") setConstraintFn("RANGE default", 7, 7, "") testutils.SucceedsSoon(t, func() error { if err := forceScanOnAllReplicationQueues(tc); err != nil { diff --git a/pkg/kv/kvserver/reports/reporter_test.go b/pkg/kv/kvserver/reports/reporter_test.go index c64f010cb1a5..74b7491685f0 100644 --- a/pkg/kv/kvserver/reports/reporter_test.go +++ b/pkg/kv/kvserver/reports/reporter_test.go @@ -196,9 +196,13 @@ func TestCriticalLocalitiesReportIntegration(t *testing.T) { } require.Greater(t, len(systemZoneIDs), 0, "expected some system zones, got none") // Remove the entries in systemZoneIDs that don't get critical locality reports. + // The default range (zoneID=0) does get reports, but since there is no system + // data in the default range that is not part of a more granular zone (e.g., + // the meta, liveness, timeseries, or system range), there won't be a report + // for it. i := 0 for j, zid := range systemZoneIDs { - if zoneChangesReplication(&systemZones[j]) { + if zoneChangesReplication(&systemZones[j]) && zid != keys.RootNamespaceID { systemZoneIDs[i] = zid i++ } diff --git a/pkg/sql/catalog/bootstrap/metadata.go b/pkg/sql/catalog/bootstrap/metadata.go index 1d461a306985..3b5de92eb52a 100644 --- a/pkg/sql/catalog/bootstrap/metadata.go +++ b/pkg/sql/catalog/bootstrap/metadata.go @@ -511,6 +511,10 @@ func InitialZoneConfigKVs( metaRangeZoneConf := protoutil.Clone(defaultSystemZoneConfig).(*zonepb.ZoneConfig) livenessZoneConf := protoutil.Clone(defaultSystemZoneConfig).(*zonepb.ZoneConfig) + // The timeseries zone is the same as the default zone, but we create it + // explicitly here so it's more clearly reported with SHOW ALL ZONE CONFIGURATIONS. + timeseriesZoneConf := protoutil.Clone(defaultZoneConfig).(*zonepb.ZoneConfig) + // .meta zone config entry with a shorter GC time. metaRangeZoneConf.GC.TTLSeconds = 60 * 60 // 1h @@ -534,6 +538,7 @@ func InitialZoneConfigKVs( add(keys.MetaRangesID, metaRangeZoneConf) add(keys.LivenessRangesID, livenessZoneConf) add(keys.SystemRangesID, systemZoneConf) + add(keys.TimeseriesRangesID, timeseriesZoneConf) add(keys.SystemDatabaseID, systemZoneConf) add(keys.ReplicationConstraintStatsTableID, replicationConstraintStatsZoneConf) add(keys.ReplicationStatsTableID, replicationStatsZoneConf) diff --git a/pkg/sql/catalog/bootstrap/testdata/testdata b/pkg/sql/catalog/bootstrap/testdata/testdata index f48d2cf50373..57e2a48f48b7 100644 --- a/pkg/sql/catalog/bootstrap/testdata/testdata +++ b/pkg/sql/catalog/bootstrap/testdata/testdata @@ -1,4 +1,4 @@ -system hash=ccfcf37e550c9bceebe02e444e7b9269f7565df315eccde68b1ff384e742c8b8 +system hash=4f192396c59085346e3e43f000323964a38c038f3e4b28bab0c09d92bb0ae6eb ---- [{"key":"8b"} ,{"key":"8b89898a89","value":"0312450a0673797374656d10011a250a0d0a0561646d696e1080101880100a0c0a04726f6f7410801018801012046e6f646518032200280140004a006a0a08d8843d10011800200a"} @@ -65,6 +65,7 @@ system hash=ccfcf37e550c9bceebe02e444e7b9269f7565df315eccde68b1ff384e742c8b8 ,{"key":"8d89938a89","value":"03220308d804500158017800"} ,{"key":"8d89988a89","value":"031080808040188080808002220308901c2805500058007801"} ,{"key":"8d89998a89","value":"031080808040188080808002220308c0702805500058007801"} +,{"key":"8d899a8a89","value":"031080808040188080808002220308c0702803500058007801"} ,{"key":"8d899e8a89","value":"031080808040188080808002220308d8042805500058007801"} ,{"key":"8d89a18a89","value":"03220308d804500158017800"} ,{"key":"8d89a38a89","value":"03220308d804500158017800"} diff --git a/pkg/sql/catalog/internal/catkv/testdata/testdata_system b/pkg/sql/catalog/internal/catkv/testdata/testdata_system index 2dd11cd354bb..1ee37bf844bc 100644 --- a/pkg/sql/catalog/internal/catkv/testdata/testdata_system +++ b/pkg/sql/catalog/internal/catkv/testdata/testdata_system @@ -247,6 +247,8 @@ catalog: zone: gc.ttlseconds=3600 "017": zone: gc.ttlseconds=14400 + "018": + zone: gc.ttlseconds=14400 "019": descriptor: relation namespace: (1, 29, "web_sessions") diff --git a/pkg/sql/logictest/testdata/logic_test/crdb_internal b/pkg/sql/logictest/testdata/logic_test/crdb_internal index 73e737ce2e16..a51dd026be18 100644 --- a/pkg/sql/logictest/testdata/logic_test/crdb_internal +++ b/pkg/sql/logictest/testdata/logic_test/crdb_internal @@ -567,7 +567,6 @@ SELECT 'schema.bar'::regclass::int statement ok INSERT INTO system.zones (id, config) VALUES - (18, (SELECT raw_config_protobuf FROM crdb_internal.zones WHERE zone_id = 0)), ($testdb_id, (SELECT raw_config_protobuf FROM crdb_internal.zones WHERE zone_id = 0)), ($testdb_foo_id, (SELECT raw_config_protobuf FROM crdb_internal.zones WHERE zone_id = 0)), ($schema_bar_id, (SELECT raw_config_protobuf FROM crdb_internal.zones WHERE zone_id = 0)) diff --git a/pkg/sql/logictest/testdata/logic_test/zone_config_system_tenant b/pkg/sql/logictest/testdata/logic_test/zone_config_system_tenant index 29312a6a501c..14747a5d08d0 100644 --- a/pkg/sql/logictest/testdata/logic_test/zone_config_system_tenant +++ b/pkg/sql/logictest/testdata/logic_test/zone_config_system_tenant @@ -30,6 +30,7 @@ SELECT zone_id, target FROM crdb_internal.zones ORDER BY 1 11 TABLE system.public.lease 16 RANGE meta 17 RANGE system +18 RANGE timeseries 22 RANGE liveness 25 TABLE system.public.replication_constraint_stats 27 TABLE system.public.replication_stats diff --git a/pkg/sql/tests/system_table_test.go b/pkg/sql/tests/system_table_test.go index 0d01db593701..075bff3b7677 100644 --- a/pkg/sql/tests/system_table_test.go +++ b/pkg/sql/tests/system_table_test.go @@ -53,7 +53,7 @@ func TestInitialKeys(t *testing.T) { var nonDescKeys int if systemTenant { codec = keys.SystemSQLCodec - nonDescKeys = 16 + nonDescKeys = 17 } else { codec = keys.MakeSQLCodec(roachpb.MustMakeTenantID(5)) nonDescKeys = 8 diff --git a/pkg/sql/tests/testdata/initial_keys b/pkg/sql/tests/testdata/initial_keys index e9ccf3e3a787..a7d1396c85eb 100644 --- a/pkg/sql/tests/testdata/initial_keys +++ b/pkg/sql/tests/testdata/initial_keys @@ -1,6 +1,6 @@ initial-keys tenant=system ---- -130 keys: +131 keys: /Table/3/1/1/2/1 /Table/3/1/3/2/1 /Table/3/1/4/2/1 @@ -63,6 +63,7 @@ initial-keys tenant=system /Table/5/1/11/2/1 /Table/5/1/16/2/1 /Table/5/1/17/2/1 + /Table/5/1/18/2/1 /Table/5/1/22/2/1 /Table/5/1/25/2/1 /Table/5/1/27/2/1