Skip to content

Commit

Permalink
spanconfigsqltranslator: populate protected_timestamps in SpanConfig
Browse files Browse the repository at this point in the history
This change teaches the SQLTranslator to hydrate the SpanConfigs
for a table with protected timestamps that apply to that table.

Concretely, this change initializes a spanconfig.ProtectedTimestampStateReader
in the txn in which the translation is taking place, thereby
providing a transactional view of the system.protected_ts_records
table. After generating the span configurations based on the zone
configurations that apply to the table, we hydrate the newly
introduced protected_timestamps field on each span configuration
with all the protected timestamps that apply to this table. This
includes protected timestamp records that directly target this
table, as well as records targetting the table's parent database.
This information is obtained from the ProtectedTimestampStateReader
mentioned above.

Additionally, this change modifies StartTenant to allow secondary
tenants to interact with the protected timestamp subsystem using a
"real" protectedts.Provider provided the migration
EnableProtectedTimestampsForTenant has run.

For testing purposes, this change teaches the data driven framework
of two additional commands protect and release.

Informs: #73727

Release note: None
  • Loading branch information
adityamaru committed Jan 20, 2022
1 parent fb72381 commit 2e87a4b
Show file tree
Hide file tree
Showing 18 changed files with 407 additions and 36 deletions.
1 change: 1 addition & 0 deletions pkg/ccl/spanconfigccl/spanconfigcomparedccl/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ go_test(
"//pkg/ccl/utilccl",
"//pkg/gossip",
"//pkg/jobs",
"//pkg/kv/kvserver/protectedts",
"//pkg/roachpb:with-mocks",
"//pkg/security",
"//pkg/security/securitytest",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
_ "github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl"
"github.com/cockroachdb/cockroach/pkg/gossip"
"github.com/cockroachdb/cockroach/pkg/jobs"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/spanconfig"
"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigtestutils"
Expand Down Expand Up @@ -109,7 +110,7 @@ func TestDataDriven(t *testing.T) {
tdb.Exec(t, `SET CLUSTER SETTING kv.closed_timestamp.target_duration = '100ms'`)
}

spanConfigTestCluster := spanconfigtestcluster.NewHandle(t, tc, scKnobs)
spanConfigTestCluster := spanconfigtestcluster.NewHandle(t, tc, scKnobs, &protectedts.TestingKnobs{})
defer spanConfigTestCluster.Cleanup()

kvSubscriber := tc.Server(0).SpanConfigKVSubscriber().(spanconfig.KVSubscriber)
Expand Down
1 change: 1 addition & 0 deletions pkg/ccl/spanconfigccl/spanconfigreconcilerccl/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ go_test(
"//pkg/ccl/utilccl",
"//pkg/jobs",
"//pkg/keys",
"//pkg/kv/kvserver/protectedts",
"//pkg/roachpb:with-mocks",
"//pkg/security",
"//pkg/security/securitytest",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
_ "github.com/cockroachdb/cockroach/pkg/ccl/partitionccl"
"github.com/cockroachdb/cockroach/pkg/jobs"
"github.com/cockroachdb/cockroach/pkg/keys"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/spanconfig"
"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigtestutils"
Expand Down Expand Up @@ -102,7 +103,7 @@ func TestDataDriven(t *testing.T) {
tdb.Exec(t, `SET CLUSTER SETTING kv.closed_timestamp.target_duration = '100ms'`)
}

spanConfigTestCluster := spanconfigtestcluster.NewHandle(t, tc, scKnobs)
spanConfigTestCluster := spanconfigtestcluster.NewHandle(t, tc, scKnobs, &protectedts.TestingKnobs{})
defer spanConfigTestCluster.Cleanup()

systemTenant := spanConfigTestCluster.InitializeTenant(ctx, roachpb.SystemTenantID)
Expand Down
9 changes: 9 additions & 0 deletions pkg/ccl/spanconfigccl/spanconfigsqltranslatorccl/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -13,22 +13,31 @@ go_test(
"//pkg/ccl/partitionccl",
"//pkg/ccl/utilccl",
"//pkg/config/zonepb",
"//pkg/jobs",
"//pkg/jobs/jobsprotectedts",
"//pkg/kv",
"//pkg/kv/kvserver/protectedts",
"//pkg/kv/kvserver/protectedts/ptpb",
"//pkg/roachpb:with-mocks",
"//pkg/security",
"//pkg/security/securitytest",
"//pkg/server",
"//pkg/spanconfig",
"//pkg/spanconfig/spanconfigtestutils",
"//pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster",
"//pkg/sql",
"//pkg/sql/catalog/descpb",
"//pkg/sql/catalog/tabledesc",
"//pkg/sql/distsql",
"//pkg/testutils",
"//pkg/testutils/serverutils",
"//pkg/testutils/sqlutils",
"//pkg/testutils/testcluster",
"//pkg/util/hlc",
"//pkg/util/leaktest",
"//pkg/util/log",
"//pkg/util/randutil",
"//pkg/util/uuid",
"@com_github_cockroachdb_datadriven//:datadriven",
"@com_github_stretchr_testify//require",
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,26 @@ import (
_ "github.com/cockroachdb/cockroach/pkg/ccl/kvccl/kvtenantccl"
_ "github.com/cockroachdb/cockroach/pkg/ccl/partitionccl"
"github.com/cockroachdb/cockroach/pkg/config/zonepb"
"github.com/cockroachdb/cockroach/pkg/jobs"
"github.com/cockroachdb/cockroach/pkg/jobs/jobsprotectedts"
"github.com/cockroachdb/cockroach/pkg/kv"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts"
"github.com/cockroachdb/cockroach/pkg/kv/kvserver/protectedts/ptpb"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/spanconfig"
"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigtestutils"
"github.com/cockroachdb/cockroach/pkg/spanconfig/spanconfigtestutils/spanconfigtestcluster"
"github.com/cockroachdb/cockroach/pkg/sql"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/descpb"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/tabledesc"
"github.com/cockroachdb/cockroach/pkg/sql/distsql"
"github.com/cockroachdb/cockroach/pkg/testutils"
"github.com/cockroachdb/cockroach/pkg/testutils/sqlutils"
"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/leaktest"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/uuid"
"github.com/cockroachdb/datadriven"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -61,6 +70,15 @@ import (
// - "mark-table-public" [database=<str>] [table=<str>]
// Marks the given table as public.
//
// - "protect" [id=<int>] [ts=<int>]
// cluster OR
// tenants id1,id2... OR
// schemaObject id1,id2...
// Creates and writes a protected timestamp record with id and ts with an
// appropriate ptpb.Target.
//
// - "release" [id=<int>]
// Releases the protected timestamp record with id.
func TestDataDriven(t *testing.T) {
defer leaktest.AfterTest(t)()
defer log.Scope(t).Close(t)
Expand All @@ -77,17 +95,21 @@ func TestDataDriven(t *testing.T) {
// test cluster).
ManagerDisableJobCreation: true,
}
ptsKnobs := &protectedts.TestingKnobs{
EnableProtectedTimestampForMultiTenant: true,
}
datadriven.Walk(t, testutils.TestDataPath(t), func(t *testing.T, path string) {
tc := testcluster.StartTestCluster(t, 1, base.TestClusterArgs{
ServerArgs: base.TestServerArgs{
Knobs: base.TestingKnobs{
SpanConfig: scKnobs,
SpanConfig: scKnobs,
ProtectedTS: ptsKnobs,
},
},
})
defer tc.Stopper().Stop(ctx)

spanConfigTestCluster := spanconfigtestcluster.NewHandle(t, tc, scKnobs)
spanConfigTestCluster := spanconfigtestcluster.NewHandle(t, tc, scKnobs, ptsKnobs)
defer spanConfigTestCluster.Cleanup()

var tenant *spanconfigtestcluster.Tenant
Expand All @@ -98,6 +120,29 @@ func TestDataDriven(t *testing.T) {
tenant = spanConfigTestCluster.InitializeTenant(ctx, roachpb.SystemTenantID)
}

execCfg := tenant.ExecutorConfig().(sql.ExecutorConfig)
ptp := tenant.DistSQLServer().(*distsql.ServerImpl).ServerConfig.ProtectedTimestampProvider
jr := tenant.JobRegistry().(*jobs.Registry)

mkRecordAndProtect := func(recordID string, ts hlc.Timestamp, target *ptpb.Target) {
jobID := jr.MakeJobID()
require.NoError(t, execCfg.DB.Txn(ctx, func(ctx context.Context, txn *kv.Txn) (err error) {
recID, err := uuid.FromBytes([]byte(strings.Repeat(recordID, 16)))
require.NoError(t, err)
rec := jobsprotectedts.MakeRecord(recID, int64(jobID), ts,
nil /* deprecatedSpans */, jobsprotectedts.Jobs, target)
return ptp.Protect(ctx, txn, rec)
}))
}

releaseRecord := func(recordID string) {
require.NoError(t, execCfg.DB.Txn(ctx, func(ctx context.Context, txn *kv.Txn) error {
recID, err := uuid.FromBytes([]byte(strings.Repeat(recordID, 16)))
require.NoError(t, err)
return ptp.Release(ctx, txn, recID)
}))
}

datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
switch d.Cmd {
case "exec-sql":
Expand Down Expand Up @@ -182,7 +227,18 @@ func TestDataDriven(t *testing.T) {
tenant.WithMutableTableDescriptor(ctx, dbName, tbName, func(mutable *tabledesc.Mutable) {
mutable.SetPublic()
})

case "protect":
var recordID string
var protectTS int
d.ScanArgs(t, "id", &recordID)
d.ScanArgs(t, "ts", &protectTS)
target := spanconfigtestutils.ParseProtectionTarget(t, d.Input)
mkRecordAndProtect(recordID, hlc.Timestamp{WallTime: int64(protectTS)}, target)
return ""
case "release":
var recordID string
d.ScanArgs(t, "id", &recordID)
releaseRecord(recordID)
default:
t.Fatalf("unknown command: %s", d.Cmd)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
# Create a database with some tables and write protected timestamps on the
# tables and database. Check that span configurations are as we expect.

exec-sql
CREATE DATABASE db;
CREATE TABLE db.t1(id INT);
CREATE TABLE db.t2();
----

# Scheme object IDs
# db: 54
# t1: 56
# t2: 57

# Alter zone config fields on the database and one of the tables to ensure
# things are cascading.
exec-sql
ALTER DATABASE db CONFIGURE ZONE USING num_replicas=7;
ALTER TABLE db.t1 CONFIGURE ZONE USING num_voters=5;
----

# Write a protected timestamp on t1.
protect id=1 ts=1
schemaObject 56
----

translate database=db
----
/Table/5{6-7} num_replicas=7 num_voters=5 pts=[1]
/Table/5{7-8} num_replicas=7

# Write a protected timestamp on db, so we should see it on both t1 and t2.
protect id=2 ts=2
schemaObject 54
----

translate database=db
----
/Table/5{6-7} num_replicas=7 num_voters=5 pts=[1 2]
/Table/5{7-8} num_replicas=7 pts=[2]

# Release the protected timestamp on table t1
release id=1
----

translate database=db
----
/Table/5{6-7} num_replicas=7 num_voters=5 pts=[2]
/Table/5{7-8} num_replicas=7 pts=[2]

# Release the protected timestamp on database db
release id=2
----

translate database=db
----
/Table/5{6-7} num_replicas=7 num_voters=5
/Table/5{7-8} num_replicas=7

# Create an index on t1 to ensure that subzones also see protected timestamps.
exec-sql
CREATE INDEX idx ON db.t1(id);
ALTER INDEX db.t1@idx CONFIGURE ZONE USING gc.ttlseconds = 1;
----

protect id=3 ts=3
schemaObject 56
----

translate database=db
----
/Table/56{-/2} num_replicas=7 num_voters=5 pts=[3]
/Table/56/{2-3} ttl_seconds=1 num_replicas=7 num_voters=5 pts=[3]
/Table/5{6/3-7} num_replicas=7 num_voters=5 pts=[3]
/Table/5{7-8} num_replicas=7

Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
# Create a database with some tables and write protected timestamps on the
# tables and database. Check that span configurations are as we expect.

exec-sql
CREATE DATABASE db;
CREATE TABLE db.t1(id INT);
CREATE TABLE db.t2();
----

# Scheme object IDs
# db: 54
# t1: 56
# t2: 57

# Alter zone config fields on the database and one of the tables to ensure
# things are cascading.
exec-sql
ALTER DATABASE db CONFIGURE ZONE USING num_replicas=7;
ALTER TABLE db.t1 CONFIGURE ZONE USING num_voters=5;
----

# Write a protected timestamp on t1.
protect id=1 ts=1
schemaObject 56
----

translate database=db
----
/Tenant/10/Table/5{6-7} num_replicas=7 num_voters=5 pts=[1]
/Tenant/10/Table/5{7-8} num_replicas=7

# Write a protected timestamp on db, so we should see it on both t1 and t2.
protect id=2 ts=2
schemaObject 54
----

translate database=db
----
/Tenant/10/Table/5{6-7} num_replicas=7 num_voters=5 pts=[1 2]
/Tenant/10/Table/5{7-8} num_replicas=7 pts=[2]

# Release the protected timestamp on table t1
release id=1
----

translate database=db
----
/Tenant/10/Table/5{6-7} num_replicas=7 num_voters=5 pts=[2]
/Tenant/10/Table/5{7-8} num_replicas=7 pts=[2]

# Release the protected timestamp on database db
release id=2
----

translate database=db
----
/Tenant/10/Table/5{6-7} num_replicas=7 num_voters=5
/Tenant/10/Table/5{7-8} num_replicas=7

# Create an index on t1 to ensure that subzones also see protected timestamps.
exec-sql
CREATE INDEX idx ON db.t1(id);
ALTER INDEX db.t1@idx CONFIGURE ZONE USING gc.ttlseconds = 1;
----

protect id=3 ts=3
schemaObject 56
----

translate database=db
----
/Tenant/10/Table/56{-/2} num_replicas=7 num_voters=5 pts=[3]
/Tenant/10/Table/56/{2-3} ttl_seconds=1 num_replicas=7 num_voters=5 pts=[3]
/Tenant/10/Table/5{6/3-7} num_replicas=7 num_voters=5 pts=[3]
/Tenant/10/Table/5{7-8} num_replicas=7
Loading

0 comments on commit 2e87a4b

Please sign in to comment.