Skip to content

Commit

Permalink
Merge #96805 #96872 #96954
Browse files Browse the repository at this point in the history 
96805: storage: remove pre-22.2 code r=RaduBerinde a=RaduBerinde

This change removes any code that handles pre-22.2 versions. Features
that are in 22.2 (like range keys) are now always enabled.

Any new store always uses at least the 22.2 pebble format version.

Release note: None
Epic: none

Fixes: #96764

96872: ui: add statement fingerprint to insights r=j82w a=j82w

1. Removes contention events from insights. This avoids tracking and storing duplicate information.

2. Add database_name, schema_name, table_name, index_name, contending_pretty_key to crdb_internal.transaction_contention_events. This avoids needing to join with multiple tables and makes it easier for users to understand.

3. Add waiting statement info to the insights transaction details page. This way if users have a large transaction with multiple statements with multiple contention events they can see which specific statement was waited on.

4. Add blocking transaction fingerprint to the insight statement details page. The user can now see which transaction is blocking the statement ,and it has a link to the blocking transaction activity page.

https://www.loom.com/share/de37b2e3007d48c2a1e3ad39ed334e20

closes: #91665

Release note (ui change): Add waiting statement id and fingerprint to insights transaction details page. Add blocking transaction id and fingerprint to the insights statement page.

96954: workload/schemachange: skip tsquery/tsvector types in mixed version envs r=fqazi a=fqazi

Previously, the randomized schema change workload in a mixed version environment attempted to create tables with the tsvector/tsquery types. This could lead to syntax errors on older nodes that do not know of this. This patch, avoids using these types until the version of the cluster hits 23.1.

Epic: none
Release note: None

Co-authored-by: Radu Berinde <[email protected]>
Co-authored-by: j82w <[email protected]>
Co-authored-by: Faizan Qazi <[email protected]>
  • Loading branch information
@RaduBerinde @fqazi
4 people committed Feb 13, 2023
4 parents 0c2cc47 + 12aec72 + e135c67 + 4607a74 commit 47331b1
Show file tree
Hide file tree
Showing 40 changed files with 601 additions and 1,005 deletions.
27 changes: 19 additions & 8 deletions pkg/ccl/cliccl/ear_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,13 @@ func TestList(t *testing.T) {
000004.log:
env type: Data, AES128_CTR
keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
nonce: 80 18 c0 79 61 c7 cf ef b4 25 4e 78
counter: 1483615076
nonce: 31 d3 cd 5a 69 e2 13 64 21 53 57 64
counter: 3952287331
000005.sst:
env type: Data, AES128_CTR
keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
nonce: 71 12 f7 22 9a fb 90 24 4e 58 27 01
counter: 3082989236
nonce: 23 d9 b2 e1 39 b0 87 ed f9 6d 49 20
counter: 3481614039
COCKROACHDB_DATA_KEYS_000001_monolith:
env type: Store, AES128_CTR
keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867
Expand All @@ -171,8 +171,8 @@ COCKROACHDB_DATA_KEYS_000001_monolith:
CURRENT:
env type: Data, AES128_CTR
keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
nonce: 18 c2 a6 23 cc 6e 2e 7c 8e bf 84 77
counter: 3159373900
nonce: 71 12 f7 22 9a fb 90 24 4e 58 27 01
counter: 3082989236
MANIFEST-000001:
env type: Data, AES128_CTR
keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
Expand All @@ -181,14 +181,25 @@ MANIFEST-000001:
OPTIONS-000003:
env type: Data, AES128_CTR
keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
nonce: d3 97 11 b3 1a ed 22 2b 74 fb 02 0c
counter: 1229228536
nonce: c3 6d b2 7b 3f 3e 67 b9 28 b9 81 b1
counter: 3050109376
marker.datakeys.000001.COCKROACHDB_DATA_KEYS_000001_monolith:
env type: Store, AES128_CTR
keyID: f594229216d81add7811c4360212eb7629b578ef4eab6e5d05679b3c5de48867
nonce: 55 d7 d4 27 6c 97 9b dd f1 5d 40 c8
counter: 467030050
marker.format-version.000009.010:
env type: Data, AES128_CTR
keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
nonce: 6e 34 f4 3c 11 43 1a f5 69 ce 33 f1
counter: 2398097086
marker.manifest.000001.MANIFEST-000001:
env type: Data, AES128_CTR
keyID: bbb65a9d114c2a18740f27b6933b74f61018bd5adf545c153b48ffe6473336ef
nonce: d3 97 11 b3 1a ed 22 2b 74 fb 02 0c
counter: 1229228536
`

require.Equal(t, want, b.String())
}

Expand Down
5 changes: 3 additions & 2 deletions pkg/ccl/storageccl/engineccl/encrypted_fs_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,9 +268,10 @@ func TestPebbleEncryption(t *testing.T) {
stats, err := db.GetEnvStats()
require.NoError(t, err)
// Opening the DB should've created OPTIONS, CURRENT, MANIFEST and the
// WAL, all under the active key.
// WAL.
require.Equal(t, uint64(4), stats.TotalFiles)
require.Equal(t, uint64(4), stats.ActiveKeyFiles)
// We also created markers for the format version and the manifest.
require.Equal(t, uint64(6), stats.ActiveKeyFiles)
var s enginepbccl.EncryptionStatus
require.NoError(t, protoutil.Unmarshal(stats.EncryptionStatus, &s))
require.Equal(t, "16.key", s.ActiveStoreKey.Source)
Expand Down
42 changes: 20 additions & 22 deletions pkg/kv/kvserver/kvstorage/cluster_version_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,14 @@ func TestStoresClusterVersionIncompatible(t *testing.T) {

ctx := context.Background()

vOneDashOne := roachpb.Version{Major: 1, Internal: 1}
vOne := roachpb.Version{Major: 1}
current := clusterversion.ByKey(clusterversion.BinaryVersionKey)
minSupported := clusterversion.ByKey(clusterversion.BinaryMinSupportedVersionKey)

future := current
future.Major++

tooOld := minSupported
tooOld.Major--

type testCase struct {
binV, minV roachpb.Version // binary version and min supported version
Expand All @@ -42,41 +48,33 @@ func TestStoresClusterVersionIncompatible(t *testing.T) {
for name, tc := range map[string]testCase{
"StoreTooNew": {
// This is what the node is running.
binV: vOneDashOne,
binV: current,
// This is what the running node requires from its stores.
minV: vOne,
minV: minSupported,
// Version is way too high for this node.
engV: roachpb.Version{Major: 9},
expErr: `cockroach version v1\.0-1 is incompatible with data in store <no-attributes>=<in-mem>; use version v9\.0 or later`,
engV: future,
expErr: `cockroach version .* is incompatible with data in store <no-attributes>=<in-mem>; use version .* or later`,
},
"StoreTooOldVersion": {
// This is what the node is running.
binV: roachpb.Version{Major: 9},
binV: current,
// This is what the running node requires from its stores.
minV: roachpb.Version{Major: 5},
minV: minSupported,
// Version is way too low.
engV: roachpb.Version{Major: 4},
expErr: `store <no-attributes>=<in-mem>, last used with cockroach version v4\.0, is too old for running version v9\.0 \(which requires data from v5\.0 or later\)`,
},
"StoreTooOldMinVersion": {
// Like the previous test case, but this time cv.MinimumVersion is the culprit.
binV: roachpb.Version{Major: 9},
minV: roachpb.Version{Major: 5},
engV: roachpb.Version{Major: 4},
expErr: `store <no-attributes>=<in-mem>, last used with cockroach version v4\.0, is too old for running version v9\.0 \(which requires data from v5\.0 or later\)`,
engV: tooOld,
expErr: `store <no-attributes>=<in-mem>, last used with cockroach version .*, is too old for running version .* \(which requires data from .* or later\)`,
},
} {
t.Run(name, func(t *testing.T) {
engs := []storage.Engine{storage.NewDefaultInMemForTesting()}
defer engs[0].Close()
// Configure versions and write.
cv := clusterversion.ClusterVersion{Version: tc.engV}
if err := WriteClusterVersionToEngines(ctx, engs, cv); err != nil {
t.Fatal(err)
err := WriteClusterVersionToEngines(ctx, engs, cv)
if err == nil {
cv, err = SynthesizeClusterVersionFromEngines(ctx, engs, tc.binV, tc.minV)
}
if cv, err := SynthesizeClusterVersionFromEngines(
ctx, engs, tc.binV, tc.minV,
); !testutils.IsError(err, tc.expErr) {
if !testutils.IsError(err, tc.expErr) {
t.Fatalf("unexpected error: %+v, got version %v", err, cv)
}
})
Expand Down
5 changes: 0 additions & 5 deletions pkg/kv/kvserver/spanset/batch.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -495,11 +495,6 @@ func (s spanSetReader) ConsistentIterators() bool {
return s.r.ConsistentIterators()
}

// SupportsRangeKeys implements the storage.Reader interface.
func (s spanSetReader) SupportsRangeKeys() bool {
return s.r.SupportsRangeKeys()
}

// PinEngineStateForIterators implements the storage.Reader interface.
func (s spanSetReader) PinEngineStateForIterators() error {
return s.r.PinEngineStateForIterators()
Expand Down
129 changes: 62 additions & 67 deletions pkg/sql/crdb_internal.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ import (
"github.com/cockroachdb/cockroach/pkg/sql/catalog/systemschema"
"github.com/cockroachdb/cockroach/pkg/sql/catalog/typedesc"
"github.com/cockroachdb/cockroach/pkg/sql/clusterunique"
"github.com/cockroachdb/cockroach/pkg/sql/contentionpb"
"github.com/cockroachdb/cockroach/pkg/sql/idxusage"
"github.com/cockroachdb/cockroach/pkg/sql/isql"
"github.com/cockroachdb/cockroach/pkg/sql/parser"
Expand Down Expand Up @@ -6473,9 +6474,15 @@ CREATE TABLE crdb_internal.transaction_contention_events (
contention_duration INTERVAL NOT NULL,
contending_key BYTES NOT NULL,
waiting_stmt_id string NOT NULL,
waiting_stmt_fingerprint_id BYTES NOT NULL
contending_pretty_key STRING NOT NULL,
waiting_stmt_id string NOT NULL,
waiting_stmt_fingerprint_id BYTES NOT NULL,
database_name STRING NOT NULL,
schema_name STRING NOT NULL,
table_name STRING NOT NULL,
index_name STRING
);`,
generator: func(ctx context.Context, p *planner, db catalog.DatabaseDescriptor, stopper *stop.Stopper) (virtualTableGenerator, cleanupFunc, error) {
// Check permission first before making RPC fanout.
Expand Down Expand Up @@ -6541,28 +6548,41 @@ CREATE TABLE crdb_internal.transaction_contention_events (
types.DefaultIntervalTypeMetadata,
)

contendingPrettyKey := tree.NewDString("")
contendingKey := tree.NewDBytes("")
if !shouldRedactContendingKey {
decodedKey, _, _ := keys.DecodeTenantPrefix(resp.Events[i].BlockingEvent.Key)
contendingPrettyKey = tree.NewDString(keys.PrettyPrint(nil /* valDirs */, decodedKey))
contendingKey = tree.NewDBytes(
tree.DBytes(resp.Events[i].BlockingEvent.Key))
tree.DBytes(decodedKey))
}

waitingStmtFingerprintID := tree.NewDBytes(
tree.DBytes(sqlstatsutil.EncodeUint64ToBytes(uint64(resp.Events[i].WaitingStmtFingerprintID))))

waitingStmtId := tree.NewDString(hex.EncodeToString(resp.Events[i].WaitingStmtID.GetBytes()))

schemaName, dbName, tableName, indexName, err := getContentionEventInfo(ctx, p, resp.Events[i])
if err != nil {
return err
}

row = row[:0]
row = append(row,
collectionTs, // collection_ts
tree.NewDUuid(tree.DUuid{UUID: resp.Events[i].BlockingEvent.TxnMeta.ID}), // blocking_txn_id
blockingFingerprintID, // blocking_fingerprint_id
tree.NewDUuid(tree.DUuid{UUID: resp.Events[i].WaitingTxnID}), // waiting_txn_id
waitingFingerprintID, // waiting_fingerprint_id
contentionDuration, // contention_duration
contendingKey, // contending_key,
waitingStmtId, // waiting_stmt_id
waitingStmtFingerprintID, // waiting_stmt_fingerprint_id
waitingFingerprintID, // waiting_fingerprint_id
contentionDuration, // contention_duration
contendingKey, // contending_key,
contendingPrettyKey, // contending_pretty_key
waitingStmtId, // waiting_stmt_id
waitingStmtFingerprintID, // waiting_stmt_fingerprint_id
tree.NewDString(dbName), // database_name
tree.NewDString(schemaName), // schema_name
tree.NewDString(tableName), // table_name
tree.NewDString(indexName), // index_name
)

if err = pusher.pushRow(row...); err != nil {
Expand Down Expand Up @@ -7207,7 +7227,6 @@ CREATE TABLE crdb_internal.%s (
last_retry_reason STRING,
exec_node_ids INT[] NOT NULL,
contention INTERVAL,
contention_events JSONB,
index_recommendations STRING[] NOT NULL,
implicit_txn BOOL NOT NULL,
cpu_sql_nanos INT8
Expand Down Expand Up @@ -7295,17 +7314,6 @@ func populateStmtInsights(
)
}

contentionEvents := tree.DNull
if len(s.ContentionEvents) > 0 {
var contentionEventsJSON json.JSON
contentionEventsJSON, err = convertContentionEventsToJSON(ctx, p, s.ContentionEvents)
if err != nil {
return err
}

contentionEvents = tree.NewDJSON(contentionEventsJSON)
}

indexRecommendations := tree.NewDArray(types.String)
for _, recommendation := range s.IndexRecommendations {
if err = indexRecommendations.Append(tree.NewDString(recommendation)); err != nil {
Expand Down Expand Up @@ -7337,7 +7345,6 @@ func populateStmtInsights(
autoRetryReason,
execNodeIDs,
contentionTime,
contentionEvents,
indexRecommendations,
tree.MakeDBool(tree.DBool(insight.Transaction.ImplicitTxn)),
tree.NewDInt(tree.DInt(s.CPUSQLNanos)),
Expand All @@ -7347,57 +7354,45 @@ func populateStmtInsights(
return
}

func convertContentionEventsToJSON(
ctx context.Context, p *planner, contentionEvents []roachpb.ContentionEvent,
) (json json.JSON, err error) {
func getContentionEventInfo(
ctx context.Context, p *planner, contentionEvent contentionpb.ExtendedContentionEvent,
) (schemaName, dbName, tableName, indexName string, err error) {

eventWithNames := make([]sqlstatsutil.ContentionEventWithNames, len(contentionEvents))
for i, contentionEvent := range contentionEvents {
_, tableID, err := p.ExecCfg().Codec.DecodeTablePrefix(contentionEvent.Key)
if err != nil {
return nil, err
}
_, _, indexID, err := p.ExecCfg().Codec.DecodeIndexPrefix(contentionEvent.Key)
if err != nil {
return nil, err
}

desc := p.Descriptors()
var tableDesc catalog.TableDescriptor
tableDesc, err = desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Table(ctx, descpb.ID(tableID))
if err != nil {
return nil, err
}
_, tableID, err := p.ExecCfg().Codec.DecodeTablePrefix(contentionEvent.BlockingEvent.Key)
if err != nil {
return "", "", "", "", err
}
_, _, indexID, err := p.ExecCfg().Codec.DecodeIndexPrefix(contentionEvent.BlockingEvent.Key)
if err != nil {
return "", "", "", "", err
}

idxDesc, err := catalog.MustFindIndexByID(tableDesc, descpb.IndexID(indexID))
if err != nil {
return nil, err
}
desc := p.Descriptors()
var tableDesc catalog.TableDescriptor
tableDesc, err = desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Table(ctx, descpb.ID(tableID))
if err != nil {
return "", "", "", "", err
}

dbDesc, err := desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Database(ctx, tableDesc.GetParentID())
if err != nil {
return nil, err
}
idxDesc, err := catalog.MustFindIndexByID(tableDesc, descpb.IndexID(indexID))
if err != nil {
return "", "", "", "", err
}

schemaDesc, err := desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Schema(ctx, tableDesc.GetParentSchemaID())
if err != nil {
return nil, err
}
dbDesc, err := desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Database(ctx, tableDesc.GetParentID())
if err != nil {
return "", "", "", "", err
}

var idxName string
if idxDesc != nil {
idxName = idxDesc.GetName()
}
schemaDesc, err := desc.ByIDWithLeased(p.txn).WithoutNonPublic().Get().Schema(ctx, tableDesc.GetParentSchemaID())
if err != nil {
return "", "", "", "", err
}

eventWithNames[i] = sqlstatsutil.ContentionEventWithNames{
BlockingTransactionID: contentionEvent.TxnMeta.ID.String(),
SchemaName: schemaDesc.GetName(),
DatabaseName: dbDesc.GetName(),
TableName: tableDesc.GetName(),
IndexName: idxName,
DurationInMs: float64(contentionEvent.Duration) / float64(time.Millisecond),
}
var idxName string
if idxDesc != nil {
idxName = idxDesc.GetName()
}

return sqlstatsutil.BuildContentionEventsJSON(eventWithNames)
return schemaDesc.GetName(), dbDesc.GetName(), tableDesc.GetName(), idxName, nil
}
Loading

0 comments on commit 47331b1

Please sign in to comment.