-
Notifications
You must be signed in to change notification settings - Fork 3.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
66639: sql,server: change indexusagestats subsystem to issue cluster RPC fanout r=Azhng a=Azhng sql: introduce crdb_internal.index_usage_stats virtual table This commit introduce crdb_internal.index_usage_stats virtual table that is backed by new clusterindexusagestats package. This new package implements a variant of the indexusagestats interface and serves the data by issuing cluster RPC fanout. Addresses #64740 Followup to #66451 Release note (sql change): introduce crdb_internal.index_usage_statistics virtual table to surface index usage statistics. sql.metrics.index_usage_stats.enabled cluster setting can be used to turn on/off the subsystem. It is default to true. 68045: kv: grab raftMu during no-op writes with local gossip triggers r=nvanbenschoten a=nvanbenschoten Fixes #68011. As of 9f8c019, it is now possible to have no-op writes that do not go through Raft but do set one of the gossip triggers. These gossip triggers require the raftMu to be held, so we were running into trouble when handling the local eval results above Raft. For instance, we see this case when a transaction sets the system config trigger and then performs a delete range over an empty span before committing. In this case, the transaction will have no intents to remove, so it can auto-GC its record during an EndTxn. If its record was never written in the first place, this is a no-op (as of 9f8c019). There appear to be three ways we could solve this: 1. we can avoid setting gossip triggers on transactions that don't perform any writes. 2. we can force EndTxn requests with gossip triggers to go through Raft even if they are otherwise no-ops. 3. we can properly handle gossip triggers on the above Raft local eval result path. This commit opts for the third option. Co-authored-by: Azhng <[email protected]> Co-authored-by: Nathan VanBenschoten <[email protected]>
- Loading branch information
Showing
23 changed files
with
1,470 additions
and
532 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,112 @@ | ||
| // Copyright 2021 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package server | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/roachpb" | ||
| "github.com/cockroachdb/cockroach/pkg/server/serverpb" | ||
| "github.com/cockroachdb/cockroach/pkg/sql/idxusage" | ||
| "github.com/cockroachdb/errors" | ||
| "google.golang.org/grpc/codes" | ||
| "google.golang.org/grpc/status" | ||
| ) | ||
|
|
||
| // IndexUsageStatistics is the GRPC handler for serving index usage statistics. | ||
| // If the NodeID in the request payload is left empty, the handler will issue | ||
| // a cluster-wide RPC fanout to aggregate all index usage statistics from all | ||
| // the nodes. If the NodeID is specified, then the handler will handle the | ||
| // request either locally (if the NodeID matches the current node's NodeID) or | ||
| // forward it to the correct node. | ||
| func (s *statusServer) IndexUsageStatistics( | ||
| ctx context.Context, req *serverpb.IndexUsageStatisticsRequest, | ||
| ) (*serverpb.IndexUsageStatisticsResponse, error) { | ||
| ctx = propagateGatewayMetadata(ctx) | ||
| ctx = s.AnnotateCtx(ctx) | ||
|
|
||
| if _, err := s.privilegeChecker.requireViewActivityPermission(ctx); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| localReq := &serverpb.IndexUsageStatisticsRequest{ | ||
| NodeID: "local", | ||
| } | ||
|
|
||
| if len(req.NodeID) > 0 { | ||
| requestedNodeID, local, err := s.parseNodeID(req.NodeID) | ||
| if err != nil { | ||
| return nil, status.Error(codes.InvalidArgument, err.Error()) | ||
| } | ||
| if local { | ||
| statsReader := s.sqlServer.pgServer.SQLServer.GetLocalIndexStatistics() | ||
| return indexUsageStatsLocal(statsReader) | ||
| } | ||
|
|
||
| statusClient, err := s.dialNode(ctx, requestedNodeID) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| // We issue a localReq instead of the incoming req to other nodes. This is | ||
| // to instruct other nodes to only return us their node-local stats and | ||
| // do not further propagates the RPC call. | ||
| return statusClient.IndexUsageStatistics(ctx, localReq) | ||
| } | ||
|
|
||
| dialFn := func(ctx context.Context, nodeID roachpb.NodeID) (interface{}, error) { | ||
| client, err := s.dialNode(ctx, nodeID) | ||
| return client, err | ||
| } | ||
|
|
||
| fetchIndexUsageStats := func(ctx context.Context, client interface{}, _ roachpb.NodeID) (interface{}, error) { | ||
| statusClient := client.(serverpb.StatusClient) | ||
| return statusClient.IndexUsageStatistics(ctx, localReq) | ||
| } | ||
|
|
||
| resp := &serverpb.IndexUsageStatisticsResponse{} | ||
| aggFn := func(_ roachpb.NodeID, nodeResp interface{}) { | ||
| stats := nodeResp.(*serverpb.IndexUsageStatisticsResponse) | ||
| resp.Statistics = append(resp.Statistics, stats.Statistics...) | ||
| } | ||
|
|
||
| var combinedError error | ||
| errFn := func(_ roachpb.NodeID, nodeFnError error) { | ||
| combinedError = errors.CombineErrors(combinedError, nodeFnError) | ||
| } | ||
|
|
||
| // It's unfortunate that we cannot use paginatedIterateNodes here because we | ||
| // need to aggregate all stats before returning. Returning a partial result | ||
| // yields an incorrect result. | ||
| if err := s.iterateNodes(ctx, | ||
| fmt.Sprintf("requesting index usage stats for node %s", req.NodeID), | ||
| dialFn, fetchIndexUsageStats, aggFn, errFn); err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| return resp, nil | ||
| } | ||
|
|
||
| func indexUsageStatsLocal( | ||
| idxUsageStats *idxusage.LocalIndexUsageStats, | ||
| ) (*serverpb.IndexUsageStatisticsResponse, error) { | ||
| resp := &serverpb.IndexUsageStatisticsResponse{} | ||
| if err := idxUsageStats.ForEach(idxusage.IteratorOptions{}, func(key *roachpb.IndexUsageKey, value *roachpb.IndexUsageStatistics) error { | ||
| resp.Statistics = append(resp.Statistics, roachpb.CollectedIndexUsageStatistics{Key: *key, | ||
| Stats: *value, | ||
| }) | ||
| return nil | ||
| }); err != nil { | ||
| return nil, err | ||
| } | ||
| return resp, nil | ||
| } |
Oops, something went wrong.