diff --git a/.changelog/2500.feature.md b/.changelog/2500.feature.md new file mode 100644 index 00000000000..f10c723d46e --- /dev/null +++ b/.changelog/2500.feature.md @@ -0,0 +1,4 @@ +Entity block signatures count tool. + +The tool uses node consensus and registry API endpoints and computes the per +entity block signature counts. diff --git a/go/Makefile b/go/Makefile index b6590407356..e4c2efb2bbb 100644 --- a/go/Makefile +++ b/go/Makefile @@ -10,7 +10,7 @@ all: build # Build. # List of Go binaries to build. -go-binaries := oasis-node oasis-test-runner oasis-net-runner +go-binaries := oasis-node oasis-test-runner oasis-net-runner extra/stats # List of test helpers to build. test-helpers := urkel # List of test vectors to generate. @@ -22,7 +22,7 @@ generate: $(go-binaries): @$(ECHO) "$(MAGENTA)*** Building $@...$(OFF)" - @$(GO) build $(GOFLAGS) $(GO_EXTRA_FLAGS) -o ./$@/$@ ./$@ + @$(GO) build $(GOFLAGS) $(GO_EXTRA_FLAGS) -o ./$@/$(notdir $@) ./$@ oasis-node: diff --git a/go/consensus/api/api.go b/go/consensus/api/api.go index caa30d4519b..2544982fac2 100644 --- a/go/consensus/api/api.go +++ b/go/consensus/api/api.go @@ -6,6 +6,7 @@ import ( "context" beacon "github.com/oasislabs/oasis-core/go/beacon/api" + "github.com/oasislabs/oasis-core/go/common/cbor" "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/errors" "github.com/oasislabs/oasis-core/go/common/node" @@ -73,7 +74,7 @@ type Block struct { // Height contains the block height. Height int64 `json:"height"` // Meta contains the consensus backend specific block metadata. - Meta interface{} `json:"meta"` + Meta cbor.RawMessage `json:"meta"` } // Backend is an interface that a consensus backend must provide. diff --git a/go/consensus/tendermint/api/api.go b/go/consensus/tendermint/api/api.go index 67d71c3093c..f23a594e9a0 100644 --- a/go/consensus/tendermint/api/api.go +++ b/go/consensus/tendermint/api/api.go @@ -12,6 +12,7 @@ import ( tmp2p "github.com/tendermint/tendermint/p2p" tmtypes "github.com/tendermint/tendermint/types" + "github.com/oasislabs/oasis-core/go/common/cbor" "github.com/oasislabs/oasis-core/go/common/crypto/signature" "github.com/oasislabs/oasis-core/go/common/node" consensus "github.com/oasislabs/oasis-core/go/consensus/api" @@ -140,11 +141,14 @@ type BlockMeta struct { // NewBlock creates a new consensus.Block from a Tendermint block. func NewBlock(blk *tmtypes.Block) *consensus.Block { + meta := BlockMeta{ + Header: &blk.Header, + LastCommit: blk.LastCommit, + } + rawMeta := cbor.Marshal(meta) + return &consensus.Block{ Height: blk.Header.Height, - Meta: BlockMeta{ - Header: &blk.Header, - LastCommit: blk.LastCommit, - }, + Meta: rawMeta, } } diff --git a/go/extra/README.md b/go/extra/README.md new file mode 100644 index 00000000000..a983eeaea08 --- /dev/null +++ b/go/extra/README.md @@ -0,0 +1,4 @@ +## Extra + +This directory contains packages depending on (but not directly part of) +oasis-core. diff --git a/go/extra/stats/.gitignore b/go/extra/stats/.gitignore new file mode 100644 index 00000000000..12ee8c78aca --- /dev/null +++ b/go/extra/stats/.gitignore @@ -0,0 +1 @@ +stats diff --git a/go/extra/stats/README.md b/go/extra/stats/README.md new file mode 100644 index 00000000000..7253fbf3486 --- /dev/null +++ b/go/extra/stats/README.md @@ -0,0 +1,19 @@ +## Stats + +Queries a node for network stats. Currently implemented per entity block +signature counts. + +## Usage + +``` +stats/stats entity-signatures \ + --address unix:/internal.sock \ + --start-block 0 \ + --end-block 100 \ + --top-n 100 + +|Rank |Entity ID |Nodes |Signatures| +------------------------------------------------------------------------------------------ +|1 |ef9ccfc825d5f0087e56937c697b46520f29f81e21d8a289218a4ebaef00509c| 6| 100| +|2 |4ea5328f943ef6f66daaed74cb0e99c3b1c45f76307b425003dbc7cb3638ed35| 1| 80| +... diff --git a/go/extra/stats/cmd/root.go b/go/extra/stats/cmd/root.go new file mode 100644 index 00000000000..db2a01fc3d5 --- /dev/null +++ b/go/extra/stats/cmd/root.go @@ -0,0 +1,53 @@ +// Package cmd implements stats cmd tool. +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/oasislabs/oasis-core/go/common/logging" + "github.com/oasislabs/oasis-core/go/common/version" + "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common" +) + +const cfgLogLevel = "log.level" + +var ( + rootCmd = &cobra.Command{ + Use: "stats", + Short: "Oasis stats", + Version: version.SoftwareVersion, + } + + rootFlags = flag.NewFlagSet("", flag.ContinueOnError) +) + +// RootCommand returns the root (top level) cobra.Command. +func RootCommand() *cobra.Command { + return rootCmd +} + +// Execute spawns the main entry point after handling the command line arguments. +func Execute() { + var logLevel logging.Level + if err := logLevel.Set(viper.GetString(cfgLogLevel)); err != nil { + common.EarlyLogAndExit(fmt.Errorf("root: failed to set log level: %w", err)) + } + + if err := rootCmd.Execute(); err != nil { + common.EarlyLogAndExit(err) + } +} + +func init() { + logLevel := logging.LevelInfo + rootFlags.Var(&logLevel, cfgLogLevel, "log level") + _ = viper.BindPFlags(rootFlags) + rootCmd.PersistentFlags().AddFlagSet(rootFlags) + + // Register all of the sub-commands. + RegisterStatsCmd(rootCmd) +} diff --git a/go/extra/stats/cmd/stats.go b/go/extra/stats/cmd/stats.go new file mode 100644 index 00000000000..b3d078a45b4 --- /dev/null +++ b/go/extra/stats/cmd/stats.go @@ -0,0 +1,322 @@ +package cmd + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + + "github.com/spf13/cobra" + flag "github.com/spf13/pflag" + "github.com/spf13/viper" + + "github.com/oasislabs/oasis-core/go/common/cbor" + "github.com/oasislabs/oasis-core/go/common/crypto/signature" + "github.com/oasislabs/oasis-core/go/common/logging" + "github.com/oasislabs/oasis-core/go/common/node" + consensusAPI "github.com/oasislabs/oasis-core/go/consensus/api" + tmApi "github.com/oasislabs/oasis-core/go/consensus/tendermint/api" + tmcrypto "github.com/oasislabs/oasis-core/go/consensus/tendermint/crypto" + nodeCmdCommon "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common" + cmdGrpc "github.com/oasislabs/oasis-core/go/oasis-node/cmd/common/grpc" + registryAPI "github.com/oasislabs/oasis-core/go/registry/api" +) + +const ( + cfgStartBlock = "start-block" + cfgEndBlock = "end-block" + cfgTopN = "top-n" +) + +var ( + printStatsFlags = flag.NewFlagSet("", flag.ContinueOnError) + + printStatsCmd = &cobra.Command{ + Use: "entity-signatures", + Short: "prints per entity block signature counts", + Run: doPrintStats, + } + + logger = logging.GetLogger("cmd/stats") +) + +// entityStats are per entity stats. +type entityStats struct { + id signature.PublicKey + nodeSignatures map[signature.PublicKey]int64 +} + +// nodeIDs are node identifiers. +type nodeIDs struct { + entityID signature.PublicKey + nodeID signature.PublicKey +} + +// stats are gathered entity stats. +type stats struct { + // Per entity stats. + entities map[signature.PublicKey]*entityStats + + // Tendermint stores the validator addresses (which are the truncated SHA-256 + // of the node consensus public keys) in Commit data instead of the actual + // public keys. + nodeAddressMap map[string]nodeIDs +} + +// printEntitySignatures prints topN entities by block signature counts. +func (s stats) printEntitySignatures(topN int) { + type results struct { + entityID signature.PublicKey + signatures int64 + nodes int + } + res := []results{} + + // Compute per entity signature counts. + for eID, eStats := range s.entities { + entity := results{entityID: eID, nodes: len(eStats.nodeSignatures)} + for _, signs := range eStats.nodeSignatures { + entity.signatures += signs + } + res = append(res, entity) + } + + sort.Slice(res, func(i, j int) bool { + return res[i].signatures > res[j].signatures + }) + + // Print results. + fmt.Printf("|%-5s|%-64s|%-6s|%10s|\n", "Rank", "Entity ID", "Nodes", "Signatures") + fmt.Println(strings.Repeat("-", 5+64+6+10+5)) + for idx, r := range res { + fmt.Printf("|%-5d|%-64s|%6d|%10d|\n", idx+1, r.entityID, r.nodes, r.signatures) + } +} + +// nodeExists returns if node with address exists. +func (s stats) nodeExists(nodeAddr string) bool { + _, ok := s.nodeAddressMap[nodeAddr] + return ok +} + +// addNodeSignature adds node signature. +func (s stats) addNodeSignature(nodeAddr string) error { + node, ok := s.nodeAddressMap[nodeAddr] + if !ok { + return fmt.Errorf("missing node address map, address: %s", nodeAddr) + } + entity, ok := s.entities[node.entityID] + if !ok { + return fmt.Errorf("missing entity for node, address: %s", nodeAddr) + } + _, ok = entity.nodeSignatures[node.nodeID] + if !ok { + return fmt.Errorf("missing entity node: %s", nodeAddr) + } + entity.nodeSignatures[node.nodeID]++ + return nil +} + +// newStats initializes empty stats. +func newStats() *stats { + b := &stats{ + entities: make(map[signature.PublicKey]*entityStats), + nodeAddressMap: make(map[string]nodeIDs), + } + return b +} + +func (s *stats) addRegistryData(ctx context.Context, registry registryAPI.Backend, height int64) error { + // Fetch entities. + entities, err := registry.GetEntities(ctx, height) + if err != nil { + return err + } + + // Fetch nodes. + nodes, err := registry.GetNodes(ctx, height) + if err != nil { + return err + } + + // Map: nodeID -> Node + nodesMap := make(map[signature.PublicKey]*node.Node) + for _, n := range nodes { + nodesMap[n.ID] = n + } + + // Store new nodes and entities info. + for _, ent := range entities { + var es *entityStats + var ok bool + + // Since registry data can be fetched at multiple heights, entities might + // already exist. + if es, ok = s.entities[ent.ID]; !ok { + es = &entityStats{ + id: ent.ID, + nodeSignatures: make(map[signature.PublicKey]int64), + } + s.entities[ent.ID] = es + } + + for _, nodeID := range ent.Nodes { + node, ok := nodesMap[nodeID] + if !ok { + return fmt.Errorf("entity's node missing: %s", nodeID) + } + + // Add missing nodes. + if _, ok := es.nodeSignatures[nodeID]; !ok { + es.nodeSignatures[nodeID] = 0 + cID := node.Consensus.ID + tmADdr := tmcrypto.PublicKeyToTendermint(&cID).Address().String() + s.nodeAddressMap[tmADdr] = nodeIDs{ + entityID: ent.ID, + nodeID: nodeID, + } + } + + } + } + + return nil +} + +// getStats queries node for entity stats between 'start' and 'end' block heights. +func getStats(ctx context.Context, consensus consensusAPI.ClientBackend, registry registryAPI.Backend, start int64, end int64) *stats { + // Init stats. + stats := newStats() + + // If latest block, query for exact block number so it doesn't change during + // the execution. + if end == consensusAPI.HeightLatest { + block, err := consensus.GetBlock(ctx, end) + if err != nil { + logger.Error("failed to query block", + "err", err, + "height", end, + ) + os.Exit(1) + } + end = block.Height + } + + // Prepopulate registry state with the state at latest height, to avoid + // querying it at every height. We only query registry at specific heights + // in case we encounter missing nodes during block traversal. + err := stats.addRegistryData(ctx, registry, consensusAPI.HeightLatest) + if err != nil { + logger.Error("failed to initialize block signatures", + "err", err, + "height", consensusAPI.HeightLatest, + ) + os.Exit(1) + } + + // Block traversal. + for height := start; height <= end; height++ { + if height%1000 == 0 { + logger.Debug("querying block", + "height", height, + ) + } + + // Get block. + block, err := consensus.GetBlock(ctx, height) + if err != nil { + logger.Error("failed to query block", + "err", err, + "height", height, + ) + os.Exit(1) + } + var tmBlockMeta tmApi.BlockMeta + if err := cbor.Unmarshal(block.Meta, &tmBlockMeta); err != nil { + logger.Error("unmarshal error", + "meta", block.Meta, + "err", err, + ) + os.Exit(1) + } + + // Go over all signatures for a block. + // XXX: In tendermint master (not yet released) the signatures are + // obtained in LastCommit.Signatures. + for _, sig := range tmBlockMeta.LastCommit.Precommits { + nodeTmAddr := sig.ValidatorAddress.String() + + // Check if node is already being tracked. + if !stats.nodeExists(nodeTmAddr) { + logger.Debug("missing node tendermint address, querying registry", + "height", height, + "addr", nodeTmAddr, + ) + + // Query registry at current height. + err := stats.addRegistryData(ctx, registry, height) + if err != nil { + logger.Error("failed to query registry", + "err", err, + "height", height, + ) + os.Exit(1) + } + } + + // Add signatures. + if err := stats.addNodeSignature(nodeTmAddr); err != nil { + logger.Error("failure adding signature", + "err", err, + ) + os.Exit(1) + } + } + } + + return stats +} + +func doPrintStats(cmd *cobra.Command, args []string) { + ctx := context.Background() + + if err := nodeCmdCommon.Init(); err != nil { + nodeCmdCommon.EarlyLogAndExit(err) + } + + // Initialize client connection. + conn, err := cmdGrpc.NewClient(cmd) + if err != nil { + logger.Error("failed to establish connection with node", + "err", err, + ) + os.Exit(1) + } + defer conn.Close() + + // Clients. + consClient := consensusAPI.NewConsensusClient(conn) + regClient := registryAPI.NewRegistryClient(conn) + + start := viper.GetInt64(cfgStartBlock) + end := viper.GetInt64(cfgEndBlock) + topN := viper.GetInt(cfgTopN) + // Load stats. + stats := getStats(ctx, consClient, regClient, start, end) + + stats.printEntitySignatures(topN) +} + +// Register stats cmd sub-command and all of it's children. +func RegisterStatsCmd(parentCmd *cobra.Command) { + printStatsFlags.Int64(cfgStartBlock, 1, "start block") + printStatsFlags.Int64(cfgEndBlock, consensusAPI.HeightLatest, "end block") + printStatsFlags.Int(cfgTopN, 50, "top N results that will be printed") + _ = viper.BindPFlags(printStatsFlags) + + printStatsCmd.Flags().AddFlagSet(printStatsFlags) + printStatsCmd.PersistentFlags().AddFlagSet(cmdGrpc.ClientFlags) + + parentCmd.AddCommand(printStatsCmd) +} diff --git a/go/extra/stats/main.go b/go/extra/stats/main.go new file mode 100644 index 00000000000..afc132ba5e9 --- /dev/null +++ b/go/extra/stats/main.go @@ -0,0 +1,10 @@ +// Stats implementation. +package main + +import ( + "github.com/oasislabs/oasis-core/go/extra/stats/cmd" +) + +func main() { + cmd.Execute() +}