From 9d3a7f7d878a666f8425a149e2d9f6f1da2ade08 Mon Sep 17 00:00:00 2001 From: ptrus Date: Fri, 18 Sep 2020 15:27:30 +0200 Subject: [PATCH] go/e2e/sentry: sanity check consensus peers --- .changelog/3291.internal.md | 4 + .../scenario/e2e/runtime/sentry.go | 124 ++++++++++++++++-- 2 files changed, 119 insertions(+), 9 deletions(-) create mode 100644 .changelog/3291.internal.md diff --git a/.changelog/3291.internal.md b/.changelog/3291.internal.md new file mode 100644 index 00000000000..f38e982c2db --- /dev/null +++ b/.changelog/3291.internal.md @@ -0,0 +1,4 @@ +go/e2e/sentry: Sanity check consensus peers + +Sentry E2E test now ensures that validators running behind sentries only +connect to their sentry nodes. diff --git a/go/oasis-test-runner/scenario/e2e/runtime/sentry.go b/go/oasis-test-runner/scenario/e2e/runtime/sentry.go index 9096db379de..6bd2a0a64a5 100644 --- a/go/oasis-test-runner/scenario/e2e/runtime/sentry.go +++ b/go/oasis-test-runner/scenario/e2e/runtime/sentry.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "errors" "fmt" + "strings" "time" "google.golang.org/grpc" @@ -14,6 +15,8 @@ import ( "github.com/oasisprotocol/oasis-core/go/common/crypto/signature" cmnGrpc "github.com/oasisprotocol/oasis-core/go/common/grpc" "github.com/oasisprotocol/oasis-core/go/common/identity" + "github.com/oasisprotocol/oasis-core/go/consensus/tendermint/crypto" + control "github.com/oasisprotocol/oasis-core/go/control/api" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/log" "github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis" @@ -168,7 +171,7 @@ func (s *sentryImpl) dial(address string, clientOpts *cmnGrpc.ClientOptions) (*g return conn, nil } -func (s *sentryImpl) Run(childEnv *env.Env) error { +func (s *sentryImpl) Run(childEnv *env.Env) error { // nolint: gocyclo // Run the basic runtime test. if err := s.runtimeImpl.Run(childEnv); err != nil { return err @@ -178,11 +181,10 @@ func (s *sentryImpl) Run(childEnv *env.Env) error { defer cancel() // Load identities and addresses used in the sanity checks. - sentry0 := s.Net.Sentries()[0] - sentry0Address := sentry0.GetSentryControlAddress() - - sentry4 := s.Net.Sentries()[4] - sentry4Address := sentry4.GetSentryAddress() + sentry0, _, sentry0CtrlAddress, sentry0P2PPubkey := loadSentryNodeInfo(s.Net.Sentries()[0]) + _, _, _, sentry1P2PPubkey := loadSentryNodeInfo(s.Net.Sentries()[1]) + _, _, _, sentry2P2PPubkey := loadSentryNodeInfo(s.Net.Sentries()[2]) + sentry4, sentry4Address, _, _ := loadSentryNodeInfo(s.Net.Sentries()[4]) storage0 := s.Net.StorageWorkers()[0] storage0Identity, err := storage0.LoadIdentity() @@ -215,12 +217,29 @@ func (s *sentryImpl) Run(childEnv *env.Env) error { if err != nil { return fmt.Errorf("sentry: error loading validator node identity: %w", err) } + validator0Ctrl, err := oasis.NewController(validator0.SocketPath()) + if err != nil { + return err + } validator1 := s.Net.Validators()[1] validator1Identity, err := validator1.LoadIdentity() if err != nil { return fmt.Errorf("sentry: error loading validator node identity: %w", err) } + validator1Ctrl, err := oasis.NewController(validator1.SocketPath()) + if err != nil { + return err + } + + validator2 := s.Net.Validators()[2] + if err != nil { + return fmt.Errorf("sentry: error loading validator node identity: %w", err) + } + validator2Ctrl, err := oasis.NewController(validator2.SocketPath()) + if err != nil { + return err + } // Sanity check sentry control endpoints. Only configured upstream nodes are // allowed to access their corresponding sentry control endpoint. @@ -233,7 +252,7 @@ func (s *sentryImpl) Run(childEnv *env.Env) error { }, Certificates: []tls.Certificate{}, } - conn, err := s.dial(sentry0Address, opts) + conn, err := s.dial(sentry0CtrlAddress, opts) if err != nil { return fmt.Errorf("sentry: dial error: %w", err) } @@ -253,7 +272,7 @@ func (s *sentryImpl) Run(childEnv *env.Env) error { }, Certificates: []tls.Certificate{*validator1Identity.TLSSentryClientCertificate}, } - conn, err = s.dial(sentry0Address, opts) + conn, err = s.dial(sentry0CtrlAddress, opts) if err != nil { return fmt.Errorf("sentry: dial error: %w", err) } @@ -273,7 +292,7 @@ func (s *sentryImpl) Run(childEnv *env.Env) error { }, Certificates: []tls.Certificate{*validator0Identity.TLSSentryClientCertificate}, } - conn, err = s.dial(sentry0Address, opts) + conn, err = s.dial(sentry0CtrlAddress, opts) if err != nil { return fmt.Errorf("sentry: dial error: %w", err) } @@ -382,5 +401,92 @@ func (s *sentryImpl) Run(childEnv *env.Env) error { return errors.New("sentry4 storage checkpoints endpoint should allow connection with storage0 certificate") } + // Sanity check validator peers - only sentry nodes should be present. + // Expected consensus peers. + validator0ExpectedPeerKeys := []string{ + strings.ToLower(crypto.PublicKeyToTendermint(&sentry0P2PPubkey).Address().String()), + strings.ToLower(crypto.PublicKeyToTendermint(&sentry1P2PPubkey).Address().String()), + } + validator12ExpectedPeerKeys := []string{ + strings.ToLower(crypto.PublicKeyToTendermint(&sentry2P2PPubkey).Address().String()), + } + + // Sanity check validator0. + validator0Status, err := validator0Ctrl.GetStatus(ctx) + if err != nil { + return fmt.Errorf("validator0.GetStatus: %w", err) + } + validator0ConsensusPeers := consensusTendermintAddrs(validator0Status) + if err = sanityCheckValidatorPeers(validator0ExpectedPeerKeys, validator0ConsensusPeers); err != nil { + s.Logger.Error("validator0 invalid consensus peers", + "expected", validator0ExpectedPeerKeys, + "actual", validator0ConsensusPeers, + ) + return err + } + + // Sanity check validator1. + validator1Status, err := validator1Ctrl.GetStatus(ctx) + if err != nil { + return fmt.Errorf("validator1.GetStatus: %w", err) + } + validator1ConsensusPeers := consensusTendermintAddrs(validator1Status) + if err = sanityCheckValidatorPeers(validator12ExpectedPeerKeys, validator1ConsensusPeers); err != nil { + s.Logger.Error("validator1 invalid consensus peers", + "expected", validator12ExpectedPeerKeys, + "actual", validator1ConsensusPeers, + ) + return err + } + + // Sanity check validator2. + validator2Status, err := validator2Ctrl.GetStatus(ctx) + if err != nil { + return fmt.Errorf("validator2.GetStatus: %w", err) + } + validator2ConsensusPeers := consensusTendermintAddrs(validator2Status) + if err = sanityCheckValidatorPeers(validator12ExpectedPeerKeys, validator2ConsensusPeers); err != nil { + s.Logger.Error("validator2 invalid consensus peers", + "expected", validator12ExpectedPeerKeys, + "actual", validator2ConsensusPeers, + ) + return err + } + return nil +} + +func loadSentryNodeInfo(s *oasis.Sentry) (*oasis.Sentry, string, string, signature.PublicKey) { + sentryCtrlAddress := s.GetSentryControlAddress() + sentryAddress := s.GetSentryAddress() + sentryIdentity, _ := s.LoadIdentity() + sentryP2PPubkey := sentryIdentity.P2PSigner.Public() + return s, sentryAddress, sentryCtrlAddress, sentryP2PPubkey +} + +func consensusTendermintAddrs(status *control.Status) (consensusPeers []string) { + for _, np := range status.Consensus.NodePeers { + consensusPeers = append(consensusPeers, strings.Split(np, "@")[0]) + } + return +} + +func sanityCheckValidatorPeers(expected, actual []string) error { + if len(expected) != len(actual) { + return fmt.Errorf("consensus peers length mismatch, expected: %d, actual: %d", + len(expected), len(actual)) + } + for _, expect := range expected { + var found bool + for _, key := range actual { + if key == expect { + found = true + break + } + } + if !found { + return fmt.Errorf("expected consensus peer missing: %s", expect) + } + } + return nil }