Merge pull request #2975 from oasisprotocol/kostko/fix/tm-init-cancel

go/oasis-node: Properly handle shutdown during startup

.changelog/2975.bugfix.md

@@ -0,0 +1 @@
+go/oasis-node: Properly handle shutdown during startup

go/consensus/tendermint/tendermint.go

@@ -1119,7 +1119,20 @@ func (t *tendermintService) lazyInit() error {
 	//
 	// Defer actually initializing the node till after everything
 	// else is setup.
-	t.startFn = func() error {
+	t.startFn = func() (err error) {
+		defer func() {
+			// The node constructor can panic early in case an error occurrs during block replay as
+			// the fail monitor is not yet initialized in that case. Propagate the error.
+			if p := recover(); p != nil {
+				switch pt := p.(type) {
+				case error:
+					err = pt
+				default:
+					err = fmt.Errorf("%v", pt)
+				}
+			}
+		}()
+
 		t.node, err = tmnode.NewNode(tenderConfig,
 			tendermintPV,
 			&tmp2p.NodeKey{PrivKey: crypto.SignerToTendermint(t.nodeSigner)},
@@ -1134,7 +1147,7 @@ func (t *tendermintService) lazyInit() error {
 		}
 		if t.stateDb == nil {
 			// Sanity check for the above wrapDbProvider hack in case the DB provider changes.
-			panic("tendermint: state database not set")
+			return fmt.Errorf("tendermint: internal error: state database not set")
 		}
 		t.client = tmcli.New(t.node)
 		t.failMonitor = newFailMonitor(t.Logger, t.node.ConsensusState().Wait)

go/oasis-node/cmd/node/node.go

@@ -82,7 +82,12 @@ func Run(cmd *cobra.Command, args []string) {
 	cmdCommon.SetIsNodeCmd(true)
 
 	node, err := NewNode()
-	if err != nil {
+	switch {
+	case err == nil:
+	case errors.Is(err, context.Canceled):
+		// Shutdown requested during startup.
+		return
+	default:
 		os.Exit(1)
 	}
 	defer node.Cleanup()
@@ -482,22 +487,26 @@ func NewTestNode() (*Node, error) {
 	return newNode(true)
 }
 
-func newNode(testNode bool) (*Node, error) { // nolint: gocyclo
+func newNode(testNode bool) (node *Node, err error) { // nolint: gocyclo
 	logger := cmdCommon.Logger()
 
-	node := &Node{
+	node = &Node{
 		svcMgr: background.NewServiceManager(logger),
 	}
 
 	var startOk bool
 	defer func() {
 		if !startOk {
+			if cErr := node.svcMgr.Ctx.Err(); cErr != nil {
+				err = cErr
+			}
+
 			node.Stop()
 			node.Cleanup()
 		}
 	}()
 
-	if err := cmdCommon.Init(); err != nil {
+	if err = cmdCommon.Init(); err != nil {
 		// Common stuff like logger not correcty initialized. Print to stderr
 		_, _ = fmt.Fprintln(os.Stderr, err)
 		return nil, err
@@ -512,8 +521,6 @@ func newNode(testNode bool) (*Node, error) { // nolint: gocyclo
 	// Load configured values for all registered crash points.
 	crash.LoadViperArgValues()
 
-	var err error
-
 	// Open the common node store.
 	node.commonStore, err = persistent.NewCommonStore(dataDir)
 	if err != nil {