main: stacktrace/coredump on error

If the proxy fails due to an internal error or if it receives
a fatal signal, write a stack trace to the log and exit.

If it was started with `--debug` also dump core.

Note: `handleVersion()` introduced to keep `gocyclo` happy.

Fixes #59.

Signed-off-by: James O. D. Hunt <[email protected]>

fatal.go

@@ -0,0 +1,81 @@
+// Copyright 2018 Intel Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/signal"
+	"runtime/pprof"
+	"strings"
+	"syscall"
+)
+
+// List of fatal signals
+var sigFatal = map[syscall.Signal]bool{
+	syscall.SIGABRT:   true,
+	syscall.SIGBUS:    true,
+	syscall.SIGILL:    true,
+	syscall.SIGQUIT:   true,
+	syscall.SIGSEGV:   true,
+	syscall.SIGSTKFLT: true,
+	syscall.SIGSYS:    true,
+	syscall.SIGTRAP:   true,
+}
+
+func handlePanic() {
+	r := recover()
+
+	if r != nil {
+		msg := fmt.Sprintf("%s", r)
+		logger().WithField("panic", msg).Error("fatal error")
+
+		die()
+	}
+}
+
+func backtrace() {
+	profiles := pprof.Profiles()
+
+	buf := &bytes.Buffer{}
+
+	for _, p := range profiles {
+		// The magic number requests a full stacktrace. See
+		// https://golang.org/pkg/runtime/pprof/#Profile.WriteTo.
+		pprof.Lookup(p.Name()).WriteTo(buf, 2)
+	}
+
+	for _, line := range strings.Split(buf.String(), "\n") {
+		logger().Error(line)
+	}
+}
+
+func fatalSignal(sig syscall.Signal) bool {
+	return sigFatal[sig]
+}
+
+func fatalSignals() []syscall.Signal {
+	var signals []syscall.Signal
+
+	for sig := range sigFatal {
+		signals = append(signals, sig)
+
+	}
+
+	return signals
+}
+
+func die() {
+	backtrace()
+
+	if crashOnError {
+		signal.Reset(syscall.SIGABRT)
+		syscall.Kill(0, syscall.SIGABRT)
+	}
+
+	os.Exit(1)
+}

proxy.go

@@ -35,6 +35,9 @@ const (
 // version is the proxy version. This variable is populated at build time.
 var version = "unknown"
 
+// if true, coredump when an internal error occurs or a fatal signal is received
+var crashOnError = false
+
 var proxyLog = logrus.New()
 
 func serve(servConn io.ReadWriteCloser, proto, addr string, results chan error) (net.Listener, error) {
@@ -196,6 +199,10 @@ func setupNotifier() chan os.Signal {
 	sigCh := make(chan os.Signal, 8)
 	signal.Notify(sigCh, termSignal)
 
+	for _, sig := range fatalSignals() {
+		signal.Notify(sigCh, sig)
+	}
+
 	return sigCh
 }
 
@@ -207,6 +214,15 @@ func handleSignal(sigCh chan os.Signal, vmConn *net.Conn, proxyListener *net.Lis
 
 	// Blocking here waiting for the signal to be received.
 	sig := <-sigCh
+
+	nativeSignal, ok := sig.(syscall.Signal)
+	if ok {
+		if fatalSignal(nativeSignal) {
+			logger().WithField("signal", sig).Error("received fatal signal")
+			die()
+		}
+	}
+
 	if sig != termSignal {
 		return fmt.Errorf("Signal received should be %q, got %q instead", termSignal.String(), sig.String())
 	}
@@ -231,10 +247,18 @@ func handleSignal(sigCh chan os.Signal, vmConn *net.Conn, proxyListener *net.Lis
 	return nil
 }
 
-func main() {
+func handleVersion(showVersion bool) {
+	if showVersion {
+		fmt.Printf("%v version %v\n", proxyName, version)
+		os.Exit(0)
+	}
+}
+
+func realMain() {
 	var channel, proxyAddr, agentLogsSocket, logLevel string
-	var showVersion bool
+	var debug, showVersion bool
 
+	flag.BoolVar(&debug, "debug", false, "enable debug mode")
 	flag.BoolVar(&showVersion, "version", false, "display program version and exit")
 	flag.StringVar(&channel, "mux-socket", "", "unix socket to multiplex on")
 	flag.StringVar(&proxyAddr, "listen-socket", "", "unix socket to listen on")
@@ -245,9 +269,10 @@ func main() {
 
 	flag.Parse()
 
-	if showVersion {
-		fmt.Printf("%v version %v\n", proxyName, version)
-		os.Exit(0)
+	handleVersion(showVersion)
+
+	if debug {
+		crashOnError = true
 	}
 
 	if channel == "" || proxyAddr == "" {
@@ -315,3 +340,8 @@ func main() {
 
 	logger().Debug("shutting down")
 }
+
+func main() {
+	defer handlePanic()
+	realMain()
+}