signals: Add standard signal handling

Add the standard project signal handling. However, since it needs to be
applied to two different binaries, encapsulate it in a new package.

Fixes kata-containers#32.

Signed-off-by: James O. D. Hunt <[email protected]>

ksm.go

@@ -21,6 +21,7 @@ import (
 	"syscall"
 	"time"
 
+	ksig "github.com/kata-containers/ksm-throttler/pkg/signals"
 	"github.com/sirupsen/logrus"
 )
 
@@ -381,10 +382,37 @@ func startKSM(root string, mode ksmMode) (*ksm, error) {
 		c := make(chan os.Signal, 2)
 		signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 
+		for _, sig := range ksig.HandledSignals() {
+			signal.Notify(c, sig)
+		}
+
 		go func() {
-			<-c
-			_ = k.restore()
-			os.Exit(0)
+			for {
+				// Block waiting for a signal
+				sig := <-c
+
+				if sig == syscall.SIGTERM {
+					_ = k.restore()
+					os.Exit(0)
+				}
+
+				nativeSignal, ok := sig.(syscall.Signal)
+				if !ok {
+					err := errors.New("unknown signal")
+					throttlerLog.WithError(err).WithField("signal", sig.String()).Error()
+					continue
+				}
+
+				if ksig.FatalSignal(nativeSignal) {
+					throttlerLog.WithField("signal", sig).Error("received fatal signal")
+					ksig.Die()
+				} else if ksig.NonFatalSignal(nativeSignal) {
+					if debug {
+						throttlerLog.WithField("signal", sig).Debug("handling signal")
+						ksig.Backtrace()
+					}
+				}
+			}
 		}()
 
 		if mode == ksmAuto {

pkg/signals/signals.go

@@ -0,0 +1,120 @@
+// Copyright 2018 Intel Corporation.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package signals
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"os/signal"
+	"runtime/pprof"
+	"strings"
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+)
+
+var signalLog = logrus.WithField("default-signal-logger", true)
+
+// CrashOnError causes a coredump to be produced when an internal error occurs
+// or a fatal signal is received.
+var CrashOnError = false
+
+// List of handled signals.
+//
+// The value is true if receiving the signal should be fatal.
+var handledSignalsMap = map[syscall.Signal]bool{
+	syscall.SIGABRT:   true,
+	syscall.SIGBUS:    true,
+	syscall.SIGILL:    true,
+	syscall.SIGQUIT:   true,
+	syscall.SIGSEGV:   true,
+	syscall.SIGSTKFLT: true,
+	syscall.SIGSYS:    true,
+	syscall.SIGTRAP:   true,
+	syscall.SIGUSR1:   false,
+}
+
+// SetLogger sets the custom logger to be used by this package. If not called,
+// the package will create its own logger.
+func SetLogger(logger *logrus.Entry) {
+	signalLog = logger
+}
+
+// HandlePanic writes a message to the logger and then calls Die().
+func HandlePanic() {
+	r := recover()
+
+	if r != nil {
+		msg := fmt.Sprintf("%s", r)
+		signalLog.WithField("panic", msg).Error("fatal error")
+
+		Die()
+	}
+}
+
+// Backtrace writes a multi-line backtrace to the logger.
+func Backtrace() {
+	profiles := pprof.Profiles()
+
+	buf := &bytes.Buffer{}
+
+	for _, p := range profiles {
+		// The magic number requests a full stacktrace. See
+		// https://golang.org/pkg/runtime/pprof/#Profile.WriteTo.
+		pprof.Lookup(p.Name()).WriteTo(buf, 2)
+	}
+
+	for _, line := range strings.Split(buf.String(), "\n") {
+		signalLog.Error(line)
+	}
+}
+
+// FatalSignal returns true if the specified signal should cause the program
+// to abort.
+func FatalSignal(sig syscall.Signal) bool {
+	s, exists := handledSignalsMap[sig]
+	if !exists {
+		return false
+	}
+
+	return s
+}
+
+// NonFatalSignal returns true if the specified signal should simply cause the
+// program to Backtrace() but continue running.
+func NonFatalSignal(sig syscall.Signal) bool {
+	s, exists := handledSignalsMap[sig]
+	if !exists {
+		return false
+	}
+
+	return !s
+}
+
+// HandledSignals returns a list of signals the package can deal with.
+func HandledSignals() []syscall.Signal {
+	var signals []syscall.Signal
+
+	for sig := range handledSignalsMap {
+		signals = append(signals, sig)
+	}
+
+	return signals
+}
+
+// Die causes a backtrace to be produced. If CrashOnError is set a coredump
+// will be produced, else the program will exit.
+func Die() {
+	Backtrace()
+
+	if CrashOnError {
+		signal.Reset(syscall.SIGABRT)
+		syscall.Kill(0, syscall.SIGABRT)
+	}
+
+	os.Exit(1)
+}

pkg/signals/signals_test.go

@@ -0,0 +1,140 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package signals
+
+import (
+	"bytes"
+	"os"
+	"reflect"
+	goruntime "runtime"
+	"sort"
+	"strings"
+	"syscall"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSignalFatalSignal(t *testing.T) {
+	assert := assert.New(t)
+
+	for sig, fatal := range handledSignalsMap {
+		result := NonFatalSignal(sig)
+		if fatal {
+			assert.False(result)
+		} else {
+			assert.True(result)
+		}
+	}
+}
+
+func TestSignalHandledSignalsMap(t *testing.T) {
+	assert := assert.New(t)
+
+	for sig, fatal := range handledSignalsMap {
+		result := FatalSignal(sig)
+		if fatal {
+			assert.True(result)
+		} else {
+			assert.False(result)
+		}
+	}
+}
+
+func TestSignalHandledSignals(t *testing.T) {
+	assert := assert.New(t)
+
+	var expected []syscall.Signal
+
+	for sig := range handledSignalsMap {
+		expected = append(expected, sig)
+	}
+
+	got := HandledSignals()
+
+	sort.Slice(expected, func(i, j int) bool {
+		return int(expected[i]) < int(expected[j])
+	})
+
+	sort.Slice(got, func(i, j int) bool {
+		return int(got[i]) < int(got[j])
+	})
+
+	assert.True(reflect.DeepEqual(expected, got))
+}
+
+func TestSignalNonFatalSignal(t *testing.T) {
+	assert := assert.New(t)
+
+	for sig, fatal := range handledSignalsMap {
+		result := NonFatalSignal(sig)
+		if fatal {
+			assert.False(result)
+		} else {
+			assert.True(result)
+		}
+	}
+}
+
+func TestSignalFatalSignalInvalidSignal(t *testing.T) {
+	assert := assert.New(t)
+
+	sig := syscall.SIGXCPU
+
+	result := FatalSignal(sig)
+	assert.False(result)
+}
+
+func TestSignalNonFatalSignalInvalidSignal(t *testing.T) {
+	assert := assert.New(t)
+
+	sig := syscall.SIGXCPU
+
+	result := NonFatalSignal(sig)
+	assert.False(result)
+}
+
+func TestSignalBacktrace(t *testing.T) {
+	assert := assert.New(t)
+
+	savedLog := signalLog
+	defer func() {
+		signalLog = savedLog
+	}()
+
+	signalLog = logrus.WithFields(logrus.Fields{
+		"name":        "name",
+		"pid":         os.Getpid(),
+		"source":      "throttler",
+		"test-logger": true})
+
+	// create buffer to save logger output
+	buf := &bytes.Buffer{}
+
+	savedOut := signalLog.Logger.Out
+	defer func() {
+		signalLog.Logger.Out = savedOut
+	}()
+
+	// capture output to buffer
+	signalLog.Logger.Out = buf
+
+	// determine name of *this* function
+	pc := make([]uintptr, 1)
+	goruntime.Callers(1, pc)
+	fn := goruntime.FuncForPC(pc[0])
+	name := fn.Name()
+
+	Backtrace()
+
+	b := buf.String()
+
+	// very basic tests to check if a backtrace was produced
+	assert.True(strings.Contains(b, "contention:"))
+	assert.True(strings.Contains(b, `level=error`))
+	assert.True(strings.Contains(b, name))
+}

throttler.go

@@ -17,6 +17,7 @@ import (
 
 	gpb "github.com/golang/protobuf/ptypes/empty"
 	kpb "github.com/kata-containers/ksm-throttler/pkg/grpc"
+	ksig "github.com/kata-containers/ksm-throttler/pkg/signals"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
@@ -33,6 +34,8 @@ var memInfo = "/proc/meminfo"
 // version is the KSM throttler version. This variable is populated at build time.
 var version = "unknown"
 
+var debug = false
+
 // DefaultURI is populated at link time with the value of:
 //   ${locatestatedir}/run/ksm-throttler/ksm.sock
 var DefaultURI string
@@ -109,6 +112,11 @@ func SetLoggingLevel(l string) error {
 
 	throttlerLog.Logger.SetLevel(level)
 
+	if levelStr == "debug" {
+		debug = true
+		ksig.CrashOnError = true
+	}
+
 	throttlerLog.Logger.Formatter = &logrus.TextFormatter{TimestampFormat: time.RFC3339Nano}
 
 	throttlerLog.WithField("version", version).Info()
@@ -184,7 +192,7 @@ func getSocketPath() (string, error) {
 	return socketURI, nil
 }
 
-func main() {
+func realMain() {
 	doVersion := flag.Bool("version", false, "display the version")
 	logLevel := flag.String("log", "warn",
 		"log messages above specified level; one of debug, warn, error, fatal or panic")
@@ -201,6 +209,8 @@ func main() {
 		os.Exit(1)
 	}
 
+	ksig.SetLogger(throttlerLog)
+
 	uri, err := getSocketPath()
 	if err != nil {
 		throttlerLog.WithError(err).Error("Could net get service socket URI")
@@ -234,3 +244,8 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func main() {
+	defer ksig.HandlePanic()
+	realMain()
+}