refactor: new api for dflog and output log to console by default

cmd/dfdaemon/app/init.go

@@ -3,14 +3,11 @@ package app
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
-	"log"
 	"os"
 	"os/exec"
 	"os/user"
 	"path/filepath"
 	"runtime"
-	"syscall"
 	"time"
 
 	"github.com/dragonflyoss/Dragonfly/dfdaemon/config"
@@ -64,70 +61,18 @@ func initLogger(cfg config.Properties) error {
 		return errors.Wrap(err, "get current user")
 	}
 
-	// Set the log level here so the following line will be output normally
-	// to the console, before setting the log file.
-	if cfg.Verbose {
-		logrus.SetLevel(logrus.DebugLevel)
-	}
 	logFilePath := filepath.Join(current.HomeDir, ".small-dragonfly/logs/dfdaemon.log")
-	logrus.Debugf("use log file %s", logFilePath)
-
-	if err := dflog.InitLog(cfg.Verbose, logFilePath, fmt.Sprintf("%d", os.Getpid())); err != nil {
-		return errors.Wrap(err, "init log file")
-	}
-
-	logFile, ok := (logrus.StandardLogger().Out).(*os.File)
-	if !ok {
-		return nil
-	}
-	go func(logFile *os.File) {
-		logrus.Infof("rotate %s every 60 seconds", logFilePath)
-		ticker := time.NewTicker(60 * time.Second)
-		for range ticker.C {
-			if err := rotateLog(logFile); err != nil {
-				logrus.Errorf("failed to rotate log %s: %v", logFile.Name(), err)
-			}
-		}
-	}(logFile)
-
-	return nil
-}
-
-// rotateLog truncates the logs file by a certain amount bytes.
-func rotateLog(logFile *os.File) error {
-	fStat, err := logFile.Stat()
-	if err != nil {
-		return err
-	}
-	logSizeLimit := int64(20 * 1024 * 1024)
 
-	if fStat.Size() <= logSizeLimit {
-		return nil
+	opts := []dflog.Option{
+		dflog.WithLogFile(logFilePath),
+		dflog.WithSign(fmt.Sprintf("%d", os.Getpid())),
+		dflog.WithDebug(cfg.Verbose),
+		dflog.WithConsole(),
 	}
 
-	// if it exceeds the 20MB limitation
-	log.SetOutput(ioutil.Discard)
-	// make sure set the output of log back to logFile when error be raised.
-	defer log.SetOutput(logFile)
-	logFile.Sync()
-	truncateSize := logSizeLimit/2 - 1
-	mem, err := syscall.Mmap(int(logFile.Fd()), 0, int(fStat.Size()),
-		syscall.PROT_READ|syscall.PROT_WRITE, syscall.MAP_SHARED)
-	if err != nil {
-		return err
-	}
-	copy(mem[0:], mem[truncateSize:])
-	if err := syscall.Munmap(mem); err != nil {
-		return err
-	}
-	if err := logFile.Truncate(fStat.Size() - truncateSize); err != nil {
-		return err
-	}
-	if _, err := logFile.Seek(truncateSize, 0); err != nil {
-		return err
-	}
+	logrus.Debugf("use log file %s", logFilePath)
 
-	return nil
+	return errors.Wrap(dflog.Init(logrus.StandardLogger(), opts...), "init log")
 }
 
 // cleanLocalRepo checks the files at local periodically, and delete the file when

cmd/dfget/app/root.go

@@ -174,13 +174,18 @@ func transParams() error {
 func initClientLog() error {
 	logFilePath := path.Join(cfg.WorkHome, "logs", "dfclient.log")
 
-	dflog.InitLog(cfg.Verbose, logFilePath, cfg.Sign)
+	opts := []dflog.Option{
+		dflog.WithLogFile(logFilePath),
+		dflog.WithSign(cfg.Sign),
+		dflog.WithDebug(cfg.Verbose),
+	}
 
 	// once cfg.Console is set, process should also output log to console
 	if cfg.Console {
-		dflog.InitConsoleLog(cfg.Verbose, cfg.Sign)
+		opts = append(opts, dflog.WithConsole())
 	}
-	return nil
+
+	return dflog.Init(logrus.StandardLogger(), opts...)
 }
 
 func initFlags() {

cmd/dfget/app/server.go

@@ -24,6 +24,7 @@ import (
 	"github.com/dragonflyoss/Dragonfly/dfget/core/uploader"
 	"github.com/dragonflyoss/Dragonfly/pkg/dflog"
 
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
@@ -77,5 +78,17 @@ func runServer() error {
 
 func initServerLog() error {
 	logFilePath := path.Join(cfg.WorkHome, "logs", "dfserver.log")
-	return dflog.InitLog(cfg.Verbose, logFilePath, cfg.Sign)
+
+	opts := []dflog.Option{
+		dflog.WithLogFile(logFilePath),
+		dflog.WithSign(cfg.Sign),
+		dflog.WithDebug(cfg.Verbose),
+	}
+
+	// once cfg.Console is set, process should also output log to console
+	if cfg.Console {
+		opts = append(opts, dflog.WithConsole())
+	}
+
+	return dflog.Init(logrus.StandardLogger(), opts...)
 }

cmd/supernode/app/root.go

@@ -145,12 +145,17 @@ func runSuperNode() error {
 
 // initLog initializes log Level and log format of daemon.
 func initLog() error {
-	logPath := path.Join(options.HomeDir, "logs", "app.log")
-	err := dflog.InitLog(options.Debug, logPath, fmt.Sprintf("%d", os.Getpid()))
-	if err != nil {
-		logrus.Errorf("failed to initialize logs: %v", err)
+	logFilePath := path.Join(options.HomeDir, "logs", "app.log")
+
+	opts := []dflog.Option{
+		dflog.WithLogFile(logFilePath),
+		dflog.WithSign(fmt.Sprintf("%d", os.Getpid())),
+		dflog.WithDebug(options.Debug),
 	}
-	return err
+
+	logrus.Debugf("use log file %s", logFilePath)
+
+	return errors.Wrap(dflog.Init(logrus.StandardLogger(), opts...), "init log")
 }
 
 // initConfig load configuration from config file.

go.mod

@@ -40,6 +40,7 @@ require (
 	github.com/willf/bitset v0.0.0-20190228212526-18bd95f470f9
 	gopkg.in/gcfg.v1 v1.2.3
 	gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/warnings.v0 v0.1.2
 	gopkg.in/yaml.v2 v2.2.2
 )

go.sum

@@ -213,6 +213,8 @@ gopkg.in/gcfg.v1 v1.2.3 h1:m8OOJ4ccYHnx2f4gQwpno8nAX5OGOh7RLaaz0pj3Ogs=
 gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o=
 gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528 h1:/saqWwm73dLmuzbNhe92F0QsZ/KiFND+esHco2v1hiY=
 gopkg.in/mgo.v2 v2.0.0-20160818020120-3f83fa500528/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
+gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
 gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=

pkg/dflog/log.go

@@ -20,113 +20,114 @@ import (
 	"bytes"
 	"fmt"
 	"os"
-	"path"
-	"path/filepath"
 	"strings"
 
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"gopkg.in/natefinch/lumberjack.v2"
 )
 
 // DefaultLogTimeFormat defines the timestamp format.
 const DefaultLogTimeFormat = "2006-01-02 15:04:05.000"
 
-// InitLog initializes the file logger for process.
-// logfile is used to stored generated log in local filesystem.
-func InitLog(debug bool, logFilePath string, sign string) error {
-	// set the log level
-	logLevel := logrus.InfoLevel
-	if debug {
-		logLevel = logrus.DebugLevel
-	}
+// Option is a functional configuration for the given logrus logger
+type Option func(l *logrus.Logger) error
 
-	// create and log file
-	if err := os.MkdirAll(filepath.Dir(logFilePath), 0755); err != nil {
-		return fmt.Errorf("failed to create log file %s: %v", logFilePath, err)
-	}
-	logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0644)
-	if err != nil {
-		return err
+// WithDebug sets the log level to debug
+func WithDebug(debug bool) Option {
+	return func(l *logrus.Logger) error {
+		if debug {
+			l.SetLevel(logrus.DebugLevel)
+		}
+		return nil
 	}
-	logFile.Seek(0, 2)
+}
 
-	// create formatter for default log Logger
-	formatter := &DragonflyFormatter{
-		TimestampFormat: DefaultLogTimeFormat,
-		Sign:            sign,
+func getLumberjack(l *logrus.Logger) *lumberjack.Logger {
+	if logger, ok := l.Out.(*lumberjack.Logger); ok {
+		return logger
 	}
-
-	// set all details in log default logger
-	logrus.SetLevel(logLevel)
-	logrus.SetOutput(logFile)
-	logrus.SetFormatter(formatter)
 	return nil
 }
 
-// InitConsoleLog initializes console logger for process.
-// console log will output the dfget client's log in console/terminal for
-// debugging usage.
-func InitConsoleLog(debug bool, sign string) {
-	formatter := &DragonflyFormatter{
-		TimestampFormat: DefaultLogTimeFormat,
-		Sign:            sign,
-	}
+// WithLogFile sets the logger to output to the given file, with log rotation.
+// If the given file is empty, nothing will be done.
+func WithLogFile(f string) Option {
+	return func(l *logrus.Logger) error {
+		if f == "" {
+			return nil
+		}
 
-	logLevel := logrus.InfoLevel
-	if debug {
-		logLevel = logrus.DebugLevel
-	}
+		if logger := getLumberjack(l); logger == nil {
+			l.SetOutput(&lumberjack.Logger{
+				Filename:   f,
+				MaxSize:    20, // mb
+				MaxBackups: 1,
+			})
+		} else {
+			logger.Filename = f
+		}
 
-	consoleLog := &logrus.Logger{
-		Out:       os.Stdout,
-		Formatter: formatter,
-		Hooks:     make(logrus.LevelHooks),
-		Level:     logLevel,
+		return nil
 	}
-	hook := &ConsoleHook{
-		logger: consoleLog,
-		levels: logrus.AllLevels,
-	}
-	logrus.AddHook(hook)
 }
 
-// CreateLogger creates a Logger.
-func CreateLogger(logPath string, logName string, logLevel string, sign string) (*logrus.Logger, error) {
-	// parse log level
-	level, err := logrus.ParseLevel(logLevel)
-	if err != nil {
-		level = logrus.InfoLevel
+// WithMaxSizeMB sets the max size of log files in MB. If the logger is not configured
+// to use a log file, an error is returned.
+func WithMaxSizeMB(max uint) Option {
+	return func(l *logrus.Logger) error {
+		if logger := getLumberjack(l); logger != nil {
+			logger.MaxSize = int(max)
+			return nil
+		}
+		return errors.Errorf("lumberjack is not configured")
 	}
+}
 
-	// create log file path
-	logFilePath := path.Join(logPath, logName)
-	if err := os.MkdirAll(filepath.Dir(logFilePath), 0755); err != nil {
-		return nil, err
+// WithConsole add a hook to output logs to stdout
+func WithConsole() Option {
+	return func(l *logrus.Logger) error {
+		consoleLog := &logrus.Logger{
+			Out:       os.Stdout,
+			Formatter: l.Formatter,
+			Hooks:     make(logrus.LevelHooks),
+			Level:     l.Level,
+		}
+		hook := &ConsoleHook{
+			logger: consoleLog,
+			levels: logrus.AllLevels,
+		}
+		l.AddHook(hook)
+		return nil
 	}
+}
 
-	// open log file
-	logFile, err := os.OpenFile(logFilePath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
-	if err != nil {
-		return nil, err
+// WithSign sets the sign in formatter
+func WithSign(sign string) Option {
+	return func(l *logrus.Logger) error {
+		l.Formatter = &DragonflyFormatter{
+			TimestampFormat: DefaultLogTimeFormat,
+			Sign:            sign,
+		}
+		return nil
 	}
-
-	logFile.Seek(0, 2)
-	Logger := logrus.New()
-	Logger.Out = logFile
-	Logger.Formatter = &DragonflyFormatter{TimestampFormat: DefaultLogTimeFormat, Sign: sign}
-	Logger.Level = level
-	return Logger, nil
 }
 
-// AddConsoleLog will add a ConsoleLog into Logger's hooks.
-// It will output logs to console when Logger's outputting logs.
-func AddConsoleLog(Logger *logrus.Logger) {
-	consoleLog := &logrus.Logger{
-		Out:       os.Stdout,
-		Formatter: Logger.Formatter,
-		Hooks:     make(logrus.LevelHooks),
-		Level:     Logger.Level,
+// Init initializes the logger with given options. If no option is provided,
+// the logger's formatter will be set with an empty sign.
+func Init(l *logrus.Logger, opts ...Option) error {
+	opts = append([]Option{
+		WithSign(""),
+	}, opts...)
+	for _, opt := range opts {
+		if err := opt(l); err != nil {
+			return err
+		}
 	}
-	Logger.Hooks.Add(&ConsoleHook{logger: consoleLog, levels: logrus.AllLevels})
+	if logger, ok := l.Out.(*lumberjack.Logger); ok {
+		return logger.Rotate()
+	}
+	return nil
 }
 
 // ConsoleHook shows logs on console.
@@ -207,10 +208,3 @@ func (f *DragonflyFormatter) appendValue(b *bytes.Buffer, value interface{}, wit
 		b.WriteByte(' ')
 	}
 }
-
-// ----------------------------------------------------------------------------
-
-// IsDebug returns the log level is debug.
-func IsDebug(level logrus.Level) bool {
-	return level >= logrus.DebugLevel
-}