Promtail: Autodetect log format

pkg/promtail/api/entry_parser.go

@@ -18,15 +18,22 @@ const (
 	Docker EntryParser = iota
 	Raw
 	CRI
+	Auto
 )
 
 var (
-	criPattern = regexp.MustCompile(`^(?s)(?P<time>\S+?) (?P<stream>stdout|stderr) (?P<flags>\S+?) (?P<content>.*)$`)
+	criPattern      = regexp.MustCompile(`^(?s)(?P<time>\S+?) (?P<stream>stdout|stderr) (?P<flags>\S+?) (?P<content>.*)$`)
+	autodetectOrder = [3]EntryParser{Docker, CRI, Raw}
+	dummyHandler    = EntryHandlerFunc(func(ls model.LabelSet, t time.Time, s string) error {
+		return nil
+	})
 )
 
 // String returns a string representation of the EntryParser.
 func (e EntryParser) String() string {
 	switch e {
+	case Auto:
+		return "auto"
 	case CRI:
 		return "cri"
 	case Docker:
@@ -41,6 +48,9 @@ func (e EntryParser) String() string {
 // Set implements flag.Value.
 func (e *EntryParser) Set(s string) error {
 	switch strings.ToLower(s) {
+	case "auto":
+		*e = Auto
+		return nil
 	case "cri":
 		*e = CRI
 		return nil
@@ -67,6 +77,20 @@ func (e *EntryParser) UnmarshalYAML(unmarshal func(interface{}) error) error {
 // Wrap implements EntryMiddleware.
 func (e EntryParser) Wrap(next EntryHandler) EntryHandler {
 	switch e {
+	case Auto:
+		autodetected := EntryHandler(nil)
+		return EntryHandlerFunc(func(labels model.LabelSet, timestamp time.Time, line string) error {
+			if autodetected == nil {
+				for _, current := range autodetectOrder {
+					if current.Wrap(dummyHandler).Handle(labels, timestamp, line) == nil {
+						autodetected = current.Wrap(next)
+						break
+					}
+				}
+			}
+
+			return autodetected.Handle(labels, timestamp, line)
+		})
 	case CRI:
 		return EntryHandlerFunc(func(labels model.LabelSet, _ time.Time, line string) error {
 			parts := criPattern.FindStringSubmatch(line)

pkg/promtail/api/entry_parser_test.go

@@ -12,6 +12,7 @@ import (
 var (
 	TestTimeStr = "2019-01-01T01:00:00.000000001Z"
 	TestTime, _ = time.Parse(time.RFC3339Nano, TestTimeStr)
+	CurrentTime = time.Now()
 )
 
 type Entry struct {
@@ -73,14 +74,32 @@ func TestDocker(t *testing.T) {
 	runTestCases(Docker, dockerTestCases, t)
 }
 
+var rawTestCases = []TestCase{
+	{"message", false, Entry{CurrentTime, "message", model.LabelSet{}}},
+}
+
+func TestRaw(t *testing.T) {
+	runTestCases(Raw, rawTestCases, t)
+}
+
+var autoTestCases = []TestCase{
+	{"{\"log\":\"message\",\"stream\":\"stdout\",\"time\":\"" + TestTimeStr + "\"}", false, NewEntry(TestTime, "message", "stdout")},
+	{TestTimeStr + " stdout F message", false, NewEntry(TestTime, "message", "stdout")},
+	{"message", false, Entry{CurrentTime, "message", model.LabelSet{}}},
+}
+
+func TestAuto(t *testing.T) {
+	runTestCases(Auto, autoTestCases, t)
+}
+
 func runTestCases(parser EntryParser, testCases []TestCase, t *testing.T) {
 	for i, tc := range testCases {
 		client := &TestClient{
 			Entries: make([]Entry, 0),
 		}
 
 		handler := parser.Wrap(client)
-		err := handler.Handle(model.LabelSet{}, time.Now(), tc.Line)
+		err := handler.Handle(model.LabelSet{}, CurrentTime, tc.Line)
 
 		if err != nil && tc.ExpectedError {
 			continue

pkg/promtail/scrape/scrape.go

@@ -19,7 +19,7 @@ type Config struct {
 
 // DefaultScrapeConfig is the default Config.
 var DefaultScrapeConfig = Config{
-	EntryParser: api.Docker,
+	EntryParser: api.Auto,
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.

pkg/promtail/targets/filetargetmanager.go

@@ -79,7 +79,8 @@ func NewFileTargetManager(
 			relabelConfig: cfg.RelabelConfigs,
 			targets:       map[string]*FileTarget{},
 			hostname:      hostname,
-			entryHandler:  cfg.EntryParser.Wrap(client),
+			entryHandler:  client,
+			entryParser:   cfg.EntryParser,
 			targetConfig:  targetConfig,
 		}
 		tm.syncers[cfg.JobName] = s
@@ -124,6 +125,7 @@ type syncer struct {
 	log          log.Logger
 	positions    *positions.Positions
 	entryHandler api.EntryHandler
+	entryParser  api.EntryParser
 	hostname     string
 
 	targets       map[string]*FileTarget
@@ -201,7 +203,7 @@ func (s *syncer) sync(groups []*targetgroup.Group) {
 }
 
 func (s *syncer) newTarget(path string, labels model.LabelSet) (*FileTarget, error) {
-	return NewFileTarget(s.log, s.entryHandler, s.positions, path, labels, s.targetConfig)
+	return NewFileTarget(s.log, s.entryParser.Wrap(s.entryHandler), s.positions, path, labels, s.targetConfig)
 }
 
 func (s *syncer) ready() bool {

production/helm/promtail/values.yaml

@@ -6,7 +6,7 @@ annotations: {}
 
 deploymentStrategy: RollingUpdate
 
-entryParser: docker
+entryParser: auto
 
 image:
   repository: grafana/promtail

production/ksonnet/promtail/config.libsonnet

@@ -12,7 +12,7 @@
       hostname: 'logs-us-west1.grafana.net',
       container_root_path: '/var/lib/docker',
       external_labels: {},
-      entry_parser: 'docker',
+      entry_parser: 'auto',
     },
 
     service_url:

tools/promtail.sh

@@ -5,7 +5,7 @@ APIKEY="${2:-}"
 INSTANCEURL="${3:-}"
 NAMESPACE="${4:-default}"
 CONTAINERROOT="${5:-/var/lib/docker}"
-PARSER="${6:-docker}"
+PARSER="${6:-auto}"
 
 if [ -z "$INSTANCEID" -o -z "$APIKEY" -o -z "$INSTANCEURL" -o -z "$NAMESPACE" -o -z "$CONTAINERROOT" -o -z "$PARSER" ]; then
     echo "usage: $0 <instanceId> <apiKey> <url> [<namespace>[<container_root_path>[<parser>]]]"