promtail: Add systemd journal support

.circleci/config.yml

@@ -45,7 +45,7 @@ workflows:
 # https://circleci.com/blog/circleci-hacks-reuse-yaml-in-your-circleci-config-with-yaml/
 defaults: &defaults
   docker:
-    - image: grafana/loki-build-image:0.2.1
+    - image: grafana/loki-build-image:0.3.0
   working_directory: /go/src/github.com/grafana/loki
 
 jobs:
@@ -156,7 +156,7 @@ jobs:
           key: v1-loki-{{ .Branch }}-{{ .Revision }}
       - restore_cache:
           key: v1-loki-plugin-{{ .Branch }}-{{ .Revision }}
-          
+
       - run:
           name: Load Images
           command: |
@@ -168,7 +168,7 @@ jobs:
           command: |
             docker login -u "$DOCKER_USER" -p "$DOCKER_PASS" &&
             make push-latest
-      
+
       - run:
           name: Push Docker Plugin
           command: |

Gopkg.toml

@@ -75,3 +75,7 @@
 [[override]]
   name = "k8s.io/client-go"
   revision = "1a26190bd76a9017e289958b9fba936430aa3704"
+
+[[constraint]]
+  name = "github.com/coreos/go-systemd"
+  version = "19.0.0"

Makefile

@@ -104,6 +104,7 @@ loki-build-image/$(UPTODATE): loki-build-image/*
 # All the boiler plate for building golang follows:
 SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E")
 BUILD_IN_CONTAINER := true
+CGO_ENABLED := 0
 # RM is parameterized to allow CircleCI to run builds, as it
 # currently disallows `docker run --rm`. This value is overridden
 # in circle.yml
@@ -149,13 +150,13 @@ $(EXES) $(DEBUG_EXES) $(PROTO_GOS) $(YACC_GOS) lint test shell check-generated-f
 else
 
 $(DEBUG_EXES): loki-build-image/$(UPTODATE)
-	CGO_ENABLED=0 go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D)
+	CGO_ENABLED=$(CGO_ENABLED) go build $(DEBUG_GO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 	# Copy the delve binary to make it easily available to put in the binary's container.
 	[ -f "/go/bin/dlv" ] && mv "/go/bin/dlv" $(@D)/dlv
 
 $(EXES): loki-build-image/$(UPTODATE)
-	CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D)
+	CGO_ENABLED=$(CGO_ENABLED) go build $(GO_FLAGS) -o $@ ./$(@D)
 	$(NETGO_CHECK)
 
 %.pb.go: loki-build-image/$(UPTODATE)

build/Dockerfile

@@ -12,12 +12,12 @@ ARG GOARCH="amd64"
 COPY . /go/src/github.com/grafana/loki
 WORKDIR /go/src/github.com/grafana/loki
 RUN touch loki-build-image/.uptodate &&\
-    mkdir /build
+  mkdir /build
 
 # production image
 FROM golang as builder-production
 ARG APP
-RUN make BUILD_IN_CONTAINER=false cmd/${APP}/${APP} &&\
+RUN make CGO_ENABLED=1 BUILD_IN_CONTAINER=false cmd/${APP}/${APP} &&\
   mv cmd/${APP}/${APP} /build/${APP}
 
 FROM scratch as production
@@ -28,7 +28,7 @@ COPY --from=builder-production /build/${APP} /usr/bin/${APP}
 FROM golang as builder-debug
 ARG APP
 RUN go get github.com/go-delve/delve/cmd/dlv &&\
-  make BUILD_IN_CONTAINER=false cmd/promtail/promtail-debug &&\
+  make CGO_ENBALED=1 BUILD_IN_CONTAINER=false cmd/promtail/promtail-debug &&\
   mv cmd/${APP}/${APP}-debug /build/app-debug &&\
   mv cmd/${APP}/dlv /build/dlv
 

docs/promtail-examples.md

@@ -10,7 +10,7 @@ This example of config promtail based on original docker [config](https://github
 and show how work with 2 and more sources:
 
 Filename for example: my-docker-config.yaml
-```
+```yaml
 server:
   http_listen_port: 9080
   grpc_listen_port: 0
@@ -45,6 +45,7 @@ scrape_configs:
       __path__: /srv/log/someone_service/*.log
 
 ```
+
 #### Description
 
 Scrape_config section of config.yaml contents contains various jobs for parsing your logs
@@ -54,15 +55,85 @@ Scrape_config section of config.yaml contents contains various jobs for parsing
 `__path__` it is path to directory where stored your logs.
 
 If you run promtail and this config.yaml in Docker container, don't forget use docker volumes for mapping real directories
-with log to those folders in the container. 
+with log to those folders in the container.
 
 #### Example Use
 1) Create folder, for example `promtail`, then new sub directory `build/conf` and place there `my-docker-config.yaml`.
 2) Create new Dockerfile in root folder `promtail`, with contents
-```
+```dockerfile
 FROM grafana/promtail:latest
 COPY build/conf /etc/promtail
 ```
 3) Create your Docker image based on original Promtail image and tag it, for example `mypromtail-image`
 3) After that you can run Docker container by this command:
 `docker run -d --name promtail --network loki_network -p 9080:9080 -v /var/log:/var/log -v /srv/log/someone_service:/srv/log/someone_service mypromtail-image -config.file=/etc/promtail/my-docker-config.yaml`
+
+## Simple Systemd Journal Config
+
+This example demonstrates how to configure promtail to listen to systemd journal
+entries and write them to Loki:
+
+Filename for example: my-systemd-journal-config.yaml
+
+```yaml
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://ip_or_hostname_where_loki_runns:3100/api/prom/push
+
+scrape_configs:
+  - job_name: journal
+    journal:
+      since: 0
+      path: /var/log/journal
+      labels:
+        job: systemd-journal
+    relabel_configs:
+      - source_labels: ['__journal__systemd_unit']
+        target_label: 'unit'
+```
+
+### Description
+
+Just like the Docker example, the `scrape_configs` sections holds various
+jobs for parsing logs. A job with a `journal` key configures it for systemd
+journal reading.
+
+`since` is an optional unsigned integer value determining the earliest
+log that should be read. If nonzero, the integer is read as a nanosecond
+offset from the current system time when the reader is initialized. Zero
+is the default value, indicating that all journal entries should be read.
+
+`path` is an optional string specifying the path to read journal entries
+from. If unspecified, defaults to the system default (`/var/log/journal`).
+
+`labels`: is a map of string values specifying labels that should always
+be associated with each log entry being read from the systemd journal.
+In our example, each log will have a label of `job=systemd-journal`.
+
+Every field written to the systemd journal is available for processing
+in the `relabel_configs` section. Label names are converted to lowercase
+and prefixed with `__journal_`. After `relabel_configs` processes all
+labels for a job entry, any label starting with `__` is deleted.
+
+Our example renames the `_SYSTEMD_UNIT` label (available as
+`__journal__systemd_unit` in promtail) to `unit` so it will be available
+in Loki. All other labels from the journal entry are dropped.
+
+### Example Use
+
+`promtail` must have access to the journal path (`/var/log/journal`)
+where journal entries are stored for journal support to work correctly.
+
+If running with Docker, that means to bind that path:
+
+```bash
+docker run -d --name promtail --network loki_network -p 9080:9080 \
+  -v /var/log/journal:/var/log/journal \
+  mypromtail-image -config.file=/etc/promtail/my-systemd-journal-config.yaml
+```

loki-build-image/Dockerfile

@@ -1,23 +1,23 @@
 FROM golang:1.11.4-stretch
-RUN apt-get update && apt-get install -y file jq unzip protobuf-compiler libprotobuf-dev && \
-    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN apt-get update && apt-get install -y file jq unzip protobuf-compiler libprotobuf-dev libsystemd-dev && \
+	rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 ENV DOCKER_VER="17.03.0-ce"
 RUN curl -L -o /tmp/docker-$DOCKER_VER.tgz https://download.docker.com/linux/static/stable/x86_64/docker-$DOCKER_VER.tgz && \
-    tar -xz -C /tmp -f /tmp/docker-$DOCKER_VER.tgz && \
-    mv /tmp/docker/* /usr/bin && \
-		rm /tmp/docker-$DOCKER_VER.tgz
+	tar -xz -C /tmp -f /tmp/docker-$DOCKER_VER.tgz && \
+	mv /tmp/docker/* /usr/bin && \
+	rm /tmp/docker-$DOCKER_VER.tgz
 ENV HELM_VER="v2.13.1"
 RUN curl -L -o /tmp/helm-$HELM_VER.tgz http://storage.googleapis.com/kubernetes-helm/helm-${HELM_VER}-linux-amd64.tar.gz && \
-    tar -xz -C /tmp -f /tmp/helm-$HELM_VER.tgz && \
-    mv /tmp/linux-amd64/helm /usr/bin/helm && \
-    rm -rf /tmp/linux-amd64 /tmp/helm-$HELM_VER.tgz
+	tar -xz -C /tmp -f /tmp/helm-$HELM_VER.tgz && \
+	mv /tmp/linux-amd64/helm /usr/bin/helm && \
+	rm -rf /tmp/linux-amd64 /tmp/helm-$HELM_VER.tgz
 RUN go get \
-		github.com/golang/protobuf/protoc-gen-go \
-		github.com/gogo/protobuf/protoc-gen-gogoslick \
-		github.com/gogo/protobuf/gogoproto \
-		github.com/go-delve/delve/cmd/dlv \
-		golang.org/x/tools/cmd/goyacc && \
-		rm -rf /go/pkg /go/src
+	github.com/golang/protobuf/protoc-gen-go \
+	github.com/gogo/protobuf/protoc-gen-gogoslick \
+	github.com/gogo/protobuf/gogoproto \
+	github.com/go-delve/delve/cmd/dlv \
+	golang.org/x/tools/cmd/goyacc && \
+	rm -rf /go/pkg /go/src
 ENV GOLANGCI_LINT_COMMIT="692dacb773b703162c091c2d8c59f9cd2d6801db"
 RUN mkdir -p $(go env GOPATH)/src/github.com/golangci/ && git clone https://github.com/golangci/golangci-lint.git $(go env GOPATH)/src/github.com/golangci/golangci-lint && \
 	cd $(go env GOPATH)/src/github.com/golangci/golangci-lint  && git checkout ${GOLANGCI_LINT_COMMIT} && cd cmd/golangci-lint/  &&\

pkg/promtail/positions/positions.go

@@ -5,6 +5,8 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -32,14 +34,14 @@ type Positions struct {
 	logger    log.Logger
 	cfg       Config
 	mtx       sync.Mutex
-	positions map[string]int64
+	positions map[string]string
 	quit      chan struct{}
 	done      chan struct{}
 }
 
 // File format for the positions data.
 type File struct {
-	Positions map[string]int64 `yaml:"positions"`
+	Positions map[string]string `yaml:"positions"`
 }
 
 // New makes a new Positions.
@@ -67,20 +69,42 @@ func (p *Positions) Stop() {
 	<-p.done
 }
 
-// Put records (asynchronously) how far we've read through a file.
-func (p *Positions) Put(path string, pos int64) {
+// PutString records (asynchronsouly) how far we've read through a file.
+// Unlike Put, it records a string offset and is only useful for
+// JournalTargets which doesn't have integer offsets.
+func (p *Positions) PutString(path string, pos string) {
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
 	p.positions[path] = pos
 }
 
-// Get returns how far we've read through a file.
-func (p *Positions) Get(path string) int64 {
+// Put records (asynchronously) how far we've read through a file.
+func (p *Positions) Put(path string, pos int64) {
+	p.PutString(path, strconv.FormatInt(pos, 10))
+}
+
+// GetString returns how far we've through a file as a string.
+// JournalTarget writes a journal cursor to the positions file, while
+// FileTarget writes an integer offset. Use Get to read the integer
+// offset.
+func (p *Positions) GetString(path string) string {
 	p.mtx.Lock()
 	defer p.mtx.Unlock()
 	return p.positions[path]
 }
 
+// Get returns how far we've read through a file. Returns an error
+// if the value stored for the file is not an integer.
+func (p *Positions) Get(path string) (int64, error) {
+	p.mtx.Lock()
+	defer p.mtx.Unlock()
+	pos, ok := p.positions[path]
+	if !ok {
+		return 0, nil
+	}
+	return strconv.ParseInt(pos, 10, 64)
+}
+
 // Remove removes the position tracking for a filepath
 func (p *Positions) Remove(path string) {
 	p.mtx.Lock()
@@ -118,7 +142,7 @@ func (p *Positions) run() {
 
 func (p *Positions) save() {
 	p.mtx.Lock()
-	positions := make(map[string]int64, len(p.positions))
+	positions := make(map[string]string, len(p.positions))
 	for k, v := range p.positions {
 		positions[k] = v
 	}
@@ -134,6 +158,12 @@ func (p *Positions) cleanup() {
 	defer p.mtx.Unlock()
 	toRemove := []string{}
 	for k := range p.positions {
+		// If the position file is prefixed with journal, it's a
+		// JournalTarget cursor and not a file on disk.
+		if strings.HasPrefix(k, "journal-") {
+			continue
+		}
+
 		if _, err := os.Stat(k); err != nil {
 			if os.IsNotExist(err) {
 				// File no longer exists.
@@ -150,11 +180,11 @@ func (p *Positions) cleanup() {
 	}
 }
 
-func readPositionsFile(filename string) (map[string]int64, error) {
+func readPositionsFile(filename string) (map[string]string, error) {
 	buf, err := ioutil.ReadFile(filepath.Clean(filename))
 	if err != nil {
 		if os.IsNotExist(err) {
-			return map[string]int64{}, nil
+			return map[string]string{}, nil
 		}
 		return nil, err
 	}
@@ -167,7 +197,7 @@ func readPositionsFile(filename string) (map[string]int64, error) {
 	return p.Positions, nil
 }
 
-func writePositionFile(filename string, positions map[string]int64) error {
+func writePositionFile(filename string, positions map[string]string) error {
 	buf, err := yaml.Marshal(File{
 		Positions: positions,
 	})