diff --git a/.circleci/config.yml b/.circleci/config.yml index 90a3e4c2604b..f54f68ca0377 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,9 +55,6 @@ jobs: steps: - checkout - setup_remote_docker - - run: - name: touch-protos - command: make touch-protos - run: name: build command: make GOOS=windows GOGC=10 promtail diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index b2efdbe2323b..eca7e77caf4c 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -210,7 +210,6 @@ local promtail(arch) = pipeline('promtail-' + arch) + arch_image(arch) { when: condition('exclude').tagMain, settings+: { dry_run: true, - build_args: ['TOUCH_PROTOS=1'], }, }, ] + [ @@ -218,9 +217,7 @@ local promtail(arch) = pipeline('promtail-' + arch) + arch_image(arch) { clients_docker(arch, 'promtail') { depends_on: ['image-tag'], when: condition('include').tagMain, - settings+: { - build_args: ['TOUCH_PROTOS=1'], - }, + settings+: {}, }, ], depends_on: ['check'], @@ -249,9 +246,7 @@ local lambda_promtail(tags='') = pipeline('lambda-promtail'){ lambda_promtail_ecr('lambda-promtail') { depends_on: ['image-tag'], when: condition('include').tagMain, - settings+: { - build_args: ['TOUCH_PROTOS=1'], - }, + settings+: {}, }, ], depends_on: ['check'], @@ -265,7 +260,6 @@ local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) { when: condition('exclude').tagMain, settings+: { dry_run: true, - build_args: ['TOUCH_PROTOS=1'], }, } for app in apps @@ -274,9 +268,7 @@ local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) { docker(arch, app) { depends_on: ['image-tag'], when: condition('include').tagMain, - settings+: { - build_args: ['TOUCH_PROTOS=1'], - }, + settings+: {}, } for app in apps ], @@ -323,9 +315,9 @@ local manifest(apps) = pipeline('manifest') { path: 'loki', }, steps: [ - make('test', container=false) { depends_on: ['clone'] }, - make('lint', container=false) { depends_on: ['clone'] }, make('check-generated-files', container=false) { depends_on: ['clone'] }, + make('test', container=false) { depends_on: ['clone','check-generated-files'] }, + make('lint', container=false) { depends_on: ['clone','check-generated-files'] }, make('check-mod', container=false) { depends_on: ['clone', 'test', 'lint'] }, { name: 'shellcheck', diff --git a/.drone/drone.yml b/.drone/drone.yml index add72a6c397d..3cbb6adbdbb1 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -3,23 +3,25 @@ kind: pipeline name: check steps: - commands: - - make BUILD_IN_CONTAINER=false test + - make BUILD_IN_CONTAINER=false check-generated-files depends_on: - clone image: grafana/loki-build-image:0.18.0 - name: test + name: check-generated-files - commands: - - make BUILD_IN_CONTAINER=false lint + - make BUILD_IN_CONTAINER=false test depends_on: - clone + - check-generated-files image: grafana/loki-build-image:0.18.0 - name: lint + name: test - commands: - - make BUILD_IN_CONTAINER=false check-generated-files + - make BUILD_IN_CONTAINER=false lint depends_on: - clone + - check-generated-files image: grafana/loki-build-image:0.18.0 - name: check-generated-files + name: lint - commands: - make BUILD_IN_CONTAINER=false check-mod depends_on: @@ -73,8 +75,6 @@ steps: image: plugins/docker name: build-loki-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki/Dockerfile dry_run: true password: @@ -93,8 +93,6 @@ steps: image: plugins/docker name: build-loki-canary-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki-canary/Dockerfile dry_run: true password: @@ -113,8 +111,6 @@ steps: image: plugins/docker name: build-logcli-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/logcli/Dockerfile dry_run: true password: @@ -133,8 +129,6 @@ steps: image: plugins/docker name: publish-loki-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki/Dockerfile dry_run: false password: @@ -153,8 +147,6 @@ steps: image: plugins/docker name: publish-loki-canary-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki-canary/Dockerfile dry_run: false password: @@ -173,8 +165,6 @@ steps: image: plugins/docker name: publish-logcli-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/logcli/Dockerfile dry_run: false password: @@ -208,8 +198,6 @@ steps: image: plugins/docker name: build-loki-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki/Dockerfile dry_run: true password: @@ -228,8 +216,6 @@ steps: image: plugins/docker name: build-loki-canary-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki-canary/Dockerfile dry_run: true password: @@ -248,8 +234,6 @@ steps: image: plugins/docker name: build-logcli-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/logcli/Dockerfile dry_run: true password: @@ -268,8 +252,6 @@ steps: image: plugins/docker name: publish-loki-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki/Dockerfile dry_run: false password: @@ -288,8 +270,6 @@ steps: image: plugins/docker name: publish-loki-canary-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki-canary/Dockerfile dry_run: false password: @@ -308,8 +288,6 @@ steps: image: plugins/docker name: publish-logcli-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/logcli/Dockerfile dry_run: false password: @@ -343,8 +321,6 @@ steps: image: plugins/docker name: build-loki-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki/Dockerfile dry_run: true password: @@ -363,8 +339,6 @@ steps: image: plugins/docker name: build-loki-canary-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki-canary/Dockerfile dry_run: true password: @@ -383,8 +357,6 @@ steps: image: plugins/docker name: build-logcli-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/logcli/Dockerfile dry_run: true password: @@ -403,8 +375,6 @@ steps: image: plugins/docker name: publish-loki-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki/Dockerfile dry_run: false password: @@ -423,8 +393,6 @@ steps: image: plugins/docker name: publish-loki-canary-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/loki-canary/Dockerfile dry_run: false password: @@ -443,8 +411,6 @@ steps: image: plugins/docker name: publish-logcli-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: cmd/logcli/Dockerfile dry_run: false password: @@ -478,8 +444,6 @@ steps: image: plugins/docker name: build-promtail-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: clients/cmd/promtail/Dockerfile dry_run: true password: @@ -498,8 +462,6 @@ steps: image: plugins/docker name: publish-promtail-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: clients/cmd/promtail/Dockerfile dry_run: false password: @@ -533,8 +495,6 @@ steps: image: plugins/docker name: build-promtail-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: clients/cmd/promtail/Dockerfile dry_run: true password: @@ -553,8 +513,6 @@ steps: image: plugins/docker name: publish-promtail-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: clients/cmd/promtail/Dockerfile dry_run: false password: @@ -588,8 +546,6 @@ steps: image: plugins/docker name: build-promtail-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: clients/cmd/promtail/Dockerfile.arm32 dry_run: true password: @@ -608,8 +564,6 @@ steps: image: plugins/docker name: publish-promtail-image settings: - build_args: - - TOUCH_PROTOS=1 dockerfile: clients/cmd/promtail/Dockerfile.arm32 dry_run: false password: @@ -925,8 +879,6 @@ steps: settings: access_key: from_secret: ecr_key - build_args: - - TOUCH_PROTOS=1 dockerfile: tools/lambda-promtail/Dockerfile dry_run: false region: us-east-1 @@ -984,6 +936,6 @@ kind: secret name: deploy_config --- kind: signature -hmac: 55440faa2728a5b8abbd213c2cf198e01f00201ba7143391924da1b9aa02c350 +hmac: b51ec8dfc84d0be83827fc851b21b81e1091886be480d675f51485b647e58001 ... diff --git a/Makefile b/Makefile index 0b9c51001770..a6ea61fecff5 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .DEFAULT_GOAL := all -.PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos touch-protobuf-sources touch-protos format +.PHONY: all images check-generated-files logcli loki loki-debug promtail promtail-debug loki-canary lint test clean yacc protos touch-protobuf-sources format .PHONY: docker-driver docker-driver-clean docker-driver-enable docker-driver-push .PHONY: fluent-bit-image, fluent-bit-push, fluent-bit-test .PHONY: fluentd-image, fluentd-push, fluentd-test @@ -8,6 +8,7 @@ .PHONY: benchmark-store, drone, check-mod .PHONY: migrate migrate-image lint-markdown ragel .PHONY: validate-example-configs generate-example-config-doc check-example-config-doc +.PHONY: clean clean-protos SHELL = /usr/bin/env bash @@ -128,10 +129,10 @@ binfmt: ################ # Main Targets # ################ -all: promtail logcli loki loki-canary check-generated-files +all: promtail logcli loki loki-canary # This is really a check for the CI to make sure generated files are built and checked in manually -check-generated-files: touch-protobuf-sources yacc ragel protos clients/pkg/promtail/server/ui/assets_vfsdata.go +check-generated-files: yacc ragel protos clients/pkg/promtail/server/ui/assets_vfsdata.go @if ! (git diff --exit-code $(YACC_GOS) $(RAGEL_GOS) $(PROTO_GOS) $(PROMTAIL_GENERATED_FILE)); then \ echo "\nChanges found in generated files"; \ echo "Run 'make check-generated-files' and commit the changes to fix this error."; \ @@ -140,14 +141,6 @@ check-generated-files: touch-protobuf-sources yacc ragel protos clients/pkg/prom exit 1; \ fi -# Trick used to ensure that protobuf files are always compiled even if not changed, because the -# tooling may have been upgraded and the compiled output may be different. We're not using a -# PHONY target so that we can control where we want to touch it. -touch-protobuf-sources: - for def in $(PROTO_DEFS); do \ - touch $$def; \ - done - ########## # Logcli # ########## @@ -165,8 +158,8 @@ cmd/logcli/logcli: $(APP_GO_FILES) cmd/logcli/main.go # Loki # ######## -loki: protos yacc ragel cmd/loki/loki -loki-debug: protos yacc ragel cmd/loki/loki-debug +loki: cmd/loki/loki +loki-debug: cmd/loki/loki-debug cmd/loki/loki: $(APP_GO_FILES) cmd/loki/main.go CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) @@ -180,7 +173,7 @@ cmd/loki/loki-debug: $(APP_GO_FILES) cmd/loki/main.go # Loki-Canary # ############### -loki-canary: protos yacc ragel cmd/loki-canary/loki-canary +loki-canary: cmd/loki-canary/loki-canary cmd/loki-canary/loki-canary: $(APP_GO_FILES) cmd/loki-canary/main.go CGO_ENABLED=0 go build $(GO_FLAGS) -o $@ ./$(@D) @@ -281,6 +274,9 @@ test: all # Clean # ######### +clean-protos: + rm -rf $(PROTO_GOS) + clean: rm -rf clients/cmd/promtail/promtail rm -rf cmd/loki/loki @@ -340,13 +336,9 @@ endif # Protobufs # ############# -protos: $(PROTO_GOS) - -# use with care. This signals to make that the proto definitions don't need recompiling. -touch-protos: - for proto in $(PROTO_GOS); do [ -f "./$${proto}" ] && touch "$${proto}" && echo "touched $${proto}"; done +protos: clean-protos $(PROTO_GOS) -%.pb.go: $(PROTO_DEFS) +%.pb.go: ifeq ($(BUILD_IN_CONTAINER),true) @mkdir -p $(shell pwd)/.pkg @mkdir -p $(shell pwd)/.cache diff --git a/clients/cmd/promtail/Dockerfile b/clients/cmd/promtail/Dockerfile index 726d7366ff3b..1d037f7c3900 100644 --- a/clients/cmd/promtail/Dockerfile +++ b/clients/cmd/promtail/Dockerfile @@ -1,13 +1,11 @@ FROM golang:1.17.2-bullseye as build -# TOUCH_PROTOS signifies if we should touch the compiled proto files and thus not regenerate them. -# This is helpful when file system timestamps can't be trusted with make -ARG TOUCH_PROTOS + COPY . /src/loki WORKDIR /src/loki # Backports repo required to get a libsystemd version 246 or newer which is required to handle journal +ZSTD compression RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list RUN apt-get update && apt-get install -t bullseye-backports -qy libsystemd-dev -RUN make clean && (if [ "${TOUCH_PROTOS}" ]; then make touch-protos; fi) && make BUILD_IN_CONTAINER=false promtail +RUN make clean && make BUILD_IN_CONTAINER=false promtail # Promtail requires debian as the base image to support systemd journal reading FROM debian:bullseye-slim diff --git a/clients/cmd/promtail/Dockerfile.arm32 b/clients/cmd/promtail/Dockerfile.arm32 index 0a5e8c759090..a0da39364c11 100644 --- a/clients/cmd/promtail/Dockerfile.arm32 +++ b/clients/cmd/promtail/Dockerfile.arm32 @@ -1,11 +1,9 @@ FROM golang:1.17.2 as build -# TOUCH_PROTOS signifies if we should touch the compiled proto files and thus not regenerate them. -# This is helpful when file system timestamps can't be trusted with make -ARG TOUCH_PROTOS + COPY . /src/loki WORKDIR /src/loki RUN apt-get update && apt-get install -qy libsystemd-dev -RUN make clean && (if [ "${TOUCH_PROTOS}" ]; then make touch-protos; fi) && make BUILD_IN_CONTAINER=false promtail +RUN make clean && make BUILD_IN_CONTAINER=false promtail # Promtail requires debian as the base image to support systemd journal reading FROM debian:stretch-slim diff --git a/cmd/logcli/Dockerfile b/cmd/logcli/Dockerfile index d804f4112809..70598177de1a 100644 --- a/cmd/logcli/Dockerfile +++ b/cmd/logcli/Dockerfile @@ -1,9 +1,8 @@ FROM golang:1.17.2 as build -ARG TOUCH_PROTOS COPY . /src/loki WORKDIR /src/loki -RUN make clean && (if [ "${TOUCH_PROTOS}" ]; then make touch-protos; fi) && make BUILD_IN_CONTAINER=false logcli +RUN make clean && make BUILD_IN_CONTAINER=false logcli FROM alpine:3.13 diff --git a/cmd/loki-canary/Dockerfile b/cmd/loki-canary/Dockerfile index e5aa0b6a8c80..7faa6450efa9 100644 --- a/cmd/loki-canary/Dockerfile +++ b/cmd/loki-canary/Dockerfile @@ -1,10 +1,8 @@ FROM golang:1.17.2 as build -# TOUCH_PROTOS signifies if we should touch the compiled proto files and thus not regenerate them. -# This is helpful when file system timestamps can't be trusted with make -ARG TOUCH_PROTOS + COPY . /src/loki WORKDIR /src/loki -RUN make clean && (if [ "${TOUCH_PROTOS}" ]; then make touch-protos; fi) && make BUILD_IN_CONTAINER=false loki-canary +RUN make clean && make BUILD_IN_CONTAINER=false loki-canary FROM alpine:3.13 RUN apk add --update --no-cache ca-certificates diff --git a/cmd/loki/Dockerfile b/cmd/loki/Dockerfile index 81b69d4ef4c8..0fad55ff9bcf 100644 --- a/cmd/loki/Dockerfile +++ b/cmd/loki/Dockerfile @@ -1,10 +1,8 @@ FROM golang:1.17.2 as build -# TOUCH_PROTOS signifies if we should touch the compiled proto files and thus not regenerate them. -# This is helpful when file system timestamps can't be trusted with make -ARG TOUCH_PROTOS + COPY . /src/loki WORKDIR /src/loki -RUN make clean && (if [ "${TOUCH_PROTOS}" ]; then make touch-protos; fi) && make BUILD_IN_CONTAINER=false loki +RUN make clean && make BUILD_IN_CONTAINER=false loki FROM alpine:3.13 diff --git a/cmd/querytee/Dockerfile b/cmd/querytee/Dockerfile index ec873807529a..61e328e0a5d2 100644 --- a/cmd/querytee/Dockerfile +++ b/cmd/querytee/Dockerfile @@ -1,6 +1,5 @@ FROM golang:1.17.2 as build -ARG TOUCH_PROTOS COPY . /src/loki WORKDIR /src/loki RUN make clean && make BUILD_IN_CONTAINER=false loki-querytee