Use go-chef for image builds (#989)

It *dramatically* reduces the amount of time it takes to re-run image building if the set of packages used hasn't changed. It's hard to get an objective measurement (because we do get *some* caching from being careful what we add), but: On my machine, for a rebuild after a change in 'pkg/api', it reduces `make docker-build` time from 249s -> 22s. And, it also slightly improves the time it takes to run a clean build. After `docker system prune -af`, it reduces `make docker-build` on my machine from 345s -> 126s.
neondatabase · Sep 12, 2024 · d42ed7c · d42ed7c
1 parent 517817d
commit d42ed7c
Show file tree

Hide file tree

Showing 8 changed files with 101 additions and 122 deletions.
diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml
@@ -58,6 +58,13 @@ env:
   ECR_DEV:                "369495373322.dkr.ecr.eu-central-1.amazonaws.com"
   ECR_PROD:               "093970136003.dkr.ecr.eu-central-1.amazonaws.com"
 
+  # Why localhost? We use a local registry so that when docker/build-push-action tries to pull the
+  # image we built locally, it'll actually have a place to pull from.
+  #
+  # Otherwise, if we just try to use a local image, it fails trying to pull it from dockerhub.
+  # See https://github.com/moby/buildkit/issues/2343 for more information.
+  GO_BASE_IMG: "localhost:5000/neondatabase/autoscaling-go-base:dev"
+
 defaults:
   run:
     shell: bash -euo pipefail {0}
@@ -96,6 +103,13 @@ jobs:
     if: ${{ format('{0}', inputs.skip) != 'true' }}
     needs: [ tags, vm-kernel ]
     runs-on: [ self-hosted, gen3, large ]
+
+    services:
+      registry:
+        image: registry:2
+        ports:
+          - 5000:5000
+
     steps:
       - uses: actions/checkout@v4
         with:
@@ -129,6 +143,8 @@ jobs:
         uses: ./.github/actions/set-docker-config-dir
 
       - uses: docker/setup-buildx-action@v3
+        with:
+          driver-opts: network=host
 
       - name: Login to Dockerhub
         uses: docker/login-action@v3
@@ -173,6 +189,18 @@ jobs:
           docker cp ${ID}:/vmlinuz neonvm/hack/kernel/vmlinuz
           docker rm -f ${ID}
 
+      - name: Build go dependencies image
+        uses: docker/build-push-action@v6
+        id: build-go-dependencies-image
+        with:
+          context: .
+          platforms: linux/amd64
+          push: true
+          file: Dockerfile.go-base
+          cache-from: type=registry,ref=cache.neon.build/autoscaling-go-base:cache
+          cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaling-go-base:cache,mode=max' || '' }}
+          tags: ${{ env.GO_BASE_IMG }}
+
       - name: Build and push neonvm-runner image
         uses: docker/build-push-action@v6
         with:
@@ -183,6 +211,8 @@ jobs:
           cache-from: type=registry,ref=cache.neon.build/neonvm-runner:cache
           cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-runner:cache,mode=max' || '' }}
           tags: ${{ needs.tags.outputs.runner }}
+          build-args: |
+            GO_BASE_IMG=${{ env.GO_BASE_IMG }}
 
       - name: Generate neonvm-controller build tags
         id: controller-build-tags
@@ -206,6 +236,7 @@ jobs:
           cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-controller:cache,mode=max' || '' }}
           tags: ${{ needs.tags.outputs.controller }}
           build-args: |
+            GO_BASE_IMG=${{ env.GO_BASE_IMG }}
             VM_RUNNER_IMAGE=${{ needs.tags.outputs.runner }}
             BUILDTAGS=${{ steps.controller-build-tags.outputs.buildtags }}
 
@@ -219,6 +250,8 @@ jobs:
           cache-from: type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache
           cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache,mode=max' || '' }}
           tags: ${{ needs.tags.outputs.vxlan-controller }}
+          build-args: |
+            GO_BASE_IMG=${{ env.GO_BASE_IMG }}
 
       - name: Build and push autoscale-scheduler image
         uses: docker/build-push-action@v6
@@ -231,6 +264,7 @@ jobs:
           cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscale-scheduler:cache,mode=max' || '' }}
           tags: ${{ needs.tags.outputs.scheduler }}
           build-args: |
+            GO_BASE_IMG=${{ env.GO_BASE_IMG }}
             GIT_INFO=${{ steps.get-git-info.outputs.info }}:${{ inputs.tag }}
 
       - name: Build and push autoscaler-agent image
@@ -244,6 +278,7 @@ jobs:
           cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaler-agent:cache,mode=max' || '' }}
           tags: ${{ needs.tags.outputs.autoscaler-agent }}
           build-args: |
+            GO_BASE_IMG=${{ env.GO_BASE_IMG }}
             GIT_INFO=${{ steps.get-git-info.outputs.info }}
 
       - name: Build and push cluster-autoscaler image

diff --git a/Dockerfile.go-base b/Dockerfile.go-base
@@ -0,0 +1,19 @@
+# Base image for go dependencies, to speed up builds when they haven't changed.
+# For more, see https://github.com/neondatabase/go-chef
+FROM golang:1.23-alpine AS chef
+
+ARG GO_CHEF_VERSION=v0.1.0
+RUN go install github.com/neondatabase/go-chef@$GO_CHEF_VERSION
+WORKDIR /workspace
+
+FROM chef AS planner
+COPY . .
+# Produce a "recipe" containing information about all the packages imported.
+# This step is usually NOT cached, but because the recipe is usually the same, follow-up steps
+# usually WILL be cached.
+RUN go-chef --prepare recipe.json
+
+FROM chef AS builder
+COPY --from=planner /workspace/recipe.json recipe.json
+# Compile the dependencies baesd on the "recipe" alone -- usually cached.
+RUN CGO_ENABLED=0 go-chef --cook recipe.json
diff --git a/Makefile b/Makefile
@@ -5,6 +5,10 @@ IMG_RUNNER ?= runner:dev
 IMG_SCHEDULER ?= autoscale-scheduler:dev
 IMG_AUTOSCALER_AGENT ?= autoscaler-agent:dev
 
+# Shared base image for caching compiled dependencies.
+# It's only used during image builds, so doesn't need to be pushed.
+GO_BASE_IMG ?= autoscaling-go-base:dev
+
 E2E_TESTS_VM_IMG ?= vm-postgres:15-bullseye
 PG16_DISK_TEST_IMG ?= pg16-disk-test:dev
 
@@ -153,41 +157,53 @@ docker-push: docker-build ## Push docker images to docker registry
 	docker push -q $(IMG_SCHEDULER)
 	docker push -q $(IMG_AUTOSCALER_AGENT)
 
+.PHONY: docker-build-go-base
+docker-build-go-base:
+	docker build \
+		--tag $(GO_BASE_IMG) \
+		--file Dockerfile.go-base \
+		.
+
 .PHONY: docker-build-controller
-docker-build-controller: ## Build docker image for NeonVM controller
+docker-build-controller: docker-build-go-base ## Build docker image for NeonVM controller
 	docker build \
 		--tag $(IMG_CONTROLLER) \
+		--build-arg GO_BASE_IMG=$(GO_BASE_IMG) \
 		--build-arg VM_RUNNER_IMAGE=$(IMG_RUNNER) \
 		--build-arg BUILDTAGS=$(if $(PRESERVE_RUNNER_PODS),nodelete) \
 		--file neonvm/Dockerfile \
 		.
 
 .PHONY: docker-build-runner
-docker-build-runner: ## Build docker image for NeonVM runner
+docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runner
 	docker build \
 		--tag $(IMG_RUNNER) \
+		--build-arg GO_BASE_IMG=$(GO_BASE_IMG) \
 		--file neonvm/runner/Dockerfile \
 		.
 
 .PHONY: docker-build-vxlan-controller
-docker-build-vxlan-controller: ## Build docker image for NeonVM vxlan controller
+docker-build-vxlan-controller: docker-build-go-base ## Build docker image for NeonVM vxlan controller
 	docker build \
 		--tag $(IMG_VXLAN_CONTROLLER) \
+		--build-arg GO_BASE_IMG=$(GO_BASE_IMG) \
 		--file neonvm/tools/vxlan/Dockerfile \
 		.
 
 .PHONY: docker-build-autoscaler-agent
-docker-build-autoscaler-agent: ## Build docker image for autoscaler-agent
+docker-build-autoscaler-agent: docker-build-go-base ## Build docker image for autoscaler-agent
 	docker buildx build \
 		--tag $(IMG_AUTOSCALER_AGENT) \
+		--build-arg GO_BASE_IMG=$(GO_BASE_IMG) \
 		--build-arg "GIT_INFO=$(GIT_INFO)" \
 		--file build/autoscaler-agent/Dockerfile \
 		.
 
 .PHONY: docker-build-scheduler
-docker-build-scheduler: ## Build docker image for (autoscaling) scheduler
+docker-build-scheduler: docker-build-go-base ## Build docker image for (autoscaling) scheduler
 	docker buildx build \
 		--tag $(IMG_SCHEDULER) \
+		--build-arg GO_BASE_IMG=$(GO_BASE_IMG) \
 		--build-arg "GIT_INFO=$(GIT_INFO)" \
 		--file build/autoscale-scheduler/Dockerfile \
 		.

diff --git a/build/autoscale-scheduler/Dockerfile b/build/autoscale-scheduler/Dockerfile
@@ -1,26 +1,12 @@
-FROM golang:1.23-alpine AS builder
-WORKDIR /workspace
-
-RUN apk add gcc musl-dev # gcc (and therefore musl-dev) is required for cgo extensions
-
-COPY go.mod go.mod
-COPY go.sum go.sum
-RUN go mod download
-
-COPY neonvm/apis             neonvm/apis
-COPY neonvm/client           neonvm/client
-COPY pkg/api                 pkg/api
-COPY pkg/plugin              pkg/plugin
-COPY pkg/util                pkg/util
-COPY cmd/autoscale-scheduler cmd/autoscale-scheduler
+ARG GO_BASE_IMG=autoscaling-go-base:dev
+FROM $GO_BASE_IMG AS builder
 
 ARG GIT_INFO
 
-RUN --mount=type=cache,target=/root/.cache/go-build \
-    go build -a \
-	# future compat: don't modify go.mod if we have a vendor directory \
-	-mod readonly \
-    # -ldflags "-X ..." allows us to overwrite the value of a variable in a package \
+COPY . .
+# NOTE: Build flags here must be the same as in the base image, otherwise we'll rebuild
+# dependencies. See /Dockerfile.go-base for detail on the "why".
+RUN CGO_ENABLED=0 go build \
     -ldflags "-X 'github.com/neondatabase/autoscaling/pkg/util.BuildGitInfo=$GIT_INFO'" \
     cmd/autoscale-scheduler/main.go
 

diff --git a/build/autoscaler-agent/Dockerfile b/build/autoscaler-agent/Dockerfile
@@ -1,27 +1,12 @@
-FROM golang:1.23-alpine AS builder
-WORKDIR /workspace
-
-RUN apk add gcc musl-dev # gcc (and therefore musl-dev) is required for cgo extensions
-
-COPY go.mod go.mod
-COPY go.sum go.sum
-RUN go mod download
-
-COPY neonvm/apis          neonvm/apis
-COPY neonvm/client        neonvm/client
-COPY pkg/agent            pkg/agent
-COPY pkg/api              pkg/api
-COPY pkg/billing          pkg/billing
-COPY pkg/util             pkg/util
-COPY cmd/autoscaler-agent cmd/autoscaler-agent
+ARG GO_BASE_IMG=autoscaling-go-base:dev
+FROM $GO_BASE_IMG AS builder
 
 ARG GIT_INFO
 
-RUN --mount=type=cache,target=/root/.cache/go-build \
-    go build -a \
-	# future compat: don't modify go.mod if we have a vendor directory \
-	-mod readonly \
-    # -ldflags "-X ..." allows us to overwrite the value of a variable in a package \
+COPY . .
+# NOTE: Build env vars here must be the same as in the base image, otherwise we'll rebuild
+# dependencies.
+RUN CGO_ENABLED=0 go build \
     -ldflags "-X 'github.com/neondatabase/autoscaling/pkg/util.BuildGitInfo=$GIT_INFO'" \
     cmd/autoscaler-agent/main.go
 

diff --git a/neonvm/Dockerfile b/neonvm/Dockerfile
@@ -1,33 +1,9 @@
-# Build the manager binary
-FROM golang:1.23 AS builder
-ARG TARGETOS
-ARG TARGETARCH
-ARG BUILDTAGS
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
+ARG GO_BASE_IMG=autoscaling-go-base:dev
+FROM $GO_BASE_IMG AS builder
 
-# Copy the go source
-COPY pkg/util            pkg/util
-COPY neonvm/main.go      neonvm/main.go
-COPY neonvm/apis/        neonvm/apis/
-COPY neonvm/controllers/ neonvm/controllers/
-COPY neonvm/pkg/         neonvm/pkg/
-COPY neonvm/client/      neonvm/client/
-COPY pkg/api/ pkg/api
-COPY pkg/util pkg/util
-
-# Build
-# the GOARCH has not a default value to allow the binary be built according to the host where the command
-# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
-# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
-# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -tags=${BUILDTAGS} -o manager neonvm/main.go
+# Build the manager binary
+COPY . .
+RUN CGO_ENABLED=0 go build -tags=${BUILDTAGS} -o manager neonvm/main.go
 
 # Use distroless as minimal base image to package the manager binary
 # Refer to https://github.com/GoogleContainerTools/distroless for more details

diff --git a/neonvm/runner/Dockerfile b/neonvm/runner/Dockerfile
@@ -1,31 +1,10 @@
-# Build the Go binary
-FROM golang:1.23 AS builder
-ARG TARGETOS
-ARG TARGETARCH
-
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
-
-# Copy the go source
-COPY neonvm/main.go      neonvm/main.go
-COPY neonvm/apis/        neonvm/apis/
-COPY neonvm/controllers/ neonvm/controllers/
-COPY neonvm/runner/      neonvm/runner/
-COPY pkg/api/ pkg/api
-COPY pkg/util pkg/util
+ARG GO_BASE_IMG=autoscaling-go-base:dev
+FROM $GO_BASE_IMG AS builder
 
+COPY . .
 # Build
-# the GOARCH has not a default value to allow the binary be built according to the host where the command
-# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
-# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
-# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /runner neonvm/runner/main.go
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /container-mgr neonvm/runner/container-mgr/*.go
+RUN CGO_ENABLED=0 go build -o /runner neonvm/runner/main.go
+RUN CGO_ENABLED=0 go build -o /container-mgr neonvm/runner/container-mgr/*.go
 
 FROM alpine:3.19 AS crictl
 

diff --git a/neonvm/tools/vxlan/Dockerfile b/neonvm/tools/vxlan/Dockerfile
@@ -1,25 +1,8 @@
-# Build the Go binary
-FROM golang:1.23 AS builder
-ARG TARGETOS
-ARG TARGETARCH
+ARG GO_BASE_IMG=autoscaling-go-base:dev
+FROM $GO_BASE_IMG AS builder
 
-WORKDIR /workspace
-# Copy the Go Modules manifests
-COPY go.mod go.mod
-COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-RUN go mod download
-
-# Copy the go source
-COPY neonvm/tools/vxlan/controller/ neonvm/tools/vxlan/controller/
-
-# Build
-# the GOARCH has not a default value to allow the binary be built according to the host where the command
-# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO
-# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore,
-# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform.
-RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /vxlan-controller neonvm/tools/vxlan/controller/main.go
+COPY . .
+RUN CGO_ENABLED=0 go build -o /vxlan-controller neonvm/tools/vxlan/controller/main.go
 
 FROM alpine:3.19