diff --git a/.github/workflows/build-images.yaml b/.github/workflows/build-images.yaml index a7e19e385..05671dde1 100644 --- a/.github/workflows/build-images.yaml +++ b/.github/workflows/build-images.yaml @@ -58,6 +58,13 @@ env: ECR_DEV: "369495373322.dkr.ecr.eu-central-1.amazonaws.com" ECR_PROD: "093970136003.dkr.ecr.eu-central-1.amazonaws.com" + # Why localhost? We use a local registry so that when docker/build-push-action tries to pull the + # image we built locally, it'll actually have a place to pull from. + # + # Otherwise, if we just try to use a local image, it fails trying to pull it from dockerhub. + # See https://github.com/moby/buildkit/issues/2343 for more information. + GO_BASE_IMG: "localhost:5000/neondatabase/autoscaling-go-base:dev" + defaults: run: shell: bash -euo pipefail {0} @@ -96,6 +103,13 @@ jobs: if: ${{ format('{0}', inputs.skip) != 'true' }} needs: [ tags, vm-kernel ] runs-on: [ self-hosted, gen3, large ] + + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: - uses: actions/checkout@v4 with: @@ -129,6 +143,8 @@ jobs: uses: ./.github/actions/set-docker-config-dir - uses: docker/setup-buildx-action@v3 + with: + driver-opts: network=host - name: Login to Dockerhub uses: docker/login-action@v3 @@ -173,6 +189,18 @@ jobs: docker cp ${ID}:/vmlinuz neonvm/hack/kernel/vmlinuz docker rm -f ${ID} + - name: Build go dependencies image + uses: docker/build-push-action@v6 + id: build-go-dependencies-image + with: + context: . + platforms: linux/amd64 + push: true + file: Dockerfile.go-base + cache-from: type=registry,ref=cache.neon.build/autoscaling-go-base:cache + cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaling-go-base:cache,mode=max' || '' }} + tags: ${{ env.GO_BASE_IMG }} + - name: Build and push neonvm-runner image uses: docker/build-push-action@v6 with: @@ -183,6 +211,8 @@ jobs: cache-from: type=registry,ref=cache.neon.build/neonvm-runner:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-runner:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.runner }} + build-args: | + GO_BASE_IMG=${{ env.GO_BASE_IMG }} - name: Generate neonvm-controller build tags id: controller-build-tags @@ -206,6 +236,7 @@ jobs: cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-controller:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.controller }} build-args: | + GO_BASE_IMG=${{ env.GO_BASE_IMG }} VM_RUNNER_IMAGE=${{ needs.tags.outputs.runner }} BUILDTAGS=${{ steps.controller-build-tags.outputs.buildtags }} @@ -219,6 +250,8 @@ jobs: cache-from: type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/neonvm-vxlan-controller:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.vxlan-controller }} + build-args: | + GO_BASE_IMG=${{ env.GO_BASE_IMG }} - name: Build and push autoscale-scheduler image uses: docker/build-push-action@v6 @@ -231,6 +264,7 @@ jobs: cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscale-scheduler:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.scheduler }} build-args: | + GO_BASE_IMG=${{ env.GO_BASE_IMG }} GIT_INFO=${{ steps.get-git-info.outputs.info }}:${{ inputs.tag }} - name: Build and push autoscaler-agent image @@ -244,6 +278,7 @@ jobs: cache-to: ${{ github.ref_name == 'main' && 'type=registry,ref=cache.neon.build/autoscaler-agent:cache,mode=max' || '' }} tags: ${{ needs.tags.outputs.autoscaler-agent }} build-args: | + GO_BASE_IMG=${{ env.GO_BASE_IMG }} GIT_INFO=${{ steps.get-git-info.outputs.info }} - name: Build and push cluster-autoscaler image diff --git a/Dockerfile.go-base b/Dockerfile.go-base new file mode 100644 index 000000000..4b72833ab --- /dev/null +++ b/Dockerfile.go-base @@ -0,0 +1,19 @@ +# Base image for go dependencies, to speed up builds when they haven't changed. +# For more, see https://github.com/neondatabase/go-chef +FROM golang:1.23-alpine AS chef + +ARG GO_CHEF_VERSION=v0.1.0 +RUN go install github.com/neondatabase/go-chef@$GO_CHEF_VERSION +WORKDIR /workspace + +FROM chef AS planner +COPY . . +# Produce a "recipe" containing information about all the packages imported. +# This step is usually NOT cached, but because the recipe is usually the same, follow-up steps +# usually WILL be cached. +RUN go-chef --prepare recipe.json + +FROM chef AS builder +COPY --from=planner /workspace/recipe.json recipe.json +# Compile the dependencies baesd on the "recipe" alone -- usually cached. +RUN CGO_ENABLED=0 go-chef --cook recipe.json diff --git a/Makefile b/Makefile index e0bf249b0..e59a6c51d 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,10 @@ IMG_RUNNER ?= runner:dev IMG_SCHEDULER ?= autoscale-scheduler:dev IMG_AUTOSCALER_AGENT ?= autoscaler-agent:dev +# Shared base image for caching compiled dependencies. +# It's only used during image builds, so doesn't need to be pushed. +GO_BASE_IMG ?= autoscaling-go-base:dev + E2E_TESTS_VM_IMG ?= vm-postgres:15-bullseye PG16_DISK_TEST_IMG ?= pg16-disk-test:dev @@ -153,41 +157,53 @@ docker-push: docker-build ## Push docker images to docker registry docker push -q $(IMG_SCHEDULER) docker push -q $(IMG_AUTOSCALER_AGENT) +.PHONY: docker-build-go-base +docker-build-go-base: + docker build \ + --tag $(GO_BASE_IMG) \ + --file Dockerfile.go-base \ + . + .PHONY: docker-build-controller -docker-build-controller: ## Build docker image for NeonVM controller +docker-build-controller: docker-build-go-base ## Build docker image for NeonVM controller docker build \ --tag $(IMG_CONTROLLER) \ + --build-arg GO_BASE_IMG=$(GO_BASE_IMG) \ --build-arg VM_RUNNER_IMAGE=$(IMG_RUNNER) \ --build-arg BUILDTAGS=$(if $(PRESERVE_RUNNER_PODS),nodelete) \ --file neonvm/Dockerfile \ . .PHONY: docker-build-runner -docker-build-runner: ## Build docker image for NeonVM runner +docker-build-runner: docker-build-go-base ## Build docker image for NeonVM runner docker build \ --tag $(IMG_RUNNER) \ + --build-arg GO_BASE_IMG=$(GO_BASE_IMG) \ --file neonvm/runner/Dockerfile \ . .PHONY: docker-build-vxlan-controller -docker-build-vxlan-controller: ## Build docker image for NeonVM vxlan controller +docker-build-vxlan-controller: docker-build-go-base ## Build docker image for NeonVM vxlan controller docker build \ --tag $(IMG_VXLAN_CONTROLLER) \ + --build-arg GO_BASE_IMG=$(GO_BASE_IMG) \ --file neonvm/tools/vxlan/Dockerfile \ . .PHONY: docker-build-autoscaler-agent -docker-build-autoscaler-agent: ## Build docker image for autoscaler-agent +docker-build-autoscaler-agent: docker-build-go-base ## Build docker image for autoscaler-agent docker buildx build \ --tag $(IMG_AUTOSCALER_AGENT) \ + --build-arg GO_BASE_IMG=$(GO_BASE_IMG) \ --build-arg "GIT_INFO=$(GIT_INFO)" \ --file build/autoscaler-agent/Dockerfile \ . .PHONY: docker-build-scheduler -docker-build-scheduler: ## Build docker image for (autoscaling) scheduler +docker-build-scheduler: docker-build-go-base ## Build docker image for (autoscaling) scheduler docker buildx build \ --tag $(IMG_SCHEDULER) \ + --build-arg GO_BASE_IMG=$(GO_BASE_IMG) \ --build-arg "GIT_INFO=$(GIT_INFO)" \ --file build/autoscale-scheduler/Dockerfile \ . diff --git a/build/autoscale-scheduler/Dockerfile b/build/autoscale-scheduler/Dockerfile index ed8bf0f51..888cac644 100644 --- a/build/autoscale-scheduler/Dockerfile +++ b/build/autoscale-scheduler/Dockerfile @@ -1,26 +1,12 @@ -FROM golang:1.23-alpine AS builder -WORKDIR /workspace - -RUN apk add gcc musl-dev # gcc (and therefore musl-dev) is required for cgo extensions - -COPY go.mod go.mod -COPY go.sum go.sum -RUN go mod download - -COPY neonvm/apis neonvm/apis -COPY neonvm/client neonvm/client -COPY pkg/api pkg/api -COPY pkg/plugin pkg/plugin -COPY pkg/util pkg/util -COPY cmd/autoscale-scheduler cmd/autoscale-scheduler +ARG GO_BASE_IMG=autoscaling-go-base:dev +FROM $GO_BASE_IMG AS builder ARG GIT_INFO -RUN --mount=type=cache,target=/root/.cache/go-build \ - go build -a \ - # future compat: don't modify go.mod if we have a vendor directory \ - -mod readonly \ - # -ldflags "-X ..." allows us to overwrite the value of a variable in a package \ +COPY . . +# NOTE: Build flags here must be the same as in the base image, otherwise we'll rebuild +# dependencies. See /Dockerfile.go-base for detail on the "why". +RUN CGO_ENABLED=0 go build \ -ldflags "-X 'github.com/neondatabase/autoscaling/pkg/util.BuildGitInfo=$GIT_INFO'" \ cmd/autoscale-scheduler/main.go diff --git a/build/autoscaler-agent/Dockerfile b/build/autoscaler-agent/Dockerfile index 8b2fb4fa3..c67f103d7 100644 --- a/build/autoscaler-agent/Dockerfile +++ b/build/autoscaler-agent/Dockerfile @@ -1,27 +1,12 @@ -FROM golang:1.23-alpine AS builder -WORKDIR /workspace - -RUN apk add gcc musl-dev # gcc (and therefore musl-dev) is required for cgo extensions - -COPY go.mod go.mod -COPY go.sum go.sum -RUN go mod download - -COPY neonvm/apis neonvm/apis -COPY neonvm/client neonvm/client -COPY pkg/agent pkg/agent -COPY pkg/api pkg/api -COPY pkg/billing pkg/billing -COPY pkg/util pkg/util -COPY cmd/autoscaler-agent cmd/autoscaler-agent +ARG GO_BASE_IMG=autoscaling-go-base:dev +FROM $GO_BASE_IMG AS builder ARG GIT_INFO -RUN --mount=type=cache,target=/root/.cache/go-build \ - go build -a \ - # future compat: don't modify go.mod if we have a vendor directory \ - -mod readonly \ - # -ldflags "-X ..." allows us to overwrite the value of a variable in a package \ +COPY . . +# NOTE: Build env vars here must be the same as in the base image, otherwise we'll rebuild +# dependencies. +RUN CGO_ENABLED=0 go build \ -ldflags "-X 'github.com/neondatabase/autoscaling/pkg/util.BuildGitInfo=$GIT_INFO'" \ cmd/autoscaler-agent/main.go diff --git a/neonvm/Dockerfile b/neonvm/Dockerfile index 67373ecc1..c98a5a35d 100644 --- a/neonvm/Dockerfile +++ b/neonvm/Dockerfile @@ -1,33 +1,9 @@ -# Build the manager binary -FROM golang:1.23 AS builder -ARG TARGETOS -ARG TARGETARCH -ARG BUILDTAGS - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download +ARG GO_BASE_IMG=autoscaling-go-base:dev +FROM $GO_BASE_IMG AS builder -# Copy the go source -COPY pkg/util pkg/util -COPY neonvm/main.go neonvm/main.go -COPY neonvm/apis/ neonvm/apis/ -COPY neonvm/controllers/ neonvm/controllers/ -COPY neonvm/pkg/ neonvm/pkg/ -COPY neonvm/client/ neonvm/client/ -COPY pkg/api/ pkg/api -COPY pkg/util pkg/util - -# Build -# the GOARCH has not a default value to allow the binary be built according to the host where the command -# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO -# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, -# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -tags=${BUILDTAGS} -o manager neonvm/main.go +# Build the manager binary +COPY . . +RUN CGO_ENABLED=0 go build -tags=${BUILDTAGS} -o manager neonvm/main.go # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details diff --git a/neonvm/runner/Dockerfile b/neonvm/runner/Dockerfile index 979e21e92..d4f4bbe63 100644 --- a/neonvm/runner/Dockerfile +++ b/neonvm/runner/Dockerfile @@ -1,31 +1,10 @@ -# Build the Go binary -FROM golang:1.23 AS builder -ARG TARGETOS -ARG TARGETARCH - -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -# Copy the go source -COPY neonvm/main.go neonvm/main.go -COPY neonvm/apis/ neonvm/apis/ -COPY neonvm/controllers/ neonvm/controllers/ -COPY neonvm/runner/ neonvm/runner/ -COPY pkg/api/ pkg/api -COPY pkg/util pkg/util +ARG GO_BASE_IMG=autoscaling-go-base:dev +FROM $GO_BASE_IMG AS builder +COPY . . # Build -# the GOARCH has not a default value to allow the binary be built according to the host where the command -# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO -# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, -# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /runner neonvm/runner/main.go -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /container-mgr neonvm/runner/container-mgr/*.go +RUN CGO_ENABLED=0 go build -o /runner neonvm/runner/main.go +RUN CGO_ENABLED=0 go build -o /container-mgr neonvm/runner/container-mgr/*.go FROM alpine:3.19 AS crictl diff --git a/neonvm/tools/vxlan/Dockerfile b/neonvm/tools/vxlan/Dockerfile index b484ad400..5e008b797 100644 --- a/neonvm/tools/vxlan/Dockerfile +++ b/neonvm/tools/vxlan/Dockerfile @@ -1,25 +1,8 @@ -# Build the Go binary -FROM golang:1.23 AS builder -ARG TARGETOS -ARG TARGETARCH +ARG GO_BASE_IMG=autoscaling-go-base:dev +FROM $GO_BASE_IMG AS builder -WORKDIR /workspace -# Copy the Go Modules manifests -COPY go.mod go.mod -COPY go.sum go.sum -# cache deps before building and copying source so that we don't need to re-download as much -# and so that source changes don't invalidate our downloaded layer -RUN go mod download - -# Copy the go source -COPY neonvm/tools/vxlan/controller/ neonvm/tools/vxlan/controller/ - -# Build -# the GOARCH has not a default value to allow the binary be built according to the host where the command -# was called. For example, if we call make docker-build in a local env which has the Apple Silicon M1 SO -# the docker BUILDPLATFORM arg will be linux/arm64 when for Apple x86 it will be linux/amd64. Therefore, -# by leaving it empty we can ensure that the container and binary shipped on it will have the same platform. -RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -a -o /vxlan-controller neonvm/tools/vxlan/controller/main.go +COPY . . +RUN CGO_ENABLED=0 go build -o /vxlan-controller neonvm/tools/vxlan/controller/main.go FROM alpine:3.19