diff --git a/Makefile-os b/Makefile-os
index d241f8d392ff..195d47abb7fd 100644
--- a/Makefile-os
+++ b/Makefile-os
@@ -51,18 +51,23 @@ create_docker_builder: ## Create a custom builder for buildkit to efficiently bu
 		--name $(DOCKER_BUILDER) \
 		--driver=docker-container
 
+DOCKER_BUILD_ARGS := -t $(DOCKER_TAG) \
+--load \
+--platform $(DOCKER_PLATFORM) \
+--progress=$(DOCKER_PROGRESS) \
+--builder=$(DOCKER_BUILDER) \
+--label git.commit=$(DOCKER_COMMIT) \
+--cache-to=type=local,dest=$(DOCKER_CACHE_DIR)-new \
+
+DOCKER_CACHE_INDEX = $(wildcard $(DOCKER_CACHE_DIR)/index.json)
+
+ifneq ($(DOCKER_CACHE_INDEX),)
+	DOCKER_BUILD_ARGS += --cache-from=type=local,src=$(DOCKER_CACHE_DIR),mode=max
+endif
+
 .PHONY: build_docker_image
 build_docker_image: create_docker_builder ## Build the docker image
-	DOCKER_BUILDKIT=1 docker buildx build \
-		-t $(DOCKER_TAG) \
-		--load \
-		--platform $(DOCKER_PLATFORM) \
-		--progress=$(DOCKER_PROGRESS) \
-		--cache-to=type=local,dest=$(DOCKER_CACHE_DIR)-new \
-		--cache-from=type=local,src=$(DOCKER_CACHE_DIR),mode=max \
-		--builder=$(DOCKER_BUILDER) \
-		--label git.commit=$(DOCKER_COMMIT) \
-		.
+	DOCKER_BUILDKIT=1 docker buildx build $(DOCKER_BUILD_ARGS) .
 	rm -rf $(DOCKER_CACHE_DIR)
 	mv $(DOCKER_CACHE_DIR)-new $(DOCKER_CACHE_DIR)