diff --git a/Dockerfile b/Dockerfile
index 864a19e8..06d356db 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,7 +23,7 @@ RUN mkdir -p ${DEST_ROOT}/usr/local/bin/
 
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
-RUN if [ ! -f $CACHITO_ENV_FILE ]; then go mod download ; fi
+RUN if [ ! -f $CACHITO_ENV_FILE ]; then rm -f go.work go.work.sum; go mod download ; fi
 
 # Build manager
 RUN if [ -f $CACHITO_ENV_FILE ] ; then source $CACHITO_ENV_FILE ; fi ; env ${GO_BUILD_EXTRA_ENV_ARGS} go build ${GO_BUILD_EXTRA_ARGS} -a -o ${DEST_ROOT}/manager main.go