diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4c8ab096bc1..6d6db808cf3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -75,7 +75,11 @@ jobs:
       - name: Test
         working-directory: ./yarn-project/end-to-end/
         timeout-minutes: 25
-        run: earthly-ci -P --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} --no-output +${{ matrix.test }}
+        run: earthly-ci -P \
+          --secret AWS_ACCESS_KEY_ID=${{ secrets.AWS_ACCESS_KEY_ID }} \
+          --secret AWS_SECRET_ACCESS_KEY=${{ secrets.AWS_SECRET_ACCESS_KEY }} \
+          --no-output \
+          +${{ matrix.test }} --skip_build=true
 
   # bench-summary:
   #   needs: e2e
@@ -108,7 +112,7 @@ jobs:
   # barretenberg (prover) native and AVM (public VM) tests
   # only ran on x86 for resource reasons (memory intensive)
   bb-native-tests:
-    needs: setup
+    needs: build
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - {
@@ -326,7 +330,7 @@ jobs:
 
   # push benchmarking binaries to dockerhub registry
   bb-bench-binaries:
-    needs: setup
+    needs: build
     runs-on: ${{ inputs.username || github.actor }}-x86
     steps:
       - {
diff --git a/yarn-project/end-to-end/Earthfile b/yarn-project/end-to-end/Earthfile
index 49dbde955ce..7085b410cc5 100644
--- a/yarn-project/end-to-end/Earthfile
+++ b/yarn-project/end-to-end/Earthfile
@@ -1,24 +1,27 @@
 VERSION 0.8
+ARG --global skip_build=false
 
 E2E_COMPOSE_TEST:
   FUNCTION
   ARG test
   ARG compose_file=./scripts/docker-compose.yml
   ARG debug="aztec:*"
-  ARG EARTHLY_TARGET_NAME
-  ARG prover_agents=10
   LOCALLY
   ENV TEST=$test
   ENV DEBUG=$debug
-  ENV PROVER_AGENTS=$prover_agents
   LET project_name=$(echo $test | sed 's/\./_/g')
   IF docker compose > /dev/null 2>&1
     LET CMD="docker compose"
   ELSE
     LET CMD="docker-compose"
   END
-  WAIT
-    BUILD ../+export-e2e-test-images
+  # In CI, we do an optimization to push these images to docker once
+  # We still want the default code path to work with no faff locally
+  # To not rebuild unnecessarily, we pass --skip_build=true in CI
+  IF [ $skip_build != "true" ]
+    WAIT
+      BUILD ../+export-e2e-test-images
+    END
   END
   # Let docker compose know about the pushed tags above
   ENV AZTEC_DOCKER_TAG=$(git rev-parse HEAD)