diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
index 3d204b8dfa68..b6294ce73e16 100644
--- a/.github/workflows/nightly.yaml
+++ b/.github/workflows/nightly.yaml
@@ -28,20 +28,14 @@ jobs:
         with:
           java-version: 11
 
-      - name: Cache gradle dependencies
+      - name: Test
         uses: burrunan/gradle-cache-action@v1
-        with:
-          job-id: jdk${{ matrix.java }}
-
-      - name: Test with Gradle
-        uses: nick-invision/retry@v1
         env:
           S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.S3_BUILD_CACHE_ACCESS_KEY_ID }}
           S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.S3_BUILD_CACHE_SECRET_KEY }}
         with:
-          command: ./gradlew testJava${{ matrix.java }} --stacktrace
-          timeout_minutes: 60
-          max_attempts: 3
+          job-id: jdk${{ matrix.java }}
+          arguments: testJava${{ matrix.java }} --stacktrace -Dorg.gradle.caching.debug=true
 
       - name: Aggregate test reports with ciMate
         if: always()
@@ -63,20 +57,14 @@ jobs:
         with:
           java-version: 11
 
-      - name: Cache gradle dependencies
+      - name: Test
         uses: burrunan/gradle-cache-action@v1
         with:
           job-id: latestDepTest
-
-      - name: Test with Gradle
-        uses: nick-invision/retry@v1
+          arguments: test -PtestLatestDeps=true --stacktrace
         env:
           S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ secrets.S3_BUILD_CACHE_ACCESS_KEY_ID }}
           S3_BUILD_CACHE_SECRET_KEY: ${{ secrets.S3_BUILD_CACHE_SECRET_KEY }}
-        with:
-          command: ./gradlew test -PtestLatestDeps=true --stacktrace
-          timeout_minutes: 60
-          max_attempts: 3
 
       - name: Aggregate test reports with ciMate
         if: always()
diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml
index 4b7d1b2ae2e6..b148a6602cef 100644
--- a/.github/workflows/pr.yaml
+++ b/.github/workflows/pr.yaml
@@ -24,17 +24,11 @@ jobs:
         with:
           java-version: 11
 
-      - name: Cache gradle dependencies
+      - name: Test
         uses: burrunan/gradle-cache-action@v1
         with:
           job-id: jdk${{ matrix.java }}
-
-      - name: Test with Gradle
-        uses: nick-invision/retry@v1
-        with:
-          command: ./gradlew testJava${{ matrix.java }} --stacktrace
-          timeout_minutes: 60
-          max_attempts: 3
+          arguments: testJava${{ matrix.java }} --stacktrace
 
       - name: Aggregate test reports with ciMate
         if: always()
diff --git a/settings.gradle b/settings.gradle
index 152f6b1d841f..facf7e008e58 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -23,7 +23,7 @@ buildCache {
   remote(com.github.burrunan.s3cache.AwsS3BuildCache) {
     region = 'us-west-2'
     bucket = 'opentelemetry-java-instrumentation-gradle-test1'
-    push = System.getenv('S3_BUILD_CACHE_ACCESS_KEY_ID') != null
+    push = isCI && !System.getenv('S3_BUILD_CACHE_ACCESS_KEY_ID')?.isBlank()
   }
 }