diff --git a/nix/bazel-retry-cache.patch b/nix/bazel-retry-cache.patch index 7b3e3b3c9627..f49c640cb0ef 100644 --- a/nix/bazel-retry-cache.patch +++ b/nix/bazel-retry-cache.patch @@ -112,7 +112,7 @@ index 9ce71c7c52..5c7f2d0728 100644 "//src/main/java/com/google/devtools/build/lib/vfs", "//third_party:auth", diff --git a/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java b/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java -index 1efecd3bb1..d0a1fb902c 100644 +index 1efecd3bb1..98a2866200 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java +++ b/src/main/java/com/google/devtools/build/lib/remote/http/HttpCacheClient.java @@ -19,12 +19,14 @@ import com.google.auth.Credentials; @@ -145,11 +145,7 @@ index 1efecd3bb1..d0a1fb902c 100644 import java.util.function.Function; import java.util.regex.Pattern; import javax.annotation.Nullable; -@@ -129,9 +133,11 @@ public final class HttpCacheClient implements RemoteCacheClient { - private final URI uri; - private final int timeoutSeconds; - private final ImmutableList> extraHttpHeaders; -+ private AtomicLong bytesDownloaded = new AtomicLong(); +@@ -132,6 +136,7 @@ public final class HttpCacheClient implements RemoteCacheClient { private final boolean useTls; private final boolean verifyDownloads; private final DigestUtil digestUtil; @@ -157,7 +153,7 @@ index 1efecd3bb1..d0a1fb902c 100644 private final Object closeLock = new Object(); -@@ -153,6 +159,7 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -153,6 +158,7 @@ public final class HttpCacheClient implements RemoteCacheClient { boolean verifyDownloads, ImmutableList> extraHttpHeaders, DigestUtil digestUtil, @@ -165,7 +161,7 @@ index 1efecd3bb1..d0a1fb902c 100644 @Nullable final Credentials creds) throws Exception { return new HttpCacheClient( -@@ -164,6 +171,7 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -164,6 +170,7 @@ public final class HttpCacheClient implements RemoteCacheClient { verifyDownloads, extraHttpHeaders, digestUtil, @@ -173,7 +169,7 @@ index 1efecd3bb1..d0a1fb902c 100644 creds, null); } -@@ -176,6 +184,7 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -176,6 +183,7 @@ public final class HttpCacheClient implements RemoteCacheClient { boolean verifyDownloads, ImmutableList> extraHttpHeaders, DigestUtil digestUtil, @@ -181,7 +177,7 @@ index 1efecd3bb1..d0a1fb902c 100644 @Nullable final Credentials creds) throws Exception { -@@ -189,6 +198,7 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -189,6 +197,7 @@ public final class HttpCacheClient implements RemoteCacheClient { verifyDownloads, extraHttpHeaders, digestUtil, @@ -189,7 +185,7 @@ index 1efecd3bb1..d0a1fb902c 100644 creds, domainSocketAddress); } else if (Epoll.isAvailable()) { -@@ -201,6 +211,7 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -201,6 +210,7 @@ public final class HttpCacheClient implements RemoteCacheClient { verifyDownloads, extraHttpHeaders, digestUtil, @@ -197,7 +193,7 @@ index 1efecd3bb1..d0a1fb902c 100644 creds, domainSocketAddress); } else { -@@ -217,6 +228,7 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -217,6 +227,7 @@ public final class HttpCacheClient implements RemoteCacheClient { boolean verifyDownloads, ImmutableList> extraHttpHeaders, DigestUtil digestUtil, @@ -205,7 +201,7 @@ index 1efecd3bb1..d0a1fb902c 100644 @Nullable final Credentials creds, @Nullable SocketAddress socketAddress) throws Exception { -@@ -285,6 +297,7 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -285,6 +296,7 @@ public final class HttpCacheClient implements RemoteCacheClient { this.extraHttpHeaders = extraHttpHeaders; this.verifyDownloads = verifyDownloads; this.digestUtil = digestUtil; @@ -213,13 +209,21 @@ index 1efecd3bb1..d0a1fb902c 100644 } @SuppressWarnings("FutureReturnValueIgnored") -@@ -388,8 +401,18 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -363,7 +375,7 @@ public final class HttpCacheClient implements RemoteCacheClient { + } + + @SuppressWarnings("FutureReturnValueIgnored") +- private Future acquireDownloadChannel() { ++ private Future acquireDownloadChannel(long offset) { + Promise channelReady = eventLoop.next().newPromise(); + channelPool + .acquire() +@@ -388,8 +400,17 @@ public final class HttpCacheClient implements RemoteCacheClient { new IdleTimeoutHandler(timeoutSeconds, ReadTimeoutException.INSTANCE)); pipeline.addLast(new HttpClientCodec()); pipeline.addLast("inflater", new HttpContentDecompressor()); + ImmutableList.Builder> headersBuilder = ImmutableList.>builder(); + headersBuilder.addAll(extraHttpHeaders); -+ long offset = bytesDownloaded.get(); + if (offset > 0) { + logger.atInfo().log("Resuming download at offset %d at %s", offset, uri); + System.err.println(String.format("RETRY Resuming download at offset %d at %s", offset, uri)); @@ -233,7 +237,15 @@ index 1efecd3bb1..d0a1fb902c 100644 } if (!channel.eventLoop().inEventLoop()) { -@@ -470,12 +493,14 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -460,6 +481,7 @@ public final class HttpCacheClient implements RemoteCacheClient { + + @SuppressWarnings("FutureReturnValueIgnored") + private ListenableFuture get(Digest digest, final OutputStream out, boolean casDownload) { ++ final AtomicLong bytesDownloaded = new AtomicLong(); + final AtomicBoolean dataWritten = new AtomicBoolean(); + OutputStream wrappedOut = + new OutputStream() { +@@ -470,12 +492,14 @@ public final class HttpCacheClient implements RemoteCacheClient { @Override public void write(byte[] b, int offset, int length) throws IOException { dataWritten.set(true); @@ -248,7 +260,7 @@ index 1efecd3bb1..d0a1fb902c 100644 out.write(b); } -@@ -485,56 +510,58 @@ public final class HttpCacheClient implements RemoteCacheClient { +@@ -485,61 +509,63 @@ public final class HttpCacheClient implements RemoteCacheClient { } }; DownloadCommand downloadCmd = new DownloadCommand(uri, casDownload, digest, wrappedOut); @@ -300,7 +312,7 @@ index 1efecd3bb1..d0a1fb902c 100644 - releaseDownloadChannel(ch); + return retrier.executeAsync(() -> { + SettableFuture outerF = SettableFuture.create(); -+ acquireDownloadChannel() ++ acquireDownloadChannel(bytesDownloaded.get()) + .addListener( + (Future channelPromise) -> { + if (!channelPromise.isSuccess()) { @@ -355,7 +367,13 @@ index 1efecd3bb1..d0a1fb902c 100644 } @SuppressWarnings("FutureReturnValueIgnored") -@@ -673,20 +700,24 @@ public final class HttpCacheClient implements RemoteCacheClient { + private void getAfterCredentialRefresh(DownloadCommand cmd, SettableFuture outerF) { +- acquireDownloadChannel() ++ acquireDownloadChannel(0) + .addListener( + (Future channelPromise) -> { + if (!channelPromise.isSuccess()) { +@@ -673,20 +699,24 @@ public final class HttpCacheClient implements RemoteCacheClient { @Override public ListenableFuture uploadFile( RemoteActionExecutionContext context, Digest digest, Path file) {