Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

"exporting to image" hangs with v0.9.0, works with v0.9.0-rc1 #2357

Closed
olegy2008 opened this issue Sep 9, 2021 · 0 comments · Fixed by #2403
Closed

"exporting to image" hangs with v0.9.0, works with v0.9.0-rc1 #2357

olegy2008 opened this issue Sep 9, 2021 · 0 comments · Fixed by #2403
Labels
kind/bug

Comments

@olegy2008
Copy link

The same issue as #1988.
In our case, we are only facing this while building a huge image (~4.8Gb) with v0.9.0 buildkit release, with v0.9.0-rc1 works fine.
Command used for builds (only replaced registries addresses, we are using 2 google artifact registries, one for images and one for their cache):

buildctl-daemonless.sh \
--debug \
build \
--metadata-file \
./Dockerfile.metadata.json \
--progress=plain \
--frontend=dockerfile.v0 \
--opt filename=./Dockerfile \
--local context=./ \
--local dockerfile=. \
--output type=image,name=images-registry/app-name:tag,push=true \
--export-cache type=registry,ref=cache-registry/app-name:tag \
--import-cache type=registry,ref=cache-registry/app-name:tag
v0.9.0 stack trace after SIGQUIT 
SIGQUIT: quit
PC=0x4740c1 m=0 sigcode=0

goroutine 0 [idle]:
runtime.futex(0x1ea9030, 0x80, 0x0, 0x0, 0x0, 0x7ffcced46630, 0x43f1f9, 0xc00006ad50, 0x7ffcced46638, 0x40c43f, ...)
	/usr/local/go/src/runtime/sys_linux_amd64.s:579 +0x21
runtime.futexsleep(0x1ea9030, 0xc000000000, 0xffffffffffffffff)
	/usr/local/go/src/runtime/os_linux.go:44 +0x46
runtime.notesleep(0x1ea9030)
	/usr/local/go/src/runtime/lock_futex.go:159 +0x9f
runtime.mPark()
	/usr/local/go/src/runtime/proc.go:1340 +0x39
runtime.stoplockedm()
	/usr/local/go/src/runtime/proc.go:2495 +0x6e
runtime.schedule()
	/usr/local/go/src/runtime/proc.go:3103 +0x48c
runtime.park_m(0xc000001680)
	/usr/local/go/src/runtime/proc.go:3318 +0x9d
runtime.mcall(0x0)
	/usr/local/go/src/runtime/asm_amd64.s:327 +0x5b

goroutine 1 [select, 6 minutes]:
main.main.func3(0xc000169080, 0x0, 0x0)
	/src/cmd/buildkitd/main.go:295 +0xc05
github.com/urfave/cli.HandleAction(0x12c3240, 0x1558730, 0xc000169080, 0xc000169080, 0x0)
	/src/vendor/github.com/urfave/cli/app.go:523 +0x105
github.com/urfave/cli.(*App).Run(0xc00060c380, 0xc000134000, 0x2, 0x2, 0x0, 0x0)
	/src/vendor/github.com/urfave/cli/app.go:285 +0x655
main.main()
	/src/cmd/buildkitd/main.go:319 +0xe25

goroutine 22 [chan receive, 6 minutes]:
github.com/moby/buildkit/solver.(*scheduler).loop.func2(0xc00031ed90)
	/src/solver/scheduler.go:76 +0x3c
created by github.com/moby/buildkit/solver.(*scheduler).loop
	/src/solver/scheduler.go:75 +0x66

goroutine 35 [syscall, 6 minutes]:
os/signal.signal_recv(0x0)
	/usr/local/go/src/runtime/sigqueue.go:168 +0xa5
os/signal.loop()
	/usr/local/go/src/os/signal/signal_unix.go:23 +0x25
created by os/signal.Notify.func1.1
	/usr/local/go/src/os/signal/signal.go:151 +0x45

goroutine 49 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/appcontext.Context.func1.1(0xc0003859e0, 0xc000494010, 0xc000492010)
	/src/util/appcontext/appcontext.go:35 +0x38
created by github.com/moby/buildkit/util/appcontext.Context.func1
	/src/util/appcontext/appcontext.go:33 +0x15d

goroutine 50 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000422890, 0x17)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000422880)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/cond.(*StatefulCond).Wait(0xc0003ac420)
	/src/util/cond/cond.go:28 +0xaf
github.com/moby/buildkit/solver.(*scheduler).loop(0xc00031ed90)
	/src/solver/scheduler.go:101 +0x179
created by github.com/moby/buildkit/solver.newScheduler
	/src/solver/scheduler.go:35 +0x1b3

goroutine 51 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff22a1175f0, 0x72, 0x0)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc00058c518, 0x72, 0x0, 0x0, 0x14f7f39)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc00058c500, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:401 +0x212
net.(*netFD).accept(0xc00058c500, 0xc00000e020, 0x0, 0x0)
	/usr/local/go/src/net/fd_unix.go:172 +0x45
net.(*UnixListener).accept(0xc0003ac810, 0xc00009ade8, 0xc00009adf0, 0x28)
	/usr/local/go/src/net/unixsock_posix.go:162 +0x32
net.(*UnixListener).Accept(0xc0003ac810, 0x15572a0, 0xc00051ec40, 0x16a5d50, 0xc0003ac810)
	/usr/local/go/src/net/unixsock.go:260 +0x65
google.golang.org/grpc.(*Server).Serve(0xc00051ec40, 0x16a5d50, 0xc0003ac810, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/server.go:786 +0x27f
main.serveGRPC.func1.1(0x0, 0x0)
	/src/cmd/buildkitd/main.go:356 +0x107
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0003ac750, 0xc000134920)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 52 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc0003ac760)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc0003ac758)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc0003ac750, 0x0, 0x0)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
main.serveGRPC.func2(0xc00061ef00, 0xc0003ac750)
	/src/cmd/buildkitd/main.go:361 +0x2b
created by main.serveGRPC
	/src/cmd/buildkitd/main.go:360 +0x389

goroutine 1022 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0007427e0, 0xc000495570, 0xc000ad8f50, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 11 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc000507e40)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc000507e38)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc000507e30, 0xc00003a080, 0x16ab640)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/moby/buildkit/control.(*Controller).Status(0xc0002b0780, 0xc00002d9b0, 0x16b5c98, 0xc0004941f0, 0xc0002b0780, 0x9cd4ac)
	/src/control/control.go:388 +0x1a5
github.com/moby/buildkit/api/services/control._Control_Status_Handler(0x143dde0, 0xc0002b0780, 0x16b31a0, 0xc00002d950, 0x20, 0x30)
	/src/api/services/control/control.pb.go:1499 +0x113
github.com/moby/buildkit/util/grpcerrors.StreamServerInterceptor(0x143dde0, 0xc0002b0780, 0x16b31a0, 0xc00002d950, 0xc000613380, 0x15563b8, 0x16b9b01, 0xc00002d950)
	/src/util/grpcerrors/intercept.go:33 +0x52
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x143dde0, 0xc0002b0780, 0x16b31a0, 0xc00002d950, 0x1f, 0xc000134e20)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1(0x143dde0, 0xc0002b0780, 0x16b3308, 0xc00062e540, 0xc000613380, 0xc000134d60, 0x0, 0x0)
	/src/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:414 +0x6ef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x143dde0, 0xc0002b0780, 0x16b3308, 0xc00062e540, 0xc000099c70, 0x40e338)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x143dde0, 0xc0002b0780, 0x16b3308, 0xc00062e540, 0xc000613380, 0x15563b8, 0x16ab6e8, 0xc00002d560)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xd1
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc000512c60, 0xc0003ac4e0, 0x1e90bc0, 0x0, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/server.go:1540 +0x535
google.golang.org/grpc.(*Server).handleStream(0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc000512c60, 0x0)
	/src/vendor/google.golang.org/grpc/server.go:1613 +0xca5
google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00038a080, 0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc000512c60)
	/src/vendor/google.golang.org/grpc/server.go:934 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/src/vendor/google.golang.org/grpc/server.go:932 +0x1fd

goroutine 29 [select, 6 minutes]:
main.unaryInterceptor.func1.1(0x16ab640, 0xc00004ccc0, 0x16ab640, 0xc000422040, 0xc0000de850)
	/src/cmd/buildkitd/main.go:561 +0xab
created by main.unaryInterceptor.func1
	/src/cmd/buildkitd/main.go:560 +0xf6

goroutine 924 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc00074e440, 0xc0007c4ba0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 13 [chan receive, 6 minutes]:
github.com/moby/buildkit/control.(*Controller).Session.func1(0xc000616420, 0xc000591680)
	/src/control/control.go:399 +0x34
created by github.com/moby/buildkit/control.(*Controller).Session
	/src/control/control.go:398 +0x19a

goroutine 12 [chan receive, 6 minutes]:
github.com/moby/buildkit/session.(*Manager).handleConn(0xc000612d20, 0x16ab640, 0xc00058f340, 0x16b8560, 0xc00056b880, 0xc00002db90, 0x0, 0x0)
	/src/session/manager.go:144 +0x49c
github.com/moby/buildkit/session.(*Manager).HandleConn(0xc000612d20, 0x16ab640, 0xc00058f300, 0x16b8560, 0xc00056b880, 0xc00002db90, 0x0, 0x10)
	/src/session/manager.go:97 +0x74
github.com/moby/buildkit/control.(*Controller).Session(0xc0002b0780, 0x16b8248, 0xc000591670, 0x0, 0x0)
	/src/control/control.go:403 +0x1e5
github.com/moby/buildkit/api/services/control._Control_Session_Handler(0x143dde0, 0xc0002b0780, 0x16b31a0, 0xc00002de00, 0x20, 0x30)
	/src/api/services/control/control.pb.go:1516 +0xad
github.com/moby/buildkit/util/grpcerrors.StreamServerInterceptor(0x143dde0, 0xc0002b0780, 0x16b31a0, 0xc00002de00, 0xc0006134a0, 0x15563a8, 0x16b9b01, 0xc00002de00)
	/src/util/grpcerrors/intercept.go:33 +0x52
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x143dde0, 0xc0002b0780, 0x16b31a0, 0xc00002de00, 0x20, 0xc000134f60)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.StreamServerInterceptor.func1(0x143dde0, 0xc0002b0780, 0x16b3308, 0xc00062e6c0, 0xc0006134a0, 0xc000134ea0, 0x0, 0x0)
	/src/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:414 +0x6ef
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1.1.1(0x143dde0, 0xc0002b0780, 0x16b3308, 0xc00062e6c0, 0xc000099c70, 0x40e338)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:49 +0x5f
github.com/grpc-ecosystem/go-grpc-middleware.ChainStreamServer.func1(0x143dde0, 0xc0002b0780, 0x16b3308, 0xc00062e6c0, 0xc0006134a0, 0x15563a8, 0x16ab6e8, 0xc00002dcb0)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:58 +0xd1
google.golang.org/grpc.(*Server).processStreamingRPC(0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc000512d80, 0xc0003ac4e0, 0x1e90be0, 0x0, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/server.go:1540 +0x535
google.golang.org/grpc.(*Server).handleStream(0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc000512d80, 0x0)
	/src/vendor/google.golang.org/grpc/server.go:1613 +0xca5
google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00038a080, 0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc000512d80)
	/src/vendor/google.golang.org/grpc/server.go:934 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/src/vendor/google.golang.org/grpc/server.go:932 +0x1fd

goroutine 99 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc00004d180, 0xc0003cda40)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 67 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00014db30, 0x1, 0x0, 0x0, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0xff
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000131020, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:527 +0x1dd
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000103b00)
	/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:292 +0xd7
created by google.golang.org/grpc/internal/transport.newHTTP2Server
	/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:289 +0x1110

goroutine 68 [select, 6 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000103b00)
	/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:993 +0x265
created by google.golang.org/grpc/internal/transport.newHTTP2Server
	/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:300 +0x1135

goroutine 69 [IO wait]:
internal/poll.runtime_pollWait(0x7ff22a1176d8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc0001a8098, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0001a8080, 0xc00027a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc0001a8080, 0xc00027a000, 0x8000, 0x8000, 0x90d09e, 0x800010601, 0x0)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc00000e020, 0xc00027a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
bufio.(*Reader).Read(0xc0003840c0, 0xc00050a038, 0x9, 0x9, 0x0, 0x0, 0xc00009bcc8)
	/usr/local/go/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x1684100, 0xc0003840c0, 0xc00050a038, 0x9, 0x9, 0x9, 0xc00009bd58, 0x91264e, 0xc046a0dce10785f4)
	/usr/local/go/src/io/io.go:328 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader(0xc00050a038, 0x9, 0x9, 0x1684100, 0xc0003840c0, 0x0, 0x0, 0xc0001c6780, 0x0)
	/src/vendor/golang.org/x/net/http2/frame.go:237 +0x89
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00050a000, 0xc0001c6780, 0x1ea85c0, 0x0, 0x0)
	/src/vendor/golang.org/x/net/http2/frame.go:492 +0xa5
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000103b00, 0xc00057c300, 0x15572e8)
	/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:473 +0x9b
google.golang.org/grpc.(*Server).serveStreams(0xc00051ec40, 0x16b9698, 0xc000103b00)
	/src/vendor/google.golang.org/grpc/server.go:918 +0x103
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc00049e480, 0x1c)
	/src/vendor/google.golang.org/grpc/server.go:868 +0x3f
created by google.golang.org/grpc.(*Server).handleRawConn
	/src/vendor/google.golang.org/grpc/server.go:867 +0x555

goroutine 36 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc000b87810)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc000b87808)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc000b87800, 0xc000438080, 0x16ab640)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/containerd/containerd/images.Dispatch(0x16ab6e8, 0xc0007abb60, 0x16876c0, 0xc000430ae0, 0x0, 0xc0001f44e8, 0x1, 0x1, 0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:153 +0x205
github.com/moby/buildkit/util/push.Push(0x16ab6e8, 0xc0007abb60, 0xc000612d20, 0xc0003ecfe0, 0x19, 0x16850c0, 0xc00097a690, 0x7ff229eecbe0, 0xc0001344c0, 0xc0008fe280, ...)
	/src/util/push/push.go:112 +0x7ba
github.com/moby/buildkit/exporter/containerimage.(*imageExporterInstance).Export(0xc000428be0, 0x16ab6e8, 0xc0007abb60, 0x16bba18, 0xc000885140, 0x0, 0xc0005b2690, 0xc0003ecfe0, 0x19, 0x0, ...)
	/src/exporter/containerimage/export.go:280 +0x905
github.com/moby/buildkit/solver/llbsolver.(*Solver).Solve.func3(0x16ab6e8, 0xc0007ab5f0, 0x1684cc0, 0xc00082b4b8, 0x1, 0x1698a48)
	/src/solver/llbsolver/solver.go:213 +0x99
github.com/moby/buildkit/solver/llbsolver.inBuilderContext.func1(0x16ab6e8, 0xc0007ab5f0, 0x1684cc0, 0xc00082b4b8, 0x0, 0x0)
	/src/solver/llbsolver/solver.go:358 +0x1c7
github.com/moby/buildkit/solver.(*Job).InContext(0xc0004a6e00, 0x16ab6e8, 0xc0003cd560, 0xc00082b4a0, 0x18, 0xc00070ad20)
	/src/solver/jobs.go:542 +0x175
github.com/moby/buildkit/solver/llbsolver.inBuilderContext(0x16ab6e8, 0xc0003cd560, 0x16a4700, 0xc0004a6e00, 0x1505dd4, 0x12, 0x0, 0x0, 0xc00059f270, 0x0, ...)
	/src/solver/llbsolver/solver.go:354 +0x18e
github.com/moby/buildkit/solver/llbsolver.(*Solver).Solve(0xc00061ee40, 0x16ab6e8, 0xc0003cd560, 0xc0003ecfa0, 0x19, 0xc0003ecfe0, 0x19, 0x0, 0x0, 0xc0004f1940, ...)
	/src/solver/llbsolver/solver.go:212 +0xd09
github.com/moby/buildkit/control.(*Controller).Solve(0xc0002b0780, 0x16ab6e8, 0xc0003cd560, 0xc0000dc000, 0x0, 0x0, 0x0)
	/src/control/control.go:301 +0x4fa
github.com/moby/buildkit/api/services/control._Control_Solve_Handler.func1(0x16ab6e8, 0xc0003cd560, 0x14b8820, 0xc0000dc000, 0x14feb92, 0xc, 0x4, 0x0)
	/src/api/services/control/control.pb.go:1489 +0x89
github.com/moby/buildkit/util/grpcerrors.UnaryServerInterceptor(0x16ab6e8, 0xc0003cd560, 0x14b8820, 0xc0000dc000, 0xc000430420, 0xc00000c4b0, 0x0, 0x0, 0x0, 0x0)
	/src/util/grpcerrors/intercept.go:14 +0x6f
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x16ab6e8, 0xc0003cd560, 0x14b8820, 0xc0000dc000, 0x14f9587, 0x8, 0x0, 0x0)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x63
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.UnaryServerInterceptor.func1(0x16ab640, 0xc0003cd560, 0x14b8820, 0xc0000dc000, 0xc000430420, 0xc000430440, 0x0, 0x0, 0x0, 0x0)
	/src/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:327 +0x6db
main.unaryInterceptor.func1(0x16ab6e8, 0xc00004ccc0, 0x14b8820, 0xc0000dc000, 0xc000430420, 0xc000430440, 0x0, 0x0, 0x0, 0x0)
	/src/cmd/buildkitd/main.go:572 +0x169
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1.1.1(0x16ab6e8, 0xc0003cca20, 0x14b8820, 0xc0000dc000, 0xc00006a400, 0x0, 0xc0000f5b30, 0x40e338)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:25 +0x63
github.com/grpc-ecosystem/go-grpc-middleware.ChainUnaryServer.func1(0x16ab6e8, 0xc0003cca20, 0x14b8820, 0xc0000dc000, 0xc000430420, 0xc00000c4b0, 0xc000307ba0, 0x535766, 0x14266e0, 0xc0003cca20)
	/src/vendor/github.com/grpc-ecosystem/go-grpc-middleware/chain.go:34 +0xd7
github.com/moby/buildkit/api/services/control._Control_Solve_Handler(0x143dde0, 0xc0002b0780, 0x16ab6e8, 0xc0003cca20, 0xc000131080, 0xc00002cd50, 0x16ab6e8, 0xc0003cca20, 0xc00054c500, 0x244)
	/src/api/services/control/control.pb.go:1491 +0x150
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc0005a4000, 0xc0003ac4e0, 0x1e8dfd8, 0x0, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/server.go:1286 +0x52b
google.golang.org/grpc.(*Server).handleStream(0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc0005a4000, 0x0)
	/src/vendor/google.golang.org/grpc/server.go:1609 +0xd0c
google.golang.org/grpc.(*Server).serveStreams.func1.2(0xc00038a080, 0xc00051ec40, 0x16b9698, 0xc000103b00, 0xc0005a4000)
	/src/vendor/google.golang.org/grpc/server.go:934 +0xab
created by google.golang.org/grpc.(*Server).serveStreams.func1
	/src/vendor/google.golang.org/grpc/server.go:932 +0x1fd

goroutine 37 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc00058f790, 0x25)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00058f780)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0002921b0, 0x16ab640, 0xc000422300, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/solver.(*Job).Status(0xc0004a6e00, 0x16ab640, 0xc000422300, 0xc00061e480, 0x0, 0x0)
	/src/solver/progress.go:26 +0xc1d
github.com/moby/buildkit/solver/llbsolver.(*Solver).Status(0xc00061ee40, 0x16ab640, 0xc000422300, 0xc00049e0e0, 0x19, 0xc00061e480, 0x0, 0x0)
	/src/solver/llbsolver/solver.go:306 +0xc5
github.com/moby/buildkit/control.(*Controller).Status.func1(0x0, 0x0)
	/src/control/control.go:325 +0x5e
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc000507e30, 0xc000507f80)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 38 [chan receive, 6 minutes]:
github.com/moby/buildkit/control.(*Controller).Status.func2(0x0, 0x0)
	/src/control/control.go:330 +0x7b2
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc000507e30, 0xc00003a080)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 14 [select, 6 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00058f4c0)
	/src/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:69 +0xac
created by google.golang.org/grpc.newCCBalancerWrapper
	/src/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:60 +0x172

goroutine 15 [chan receive, 6 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc0002dc580)
	/src/vendor/google.golang.org/grpc/clientconn.go:1214 +0x465
created by google.golang.org/grpc.(*addrConn).connect
	/src/vendor/google.golang.org/grpc/clientconn.go:844 +0x12a

goroutine 16 [select]:
github.com/moby/buildkit/session.monitorHealth(0x16ab640, 0xc00058f540, 0xc000029500, 0xc000591940)
	/src/session/grpc.go:86 +0x165
created by github.com/moby/buildkit/session.grpcClientConn
	/src/session/grpc.go:72 +0x45c

goroutine 921 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0000e5950, 0xc000495570, 0xc000ad9400, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 81 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc000093860, 0xc000613498, 0x5, 0x5, 0x0, 0x0, 0xc00024c2c0)
	/src/vendor/google.golang.org/grpc/internal/transport/transport.go:177 +0xa5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc000093860, 0xc000613498, 0x5, 0x5, 0x0, 0xc00012c570, 0x1559180)
	/src/vendor/google.golang.org/grpc/internal/transport/transport.go:171 +0x211
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc00002dc50, 0xc000613498, 0x5, 0x5, 0x2, 0xc0006cf728, 0x9391fc)
	/src/vendor/google.golang.org/grpc/internal/transport/transport.go:483 +0x55
io.ReadAtLeast(0x16863e0, 0xc00002dc50, 0xc000613498, 0x5, 0x5, 0x5, 0x16bf670, 0xc00042f450, 0xc0001c6708)
	/usr/local/go/src/io/io.go:328 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc000512d80, 0xc000613498, 0x5, 0x5, 0xc, 0x4, 0x0)
	/src/vendor/google.golang.org/grpc/internal/transport/transport.go:467 +0xd2
google.golang.org/grpc.(*parser).recvMsg(0xc000613488, 0x400000, 0xc0006cf888, 0x94004e, 0xc0001c6708, 0x13, 0x13, 0x7ff229dc8880)
	/src/vendor/google.golang.org/grpc/rpc_util.go:558 +0x63
google.golang.org/grpc.recvAndDecompress(0xc000613488, 0xc000512d80, 0x0, 0x0, 0x400000, 0x0, 0x0, 0x0, 0xc0001c6708, 0x13, ...)
	/src/vendor/google.golang.org/grpc/rpc_util.go:689 +0x4d
google.golang.org/grpc.recv(0xc000613488, 0x7ff229d52118, 0x1ed8590, 0xc000512d80, 0x0, 0x0, 0x1488340, 0xc000906200, 0x400000, 0x0, ...)
	/src/vendor/google.golang.org/grpc/rpc_util.go:757 +0x9b
google.golang.org/grpc.(*serverStream).RecvMsg(0xc00062e6c0, 0x1488340, 0xc000906200, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/stream.go:1545 +0x156
go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc.(*serverStream).RecvMsg(0xc00002de00, 0x1488340, 0xc000906200, 0x1, 0xc000906200)
	/src/vendor/go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/interceptor.go:357 +0x54
github.com/moby/buildkit/session/grpchijack.(*conn).Read(0xc00056b880, 0xc00034a000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
	/src/session/grpchijack/dial.go:74 +0x1f0
bufio.(*Reader).Read(0xc000089920, 0xc0003ca498, 0x9, 0x9, 0x14, 0x7ff250d9e108, 0x18)
	/usr/local/go/src/bufio/bufio.go:227 +0x222
io.ReadAtLeast(0x1684100, 0xc000089920, 0xc0003ca498, 0x9, 0x9, 0x9, 0xc0006cfce8, 0x45b0f3, 0x30c762ad78f)
	/usr/local/go/src/io/io.go:328 +0x87
io.ReadFull(...)
	/usr/local/go/src/io/io.go:347
golang.org/x/net/http2.readFrameHeader(0xc0003ca498, 0x9, 0x9, 0x1684100, 0xc000089920, 0x0, 0x0, 0xc046a0dce105303b, 0x5a24236289)
	/src/vendor/golang.org/x/net/http2/frame.go:237 +0x89
golang.org/x/net/http2.(*Framer).ReadFrame(0xc0003ca460, 0xc0001c6750, 0xc0001c6750, 0x0, 0x0)
	/src/vendor/golang.org/x/net/http2/frame.go:492 +0xa5
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00000b680)
	/src/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1347 +0x1a5
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/src/vendor/google.golang.org/grpc/internal/transport/http2_client.go:346 +0xdd1

goroutine 82 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000093ae0, 0x1, 0x0, 0x0, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:407 +0xff
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000089b00, 0x0, 0x0)
	/src/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:527 +0x1dd
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00000b680)
	/src/vendor/google.golang.org/grpc/internal/transport/http2_client.go:396 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
	/src/vendor/google.golang.org/grpc/internal/transport/http2_client.go:394 +0x12ae

goroutine 960 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc0009bdba0)
	/usr/local/go/src/runtime/sema.go:56 +0x45
sync.(*WaitGroup).Wait(0xc0009bdb98)
	/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc0009bdb90, 0xc000438b80, 0x16ab640)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:40 +0x31
github.com/containerd/containerd/images.Dispatch(0x16ab640, 0xc00097f600, 0x16876c0, 0xc000430ae0, 0x0, 0xc0007d4b00, 0x11, 0x11, 0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:153 +0x205
github.com/containerd/containerd/images.Dispatch.func1(0xc00087fe98, 0x1559180)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:146 +0x185
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc000b87800, 0xc000438080)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 926 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0000e59e0, 0xc000495570, 0xc000ad93b0, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 83 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc00058f740, 0xc0002921b0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 84 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*MultiReader).Reader.func1(0x16ab6e8, 0xc000292270, 0xc0003cdb90, 0xc000613608)
	/src/util/progress/multireader.go:37 +0xa9
created by github.com/moby/buildkit/util/progress.(*MultiReader).Reader
	/src/util/progress/multireader.go:36 +0x1aa

goroutine 85 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc00004d1d0, 0x4b)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00004d1c0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0003cda40, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/progress.(*MultiReader).handle(0xc0003cdb90, 0xc000292270, 0xc0003cdb90)
	/src/util/progress/multireader.go:56 +0x124
created by github.com/moby/buildkit/util/progress.(*MultiReader).Reader
	/src/util/progress/multireader.go:47 +0x215

goroutine 923 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000748100, 0x16976c0, 0xc00074e400)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 977 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0008923f0, 0x16ab640, 0xc0009bdd70, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9590, 0x47, 0xc00056bce0, 0x38, 0xc000400000, 0x0, 0xc000adeb60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9590, 0x47, 0xc00056bce0, 0x38, 0xc000b345c0, 0x38, 0x38)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438b80)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 783 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc000692000, 0xc000495570, 0xc000ad9360, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 939 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc00097ae70, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc00097ae70, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc00097ae70, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc00097ae70, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc00093fa80, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc00093fa80, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc00093fa80, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc00093fa80, 0x16ab640, 0xc00093fa80, 0xc0004275d0, 0x2)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0005ab200)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc0005ab260, 0xc0004275c0)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc0005ab260, 0xc0004275c0)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1165 [select, 6 minutes]:
net/http.(*persistConn).roundTrip(0xc00026e480, 0xc000653800, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:2610 +0x765
net/http.(*Transport).roundTrip(0xc0004963c0, 0xc0002a1200, 0xc000b86ea0, 0x16a6770, 0xc000b86810)
	/usr/local/go/src/net/http/transport.go:592 +0xacb
net/http.(*Transport).RoundTrip(0xc0004963c0, 0xc0002a1200, 0xc000b86ea0, 0x16a6770, 0xc000b86810)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x35
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*Transport).RoundTrip(0xc00031e7e0, 0xc0002a1100, 0xc00031e7e0, 0x0, 0x0)
	/src/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/transport.go:115 +0x3d9
net/http.send(0xc0002a1100, 0x16854e0, 0xc00031e7e0, 0x0, 0x0, 0x0, 0xc00040a118, 0x16ab6e8, 0x1, 0x0)
	/usr/local/go/src/net/http/client.go:251 +0x454
net/http.(*Client).send(0xc00002d6e0, 0xc0002a1100, 0x0, 0x0, 0x0, 0xc00040a118, 0x0, 0x1, 0x16ab6e8)
	/usr/local/go/src/net/http/client.go:175 +0xff
net/http.(*Client).do(0xc00002d6e0, 0xc0002a1100, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:717 +0x45f
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:585
golang.org/x/net/context/ctxhttp.Do(0x16ab6e8, 0xc000b86930, 0xc00002d6e0, 0xc0002a1000, 0x0, 0x0, 0x16ab6e8)
	/src/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:27 +0x10f
github.com/containerd/containerd/remotes/docker.(*request).do(0xc000743710, 0x16ab6e8, 0xc0007c5f20, 0x0, 0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/remotes/docker/resolver.go:542 +0x529
github.com/containerd/containerd/remotes/docker.(*request).doWithRetries(0xc000743710, 0x16ab6e8, 0xc0007c5f20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/remotes/docker/resolver.go:551 +0x46
github.com/containerd/containerd/remotes/docker.dockerPusher.push.func2(0xc00042ce40, 0xc000743710, 0x16ab6e8, 0xc0007c5f20, 0xc0003b81a0)
	/src/vendor/github.com/containerd/containerd/remotes/docker/pusher.go:278 +0xa5
created by github.com/containerd/containerd/remotes/docker.dockerPusher.push
	/src/vendor/github.com/containerd/containerd/remotes/docker/pusher.go:276 +0xd56

goroutine 1164 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc00072e8c0, 0x16976c0, 0xc000816800)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 920 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc00074e390, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00074e380)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0007c4900, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc0007c4870, 0x16851e0, 0xc0007c4900)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 982 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000431000, 0x16976c0, 0xc0009569c0)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 969 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0005ab200, 0x16ab640, 0xc00097ade0, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9130, 0x47, 0xc0003902a0, 0x10, 0xc00006ac00, 0x0, 0xc000307b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9130, 0x47, 0xc0003902a0, 0x1, 0xc000307bc8, 0x1, 0xc000816080)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34940, 0x31, 0xc000ad9130, 0x47, 0x10cf919f, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0xc0006de418, 0x4425d6)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438700)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 1017 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc000958f60, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc000958f60, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc000958f60, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc000958f60, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc000816c80, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc000816c80, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc000816c80, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc000816c80, 0x16ab640, 0xc000816c80, 0xc00072ce00, 0xc00097a700)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc000742750)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc0007427b0, 0xc00072cdf0)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc0007427b0, 0xc00072cdf0)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1176 [select, 6 minutes]:
net/http.(*persistConn).roundTrip(0xc00082e7e0, 0xc00074f980, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:2610 +0x765
net/http.(*Transport).roundTrip(0xc0004963c0, 0xc000691300, 0xc00041b3b0, 0x16a6770, 0xc00041acc0)
	/usr/local/go/src/net/http/transport.go:592 +0xacb
net/http.(*Transport).RoundTrip(0xc0004963c0, 0xc000691300, 0xc00041b3b0, 0x16a6770, 0xc00041acc0)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x35
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*Transport).RoundTrip(0xc00031e7e0, 0xc000691200, 0xc00031e7e0, 0x0, 0x0)
	/src/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/transport.go:115 +0x3d9
net/http.send(0xc000691200, 0x16854e0, 0xc00031e7e0, 0x0, 0x0, 0x0, 0xc00000e1f0, 0x16ab6e8, 0x1, 0x0)
	/usr/local/go/src/net/http/client.go:251 +0x454
net/http.(*Client).send(0xc00002d6e0, 0xc000691200, 0x0, 0x0, 0x0, 0xc00000e1f0, 0x0, 0x1, 0x16ab6e8)
	/usr/local/go/src/net/http/client.go:175 +0xff
net/http.(*Client).do(0xc00002d6e0, 0xc000691200, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:717 +0x45f
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:585
golang.org/x/net/context/ctxhttp.Do(0x16ab6e8, 0xc00041adb0, 0xc00002d6e0, 0xc000691100, 0x0, 0x0, 0x16ab6e8)
	/src/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:27 +0x10f
github.com/containerd/containerd/remotes/docker.(*request).do(0xc000692630, 0x16ab6e8, 0xc0009bdf20, 0x8affca, 0xd59db5, 0x16ab640)
	/src/vendor/github.com/containerd/containerd/remotes/docker/resolver.go:542 +0x529
github.com/containerd/containerd/remotes/docker.(*request).doWithRetries(0xc000692630, 0x16ab6e8, 0xc0009bdf20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/remotes/docker/resolver.go:551 +0x46
github.com/containerd/containerd/remotes/docker.dockerPusher.push.func2(0xc000088f60, 0xc000692630, 0x16ab6e8, 0xc0009bdf20, 0xc00040a078)
	/src/vendor/github.com/containerd/containerd/remotes/docker/pusher.go:278 +0xa5
created by github.com/containerd/containerd/remotes/docker.dockerPusher.push
	/src/vendor/github.com/containerd/containerd/remotes/docker/pusher.go:276 +0xd56

goroutine 927 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc0007c4cf0, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc0007c4cf0, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc0007c4cf0, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc0007c4cf0, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc00074e500, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc00074e500, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc00074e500, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc00074e500, 0x16ab640, 0xc00074e500, 0xc000b24530, 0x0)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0000e59e0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc0000e5a40, 0xc000b24520)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc0000e5a40, 0xc000b24520)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1166 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc0008173c0, 0xc000b1db90)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 925 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc00074e490, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00074e480)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0007c4ba0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc0007c4b10, 0x16851e0, 0xc0007c4ba0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 782 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000652250, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000652240)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0007281b0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000728120, 0x16851e0, 0xc0007281b0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 894 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0007425a0, 0xc000495570, 0xc000ad94f0, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1041 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000115c80, 0x16ab678, 0xc00012a018, 0xc0007c4900)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 912 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc000a241b0, 0xc000495570, 0xc000ad9540, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 971 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0000e59e0, 0x16ab640, 0xc0007c4c60, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad93b0, 0x47, 0xc0003a4620, 0x0, 0xc000100400, 0x0, 0xc000962b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad93b0, 0x47, 0xc0003a4620, 0x1, 0x0, 0x0, 0xc000962c90)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b349c0, 0x31, 0xc000ad93b0, 0x47, 0x81, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0x47b200, 0x4425d6)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438880)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 976 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc000a241b0, 0x16ab640, 0xc000861980, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9540, 0x47, 0xc0005084d0, 0x10, 0xc0006c0000, 0x0, 0xc00099bb60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9540, 0x47, 0xc0005084d0, 0xc00099bd20, 0x756ea1194bae1e, 0x12b7d032a85c938, 0x0)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438b00)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 975 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0007425a0, 0x16ab640, 0xc000958990, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad94f0, 0x47, 0xc0003ae000, 0x10, 0xc00006a400, 0x0, 0xc000965b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad94f0, 0x47, 0xc0003ae000, 0xc00087ff20, 0xc000965bc8, 0x1, 0xc0008fa300)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0x47b2d7, 0xc00058e250)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438a80)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 970 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc000692000, 0x16ab640, 0xc000728360, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9360, 0x47, 0xc0003d4000, 0x10, 0xc00006a800, 0x0, 0xc000963b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9360, 0x47, 0xc0003d4000, 0xc000958450, 0xc000963bc8, 0x1, 0xc0008166c0)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0x0, 0x4425d6)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438800)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 1026 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc0009281c0, 0x16976c0, 0xc000906880)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 974 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc000742750, 0x16ab640, 0xc000958ed0, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad94a0, 0x47, 0xc0003ae4d0, 0xc000300b48, 0xc00006a400, 0x0, 0xc000300b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad94a0, 0x47, 0xc0003ae4d0, 0x1, 0x1491e20, 0xc00076f470, 0xc000300c90)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34a80, 0x31, 0xc000ad94a0, 0x47, 0x1c18999, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0xc00003a360, 0x126d0e0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438a00)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 1151 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff22a116dc8, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc00028f118, 0x72, 0x2700, 0x27f4, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00028f100, 0xc0009cc000, 0x27f4, 0x27f4, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc00028f100, 0xc0009cc000, 0x27f4, 0x27f4, 0x377, 0xc000adf880, 0x418e13)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc0003a00c8, 0xc0009cc000, 0x27f4, 0x27f4, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
crypto/tls.(*atLeastReader).Read(0xc0006bccd8, 0xc0009cc000, 0x27f4, 0x27f4, 0xc000adf9f8, 0xc000100000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:776 +0x63
bytes.(*Buffer).ReadFrom(0xc0005ce978, 0x1684320, 0xc0006bccd8, 0x40b965, 0x133dbe0, 0x14b5aa0)
	/usr/local/go/src/bytes/buffer.go:204 +0xbe
crypto/tls.(*Conn).readFromUntil(0xc0005ce700, 0x1686ba0, 0xc0003a00c8, 0x5, 0xc0003a00c8, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:798 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc0005ce700, 0x0, 0x0, 0xc000adfe88)
	/usr/local/go/src/crypto/tls/conn.go:605 +0x115
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:573
crypto/tls.(*Conn).Read(0xc0005ce700, 0xc00085d000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1276 +0x165
net/http.(*persistConn).Read(0xc000966000, 0xc00085d000, 0x1000, 0x1000, 0xc000adfdc4, 0x2, 0x2)
	/usr/local/go/src/net/http/transport.go:1922 +0x77
bufio.(*Reader).fill(0xc000131bc0)
	/usr/local/go/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).Peek(0xc000131bc0, 0x1, 0x0, 0x1, 0x1, 0x1, 0x0)
	/usr/local/go/src/bufio/bufio.go:139 +0x4f
net/http.(*persistConn).readLoop(0xc000966000)
	/usr/local/go/src/net/http/transport.go:2083 +0x1a8
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1743 +0xc77

goroutine 893 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000816a10, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000816a00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0009588d0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000958840, 0x16851e0, 0xc0009588d0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1018 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc00072e320, 0x16976c0, 0xc000816c80)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1119 [select, 6 minutes]:
net/http.(*persistConn).roundTrip(0xc000966b40, 0xc000956240, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:2610 +0x765
net/http.(*Transport).roundTrip(0xc0004963c0, 0xc000703300, 0xc00087ea20, 0x16a6770, 0xc00087e450)
	/usr/local/go/src/net/http/transport.go:592 +0xacb
net/http.(*Transport).RoundTrip(0xc0004963c0, 0xc000703300, 0xc00087ea20, 0x16a6770, 0xc00087e450)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x35
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*Transport).RoundTrip(0xc00031e7e0, 0xc000703200, 0xc00031e7e0, 0x0, 0x0)
	/src/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/transport.go:115 +0x3d9
net/http.send(0xc000703200, 0x16854e0, 0xc00031e7e0, 0x0, 0x0, 0x0, 0xc0003a0108, 0x16ab6e8, 0x1, 0x0)
	/usr/local/go/src/net/http/client.go:251 +0x454
net/http.(*Client).send(0xc00002d6e0, 0xc000703200, 0x0, 0x0, 0x0, 0xc0003a0108, 0x0, 0x1, 0x16ab6e8)
	/usr/local/go/src/net/http/client.go:175 +0xff
net/http.(*Client).do(0xc00002d6e0, 0xc000703200, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:717 +0x45f
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:585
golang.org/x/net/context/ctxhttp.Do(0x16ab6e8, 0xc00087e4e0, 0xc00002d6e0, 0xc000703100, 0x0, 0x0, 0x16ab6e8)
	/src/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:27 +0x10f
github.com/containerd/containerd/remotes/docker.(*request).do(0xc0000e5cb0, 0x16ab6e8, 0xc000728630, 0xc000a8be30, 0xd2b7bf, 0x13931a0)
	/src/vendor/github.com/containerd/containerd/remotes/docker/resolver.go:542 +0x529
github.com/containerd/containerd/remotes/docker.(*request).doWithRetries(0xc0000e5cb0, 0x16ab6e8, 0xc000728630, 0x0, 0x0, 0x0, 0x13bb240, 0xc000a8bc78, 0xc000a8bc78)
	/src/vendor/github.com/containerd/containerd/remotes/docker/resolver.go:551 +0x46
github.com/containerd/containerd/remotes/docker.dockerPusher.push.func2(0xc0008acf60, 0xc0000e5cb0, 0x16ab6e8, 0xc000728630, 0xc00000e168)
	/src/vendor/github.com/containerd/containerd/remotes/docker/pusher.go:278 +0xa5
created by github.com/containerd/containerd/remotes/docker.dockerPusher.push
	/src/vendor/github.com/containerd/containerd/remotes/docker/pusher.go:276 +0xd56

goroutine 1203 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc00081f6b0, 0x1, 0xc00036c098, 0xc000242101)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc00021c0d8, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*fetcher).Fetch(0xc00021c0f0, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:143 +0x7e
github.com/moby/buildkit/util/contentutil.(*fetchedProvider).ReaderAt(0xc0004ca330, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/fetcher.go:24 +0x95
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484330, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484390, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc0004843f0, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484450, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc0004844b0, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484510, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484570, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc0004845d0, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/pull/pullprogress.(*ProviderWithProgress).ReaderAt(0xc0007487c0, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/pull/pullprogress/progress.go:28 +0x95
github.com/moby/buildkit/util/contentutil.(*localFetcher).Fetch(0xc0005903a0, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/copy.go:28 +0x95
github.com/moby/buildkit/util/resolver/limited.(*fetcher).Fetch(0xc000884810, 0x16ab6e8, 0xc00081f6b0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:147 +0xf5
github.com/containerd/containerd/remotes.fetch(0x16ab6e8, 0xc00081f6b0, 0x7ff229f39cb0, 0xc0001344c0, 0x1685260, 0xc000884810, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:141 +0x5b5
github.com/containerd/containerd/remotes.FetchHandler.func1(0x16ab6e8, 0xc00081f590, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:103 +0x378
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab6e8, 0xc00081f590, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/contentutil.Copy(0x16ab6e8, 0xc00081f590, 0x7ff229f39cb0, 0xc0001344c0, 0x1685240, 0xc0007487c0, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, ...)
	/src/util/contentutil/copy.go:17 +0x115
github.com/moby/buildkit/cache.lazyRefProvider.Unlazy.func1(0x16ab6e8, 0xc00081f590, 0x0, 0x0, 0x0, 0x0)
	/src/cache/remote.go:208 +0x318
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0000e5d40)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc0000e5da0, 0xc000590280)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc0000e5da0, 0xc000590280)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 928 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000748120, 0x16976c0, 0xc00074e500)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 979 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000956950, 0x3)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000956940)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0009bdcb0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc0009bdc20, 0x16851e0, 0xc0009bdcb0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 973 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0007426c0, 0x16ab640, 0xc000958c30, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9450, 0x47, 0xc0003ae1c0, 0x10, 0xc00006a400, 0x0, 0xc000964b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9450, 0x47, 0xc0003ae1c0, 0x1, 0xc000964bc8, 0x1, 0xc000964c90)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438980)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 937 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc00093fa10, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00093fa00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc00097ad20, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc00097ac90, 0x16851e0, 0xc00097ad20)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1010 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000816b10, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000816b00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc000958b70, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000958ae0, 0x16851e0, 0xc000958b70)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 870 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000769c80, 0x16ab640, 0xc000422300, 0xc0002921b0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 910 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc0009067c0, 0xc0008618c0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1021 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000816d10, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000816d00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0009590b0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000959020, 0x16851e0, 0xc0009590b0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 931 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000904b40, 0x16ab678, 0xc00012a018, 0xc0008618c0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 980 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0008923f0, 0xc000495570, 0xc000ad9590, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1009 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000816ac0, 0xc000958b70)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 919 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc00074e340, 0xc0007c4900)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 938 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0005ab200, 0xc000495570, 0xc000ad9130, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1024 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc00072e340, 0x16976c0, 0xc000816d80)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1121 [select, 6 minutes]:
io.(*pipe).Read(0xc0008acf00, 0xc00063e000, 0x8000, 0x8000, 0x7ff250d9e108, 0x1215087, 0xc00063e000)
	/usr/local/go/src/io/pipe.go:57 +0xcb
io.(*PipeReader).Read(0xc00000e168, 0xc00063e000, 0x8000, 0x8000, 0x7ff250d9e108, 0x8000, 0xc00063e000)
	/usr/local/go/src/io/pipe.go:134 +0x4c
io.(*LimitedReader).Read(0xc00095de48, 0xc00063e000, 0x8000, 0x8000, 0x0, 0x0, 0x11f2d02)
	/usr/local/go/src/io/io.go:473 +0x63
io.copyBuffer(0x7ff229c115f0, 0xc0005be700, 0x16869a0, 0xc00095de48, 0xc00063e000, 0x8000, 0x8000, 0x13a95a0, 0x133dd01, 0x7ff229c115d0)
	/usr/local/go/src/io/io.go:423 +0x12c
io.Copy(...)
	/usr/local/go/src/io/io.go:382
net/http.persistConnWriter.ReadFrom(0xc000966b40, 0x16869a0, 0xc00095de48, 0x7ff229c115d0, 0xc000966b40, 0x634301)
	/usr/local/go/src/net/http/transport.go:1768 +0x85
bufio.(*Writer).ReadFrom(0xc000956d40, 0x16869a0, 0xc00095de48, 0x7ff229c115b0, 0xc000956d40, 0x7ff229c11501)
	/usr/local/go/src/bufio/bufio.go:730 +0x37e
io.copyBuffer(0x1684120, 0xc000956d40, 0x16869a0, 0xc00095de48, 0x0, 0x0, 0x0, 0xc000658c38, 0x40e338, 0x18)
	/usr/local/go/src/io/io.go:409 +0x357
io.Copy(...)
	/usr/local/go/src/io/io.go:382
net/http.(*transferWriter).doBodyCopy(0xc00002f400, 0x1684120, 0xc000956d40, 0x16869a0, 0xc00095de48, 0x70e1f2, 0xc0005be700, 0xc0004c5000)
	/usr/local/go/src/net/http/transfer.go:409 +0x6a
net/http.(*transferWriter).writeBody(0xc00002f400, 0x1684120, 0xc000956d40, 0x0, 0x0)
	/usr/local/go/src/net/http/transfer.go:367 +0x825
net/http.(*Request).write(0xc000703500, 0x1684120, 0xc000956d40, 0x0, 0xc00087ea50, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/request.go:697 +0x7c6
net/http.(*persistConn).writeLoop(0xc000966b40)
	/usr/local/go/src/net/http/transport.go:2385 +0x1a7
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1744 +0xc9c

goroutine 972 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0000e5950, 0x16ab640, 0xc0007c49c0, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9400, 0x47, 0xc0003a4460, 0x10, 0xc000100400, 0x0, 0xc000961b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9400, 0x47, 0xc0003a4460, 0xc0008615c0, 0xc000961bc8, 0x1, 0xc00074e1c0)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0x47b2d7, 0xc00097e150)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438900)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 978 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000956900, 0xc0009bdcb0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 942 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff22a116f98, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc000439518, 0x72, 0x2700, 0x27f3, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc000439500, 0xc0009ce800, 0x27f3, 0x27f3, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc000439500, 0xc0009ce800, 0x27f3, 0x27f3, 0x0, 0x64, 0xc000096878)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc0006105c0, 0xc0009ce800, 0x27f3, 0x27f3, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
crypto/tls.(*atLeastReader).Read(0xc0006bd2f0, 0xc0009ce800, 0x27f3, 0x27f3, 0xc0000969f8, 0xc0004a0000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:776 +0x63
bytes.(*Buffer).ReadFrom(0xc0000285f8, 0x1684320, 0xc0006bd2f0, 0x40b965, 0x133dbe0, 0x14b5aa0)
	/usr/local/go/src/bytes/buffer.go:204 +0xbe
crypto/tls.(*Conn).readFromUntil(0xc000028380, 0x1686ba0, 0xc0006105c0, 0x5, 0xc0006105c0, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:798 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc000028380, 0x0, 0x0, 0xc000096e88)
	/usr/local/go/src/crypto/tls/conn.go:605 +0x115
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:573
crypto/tls.(*Conn).Read(0xc000028380, 0xc000792000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1276 +0x165
net/http.(*persistConn).Read(0xc00026e480, 0xc000792000, 0x1000, 0x1000, 0xc000096dc4, 0x2, 0x2)
	/usr/local/go/src/net/http/transport.go:1922 +0x77
bufio.(*Reader).fill(0xc0004092c0)
	/usr/local/go/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).Peek(0xc0004092c0, 0x1, 0x0, 0x1, 0x1, 0x1, 0x0)
	/usr/local/go/src/bufio/bufio.go:139 +0x4f
net/http.(*persistConn).readLoop(0xc00026e480)
	/usr/local/go/src/net/http/transport.go:2083 +0x1a8
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1743 +0xc77

goroutine 981 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0006926c0, 0x16ab6e8, 0xc000844de0, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc0002b0588, 0x16ab6e8, 0xc0009bdec0, 0xc000ad8cd0, 0x47, 0xc00028f180, 0x7ff229e8a480, 0xc000100000, 0x0, 0xc0004d7408)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc0002b0588, 0x16ab6e8, 0xc0009bdec0, 0xc000ad8cd0, 0x47, 0xc00028f180, 0x4d, 0xc000ad9b30, 0x4d, 0xc000ad9b30)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/cache.lazyRefProvider.Unlazy(0xc000885140, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, 0x0, 0xc000b86cf0, ...)
	/src/cache/remote.go:186 +0xc9
github.com/moby/buildkit/cache.lazyRefProvider.ReaderAt(0xc000885140, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, 0x0, 0xc000b86cf0, ...)
	/src/cache/remote.go:179 +0xf2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc00041a870, 0x16ab6e8, 0xc0009bdec0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/moby/buildkit/cache.(*lazyMultiProvider).ReaderAt(0xc000430060, 0x16ab6e8, 0xc0009bdec0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/cache/remote.go:154 +0x7d
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc00097a690, 0x16ab6e8, 0xc0009bdec0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc0009bdec0, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:186 +0x395
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc0009569c0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc0009569c0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc0009569c0, 0xc000b34b40, 0x31, 0xc000ad9590, 0x47, 0xa8e3, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc0009569c0, 0x16ab640, 0xc0009569c0, 0xc000495ea0, 0x2)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0008923f0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000892450, 0xc000495e90)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000892450, 0xc000495e90)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1014 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000816bc0, 0xc000958e10)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 784 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0000e5d40, 0x16ab6e8, 0xc00081f350, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc0002b0588, 0x16ab6e8, 0xc000728450, 0xc000ad8140, 0x47, 0xc0005ad500, 0x7ff229ce9318, 0xc000100000, 0x0, 0xc000541408)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc0002b0588, 0x16ab6e8, 0xc000728450, 0xc000ad8140, 0x47, 0xc0005ad500, 0x4d, 0xc000a96050, 0x4d, 0xc000a96050)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/cache.lazyRefProvider.Unlazy(0xc00043dc98, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, 0xc00041bbf0, ...)
	/src/cache/remote.go:186 +0xc9
github.com/moby/buildkit/cache.lazyRefProvider.ReaderAt(0xc00043dc98, 0xc000504ab0, 0x2b, 0xc000ad8140, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, 0xc00041bbf0, ...)
	/src/cache/remote.go:179 +0xf2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc00041a870, 0x16ab6e8, 0xc000728450, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/moby/buildkit/cache.(*lazyMultiProvider).ReaderAt(0xc000430060, 0x16ab6e8, 0xc000728450, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/cache/remote.go:154 +0x7d
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc00097a690, 0x16ab6e8, 0xc000728450, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc000728450, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:186 +0x395
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc0006522c0, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc0006522c0, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc0006522c0, 0xc000b34980, 0x31, 0xc000ad9360, 0x47, 0x661d7e2, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc0006522c0, 0x16ab640, 0xc0006522c0, 0xc0004ca2a0, 0xc000a0c660)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc000692000)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000692060, 0xc0004ca270)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000692060, 0xc0004ca270)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 962 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0000e5b00, 0x16ab640, 0xc0007c51a0, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad8f00, 0x47, 0xc000390230, 0x10, 0xc00006ac00, 0x0, 0xc000302b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad8f00, 0x47, 0xc000390230, 0xc00092a030, 0xc000302bc8, 0x1, 0xc0008fa400)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0xc00087fe98, 0x1559180)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438300)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 1025 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc000861a10, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc000861a10, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc000861a10, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc000861a10, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc000906880, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc000906880, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc000906880, 0xc000b34b00, 0x31, 0xc000ad9540, 0x47, 0x42465513, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc000906880, 0x16ab640, 0xc000906880, 0xc0005f2770, 0x0)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc000a241b0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000a24210, 0xc0005f2760)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000a24210, 0xc0005f2760)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1178 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000652450, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000652440)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc000844d20, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000844c90, 0x16851e0, 0xc000844d20)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 895 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc000742fc0, 0x16ab6e8, 0xc000b1cc60, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc0002b0588, 0x16ab6e8, 0xc000958a20, 0xc000ad8a50, 0x47, 0xc0001cc280, 0x7ff229e89270, 0xc000100000, 0x0, 0xc0004d3408)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc0002b0588, 0x16ab6e8, 0xc000958a20, 0xc000ad8a50, 0x47, 0xc0001cc280, 0x4d, 0xc00077ec80, 0x4d, 0xc00077ec80)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/cache.lazyRefProvider.Unlazy(0xc00082b920, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, 0x0, 0xc000b86960, ...)
	/src/cache/remote.go:186 +0xc9
github.com/moby/buildkit/cache.lazyRefProvider.ReaderAt(0xc00082b920, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, 0x0, 0xc000b86960, ...)
	/src/cache/remote.go:179 +0xf2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc00041a870, 0x16ab6e8, 0xc000958a20, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/moby/buildkit/cache.(*lazyMultiProvider).ReaderAt(0xc000430060, 0x16ab6e8, 0xc000958a20, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/cache/remote.go:154 +0x7d
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc00097a690, 0x16ab6e8, 0xc000958a20, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc000958a20, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:186 +0x395
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc000816a80, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc000816a80, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc000816a80, 0xc000b34ac0, 0x31, 0xc000ad94f0, 0x47, 0xe601, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc000816a80, 0x16ab640, 0xc000816a80, 0xc00072cc70, 0xc00097a700)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0007425a0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000742600, 0xc00072cc60)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000742600, 0xc00072cc60)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 896 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc00072e280, 0x16976c0, 0xc000816a80)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1020 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000816cc0, 0xc0009590b0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 253 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000930000, 0xc000648120)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1013 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc00072e2c0, 0x16976c0, 0xc000816b80)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 966 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc000692480, 0x16ab640, 0xc000728d80, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9040, 0x47, 0xc0003d50a0, 0x10, 0xc00006a800, 0x0, 0xc000adcb60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9040, 0x47, 0xc0003d50a0, 0x1, 0xc000adcbc8, 0x1, 0xc000adcc90)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0xc000958488, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438580)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 1023 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc000959230, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc000959230, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc000959230, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc000959230, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc000816d80, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc000816d80, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc000816d80, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc000816d80, 0x16ab640, 0xc000816d80, 0xc00072cec0, 0x0)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0007427e0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000742840, 0xc00072ceb0)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000742840, 0xc00072ceb0)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1177 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000652400, 0xc000844d20)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 932 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000904c00, 0x16ab678, 0xc00012a018, 0xc000958b70)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1265 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc000b1dec0, 0x1, 0xc00036c098, 0x203001)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc00021c0d8, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*fetcher).Fetch(0xc00021c0f0, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:143 +0x7e
github.com/moby/buildkit/util/contentutil.(*fetchedProvider).ReaderAt(0xc0004ca330, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/fetcher.go:24 +0x95
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc0004843f0, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484450, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc0004844b0, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484510, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484570, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc0004845d0, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/pull/pullprogress.(*ProviderWithProgress).ReaderAt(0xc00072ea60, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/pull/pullprogress/progress.go:28 +0x95
github.com/moby/buildkit/util/contentutil.(*localFetcher).Fetch(0xc000b256a0, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/copy.go:28 +0x95
github.com/moby/buildkit/util/resolver/limited.(*fetcher).Fetch(0xc0006bd458, 0x16ab6e8, 0xc000b1dec0, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:147 +0xf5
github.com/containerd/containerd/remotes.fetch(0x16ab6e8, 0xc000b1dec0, 0x7ff229f39cb0, 0xc0001344c0, 0x1685260, 0xc0006bd458, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:141 +0x5b5
github.com/containerd/containerd/remotes.FetchHandler.func1(0x16ab6e8, 0xc000b1de00, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:103 +0x378
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab6e8, 0xc000b1de00, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/contentutil.Copy(0x16ab6e8, 0xc000b1de00, 0x7ff229f39cb0, 0xc0001344c0, 0x1685240, 0xc00072ea60, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, ...)
	/src/util/contentutil/copy.go:17 +0x115
github.com/moby/buildkit/cache.lazyRefProvider.Unlazy.func1(0x16ab6e8, 0xc000b1de00, 0x0, 0x0, 0x0, 0x0)
	/src/cache/remote.go:208 +0x318
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0007437a0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000743800, 0xc000b255d0)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000743800, 0xc000b255d0)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1266 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc00072ea00, 0x16976c0, 0xc000817480)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1168 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0007437a0, 0xc0002b0588, 0xc000ad8410, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 992 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff22a117080, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc0001a9198, 0x72, 0x2700, 0x27f4, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0001a9180, 0xc0008e0000, 0x27f4, 0x27f4, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc0001a9180, 0xc0008e0000, 0x27f4, 0x27f4, 0x2a8, 0xc000997880, 0x418e13)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc000b1a078, 0xc0008e0000, 0x27f4, 0x27f4, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
crypto/tls.(*atLeastReader).Read(0xc0008842d0, 0xc0008e0000, 0x27f4, 0x27f4, 0xc0009979f8, 0xc000100000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:776 +0x63
bytes.(*Buffer).ReadFrom(0xc0005be978, 0x1684320, 0xc0008842d0, 0x40b965, 0x133dbe0, 0x14b5aa0)
	/usr/local/go/src/bytes/buffer.go:204 +0xbe
crypto/tls.(*Conn).readFromUntil(0xc0005be700, 0x1686ba0, 0xc000b1a078, 0x5, 0xc000b1a078, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:798 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc0005be700, 0x0, 0x0, 0xc000997e88)
	/usr/local/go/src/crypto/tls/conn.go:605 +0x115
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:573
crypto/tls.(*Conn).Read(0xc0005be700, 0xc0004c4000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1276 +0x165
net/http.(*persistConn).Read(0xc000966b40, 0xc0004c4000, 0x1000, 0x1000, 0xc000997dc4, 0x2, 0x2)
	/usr/local/go/src/net/http/transport.go:1922 +0x77
bufio.(*Reader).fill(0xc0001311a0)
	/usr/local/go/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).Peek(0xc0001311a0, 0x1, 0x0, 0x1, 0x1, 0x1, 0x0)
	/usr/local/go/src/bufio/bufio.go:139 +0x4f
net/http.(*persistConn).readLoop(0xc000966b40)
	/usr/local/go/src/net/http/transport.go:2083 +0x1a8
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1743 +0xc77

goroutine 1015 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000816c10, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000816c00)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc000958e10, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000958d80, 0x16851e0, 0xc000958e10)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1161 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000816790, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000816780)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc000b1cba0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000b1cb10, 0x16851e0, 0xc000b1cba0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1167 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000817410, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000817400)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc000b1db90, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000b1db00, 0x16851e0, 0xc000b1db90)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1019 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc0007993e0, 0x16ab678, 0xc00012a018, 0xc000958e10)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 940 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc00003ab00, 0x16976c0, 0xc00093fa80)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1016 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc000742750, 0xc000495570, 0xc000ad94a0, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1012 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc000958cc0, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc000958cc0, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc000958cc0, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc000958cc0, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc000816b80, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc000816b80, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc000816b80, 0xc000b34a40, 0x31, 0xc000ad9450, 0x47, 0x391, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc000816b80, 0x16ab640, 0xc000816b80, 0xc00072cd40, 0xc00002e000)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0007426c0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000742720, 0xc00072cd30)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000742720, 0xc00072cd30)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1181 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000a124e0, 0x16976c0, 0xc0006524c0)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1011 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0007426c0, 0xc000495570, 0xc000ad9450, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 911 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000906810, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000906800)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0008618c0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000861830, 0x16851e0, 0xc0008618c0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 963 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0007427e0, 0x16ab640, 0xc000959170, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad8f50, 0x47, 0xc0003ae700, 0x7ff22a2a9c00, 0xc00006a400, 0x0, 0xc000659b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad8f50, 0x47, 0xc0003ae700, 0x20300000000000, 0x400, 0x400, 0xc000659c90)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b347c0, 0x31, 0xc000ad8f50, 0x47, 0x6e1dd4, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0xc00092a128, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438400)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 935 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000904de0, 0x16ab678, 0xc00012a018, 0xc0007281b0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 781 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000652200, 0xc0007281b0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1179 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0006926c0, 0xc0002b0588, 0xc000ad8cd0, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 880 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc0009429c0, 0x16ab678, 0xc00012a018, 0xc000728cc0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1201 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc00074edd0, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00074edc0)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc00081f290, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc00081f200, 0x16851e0, 0xc00081f290)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 933 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000904d20, 0x16ab678, 0xc00012a018, 0xc0009588d0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 934 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000904d80, 0x16ab678, 0xc00012a018, 0xc0009bdcb0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 965 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc000462120, 0x16ab640, 0xc000648510, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad8ff0, 0x47, 0xc00056eaf0, 0xc00090a210, 0xc000100000, 0x0, 0xc000665b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad8ff0, 0x47, 0xc00056eaf0, 0xc000665d20, 0x756ea1b8ad8a39, 0xa67443186532b814, 0xc000665c90)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0xc000958428, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438500)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 922 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0007437a0, 0x16ab6e8, 0xc000b1dc50, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc0002b0588, 0x16ab6e8, 0xc0007c4a50, 0xc000ad8410, 0x47, 0xc0001cc900, 0x7ff229ce6ef8, 0xc0004a0000, 0x0, 0xc0008d7408)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc0002b0588, 0x16ab6e8, 0xc0007c4a50, 0xc000ad8410, 0x47, 0xc0001cc900, 0x4d, 0xc000379270, 0x4d, 0xc000379270)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/cache.lazyRefProvider.Unlazy(0xc00000cd50, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, 0x0, 0xc000b86180, ...)
	/src/cache/remote.go:186 +0xc9
github.com/moby/buildkit/cache.lazyRefProvider.ReaderAt(0xc00000cd50, 0xc000504ba0, 0x2b, 0xc000ad8410, 0x47, 0x63, 0x0, 0x0, 0x0, 0xc000b86180, ...)
	/src/cache/remote.go:179 +0xf2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc00041a870, 0x16ab6e8, 0xc0007c4a50, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/moby/buildkit/cache.(*lazyMultiProvider).ReaderAt(0xc000430060, 0x16ab6e8, 0xc0007c4a50, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, ...)
	/src/cache/remote.go:154 +0x7d
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc00097a690, 0x16ab6e8, 0xc0007c4a50, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc0007c4a50, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:186 +0x395
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc00074e400, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc00074e400, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc00074e400, 0xc000b34a00, 0x31, 0xc000ad9400, 0x47, 0x63, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc00074e400, 0x16ab640, 0xc00074e400, 0xc000b24470, 0xc0008c7f00)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0000e5950)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc0000e59b0, 0xc000b24460)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc0000e59b0, 0xc000b24460)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 892 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc0008169c0, 0xc0009588d0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 879 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000942900, 0x16ab678, 0xc00012a018, 0xc000648450)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1120 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc00074ed80, 0xc00081f290)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1180 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc000845080, 0x1, 0xc00036c098, 0xc000597101)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc00021c0d8, 0x16ab6e8, 0xc000845080, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*fetcher).Fetch(0xc00021c0f0, 0x16ab6e8, 0xc000845080, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:143 +0x7e
github.com/moby/buildkit/util/contentutil.(*fetchedProvider).ReaderAt(0xc0004ca330, 0x16ab6e8, 0xc000845080, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/contentutil/fetcher.go:24 +0x95
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc0004845d0, 0x16ab6e8, 0xc000845080, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/moby/buildkit/util/pull/pullprogress.(*ProviderWithProgress).ReaderAt(0xc000a12540, 0x16ab6e8, 0xc000845080, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/pull/pullprogress/progress.go:28 +0x95
github.com/moby/buildkit/util/contentutil.(*localFetcher).Fetch(0xc000700720, 0x16ab6e8, 0xc000845080, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/contentutil/copy.go:28 +0x95
github.com/moby/buildkit/util/resolver/limited.(*fetcher).Fetch(0xc00040cf30, 0x16ab6e8, 0xc000845080, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:147 +0xf5
github.com/containerd/containerd/remotes.fetch(0x16ab6e8, 0xc000845080, 0x7ff229f39cb0, 0xc0001344c0, 0x1685260, 0xc00040cf30, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:141 +0x5b5
github.com/containerd/containerd/remotes.FetchHandler.func1(0x16ab6e8, 0xc000844fc0, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:103 +0x378
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab6e8, 0xc000844fc0, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, 0xa8e3, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/contentutil.Copy(0x16ab6e8, 0xc000844fc0, 0x7ff229f39cb0, 0xc0001344c0, 0x1685240, 0xc000a12540, 0xc000504de0, 0x2b, 0xc000ad8cd0, 0x47, ...)
	/src/util/contentutil/copy.go:17 +0x115
github.com/moby/buildkit/cache.lazyRefProvider.Unlazy.func1(0x16ab6e8, 0xc000844fc0, 0x0, 0x0, 0x0, 0x0)
	/src/cache/remote.go:208 +0x318
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0006926c0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000692720, 0xc000700660)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000692720, 0xc000700660)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1204 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000748720, 0x16976c0, 0xc00074ee80)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1202 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0000e5d40, 0xc0002b0588, 0xc000ad8140, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 967 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc000462000, 0x16ab640, 0xc000648240, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9090, 0x47, 0xc00056e070, 0x10, 0xc000100000, 0x0, 0xc000664b60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad9090, 0x47, 0xc00056e070, 0xc000861380, 0xc000664bc8, 0x1, 0xc000816740)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0xc000046778, 0x1559180)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438600)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 936 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc00093f9c0, 0xc00097ad20)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 968 [select, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*call).wait(0xc0000e5a70, 0x16ab640, 0xc0007c4f00, 0x0, 0x0, 0x0, 0x0)
	/src/util/flightcontrol/flightcontrol.go:166 +0x471
github.com/moby/buildkit/util/flightcontrol.(*Group).do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad90e0, 0x47, 0xc0003a4850, 0x0, 0xc000100400, 0x0, 0xc00095eb60)
	/src/util/flightcontrol/flightcontrol.go:78 +0x1f3
github.com/moby/buildkit/util/flightcontrol.(*Group).Do(0xc000495570, 0x16ab640, 0xc0009568c0, 0xc000ad90e0, 0x47, 0xc0003a4850, 0x0, 0x0, 0x0, 0xc00095ec90)
	/src/util/flightcontrol/flightcontrol.go:37 +0x7a
github.com/moby/buildkit/util/push.dedupeHandler.func1(0x16ab640, 0xc0009568c0, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:291 +0x125
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000b873e0, 0x16ab640, 0xc0009568c0, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Handlers.func1(0x16ab640, 0xc0009568c0, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:65 +0xf8
github.com/containerd/containerd/images.HandlerFunc.Handle(0xc000430ae0, 0x16ab640, 0xc0009568c0, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:55 +0x75
github.com/containerd/containerd/images.Dispatch.func1(0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/images/handlers.go:134 +0xd5
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0009bdb90, 0xc000438680)
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:57 +0x59
created by golang.org/x/sync/errgroup.(*Group).Go
	/src/vendor/golang.org/x/sync/errgroup/errgroup.go:54 +0x66

goroutine 869 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000769c20, 0x16ab678, 0xc00012a018, 0xc0003cda40)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 993 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000a12000, 0x16976c0, 0xc0006522c0)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1042 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000115d40, 0x16ab678, 0xc00012a018, 0xc0007c4ba0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1043 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc00074e540, 0xc0007c4e40)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1044 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc00074e590, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00074e580)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0007c4e40, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc0007c4db0, 0x16851e0, 0xc0007c4e40)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1045 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0000e5a70, 0xc000495570, 0xc000ad90e0, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1046 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc0007c4f90, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc0007c4f90, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc0007c4f90, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc0007c4f90, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc00074e600, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc00074e600, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc00074e600, 0xc000b34900, 0x31, 0xc000ad90e0, 0x47, 0xf584, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc00074e600, 0x16ab640, 0xc00074e600, 0xc000b245f0, 0x16b30c8)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0000e5a70)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc0000e5ad0, 0xc000b245e0)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc0000e5ad0, 0xc000b245e0)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1047 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000748140, 0x16976c0, 0xc00074e600)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 254 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000930090, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000930080)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc000648120, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000648030, 0x16851e0, 0xc000648120)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 255 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc000462000, 0xc000495570, 0xc000ad9090, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 256 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc0006482d0, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc0006482d0, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc0006482d0, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc0006482d0, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc000930100, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc000930100, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc000930100, 0xc000b348c0, 0x31, 0xc000ad9090, 0x47, 0x279c5df0, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc000930100, 0x16ab640, 0xc000930100, 0xc00042e090, 0x0)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc000462000)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000462060, 0xc00042e080)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000462060, 0xc00042e080)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1057 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000b20000, 0x16976c0, 0xc000930100)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 995 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc00092ec60, 0x16ab678, 0xc00012a018, 0xc00097ad20)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 996 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc00092ed20, 0x16ab678, 0xc00012a018, 0xc0009590b0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1200 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc0009b93e0, 0x16ab678, 0xc00012a018, 0xc000844d20)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 998 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc00092ee40, 0x16ab678, 0xc00012a018, 0xc0007c4e40)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1058 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000bfa300, 0x16ab678, 0xc00012a018, 0xc000648120)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1048 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc00074e640, 0xc0007c50e0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1049 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc00074e690, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc00074e680)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc0007c50e0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc0007c5050, 0x16851e0, 0xc0007c50e0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1050 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc0000e5b00, 0xc000495570, 0xc000ad8f00, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1059 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc000bfa360, 0x16ab678, 0xc00012a018, 0xc0007c50e0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 999 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000652500, 0xc000728cc0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1000 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000652550, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000652540)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc000728cc0, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc000728c00, 0x16851e0, 0xc000728cc0)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1051 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc0007c5230, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc0007c5230, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc0007c5230, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc0007c5230, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc00074e700, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc00074e700, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc00074e700, 0xc000b34780, 0x31, 0xc000ad8f00, 0x47, 0x1978c2f, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc00074e700, 0x16ab640, 0xc00074e700, 0xc000b246b0, 0x16b30c8)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc0000e5b00)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc0000e5b60, 0xc000b246a0)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc0000e5b60, 0xc000b246a0)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1001 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc000692480, 0xc000495570, 0xc000ad9040, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1002 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc000728e10, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc000728e10, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc000728e10, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc000728e10, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc0006525c0, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc0006525c0, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc0006525c0, 0xc000b34880, 0x31, 0xc000ad9040, 0x47, 0x1920, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc0006525c0, 0x16ab640, 0xc0006525c0, 0xc0004cac40, 0x16b30c8)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc000692480)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc0006924e0, 0xc0004cac30)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc0006924e0, 0xc0004cac30)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1060 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000930140, 0xc000648450)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1061 [sync.Cond.Wait, 6 minutes]:
sync.runtime_notifyListWait(0xc000930190, 0x0)
	/usr/local/go/src/runtime/sema.go:513 +0xf8
sync.(*Cond).Wait(0xc000930180)
	/usr/local/go/src/sync/cond.go:56 +0x99
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc000648450, 0x16ab678, 0xc00012a018, 0x0, 0x0, 0x0, 0x0, 0x0)
	/src/util/progress/progress.go:147 +0x116
github.com/moby/buildkit/util/flightcontrol.(*progressState).run(0xc0006483c0, 0x16851e0, 0xc000648450)
	/src/util/flightcontrol/flightcontrol.go:296 +0x168
created by github.com/moby/buildkit/util/flightcontrol.newCall
	/src/util/flightcontrol/flightcontrol.go:112 +0x1d8

goroutine 1062 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc000462120, 0xc000495570, 0xc000ad8ff0, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1063 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc0006485a0, 0x1, 0xc00036c098, 0x1)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc000670bb8, 0x16ab6e8, 0xc0006485a0, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*pusher).Push(0xc000670bd0, 0x16ab6e8, 0xc0006485a0, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:91 +0x7e
github.com/containerd/containerd/remotes.push(0x16ab6e8, 0xc0006485a0, 0x16850c0, 0xc00097a690, 0x1685280, 0xc000670bd0, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:175 +0x635
github.com/containerd/containerd/remotes.PushHandler.func1(0x16ab640, 0xc000930200, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:160 +0x358
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab640, 0xc000930200, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/push.updateDistributionSourceHandler.func1(0x16ab640, 0xc000930200, 0xc000b34840, 0x31, 0xc000ad8ff0, 0x47, 0xba, 0x0, 0x0, 0x0, ...)
	/src/util/push/push.go:270 +0xd5
github.com/moby/buildkit/util/push.dedupeHandler.func1.1(0x16ab640, 0xc000930200, 0x16ab640, 0xc000930200, 0xc00042e160, 0x0)
	/src/util/push/push.go:299 +0x1f9
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc000462120)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000462180, 0xc00042e150)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000462180, 0xc00042e150)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1064 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000b20020, 0x16976c0, 0xc000930200)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1052 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000748160, 0x16976c0, 0xc00074e700)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1003 [select, 6 minutes]:
context.propagateCancel.func1(0x16abec8, 0xc000a12120, 0x16976c0, 0xc0006525c0)
	/usr/local/go/src/context/context.go:279 +0xab
created by context.propagateCancel
	/usr/local/go/src/context/context.go:278 +0x1fb

goroutine 1092 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7ff22a117250, 0x72, 0xffffffffffffffff)
	/usr/local/go/src/runtime/netpoll.go:222 +0x55
internal/poll.(*pollDesc).wait(0xc0005acb98, 0x72, 0x2700, 0x27f3, 0xffffffffffffffff)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45
internal/poll.(*pollDesc).waitRead(...)
	/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0005acb80, 0xc0009a6000, 0x27f3, 0x27f3, 0x0, 0x0, 0x0)
	/usr/local/go/src/internal/poll/fd_unix.go:166 +0x1d5
net.(*netFD).Read(0xc0005acb80, 0xc0009a6000, 0x27f3, 0x27f3, 0x30b, 0xc000667880, 0x418e13)
	/usr/local/go/src/net/fd_posix.go:55 +0x4f
net.(*conn).Read(0xc0003a0000, 0xc0009a6000, 0x27f3, 0x27f3, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/net.go:183 +0x91
crypto/tls.(*atLeastReader).Read(0xc00040cd80, 0xc0009a6000, 0x27f3, 0x27f3, 0xc0006679f8, 0xc000100000, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:776 +0x63
bytes.(*Buffer).ReadFrom(0xc0005ce5f8, 0x1684320, 0xc00040cd80, 0x40b965, 0x133dbe0, 0x14b5aa0)
	/usr/local/go/src/bytes/buffer.go:204 +0xbe
crypto/tls.(*Conn).readFromUntil(0xc0005ce380, 0x1686ba0, 0xc0003a0000, 0x5, 0xc0003a0000, 0x11)
	/usr/local/go/src/crypto/tls/conn.go:798 +0xf3
crypto/tls.(*Conn).readRecordOrCCS(0xc0005ce380, 0x0, 0x0, 0xc000667e88)
	/usr/local/go/src/crypto/tls/conn.go:605 +0x115
crypto/tls.(*Conn).readRecord(...)
	/usr/local/go/src/crypto/tls/conn.go:573
crypto/tls.(*Conn).Read(0xc0005ce380, 0xc000835000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
	/usr/local/go/src/crypto/tls/conn.go:1276 +0x165
net/http.(*persistConn).Read(0xc00082e7e0, 0xc000835000, 0x1000, 0x1000, 0xc000667dc4, 0x2, 0x2)
	/usr/local/go/src/net/http/transport.go:1922 +0x77
bufio.(*Reader).fill(0xc0001bd200)
	/usr/local/go/src/bufio/bufio.go:101 +0x108
bufio.(*Reader).Peek(0xc0001bd200, 0x1, 0x0, 0x1, 0x1, 0x1, 0x0)
	/usr/local/go/src/bufio/bufio.go:139 +0x4f
net/http.(*persistConn).readLoop(0xc00082e7e0)
	/usr/local/go/src/net/http/transport.go:2083 +0x1a8
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1743 +0xc77

goroutine 1093 [select, 6 minutes]:
io.(*pipe).Read(0xc000a0df80, 0xc0008f2000, 0x8000, 0x8000, 0x7ff250d9f3c8, 0xc0002f7987, 0xc0008f2000)
	/usr/local/go/src/io/pipe.go:57 +0xcb
io.(*PipeReader).Read(0xc00040a078, 0xc0008f2000, 0x8000, 0x8000, 0x7ff250d9f3c8, 0x8000, 0xc0008f2000)
	/usr/local/go/src/io/pipe.go:134 +0x4c
io.(*LimitedReader).Read(0xc000884f48, 0xc0008f2000, 0x8000, 0x8000, 0x0, 0xc000030000, 0x8)
	/usr/local/go/src/io/io.go:473 +0x63
io.copyBuffer(0x7ff229c115f0, 0xc0005ce380, 0x16869a0, 0xc000884f48, 0xc0008f2000, 0x8000, 0x8000, 0x13a95a0, 0x1e86301, 0x7ff229c115d0)
	/usr/local/go/src/io/io.go:423 +0x12c
io.Copy(...)
	/usr/local/go/src/io/io.go:382
net/http.persistConnWriter.ReadFrom(0xc00082e7e0, 0x16869a0, 0xc000884f48, 0x7ff229c115d0, 0xc00082e7e0, 0x634301)
	/usr/local/go/src/net/http/transport.go:1768 +0x85
bufio.(*Writer).ReadFrom(0xc0008faec0, 0x16869a0, 0xc000884f48, 0x7ff229c115b0, 0xc0008faec0, 0x7ff229c11501)
	/usr/local/go/src/bufio/bufio.go:730 +0x37e
io.copyBuffer(0x1684120, 0xc0008faec0, 0x16869a0, 0xc000884f48, 0x0, 0x0, 0x0, 0xc000661c38, 0x40e338, 0x18)
	/usr/local/go/src/io/io.go:409 +0x357
io.Copy(...)
	/usr/local/go/src/io/io.go:382
net/http.(*transferWriter).doBodyCopy(0xc0004046e0, 0x1684120, 0xc0008faec0, 0x16869a0, 0xc000884f48, 0x70e1f2, 0xc0005ce380, 0xc000846000)
	/usr/local/go/src/net/http/transfer.go:409 +0x6a
net/http.(*transferWriter).writeBody(0xc0004046e0, 0x1684120, 0xc0008faec0, 0x0, 0x0)
	/usr/local/go/src/net/http/transfer.go:367 +0x825
net/http.(*Request).write(0xc000691500, 0x1684120, 0xc0008faec0, 0x0, 0xc00041b440, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/request.go:697 +0x7c6
net/http.(*persistConn).writeLoop(0xc00082e7e0)
	/usr/local/go/src/net/http/transport.go:2385 +0x1a7
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1744 +0xc9c

goroutine 943 [select, 6 minutes]:
io.(*pipe).Read(0xc00042cde0, 0xc0001c9180, 0x63, 0x63, 0xc0001c9180, 0x70, 0x364)
	/usr/local/go/src/io/pipe.go:57 +0xcb
io.(*PipeReader).Read(0xc0003b81a0, 0xc0001c9180, 0x63, 0x63, 0x7ff250d9f878, 0x70, 0xc0001c9180)
	/usr/local/go/src/io/pipe.go:134 +0x4c
io.(*LimitedReader).Read(0xc00040d830, 0xc0001c9180, 0x63, 0x63, 0x0, 0xc000030000, 0x8)
	/usr/local/go/src/io/io.go:473 +0x63
io.copyBuffer(0x7ff229c115f0, 0xc000028380, 0x16869a0, 0xc00040d830, 0xc0001c9180, 0x63, 0x63, 0x13a95a0, 0x1e86301, 0x7ff229c115d0)
	/usr/local/go/src/io/io.go:423 +0x12c
io.Copy(...)
	/usr/local/go/src/io/io.go:382
net/http.persistConnWriter.ReadFrom(0xc00026e480, 0x16869a0, 0xc00040d830, 0x7ff229c115d0, 0xc00026e480, 0x3200000000634301)
	/usr/local/go/src/net/http/transport.go:1768 +0x85
bufio.(*Writer).ReadFrom(0xc00093fac0, 0x16869a0, 0xc00040d830, 0x7ff229c115b0, 0xc00093fac0, 0x7ff229c11501)
	/usr/local/go/src/bufio/bufio.go:730 +0x37e
io.copyBuffer(0x1684120, 0xc00093fac0, 0x16869a0, 0xc00040d830, 0x0, 0x0, 0x0, 0xc000998c38, 0x40e338, 0x18)
	/usr/local/go/src/io/io.go:409 +0x357
io.Copy(...)
	/usr/local/go/src/io/io.go:382
net/http.(*transferWriter).doBodyCopy(0xc0005f4500, 0x1684120, 0xc00093fac0, 0x16869a0, 0xc00040d830, 0x70e1f2, 0xc000028380, 0xc000793000)
	/usr/local/go/src/net/http/transfer.go:409 +0x6a
net/http.(*transferWriter).writeBody(0xc0005f4500, 0x1684120, 0xc00093fac0, 0x0, 0x0)
	/usr/local/go/src/net/http/transfer.go:367 +0x825
net/http.(*Request).write(0xc0002a1400, 0x1684120, 0xc00093fac0, 0x0, 0xc000b86ed0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/request.go:697 +0x7c6
net/http.(*persistConn).writeLoop(0xc00026e480)
	/usr/local/go/src/net/http/transport.go:2385 +0x1a7
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1744 +0xc9c

goroutine 1152 [select, 6 minutes]:
io.(*pipe).Read(0xc00042c120, 0xc000852000, 0x8000, 0x8000, 0x7ff250d9f878, 0xc000784087, 0xc000852000)
	/usr/local/go/src/io/pipe.go:57 +0xcb
io.(*PipeReader).Read(0xc0003b8070, 0xc000852000, 0x8000, 0x8000, 0x7ff250d9f878, 0x8000, 0xc000852000)
	/usr/local/go/src/io/pipe.go:134 +0x4c
io.(*LimitedReader).Read(0xc00040d6b0, 0xc000852000, 0x8000, 0x8000, 0x0, 0xc000030000, 0x8)
	/usr/local/go/src/io/io.go:473 +0x63
io.copyBuffer(0x7ff229c115f0, 0xc0005ce700, 0x16869a0, 0xc00040d6b0, 0xc000852000, 0x8000, 0x8000, 0x13a95a0, 0x1e86301, 0x7ff229c115d0)
	/usr/local/go/src/io/io.go:423 +0x12c
io.Copy(...)
	/usr/local/go/src/io/io.go:382
net/http.persistConnWriter.ReadFrom(0xc000966000, 0x16869a0, 0xc00040d6b0, 0x7ff229c115d0, 0xc000966000, 0x3200000000634301)
	/usr/local/go/src/net/http/transport.go:1768 +0x85
bufio.(*Writer).ReadFrom(0xc000931a40, 0x16869a0, 0xc00040d6b0, 0x7ff229c115b0, 0xc000931a40, 0x7ff229c11501)
	/usr/local/go/src/bufio/bufio.go:730 +0x37e
io.copyBuffer(0x1684120, 0xc000931a40, 0x16869a0, 0xc00040d6b0, 0x0, 0x0, 0x0, 0xc000ae0c38, 0x40e338, 0x18)
	/usr/local/go/src/io/io.go:409 +0x357
io.Copy(...)
	/usr/local/go/src/io/io.go:382
net/http.(*transferWriter).doBodyCopy(0xc0005f4320, 0x1684120, 0xc000931a40, 0x16869a0, 0xc00040d6b0, 0x70e1f2, 0xc0005ce700, 0xc000697000)
	/usr/local/go/src/net/http/transfer.go:409 +0x6a
net/http.(*transferWriter).writeBody(0xc0005f4320, 0x1684120, 0xc000931a40, 0x0, 0x0)
	/usr/local/go/src/net/http/transfer.go:367 +0x825
net/http.(*Request).write(0xc0002a0e00, 0x1684120, 0xc000931a40, 0x0, 0xc000b867b0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/request.go:697 +0x7c6
net/http.(*persistConn).writeLoop(0xc000966000)
	/usr/local/go/src/net/http/transport.go:2385 +0x1a7
created by net/http.(*Transport).dialConn
	/usr/local/go/src/net/http/transport.go:1744 +0xc9c

goroutine 1173 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc00092e4e0, 0x16ab678, 0xc00012a018, 0xc00081f290)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1216 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc00026bf80, 0x16ab678, 0xc00012a018, 0xc000b1cba0)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

goroutine 1162 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/flightcontrol.(*Group).do.func1(0xc000742fc0, 0xc0002b0588, 0xc000ad8a50, 0x47)
	/src/util/flightcontrol/flightcontrol.go:71 +0x3c
created by github.com/moby/buildkit/util/flightcontrol.(*Group).do
	/src/util/flightcontrol/flightcontrol.go:69 +0x1bc

goroutine 1163 [select, 6 minutes]:
golang.org/x/sync/semaphore.(*Weighted).Acquire(0xc000b2a1e0, 0x16ab6e8, 0xc000b1d140, 0x1, 0xc00036c098, 0xc000055801)
	/src/vendor/golang.org/x/sync/semaphore/semaphore.go:60 +0x345
github.com/moby/buildkit/util/resolver/limited.(*req).acquire(0xc00021c0d8, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:48 +0x25d
github.com/moby/buildkit/util/resolver/limited.(*fetcher).Fetch(0xc00021c0f0, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:143 +0x7e
github.com/moby/buildkit/util/contentutil.(*fetchedProvider).ReaderAt(0xc0004ca330, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/contentutil/fetcher.go:24 +0x95
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484510, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:34 +0xf6
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc000484570, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/contentutil.(*MultiProvider).ReaderAt(0xc0004845d0, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/contentutil/multiprovider.go:40 +0x1e2
github.com/moby/buildkit/util/pull/pullprogress.(*ProviderWithProgress).ReaderAt(0xc00072e920, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/pull/pullprogress/progress.go:28 +0x95
github.com/moby/buildkit/util/contentutil.(*localFetcher).Fetch(0xc000b25210, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/contentutil/copy.go:28 +0x95
github.com/moby/buildkit/util/resolver/limited.(*fetcher).Fetch(0xc0006bce40, 0x16ab6e8, 0xc000b1d140, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, ...)
	/src/util/resolver/limited/group.go:147 +0xf5
github.com/containerd/containerd/remotes.fetch(0x16ab6e8, 0xc000b1d140, 0x7ff229f39cb0, 0xc0001344c0, 0x1685260, 0xc0006bce40, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:141 +0x5b5
github.com/containerd/containerd/remotes.FetchHandler.func1(0x16ab6e8, 0xc000b1cf00, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, 0x0, ...)
	/src/vendor/github.com/containerd/containerd/remotes/handlers.go:103 +0x378
github.com/moby/buildkit/util/resolver/retryhandler.New.func1(0x16ab6e8, 0xc000b1cf00, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, 0xe601, 0x0, 0x0, 0x0, ...)
	/src/util/resolver/retryhandler/retry.go:21 +0xb5
github.com/moby/buildkit/util/contentutil.Copy(0x16ab6e8, 0xc000b1cf00, 0x7ff229f39cb0, 0xc0001344c0, 0x1685240, 0xc00072e920, 0xc000504cf0, 0x2b, 0xc000ad8a50, 0x47, ...)
	/src/util/contentutil/copy.go:17 +0x115
github.com/moby/buildkit/cache.lazyRefProvider.Unlazy.func1(0x16ab6e8, 0xc000b1cf00, 0x0, 0x0, 0x0, 0x0)
	/src/cache/remote.go:208 +0x318
github.com/moby/buildkit/util/flightcontrol.(*call).run(0xc000742fc0)
	/src/util/flightcontrol/flightcontrol.go:121 +0x8d
sync.(*Once).doSlow(0xc000743020, 0xc000b25140)
	/usr/local/go/src/sync/once.go:68 +0xec
sync.(*Once).Do(0xc000743020, 0xc000b25140)
	/usr/local/go/src/sync/once.go:59 +0x45
created by github.com/moby/buildkit/util/flightcontrol.(*call).wait
	/src/util/flightcontrol/flightcontrol.go:164 +0x3f8

goroutine 1160 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x16ab640, 0xc000816740, 0xc000b1cba0)
	/src/util/progress/progress.go:187 +0x4c
created by github.com/moby/buildkit/util/progress.pipe
	/src/util/progress/progress.go:186 +0x134

goroutine 1159 [select, 6 minutes]:
net/http.(*persistConn).roundTrip(0xc000966000, 0xc0006533c0, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/transport.go:2610 +0x765
net/http.(*Transport).roundTrip(0xc0004963c0, 0xc0002a0c00, 0xc000b86750, 0x16a6770, 0xc000845e60)
	/usr/local/go/src/net/http/transport.go:592 +0xacb
net/http.(*Transport).RoundTrip(0xc0004963c0, 0xc0002a0c00, 0xc000b86750, 0x16a6770, 0xc000845e60)
	/usr/local/go/src/net/http/roundtrip.go:17 +0x35
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*Transport).RoundTrip(0xc00031e7e0, 0xc0002a0b00, 0xc00031e7e0, 0x0, 0x0)
	/src/vendor/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp/transport.go:115 +0x3d9
net/http.send(0xc0002a0b00, 0x16854e0, 0xc00031e7e0, 0x0, 0x0, 0x0, 0xc00040a108, 0x16ab6e8, 0x1, 0x0)
	/usr/local/go/src/net/http/client.go:251 +0x454
net/http.(*Client).send(0xc00002d6e0, 0xc0002a0b00, 0x0, 0x0, 0x0, 0xc00040a108, 0x0, 0x1, 0x16ab6e8)
	/usr/local/go/src/net/http/client.go:175 +0xff
net/http.(*Client).do(0xc00002d6e0, 0xc0002a0b00, 0x0, 0x0, 0x0)
	/usr/local/go/src/net/http/client.go:717 +0x45f
net/http.(*Client).Do(...)
	/usr/local/go/src/net/http/client.go:585
golang.org/x/net/context/ctxhttp.Do(0x16ab6e8, 0xc000845ef0, 0xc00002d6e0, 0xc0002a0a00, 0x0, 0x0, 0x16ab6e8)
	/src/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:27 +0x10f
github.com/containerd/containerd/remotes/docker.(*request).do(0xc000742f30, 0x16ab6e8, 0xc000958a80, 0x0, 0x0, 0x0)
	/src/vendor/github.com/containerd/containerd/remotes/docker/resolver.go:542 +0x529
github.com/containerd/containerd/remotes/docker.(*request).doWithRetries(0xc000742f30, 0x16ab6e8, 0xc000958a80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4)
	/src/vendor/github.com/containerd/containerd/remotes/docker/resolver.go:551 +0x46
github.com/containerd/containerd/remotes/docker.dockerPusher.push.func2(0xc00042c180, 0xc000742f30, 0x16ab6e8, 0xc000958a80, 0xc0003b8070)
	/src/vendor/github.com/containerd/containerd/remotes/docker/pusher.go:278 +0xa5
created by github.com/containerd/containerd/remotes/docker.dockerPusher.push
	/src/vendor/github.com/containerd/containerd/remotes/docker/pusher.go:276 +0xd56

goroutine 1218 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc0009b9500, 0x16ab678, 0xc00012a018, 0xc000b1db90)
	/src/util/progress/progress.go:121 +0x8a
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
	/src/util/progress/progress.go:120 +0xdf

rax    0xca
rbx    0x1ea8ee0
rcx    0x4740c3
rdx    0x0
rdi    0x1ea9030
rsi    0x80
rbp    0x7ffcced46600
rsp    0x7ffcced465b8
r8     0x0
r9     0x0
r10    0x0
r11    0x286
r12    0xc00058afa4
r13    0x0
r14    0x1665774
r15    0x0
rip    0x4740c1
rflags 0x286
cs     0x33
fs     0x0
gs     0x0
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
kind/bug
Projects
None yet
Milestone
No milestone
Development

Successfully merging a pull request may close this issue.

limited: fix possible deadlock when pushhandler calls fetcher tonistiigi/buildkit
2 participants
@AkihiroSuda @olegy2008