fatal error: concurrent map writes #1250
Labels
Comments
|
Thanks for the report. I had a quick look and noticed one possible case (https://github.com/moby/buildkit/blob/v0.6.2/cache/manager.go#L159) that may cause this but this stacktrace is a bit weird and doesn't fully confirm it. If you have more stacktraces for this error please post them as well. |
|
Thx for answering. The issue seems to be here: https://github.com/moby/buildkit/blob/v0.6.2/cache/refs.go/#L309 But I'm not sure how to fix that (why do we have modification in an object while finalizing). |
|
Another one: https://pastebin.com/Y8YiC5F6 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Hello,
I have lot of crash with Buildkit. This is not systematic, but very common. I have it with a raw 0.6.2 install, or through buidlx.
The logs is below.
How can I help to fix that?
Thx
fatal error: concurrent map writes
goroutine 20822 [running]:
runtime.throw(0x114b515, 0x15)
/usr/local/go/src/runtime/panic.go:617 +0x72 fp=0xc000275cf8 sp=0xc000275cc8 pc=0x42ead2
runtime.mapassign(0xfbf840, 0xc002192ab0, 0xc000275db8, 0x1b22ec0)
/usr/local/go/src/runtime/map.go:590 +0x5e3 fp=0xc000275d80 sp=0xc000275cf8 pc=0x40f7a3
github.com/moby/buildkit/cache.(*cacheRecord).ref(...)
/src/cache/refs.go:73
github.com/moby/buildkit/cache.(*cacheRecord).parentRef(0xc006cd7800, 0x42d801, 0x0)
/src/cache/refs.go:143 +0xf7 fp=0xc000275dd8 sp=0xc000275d80 pc=0xb470f7
github.com/moby/buildkit/cache.(*cacheRecord).Parent(0xc006cd7800, 0x0, 0x0)
/src/cache/refs.go:130 +0x30 fp=0xc000275e00 sp=0xc000275dd8 pc=0xb46fc0
github.com/moby/buildkit/cache/blobs.isTypeWindows(0x12dd6e0, 0xc008633700, 0xc006ee8900)
/src/cache/blobs/blobs.go:151 +0x9d fp=0xc000275e58 sp=0xc000275e00 pc=0xd6580d
github.com/moby/buildkit/cache/blobs.GetDiffPairs(0x12cb6c0, 0xc006ee8980, 0x12dd800, 0xc00010a7d0, 0x12e5040, 0xc00012a340, 0x12a9ce0, 0xc0001286a0, 0x12dd6e0, 0xc008633700, ...)
/src/cache/blobs/blobs.go:40 +0xa2 fp=0xc000275ee8 sp=0xc000275e58 pc=0xd64ef2
github.com/moby/buildkit/exporter/containerimage.(*ImageWriter).exportLayers.func1.1(0x8, 0x1187000)
/src/exporter/containerimage/writer.go:160 +0xac fp=0xc000275f88 sp=0xc000275ee8 pc=0xd71b2c
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0017e6510, 0xc0094441e0)
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:58 +0x57 fp=0xc000275fd0 sp=0xc000275f88 pc=0x8b7117
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc000275fd8 sp=0xc000275fd0 pc=0x45dcc1
created by golang.org/x/sync/errgroup.(*Group).Go
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:55 +0x66
goroutine 1 [select, 38 minutes]:
main.main.func3(0xc0001a14a0, 0x0, 0x0)
/src/cmd/buildkitd/main.go:263 +0x97c
github.com/urfave/cli.HandleAction(0xf9d720, 0x1186628, 0xc0001a14a0, 0xc0001a14a0, 0xc00027d788)
/src/vendor/github.com/urfave/cli/app.go:502 +0xc8
github.com/urfave/cli.(*App).Run(0xc0001d8540, 0xc0000321e0, 0x1, 0x1, 0x0, 0x0)
/src/vendor/github.com/urfave/cli/app.go:268 +0x5aa
main.main()
/src/cmd/buildkitd/main.go:290 +0xd5e
goroutine 34 [chan receive, 38 minutes]:
github.com/moby/buildkit/util/appcontext.Context.func1.1(0xc0004489c0, 0xc000116020, 0xc00013e008)
/src/util/appcontext/appcontext.go:30 +0x38
created by github.com/moby/buildkit/util/appcontext.Context.func1
/src/util/appcontext/appcontext.go:28 +0xff
goroutine 18 [syscall, 38 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:29 +0x41
goroutine 20 [chan receive]:
github.com/moby/buildkit/util/pull.newResolverCache.func1(0xc0003be6c0, 0xc000326a50)
/src/util/pull/resolver.go:203 +0x49
created by github.com/moby/buildkit/util/pull.newResolverCache
/src/util/pull/resolver.go:201 +0x95
goroutine 23 [sync.Cond.Wait]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:307
sync.runtime_notifyListWait(0xc00012a890, 0x85)
/usr/local/go/src/runtime/sema.go:510 +0xf9
sync.(*Cond).Wait(0xc00012a880)
/usr/local/go/src/sync/cond.go:56 +0x9e
github.com/moby/buildkit/util/cond.(*StatefulCond).Wait(0xc0001e93e0)
/src/util/cond/cond.go:28 +0x98
github.com/moby/buildkit/solver.(*scheduler).loop(0xc0001522a0)
/src/solver/scheduler.go:101 +0x168
created by github.com/moby/buildkit/solver.newScheduler
/src/solver/scheduler.go:35 +0x1ad
goroutine 24 [IO wait, 6 minutes]:
internal/poll.runtime_pollWait(0x7f1d46785f90, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc000110418, 0x72, 0x0, 0x0, 0x113d221)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc000110400, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc000110400, 0xc0014ca000, 0x0, 0x0)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*UnixListener).accept(0xc0001e95c0, 0xc0003b5e20, 0xc0003b5e28, 0x18)
/usr/local/go/src/net/unixsock_posix.go:162 +0x32
net.(*UnixListener).Accept(0xc0001e95c0, 0x1186158, 0xc000084a80, 0x12db220, 0xc0014ca000)
/usr/local/go/src/net/unixsock.go:260 +0x48
google.golang.org/grpc.(*Server).Serve(0xc000084a80, 0x12c5b40, 0xc0001e95c0, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/server.go:561 +0x1e9
main.serveGRPC.func1.1(0x0, 0x0)
/src/cmd/buildkitd/main.go:323 +0x10e
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc0001e9530, 0xc000128f00)
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:58 +0x57
created by golang.org/x/sync/errgroup.(*Group).Go
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:55 +0x66
goroutine 25 [semacquire, 38 minutes]:
sync.runtime_Semacquire(0xc0001e9540)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc0001e9538)
/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc0001e9530, 0x0, 0x0)
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:41 +0x31
main.serveGRPC.func2(0xc000105140, 0xc0001e9530)
/src/cmd/buildkitd/main.go:328 +0x2b
created by main.serveGRPC
/src/cmd/buildkitd/main.go:327 +0x2ba
goroutine 26 [chan receive, 38 minutes]:
github.com/moby/buildkit/solver.(*scheduler).loop.func2(0xc0001522a0)
/src/solver/scheduler.go:76 +0x38
created by github.com/moby/buildkit/solver.(*scheduler).loop
/src/solver/scheduler.go:75 +0x6d
goroutine 14780 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000df87c0, 0x1, 0x0, 0x0, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc003b10540, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc001a82000)
/src/vendor/google.golang.org/grpc/internal/transport/http2_client.go:330 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/src/vendor/google.golang.org/grpc/internal/transport/http2_client.go:328 +0xeb2
goroutine 14721 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc000df8040, 0x1, 0x0, 0x0, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000078300, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc002bde000)
/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:276 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:273 +0xfba
goroutine 14755 [IO wait]:
internal/poll.runtime_pollWait(0x7f1d46785df0, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc002902218, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc002902200, 0xc000012000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc002902200, 0xc000012000, 0x8000, 0x8000, 0x0, 0x8, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc0014ca000, 0xc000012000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc003b10000, 0xc002f66038, 0x9, 0x9, 0xc0007ead88, 0x3, 0x0)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0x12a8d80, 0xc003b10000, 0xc002f66038, 0x9, 0x9, 0x9, 0xc0007eadff, 0x1040e00, 0x44d948)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc002f66038, 0x9, 0x9, 0x12a8d80, 0xc003b10000, 0x0, 0x0, 0xbf6a242e431cc7fb, 0x2140411b951)
/src/vendor/golang.org/x/net/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc002f66000, 0xc0053097c0, 0xc0053097c0, 0x0, 0x0)
/src/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc002bde000, 0xc000cb0270, 0x11861b0)
/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:431 +0x7c
google.golang.org/grpc.(*Server).serveStreams(0xc000084a80, 0x12dd8c0, 0xc002bde000)
/src/vendor/google.golang.org/grpc/server.go:687 +0xdd
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000084a80, 0x12dd8c0, 0xc002bde000)
/src/vendor/google.golang.org/grpc/server.go:649 +0x43
created by google.golang.org/grpc.(*Server).handleRawConn
/src/vendor/google.golang.org/grpc/server.go:648 +0x580
goroutine 14770 [semacquire, 6 minutes]:
sync.runtime_Semacquire(0xc001c06bb0)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc001c06ba8)
/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc001c06ba0, 0xc003004520, 0xc001c06ba0)
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:41 +0x31
github.com/moby/buildkit/control.(*Controller).Status(0xc000172bd0, 0xc001c06b10, 0x12d82e0, 0xc002f36490, 0xc000172bd0, 0x20)
/src/control/control.go:346 +0x174
github.com/moby/buildkit/api/services/control._Control_Status_Handler(0x109aa40, 0xc000172bd0, 0x12d3780, 0xc003004500, 0x10ecae0, 0x1b21718)
/src/api/services/control/control.pb.go:1374 +0x109
github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingStreamServerInterceptor.func1(0x109aa40, 0xc000172bd0, 0x12d3c00, 0xc001232240, 0xc0030044a0, 0x11858c8, 0x0, 0x0)
/src/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:114 +0x365
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020000, 0xc0001e9470, 0x1ab0de0, 0x0, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/server.go:1183 +0x462
google.golang.org/grpc.(*Server).handleStream(0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020000, 0x0)
/src/vendor/google.golang.org/grpc/server.go:1256 +0xd3f
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0038f0000, 0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020000)
/src/vendor/google.golang.org/grpc/server.go:691 +0x9f
created by google.golang.org/grpc.(*Server).serveStreams.func1
/src/vendor/google.golang.org/grpc/server.go:689 +0xa1
goroutine 14771 [semacquire]:
sync.runtime_Semacquire(0xc0017e6520)
/usr/local/go/src/runtime/sema.go:56 +0x39
sync.(*WaitGroup).Wait(0xc0017e6518)
/usr/local/go/src/sync/waitgroup.go:130 +0x65
golang.org/x/sync/errgroup.(*Group).Wait(0xc0017e6510, 0xc0086338e0, 0xc00012a480)
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:41 +0x31
github.com/moby/buildkit/exporter/containerimage.(*ImageWriter).exportLayers(0xc00012a480, 0x12cb780, 0xc0017e6390, 0xc0004c6928, 0x1, 0x1, 0xc0004c6750, 0x43783f, 0xc000040a00, 0xc0004c6760, ...)
/src/exporter/containerimage/writer.go:170 +0x1af
github.com/moby/buildkit/exporter/containerimage.(*ImageWriter).Commit(0xc00012a480, 0x12cb780, 0xc0017e6390, 0x12dd6e0, 0xc008633700, 0x0, 0xc00168a210, 0x12cb700, 0xc0017e6390, 0xc009444190, ...)
/src/exporter/containerimage/writer.go:56 +0x12c
github.com/moby/buildkit/exporter/containerimage.(*imageExporterInstance).Export(0xc002be4a80, 0x12cb780, 0xc0017e6390, 0x12dd6e0, 0xc008633700, 0x0, 0xc00168a210, 0x0, 0x0, 0x0)
/src/exporter/containerimage/export.go:153 +0x2a1
github.com/moby/buildkit/solver/llbsolver.(*Solver).Solve.func2(0x12cb780, 0xc00168a780, 0xc0086dcea0, 0x0)
/src/solver/llbsolver/solver.go:197 +0x7d
github.com/moby/buildkit/solver/llbsolver.inVertexContext(0x12cb780, 0xc00168a780, 0x1148414, 0x12, 0x0, 0x0, 0xc0004c7400, 0x0, 0x0)
/src/solver/llbsolver/solver.go:334 +0x233
github.com/moby/buildkit/solver/llbsolver.(*Solver).Solve(0xc000152230, 0x12cb780, 0xc002be4990, 0xc0005323c0, 0x19, 0x0, 0xc00372c020, 0xd, 0xc002be42d0, 0xc002be53b0, ...)
/src/solver/llbsolver/solver.go:196 +0xa77
github.com/moby/buildkit/control.(*Controller).Solve(0xc000172bd0, 0x12cb780, 0xc002be4660, 0xc00019e7e0, 0x0, 0x0, 0x0)
/src/control/control.go:276 +0x4a7
github.com/moby/buildkit/api/services/control._Control_Solve_Handler.func1(0x12cb780, 0xc002be4660, 0x11082a0, 0xc00019e7e0, 0x10ecae0, 0x1b21718, 0x12cb780, 0xc002be4660)
/src/api/services/control/control.pb.go:1364 +0x86
github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingServerInterceptor.func1(0x12cb6c0, 0xc000f6c000, 0x11082a0, 0xc00019e7e0, 0xc00015e060, 0xc00015e080, 0x0, 0x0, 0x12a9040, 0xc00010a3e0)
/src/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:57 +0x2eb
main.unaryInterceptor.func1(0x12cb6c0, 0xc000f6c000, 0x11082a0, 0xc00019e7e0, 0xc00015e060, 0xc00015e080, 0x0, 0x0, 0x0, 0x0)
/src/cmd/buildkitd/main.go:526 +0x15f
github.com/moby/buildkit/api/services/control._Control_Solve_Handler(0x109aa40, 0xc000172bd0, 0x12cb780, 0xc002be4030, 0xc0006b0000, 0xc0004e4600, 0x12cb780, 0xc002be4030, 0xc000508000, 0x482)
/src/api/services/control/control.pb.go:1366 +0x158
google.golang.org/grpc.(*Server).processUnaryRPC(0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020100, 0xc0001e9470, 0x1aae5b8, 0x0, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/server.go:972 +0x470
google.golang.org/grpc.(*Server).handleStream(0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020100, 0x0)
/src/vendor/google.golang.org/grpc/server.go:1252 +0xda6
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0038f0000, 0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020100)
/src/vendor/google.golang.org/grpc/server.go:691 +0x9f
created by google.golang.org/grpc.(*Server).serveStreams.func1
/src/vendor/google.golang.org/grpc/server.go:689 +0xa1
goroutine 14779 [select]:
google.golang.org/grpc/internal/transport.(*recvBufferReader).read(0xc003908280, 0xc003004290, 0x5, 0x5, 0x10000c0007e6868, 0x0, 0x20)
/src/vendor/google.golang.org/grpc/internal/transport/transport.go:146 +0xe5
google.golang.org/grpc/internal/transport.(*recvBufferReader).Read(0xc003908280, 0xc003004290, 0x5, 0x5, 0x857340, 0xc0018aa220, 0xc0007e6900)
/src/vendor/google.golang.org/grpc/internal/transport/transport.go:140 +0x1a6
google.golang.org/grpc/internal/transport.(*transportReader).Read(0xc001c06750, 0xc003004290, 0x5, 0x5, 0x13, 0xc0007e6928, 0x876ccc)
/src/vendor/google.golang.org/grpc/internal/transport/transport.go:435 +0x55
io.ReadAtLeast(0x12aa3a0, 0xc001c06750, 0xc003004290, 0x5, 0x5, 0x5, 0x13, 0x0, 0x0)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
google.golang.org/grpc/internal/transport.(*Stream).Read(0xc001020200, 0xc003004290, 0x5, 0x5, 0x10de260, 0x7f1d46fc4aa0, 0x0)
/src/vendor/google.golang.org/grpc/internal/transport/transport.go:419 +0xc8
google.golang.org/grpc.(*parser).recvMsg(0xc003004280, 0x400000, 0x13, 0x13, 0x0, 0x0, 0x7f1d46fc4a70, 0x0)
/src/vendor/google.golang.org/grpc/rpc_util.go:508 +0x63
google.golang.org/grpc.recvAndDecompress(0xc003004280, 0xc001020200, 0x0, 0x0, 0x400000, 0x0, 0x0, 0x0, 0xc008de69e0, 0xc0007e6b68, ...)
/src/vendor/google.golang.org/grpc/rpc_util.go:639 +0x4d
google.golang.org/grpc.recv(0xc003004280, 0x7f1d46fc4880, 0x1b21718, 0xc001020200, 0x0, 0x0, 0x10de260, 0xc004ae72c0, 0x400000, 0x0, ...)
/src/vendor/google.golang.org/grpc/rpc_util.go:684 +0x9b
google.golang.org/grpc.(*serverStream).RecvMsg(0xc0012320c0, 0x10de260, 0xc004ae72c0, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/stream.go:1464 +0x14e
github.com/moby/buildkit/session/grpchijack.(*conn).Read(0xc002902400, 0xc0002ca000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/src/session/grpchijack/dial.go:69 +0x1d8
bufio.(*Reader).Read(0xc003b10480, 0xc002f663b8, 0x9, 0x9, 0xc00007ae00, 0x7f1d4fa69008, 0x0)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0x12a8d80, 0xc003b10480, 0xc002f663b8, 0x9, 0x9, 0x9, 0x832c15, 0xc008de6a2c, 0xc0007e6e38)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc002f663b8, 0x9, 0x9, 0x12a8d80, 0xc003b10480, 0x0, 0xc000000000, 0x213d6e2ca1a, 0x1afe740)
/src/vendor/golang.org/x/net/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc002f66380, 0xc008de6a20, 0xc008de6a20, 0x0, 0x0)
/src/vendor/golang.org/x/net/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc001a82000)
/src/vendor/google.golang.org/grpc/internal/transport/http2_client.go:1236 +0x168
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/src/vendor/google.golang.org/grpc/internal/transport/http2_client.go:286 +0xd15
goroutine 14778 [chan receive, 6 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc00015b180)
/src/vendor/google.golang.org/grpc/clientconn.go:1040 +0x5a1
created by google.golang.org/grpc.(*addrConn).connect
/src/vendor/google.golang.org/grpc/clientconn.go:700 +0xb6
goroutine 14773 [chan receive, 6 minutes]:
github.com/moby/buildkit/control.(*Controller).Session.func1(0xc001ad4660, 0xc002f362a0)
/src/control/control.go:356 +0x34
created by github.com/moby/buildkit/control.(*Controller).Session
/src/control/control.go:355 +0x153
goroutine 14754 [select, 6 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc002bde000)
/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:935 +0x1ed
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/src/vendor/google.golang.org/grpc/internal/transport/http2_server.go:282 +0xfdf
goroutine 20836 [select]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc00571fe60, 0x12cb6c0, 0xc000df86c0, 0xc001c06f60)
/src/util/progress/progress.go:101 +0xb7
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
/src/util/progress/progress.go:100 +0xdd
goroutine 14783 [select, 6 minutes]:
github.com/moby/buildkit/util/progress.(*MultiReader).Reader.func1(0x12cb780, 0xc001c06ff0, 0xc002be5bc0, 0xc003004840)
/src/util/progress/multireader.go:37 +0xbb
created by github.com/moby/buildkit/util/progress.(*MultiReader).Reader
/src/util/progress/multireader.go:36 +0x195
goroutine 20823 [runnable]:
sort.Sort(0x12c6680, 0xc008ae90c0)
/usr/local/go/src/sort/sort.go:216 +0x88
go.etcd.io/bbolt.(*freelist).arrayMergeSpans(0xc000110100, 0xc0021e51a0, 0x6, 0x6)
/src/vendor/go.etcd.io/bbolt/freelist.go:368 +0x5d
go.etcd.io/bbolt.(*freelist).release(0xc000110100, 0xfffffffffffffffe)
/src/vendor/go.etcd.io/bbolt/freelist.go:195 +0x20d
go.etcd.io/bbolt.(*DB).freePages(0xc000156000)
/src/vendor/go.etcd.io/bbolt/db.go:617 +0x15a
go.etcd.io/bbolt.(*DB).beginRWTx(0xc000156000, 0x0, 0x0, 0x0)
/src/vendor/go.etcd.io/bbolt/db.go:604 +0x102
go.etcd.io/bbolt.(*DB).Begin(0xc000156000, 0x1ab1d01, 0xc0047a1cb0, 0xc0047a1cb0, 0xc0051f56f8)
/src/vendor/go.etcd.io/bbolt/db.go:536 +0x38
go.etcd.io/bbolt.(*DB).Update(0xc000156000, 0xc004020ed8, 0x0, 0x0)
/src/vendor/go.etcd.io/bbolt/db.go:672 +0x3c
github.com/moby/buildkit/cache/metadata.(*Store).Clear(0xc000136050, 0xc00a01bd61, 0x19, 0x0, 0x0)
/src/cache/metadata/metadata.go:135 +0x6e
github.com/moby/buildkit/cache.(*cacheRecord).remove(0xc0063d5600, 0x12cb700, 0xc00003a098, 0xc0017e6500, 0x0, 0x0)
/src/cache/refs.go:197 +0x9d
github.com/moby/buildkit/cache.(*cacheRecord).finalize.func1(0xc006cd7800, 0xc00abb42f0)
/src/cache/refs.go:309 +0x9a
created by github.com/moby/buildkit/cache.(*cacheRecord).finalize
/src/cache/refs.go:306 +0x113
goroutine 20835 [select]:
github.com/moby/buildkit/util/progress.(*progressReader).Read.func1(0xc00571fe00, 0x12cb700, 0xc00003a098, 0xc002be55c0)
/src/util/progress/progress.go:101 +0xb7
created by github.com/moby/buildkit/util/progress.(*progressReader).Read
/src/util/progress/progress.go:100 +0xdd
goroutine 14776 [sync.Cond.Wait]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:307
sync.runtime_notifyListWait(0xc000df8b10, 0xc000000383)
/usr/local/go/src/runtime/sema.go:510 +0xf9
sync.(*Cond).Wait(0xc000df8b00)
/usr/local/go/src/sync/cond.go:56 +0x9e
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc001c06f60, 0x12cb6c0, 0xc000df86c0, 0x0, 0x0, 0x0, 0x0, 0x0)
/src/util/progress/progress.go:127 +0x10b
github.com/moby/buildkit/solver.(*Job).Status(0xc000222000, 0x12cb6c0, 0xc000df86c0, 0xc003b103c0, 0x0, 0x0)
/src/solver/progress.go:25 +0xd14
github.com/moby/buildkit/solver/llbsolver.(*Solver).Status(0xc000152230, 0x12cb6c0, 0xc000df86c0, 0xc0018aa380, 0x19, 0xc003b103c0, 0xc00090d788, 0xc00090d790)
/src/solver/llbsolver/solver.go:283 +0xbe
github.com/moby/buildkit/control.(*Controller).Status.func1(0x8, 0x1187000)
/src/control/control.go:299 +0x5e
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc001c06ba0, 0xc001c06bd0)
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:58 +0x57
created by golang.org/x/sync/errgroup.(*Group).Go
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:55 +0x66
goroutine 14728 [select, 6 minutes]:
main.unaryInterceptor.func1.1(0x12cb6c0, 0xc000f6c000, 0x12cb6c0, 0xc00049e040, 0xc001718030)
/src/cmd/buildkitd/main.go:519 +0xd8
created by main.unaryInterceptor.func1
/src/cmd/buildkitd/main.go:518 +0x108
goroutine 14784 [sync.Cond.Wait]:
runtime.goparkunlock(...)
/usr/local/go/src/runtime/proc.go:307
sync.runtime_notifyListWait(0xc000f6c2d0, 0xc00000039f)
/usr/local/go/src/runtime/sema.go:510 +0xf9
sync.(*Cond).Wait(0xc000f6c2c0)
/usr/local/go/src/sync/cond.go:56 +0x9e
github.com/moby/buildkit/util/progress.(*progressReader).Read(0xc002be55c0, 0x12cb700, 0xc00003a098, 0x0, 0x0, 0x0, 0x0, 0x0)
/src/util/progress/progress.go:127 +0x10b
github.com/moby/buildkit/util/progress.(*MultiReader).handle(0xc002be5bc0, 0xc001c06ff0, 0xc002be5bc0)
/src/util/progress/multireader.go:56 +0x118
created by github.com/moby/buildkit/util/progress.(*MultiReader).Reader
/src/util/progress/multireader.go:47 +0x1ea
goroutine 14777 [chan receive]:
github.com/moby/buildkit/control.(*Controller).Status.func2(0x8, 0x1187000)
/src/control/control.go:304 +0x782
golang.org/x/sync/errgroup.(*Group).Go.func1(0xc001c06ba0, 0xc003004520)
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:58 +0x57
created by golang.org/x/sync/errgroup.(*Group).Go
/src/vendor/golang.org/x/sync/errgroup/errgroup.go:55 +0x66
goroutine 14772 [chan receive, 6 minutes]:
github.com/moby/buildkit/session.(*Manager).handleConn(0xc0004e4660, 0x12cb6c0, 0xc000df8580, 0x12dae60, 0xc002902400, 0xc001c06690, 0x0, 0x0)
/src/session/manager.go:144 +0x491
github.com/moby/buildkit/session.(*Manager).HandleConn(0xc0004e4660, 0x12cb6c0, 0xc000df8340, 0x12dae60, 0xc002902400, 0xc001c06690, 0x0, 0x0)
/src/session/manager.go:97 +0x75
github.com/moby/buildkit/control.(*Controller).Session(0xc000172bd0, 0x12dab60, 0xc002f36280, 0x0, 0x0)
/src/control/control.go:360 +0x19a
github.com/moby/buildkit/api/services/control._Control_Session_Handler(0x109aa40, 0xc000172bd0, 0x12d3780, 0xc003004300, 0x10ecae0, 0x1b21718)
/src/api/services/control/control.pb.go:1391 +0xad
github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc.OpenTracingStreamServerInterceptor.func1(0x109aa40, 0xc000172bd0, 0x12d3c00, 0xc0012320c0, 0xc0030042a0, 0x11858b8, 0x0, 0x0)
/src/vendor/github.com/grpc-ecosystem/grpc-opentracing/go/otgrpc/server.go:114 +0x365
google.golang.org/grpc.(*Server).processStreamingRPC(0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020200, 0xc0001e9470, 0x1ab0e00, 0x0, 0x0, 0x0)
/src/vendor/google.golang.org/grpc/server.go:1183 +0x462
google.golang.org/grpc.(*Server).handleStream(0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020200, 0x0)
/src/vendor/google.golang.org/grpc/server.go:1256 +0xd3f
google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0038f0000, 0xc000084a80, 0x12dd8c0, 0xc002bde000, 0xc001020200)
/src/vendor/google.golang.org/grpc/server.go:691 +0x9f
created by google.golang.org/grpc.(*Server).serveStreams.func1
/src/vendor/google.golang.org/grpc/server.go:689 +0xa1
goroutine 14782 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x12cb6c0, 0xc000df8ac0, 0xc001c06f60)
/src/util/progress/progress.go:167 +0x48
created by github.com/moby/buildkit/util/progress.pipe
/src/util/progress/progress.go:166 +0x131
goroutine 14774 [select, 6 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc000df8540)
/src/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:115 +0x110
created by google.golang.org/grpc.newCCBalancerWrapper
/src/vendor/google.golang.org/grpc/balancer_conn_wrappers.go:106 +0x14f
goroutine 14775 [select]:
github.com/moby/buildkit/session.monitorHealth(0x12cb6c0, 0xc000df8580, 0xc00019d340, 0xc002f36400)
/src/session/grpc.go:69 +0x189
created by github.com/moby/buildkit/session.grpcClientConn
/src/session/grpc.go:55 +0x270
goroutine 14729 [chan receive, 6 minutes]:
github.com/moby/buildkit/util/progress.pipe.func1(0x12cb6c0, 0xc000f6c280, 0xc002be55c0)
/src/util/progress/progress.go:167 +0x48
created by github.com/moby/buildkit/util/progress.pipe
/src/util/progress/progress.go:166 +0x131
The text was updated successfully, but these errors were encountered: