sentry: replica_command.go:1304 string

[cockroach-teamcity]

This issue was autofiled by Sentry. It represents a crash or reported error on a live cluster with telemetry enabled.

Sentry link: https://sentry.io/organizations/cockroach-labs/issues/998288431/?project=164528&referrer=webhooks_plugin

Panic message:

replica_command.go:1304 string

Stacktrace (expand for inline code snippets):

cockroach/pkg/storage/replica_command.go

Lines 1303 to 1305 in 9a9106d

	}
	log.Fatal(ctx, buf.String())
	}

in pkg/storage.evalRangeLookup

cockroach/pkg/storage/replica_command.go

Lines 195 to 197 in 9a9106d

	}
	pd, err = cmd.Eval(ctx, batch, cArgs, reply)
	} else {

in pkg/storage.evaluateCommand

cockroach/pkg/storage/replica.go

Lines 4326 to 4328 in 9a9106d

	reply := br.Responses[index].GetInner()
	curResult, pErr := evaluateCommand(ctx, idKey, index, batch, rec, ms, ba.Header, maxKeys, args, reply)

in pkg/storage.evaluateBatch

cockroach/pkg/storage/replica.go

Lines 2054 to 2056 in 9a9106d

	rec := ReplicaEvalContext{r, spans}
	br, result, pErr = evaluateBatch(ctx, storagebase.CmdIDKey(""), r.store.Engine(), rec, nil, ba)

in pkg/storage.(*Replica).executeReadOnlyBatch

cockroach/pkg/storage/replica.go

Lines 1420 to 1422 in 9a9106d

	log.Event(ctx, "read-only path")
	br, pErr = r.executeReadOnlyBatch(ctx, ba)
	} else if ba.IsAdmin() {

in pkg/storage.(*Replica).Send

cockroach/pkg/storage/store.go

Lines 2553 to 2555 in 9a9106d

	br, pErr = repl.Send(ctx, ba)
	if pErr == nil {

in pkg/storage.(*Store).Send

cockroach/pkg/storage/stores.go

Lines 186 to 188 in 9a9106d

	}
	br, pErr := store.Send(ctx, ba)
	if br != nil && br.Error != nil {

in pkg/storage.(*Stores).Send

cockroach/pkg/server/node.go

Lines 842 to 844 in 9a9106d

	var pErr *roachpb.Error
	br, pErr = n.stores.Send(ctx, *args)
	if pErr != nil {

in pkg/server.(*Node).batchInternal.func1

cockroach/pkg/util/stop/stopper.go

Lines 271 to 273 in 9a9106d

	return f(ctx)
	}

in pkg/util/stop.(*Stopper).RunTaskWithErr

cockroach/pkg/server/node.go

Lines 853 to 855 in 9a9106d

	return nil
	}); err != nil {
	return nil, err

in pkg/server.(*Node).batchInternal

cockroach/pkg/server/node.go

Lines 870 to 872 in 9a9106d

	br, err := n.batchInternal(ctx, args)

in pkg/server.(*Node).Batch

cockroach/pkg/roachpb/api.pb.go

Lines 1877 to 1879 in 9a9106d

	if interceptor == nil {
	return srv.(InternalServer).Batch(ctx, in)
	}

in pkg/roachpb._Internal_Batch_Handler
https://github.com/cockroachdb/cockroach/blob/9a9106d91c3bbe45c4a69d38eeee888f2e5e6401/vendor/google.golang.org/grpc/server.go#L737-L739 in vendor/google.golang.org/grpc.(*Server).processUnaryRPC
https://github.com/cockroachdb/cockroach/blob/9a9106d91c3bbe45c4a69d38eeee888f2e5e6401/vendor/google.golang.org/grpc/server.go#L931-L933 in vendor/google.golang.org/grpc.(*Server).handleStream
https://github.com/cockroachdb/cockroach/blob/9a9106d91c3bbe45c4a69d38eeee888f2e5e6401/vendor/google.golang.org/grpc/server.go#L496-L498 in vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1

pkg/storage/replica_command.go in pkg/storage.evalRangeLookup at line 1304
pkg/storage/replica_command.go in pkg/storage.evaluateCommand at line 196
pkg/storage/replica.go in pkg/storage.evaluateBatch at line 4327
pkg/storage/replica.go in pkg/storage.(*Replica).executeReadOnlyBatch at line 2055
pkg/storage/replica.go in pkg/storage.(*Replica).Send at line 1421
pkg/storage/store.go in pkg/storage.(*Store).Send at line 2554
pkg/storage/stores.go in pkg/storage.(*Stores).Send at line 187
pkg/server/node.go in pkg/server.(*Node).batchInternal.func1 at line 843
pkg/util/stop/stopper.go in pkg/util/stop.(*Stopper).RunTaskWithErr at line 272
pkg/server/node.go in pkg/server.(*Node).batchInternal at line 854
pkg/server/node.go in pkg/server.(*Node).Batch at line 871
pkg/roachpb/api.pb.go in pkg/roachpb._Internal_Batch_Handler at line 1878
vendor/google.golang.org/grpc/server.go in vendor/google.golang.org/grpc.(*Server).processUnaryRPC at line 738
vendor/google.golang.org/grpc/server.go in vendor/google.golang.org/grpc.(*Server).handleStream at line 932
vendor/google.golang.org/grpc/server.go in vendor/google.golang.org/grpc.(*Server).serveStreams.func1.1 at line 497

Tag	Value
Cockroach Release	9a9106d
Cockroach SHA:	9a9106d
Platform	linux amd64
Distribution	CCL
Environment	release
Command	server
Go Version	go1.8.3
# of CPUs	8
# of Goroutines	261

[nvanbenschoten]

Duplicate of #16266 which turned into #18970 and was eventually fixed by #18970.