diff --git a/pkg/cli/debug.go b/pkg/cli/debug.go index caa45090e2b2..cb88f289d9ce 100644 --- a/pkg/cli/debug.go +++ b/pkg/cli/debug.go @@ -672,21 +672,38 @@ Decode and print a hexadecimal-encoded key-value pair. var debugDecodeProtoName string var debugDecodeProtoEmitDefaults bool +var debugDecodeProtoSingleProto bool +var debugDecodeProtoBinaryOutput bool +var debugDecodeProtoOutputFile string var debugDecodeProtoCmd = &cobra.Command{ Use: "decode-proto", Short: "decode-proto --name=", Long: ` -Read from stdin and attempt to decode any hex or base64 encoded proto fields and -output them as JSON. All other fields will be outputted unchanged. Output fields -will be separated by tabs. +Read from stdin and attempt to decode any hex, base64, or C-escaped encoded +protos and output them as JSON. If --single is specified, the input is expected +to consist of a single encoded proto. Otherwise, the input can consist of +multiple fields, separated by new lines and tabs. Each field is attempted to be +decoded and, if that's unsuccessful, is echoed as is. The default value for --schema is 'cockroach.sql.sqlbase.Descriptor'. For example: -$ decode-proto < cat debug/system.decsriptor.txt +$ cat debug/system.descriptor.txt | cockroach debug decode-proto id descriptor hex_descriptor 1 \022!\012\006system\020\001\032\025\012\011\012\005admin\0200\012\010\012\004root\0200 {"database": {"id": 1, "modificationTime": {}, "name": "system", "privileges": {"users": [{"privileges": 48, "user": "admin"}, {"privileges": 48, "user": "root"}]}}} ... + +decode-proto can be used to decode protos as captured by Chrome Dev +Tools from HTTP network requests ("Copy as cURL"). Chrome captures these as +UTF8-encoded raw bytes, which are then rendered as C-escaped strings. The UTF8 +encoding breaks the proto encoding, so the curl command doesn't work as Chrome +presents it. To rectify that, take the string argument passed to "curl --data" and pass it to +"cockroach decode-proto --single --binary --out=". Then, to replay the HTTP +request, do something like: +$ curl -X POST 'http://localhost:8080/ts/query' \ + -H 'Accept: application/json' \ + -H 'Content-Type: application/x-protobuf' \ + --data-binary @ `, Args: cobra.ArbitraryArgs, RunE: runDebugDecodeProto, @@ -1798,6 +1815,12 @@ func init() { "fully qualified name of the proto to decode") f.BoolVar(&debugDecodeProtoEmitDefaults, "emit-defaults", false, "encode default values for every field") + f.BoolVar(&debugDecodeProtoSingleProto, "single", false, + "treat the input as a single field") + f.BoolVar(&debugDecodeProtoBinaryOutput, "binary", false, + "output the protos as binary instead of JSON. If specified, --out also needs to be specified.") + f.StringVar(&debugDecodeProtoOutputFile, "out", "", + "path to output file. If not specified, output goes to stdout.") f = debugCheckLogConfigCmd.Flags() f.Var(&debugLogChanSel, "only-channels", "selection of channels to include in the output diagram.") diff --git a/pkg/cli/decode.go b/pkg/cli/decode.go index f2d19fa3657b..1f2b3b05e809 100644 --- a/pkg/cli/decode.go +++ b/pkg/cli/decode.go @@ -12,29 +12,98 @@ package cli import ( "bufio" + "bytes" "encoding/base64" gohex "encoding/hex" "fmt" "io" "os" + "strconv" "strings" + "unicode/utf8" "github.com/cockroachdb/cockroach/pkg/sql/protoreflect" + "github.com/cockroachdb/cockroach/pkg/util/protoutil" "github.com/cockroachdb/errors" "github.com/mattn/go-isatty" "github.com/spf13/cobra" ) func runDebugDecodeProto(_ *cobra.Command, _ []string) error { + if debugDecodeProtoBinaryOutput && debugDecodeProtoOutputFile == "" { + return errors.Errorf("--out is required when --binary is specified. Redirecting stdout is not " + + "supported because that can introduce a trailing newline character.") + } + if debugDecodeProtoBinaryOutput && !debugDecodeProtoSingleProto { + return errors.Errorf("--single is required when --binary is specified. " + + "Outputting binary data interspersed with text fields is not supported.") + } + if isatty.IsTerminal(os.Stdin.Fd()) { fmt.Fprintln(stderr, `# Reading proto-encoded pieces of data from stdin. # Press Ctrl+C or Ctrl+D to terminate.`, ) } - return streamMap(os.Stdout, os.Stdin, + out := os.Stdout + if debugDecodeProtoOutputFile != "" { + var err error + out, err = os.OpenFile(debugDecodeProtoOutputFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, os.ModePerm) + if err != nil { + panic(err) + } + } + + if debugDecodeProtoSingleProto { + buf := bytes.NewBuffer(nil) + _, err := buf.ReadFrom(os.Stdin) + if err != nil { + return err + } + msg := tryDecodeValue(buf.String(), debugDecodeProtoName) + if msg == nil { + return errors.Errorf("decoding failed") + } + + // Output the decoded proto, either as JSON, or as binary (proto-encoded). + if debugDecodeProtoBinaryOutput { + bytes, err := protoutil.Marshal(msg) + if err != nil { + return err + } + _, err = out.Write(bytes) + if err != nil { + return err + } + } else { + j, err := protoreflect.MessageToJSON(msg, protoreflect.FmtFlags{EmitDefaults: debugDecodeProtoEmitDefaults}) + if err != nil { + // Unexpected error: the data was valid protobuf, but does not + // reflect back to JSON. We report the protobuf struct in the + // error message nonetheless. + return errors.Wrapf(err, "while JSON-encoding %#v", msg) + } + fmt.Fprint(out, j) + } + return nil + } + + // If --single was not specified, we attempt to decode individual fields. + return streamMap(out, os.Stdin, func(s string) (bool, string, error) { - return tryDecodeValue(s, debugDecodeProtoName, debugDecodeProtoEmitDefaults) + msg := tryDecodeValue(s, debugDecodeProtoName) + if msg == nil { + return false, "", nil + } + + j, err := protoreflect.MessageToJSON(msg, protoreflect.FmtFlags{EmitDefaults: debugDecodeProtoEmitDefaults}) + if err != nil { + // Unexpected error: the data was valid protobuf, but does not + // reflect back to JSON. We report the protobuf struct in the + // error message nonetheless. + return false, "", errors.Wrapf(err, "while JSON-encoding %#v", msg) + } + return true, j.String(), nil }) } @@ -63,27 +132,75 @@ func streamMap(out io.Writer, in io.Reader, fn func(string) (bool, string, error return sc.Err() } -// tryDecodeValue tries to decode the given string with the given proto name -// reports ok=false if the data was not valid proto-encoded. -func tryDecodeValue(s, protoName string, emitDefaults bool) (ok bool, val string, err error) { +// interpretString decodes s from one of a couple of supported encodings: +// - hex +// - base-64 +// - Go (or C) quoted string +func interpretString(s string) ([]byte, bool) { + // Try hex. bytes, err := gohex.DecodeString(s) - if err != nil { - b, err := base64.StdEncoding.DecodeString(s) - if err != nil { - return false, "", nil //nolint:returnerrcheck - } - bytes = b + if err == nil { + return bytes, true + } + // Try base64. + bytes, err = base64.StdEncoding.DecodeString(s) + if err == nil { + return bytes, true + } + // Try quoted string. + s = strings.TrimSpace(s) + // Remove wrapping quotes, if any. + if (strings.HasPrefix(s, "'") && strings.HasSuffix(s, "'")) || + (strings.HasPrefix(s, "\"") && strings.HasSuffix(s, "\"")) { + s = s[1 : len(s)-1] + } + // Add wrapping quotes; strconv.Unquote requires them. + s = fmt.Sprintf("\"%s\"", s) + unquoted, err := strconv.Unquote(s) + if err == nil { + return []byte(unquoted), true } + return nil, false +} + +// tryDecodeValue tries to decode the given string with the given proto name. +// Returns false if decoding fails. +func tryDecodeValue(s, protoName string) protoutil.Message { + bytes, ok := interpretString(s) + if !ok { + return nil + } + + // Try to decode the proto directly. msg, err := protoreflect.DecodeMessage(protoName, bytes) - if err != nil { - return false, "", nil //nolint:returnerrcheck + if err == nil { + return msg } - j, err := protoreflect.MessageToJSON(msg, protoreflect.FmtFlags{EmitDefaults: emitDefaults}) - if err != nil { - // Unexpected error: the data was valid protobuf, but does not - // reflect back to JSON. We report the protobuf struct in the - // error message nonetheless. - return false, "", errors.Wrapf(err, "while JSON-encoding %#v", msg) + _ = err // appease the linter + + // Try to undo UTF-8 encoding of the bytes. This compensates for how Chrome + // seems to encode the POST data through the "Export as cURL" functionality. + bytes, ok = convertFromUTF8(bytes) + if !ok { + return nil + } + msg, _ /* err */ = protoreflect.DecodeMessage(protoName, bytes) + return msg +} + +func convertFromUTF8(bytes []byte) (out []byte, ok bool) { + for len(bytes) > 0 { + // We expect only one-byte runes, which encode to one or two UTF-8 bytes. + // That's sufficient for how (I think) Chrome treats the raw bytes that it + // encodes to UTF-8: the theory is that that it goes through the raw bytes + // one by one and converts the ones above 127 into a 2-byte rune. + got, n := utf8.DecodeRune(bytes) + if got > 0xff || n > 2 { + // Unexpected multi-byte rune. + return nil, false + } + out = append(out, byte(got)) + bytes = bytes[n:] } - return true, j.String(), nil + return out, true } diff --git a/pkg/cli/decode_test.go b/pkg/cli/decode_test.go index f92c25230796..f3ddf9a3602a 100644 --- a/pkg/cli/decode_test.go +++ b/pkg/cli/decode_test.go @@ -73,7 +73,7 @@ func TestStreamMap(t *testing.T) { func TestTryDecodeValue(t *testing.T) { defer leaktest.AfterTest(t)() - protoName := "cockroach.sql.sqlbase.TableDescriptor" + const defaultProtoType = "cockroach.sql.sqlbase.TableDescriptor" marshal := func(pb protoutil.Message) []byte { s, err := protoutil.Marshal(pb) require.NoError(t, err) @@ -87,10 +87,11 @@ func TestTryDecodeValue(t *testing.T) { tableDesc := &descpb.TableDescriptor{ID: 42, ParentID: 7, Name: "foo"} tests := []struct { - name string - s string - wantOK bool - wantVal string + name string + protoType string + s string + wantOK bool + wantVal string }{ { name: "from hex", @@ -116,13 +117,35 @@ func TestTryDecodeValue(t *testing.T) { name: "base64 not proto", s: base64.StdEncoding.EncodeToString([]byte("@#$@#%$%@")), }, + { + // This is the POST data of an HTTP tsdb query taken from Chrome using + // "Copy as cURL". It is a quoted string, containing UTF-8 encoded bytes. + name: "Chrome-encoded", + s: `\u0008\u0080ì¿ùÛ\u008bù\u0083\u0017\u0010\u0080¬¢ÿ¾ôù\u0083\u0017\u001a \n\u0018cr.node.sql.select.count\u0010\u0001\u0018\u0002 \u0002\u001a \n\u0018cr.node.sql.update.count\u0010\u0001\u0018\u0002 \u0002\u001a \n\u0018cr.node.sql.insert.count\u0010\u0001\u0018\u0002 \u0002\u001a \n\u0018cr.node.sql.delete.count\u0010\u0001\u0018\u0002 \u0002\u001a*\n\u001fcr.node.sql.service.latency-p99\u0010\u0003\u0018\u0002 \u0000*\u00011\u001a3\n+cr.node.sql.distsql.contended_queries.count\u0010\u0001\u0018\u0002 \u0002\u001a\u001c\n\u0011cr.store.replicas\u0010\u0001\u0018\u0002 \u0000*\u00011\u001a\u0019\n\u0011cr.store.capacity\u0010\u0001\u0018\u0002 \u0000\u001a#\n\u001bcr.store.capacity.available\u0010\u0001\u0018\u0002 \u0000\u001a\u001e\n\u0016cr.store.capacity.used\u0010\u0001\u0018\u0002 \u0000 \u0080Ø\u008eáo`, + wantOK: true, + protoType: "cockroach.ts.tspb.TimeSeriesQueryRequest", + wantVal: `{"endNanos": "1659549679000000000", "queries": [{"derivative": "NON_NEGATIVE_DERIVATIVE", "downsampler": "AVG", "name": "cr.node.sql.select.count", "sourceAggregator": "SUM", "sources": []}, {"derivative": "NON_NEGATIVE_DERIVATIVE", "downsampler": "AVG", "name": "cr.node.sql.update.count", "sourceAggregator": "SUM", "sources": []}, {"derivative": "NON_NEGATIVE_DERIVATIVE", "downsampler": "AVG", "name": "cr.node.sql.insert.count", "sourceAggregator": "SUM", "sources": []}, {"derivative": "NON_NEGATIVE_DERIVATIVE", "downsampler": "AVG", "name": "cr.node.sql.delete.count", "sourceAggregator": "SUM", "sources": []}, {"derivative": "NONE", "downsampler": "MAX", "name": "cr.node.sql.service.latency-p99", "sourceAggregator": "SUM", "sources": ["1"]}, {"derivative": "NON_NEGATIVE_DERIVATIVE", "downsampler": "AVG", "name": "cr.node.sql.distsql.contended_queries.count", "sourceAggregator": "SUM", "sources": []}, {"derivative": "NONE", "downsampler": "AVG", "name": "cr.store.replicas", "sourceAggregator": "SUM", "sources": ["1"]}, {"derivative": "NONE", "downsampler": "AVG", "name": "cr.store.capacity", "sourceAggregator": "SUM", "sources": []}, {"derivative": "NONE", "downsampler": "AVG", "name": "cr.store.capacity.available", "sourceAggregator": "SUM", "sources": []}, {"derivative": "NONE", "downsampler": "AVG", "name": "cr.store.capacity.used", "sourceAggregator": "SUM", "sources": []}], "sampleNanos": "30000000000", "startNanos": "1659546079000000000"}`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - gotOk, gotVal, err := tryDecodeValue(tt.s, protoName, true /* emitDefaults */) - require.Equal(t, tt.wantOK, gotOk) + protoType := tt.protoType + if protoType == "" { + protoType = defaultProtoType + } + msg := tryDecodeValue(tt.s, protoType) + if !tt.wantOK { + if msg != nil { + t.Fatal("decoding succeeded unexpectedly") + } + return + } + if msg == nil { + t.Fatal("decoding failed") + } + json, err := protoreflect.MessageToJSON(msg, protoreflect.FmtFlags{EmitDefaults: true}) require.NoError(t, err) - require.Equal(t, gotVal, tt.wantVal) + require.Equal(t, tt.wantVal, json.String()) }) } }