diff --git a/docs/generated/http/full.md b/docs/generated/http/full.md index 5635b2e32473..9a2dd4124cf6 100644 --- a/docs/generated/http/full.md +++ b/docs/generated/http/full.md @@ -649,7 +649,7 @@ Closely mirrors the upstream definitions in github.com/etcd-io/etcd/raft. #### RangeProblems - +RangeProblems describes issues reported by a range. For internal use only. | Field | Type | Label | Description | Support status | | ----- | ---- | ----- | ----------- | -------------- | @@ -669,12 +669,13 @@ Closely mirrors the upstream definitions in github.com/etcd-io/etcd/raft. #### RangeStatistics - +RangeStatistics describes statistics reported by a range. For internal use +only. | Field | Type | Label | Description | Support status | | ----- | ---- | ----- | ----------- | -------------- | -| queries_per_second | [double](#cockroach.server.serverpb.RaftDebugResponse-double) | | Note that queries per second will only be known by the leaseholder. All other replicas will report it as 0. | [reserved](#support-status) | -| writes_per_second | [double](#cockroach.server.serverpb.RaftDebugResponse-double) | | | [reserved](#support-status) | +| queries_per_second | [double](#cockroach.server.serverpb.RaftDebugResponse-double) | | Queries per second served by this range.

Note that queries per second will only be known by the leaseholder. All other replicas will report it as 0. | [reserved](#support-status) | +| writes_per_second | [double](#cockroach.server.serverpb.RaftDebugResponse-double) | | Writes per second served by this range. | [reserved](#support-status) | @@ -832,7 +833,7 @@ Closely mirrors the upstream definitions in github.com/etcd-io/etcd/raft. #### RangeProblems - +RangeProblems describes issues reported by a range. For internal use only. | Field | Type | Label | Description | Support status | | ----- | ---- | ----- | ----------- | -------------- | @@ -852,12 +853,13 @@ Closely mirrors the upstream definitions in github.com/etcd-io/etcd/raft. #### RangeStatistics - +RangeStatistics describes statistics reported by a range. For internal use +only. | Field | Type | Label | Description | Support status | | ----- | ---- | ----- | ----------- | -------------- | -| queries_per_second | [double](#cockroach.server.serverpb.RangesResponse-double) | | Note that queries per second will only be known by the leaseholder. All other replicas will report it as 0. | [reserved](#support-status) | -| writes_per_second | [double](#cockroach.server.serverpb.RangesResponse-double) | | | [reserved](#support-status) | +| queries_per_second | [double](#cockroach.server.serverpb.RangesResponse-double) | | Queries per second served by this range.

Note that queries per second will only be known by the leaseholder. All other replicas will report it as 0. | [reserved](#support-status) | +| writes_per_second | [double](#cockroach.server.serverpb.RangesResponse-double) | | Writes per second served by this range. | [reserved](#support-status) | @@ -1181,7 +1183,7 @@ ActiveQuery represents a query in flight on some Session. | start | [google.protobuf.Timestamp](#cockroach.server.serverpb.ListSessionsResponse-google.protobuf.Timestamp) | | Start timestamp of this query. | [reserved](#support-status) | | is_distributed | [bool](#cockroach.server.serverpb.ListSessionsResponse-bool) | | True if this query is distributed. | [reserved](#support-status) | | phase | [ActiveQuery.Phase](#cockroach.server.serverpb.ListSessionsResponse-cockroach.server.serverpb.ActiveQuery.Phase) | | phase stores the current phase of execution for this query. | [reserved](#support-status) | -| progress | [float](#cockroach.server.serverpb.ListSessionsResponse-float) | | | [reserved](#support-status) | +| progress | [float](#cockroach.server.serverpb.ListSessionsResponse-float) | | progress is an estimate of the fraction of this query that has been processed. | [reserved](#support-status) | | sql_anon | [string](#cockroach.server.serverpb.ListSessionsResponse-string) | | The SQL statement fingerprint, compatible with StatementStatisticsKey. | [reserved](#support-status) | @@ -1309,7 +1311,7 @@ ActiveQuery represents a query in flight on some Session. | start | [google.protobuf.Timestamp](#cockroach.server.serverpb.ListSessionsResponse-google.protobuf.Timestamp) | | Start timestamp of this query. | [reserved](#support-status) | | is_distributed | [bool](#cockroach.server.serverpb.ListSessionsResponse-bool) | | True if this query is distributed. | [reserved](#support-status) | | phase | [ActiveQuery.Phase](#cockroach.server.serverpb.ListSessionsResponse-cockroach.server.serverpb.ActiveQuery.Phase) | | phase stores the current phase of execution for this query. | [reserved](#support-status) | -| progress | [float](#cockroach.server.serverpb.ListSessionsResponse-float) | | | [reserved](#support-status) | +| progress | [float](#cockroach.server.serverpb.ListSessionsResponse-float) | | progress is an estimate of the fraction of this query that has been processed. | [reserved](#support-status) | | sql_anon | [string](#cockroach.server.serverpb.ListSessionsResponse-string) | | The SQL statement fingerprint, compatible with StatementStatisticsKey. | [reserved](#support-status) | @@ -2341,7 +2343,7 @@ Closely mirrors the upstream definitions in github.com/etcd-io/etcd/raft. #### RangeProblems - +RangeProblems describes issues reported by a range. For internal use only. | Field | Type | Label | Description | Support status | | ----- | ---- | ----- | ----------- | -------------- | @@ -2361,12 +2363,13 @@ Closely mirrors the upstream definitions in github.com/etcd-io/etcd/raft. #### RangeStatistics - +RangeStatistics describes statistics reported by a range. For internal use +only. | Field | Type | Label | Description | Support status | | ----- | ---- | ----- | ----------- | -------------- | -| queries_per_second | [double](#cockroach.server.serverpb.RangeResponse-double) | | Note that queries per second will only be known by the leaseholder. All other replicas will report it as 0. | [reserved](#support-status) | -| writes_per_second | [double](#cockroach.server.serverpb.RangeResponse-double) | | | [reserved](#support-status) | +| queries_per_second | [double](#cockroach.server.serverpb.RangeResponse-double) | | Queries per second served by this range.

Note that queries per second will only be known by the leaseholder. All other replicas will report it as 0. | [reserved](#support-status) | +| writes_per_second | [double](#cockroach.server.serverpb.RangeResponse-double) | | Writes per second served by this range. | [reserved](#support-status) | diff --git a/docs/generated/swagger/spec.json b/docs/generated/swagger/spec.json index 1fbb1e4c05fc..86a110b14403 100644 --- a/docs/generated/swagger/spec.json +++ b/docs/generated/swagger/spec.json @@ -608,6 +608,7 @@ "$ref": "#/definitions/ActiveQuery_Phase" }, "progress": { + "description": "progress is an estimate of the fraction of this query that has been\nprocessed.", "type": "number", "format": "float", "x-go-name": "Progress" @@ -936,13 +937,6 @@ "format": "int32", "x-go-package": "github.com/cockroachdb/cockroach/pkg/roachpb" }, - "NodeLivenessStatus": { - "description": "TODO(irfansharif): We should reconsider usage of NodeLivenessStatus.\nIt's unclear if the enum is well considered. It enumerates across two\ndistinct set of things: the \"membership\" status (live/active,\ndecommissioning, decommissioned), and the node \"process\" status (live,\nunavailable, available). It's possible for two of these \"states\" to be true,\nsimultaneously (consider a decommissioned, dead node). It makes for confusing\nsemantics, and the code attempting to disambiguate across these states\n(kvserver.LivenessStatus() for e.g.) seem wholly arbitrary.\n\nSee #50707 for more details.", - "type": "integer", - "format": "int32", - "title": "NodeLivenessStatus describes the status of a node from the perspective of the\nliveness system. See comment on LivenessStatus() for a description of the\nstates.", - "x-go-package": "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" - }, "PrettySpan": { "type": "object", "properties": { @@ -957,19 +951,9 @@ }, "x-go-package": "github.com/cockroachdb/cockroach/pkg/server/serverpb" }, - "RKey": { - "description": "RKey stands for \"resolved key,\" as in a key whose address has been resolved.", - "title": "RKey denotes a Key whose local addressing has been accounted for.\nA key can be transformed to an RKey by keys.Addr().", - "$ref": "#/definitions/Key" - }, - "RangeID": { - "type": "integer", - "format": "int64", - "title": "A RangeID is a unique ID associated to a Raft consensus group.", - "x-go-package": "github.com/cockroachdb/cockroach/pkg/roachpb" - }, "RangeProblems": { "type": "object", + "title": "RangeProblems describes issues reported by a range. For internal use only.", "properties": { "leader_not_lease_holder": { "type": "boolean", @@ -1009,15 +993,17 @@ "x-go-package": "github.com/cockroachdb/cockroach/pkg/server/serverpb" }, "RangeStatistics": { + "description": "RangeStatistics describes statistics reported by a range. For internal use\nonly.", "type": "object", "properties": { "queries_per_second": { - "description": "Note that queries per second will only be known by the leaseholder.\nAll other replicas will report it as 0.", + "description": "Queries per second served by this range.\n\nNote that queries per second will only be known by the leaseholder.\nAll other replicas will report it as 0.", "type": "number", "format": "double", "x-go-name": "QueriesPerSecond" }, "writes_per_second": { + "description": "Writes per second served by this range.", "type": "number", "format": "double", "x-go-name": "WritesPerSecond" @@ -1814,11 +1800,13 @@ "title": "Response struct for listNodeRanges.", "properties": { "next": { + "description": "Continuation token for the next limited run. Use in the `offset` parameter.", "type": "integer", "format": "int64", "x-go-name": "Next" }, "ranges": { + "description": "Info about retrieved ranges.", "type": "array", "items": { "$ref": "#/definitions/rangeInfo" @@ -1830,6 +1818,7 @@ }, "nodeStatus": { "type": "object", + "title": "Status about a node.", "properties": { "ServerVersion": { "$ref": "#/definitions/Version" @@ -1841,21 +1830,26 @@ "$ref": "#/definitions/Attributes" }, "build_tag": { + "description": "BuildTag is an internal build marker.", "type": "string", "x-go-name": "BuildTag" }, "cluster_name": { + "description": "ClusterName is the string name of this cluster, if set.", "type": "string", "x-go-name": "ClusterName" }, "liveness_status": { - "$ref": "#/definitions/NodeLivenessStatus" + "description": "LivenessStatus is the status of the node from the perspective of the\nliveness subsystem. For internal use only.", + "type": "integer", + "format": "int32", + "x-go-name": "LivenessStatus" }, "locality": { "$ref": "#/definitions/Locality" }, "metrics": { - "description": "Other fields that are a subset of roachpb.NodeStatus.", + "description": "Metrics contain the last sampled metrics for this node.", "type": "object", "additionalProperties": { "type": "number", @@ -1864,9 +1858,13 @@ "x-go-name": "Metrics" }, "node_id": { - "$ref": "#/definitions/NodeID" + "description": "NodeID is the integer ID of this node.", + "type": "integer", + "format": "int32", + "x-go-name": "NodeID" }, "num_cpus": { + "description": "NumCpus is the number of CPUs on this node.", "type": "integer", "format": "int32", "x-go-name": "NumCpus" @@ -1875,16 +1873,19 @@ "$ref": "#/definitions/UnresolvedAddr" }, "started_at": { + "description": "StartedAt is the time when this node was started, expressed as\nnanoseconds since Unix epoch.", "type": "integer", "format": "int64", "x-go-name": "StartedAt" }, "total_system_memory": { + "description": "TotalSystemMemory is the total amount of available system memory on this\nnode (or cgroup), in bytes.", "type": "integer", "format": "int64", "x-go-name": "TotalSystemMemory" }, "updated_at": { + "description": "UpdatedAt is the time at which the node status record was last updated,\nin nanoseconds since Unix epoch.", "type": "integer", "format": "int64", "x-go-name": "UpdatedAt" @@ -1903,6 +1904,7 @@ "x-go-name": "Next" }, "nodes": { + "description": "Status of nodes.", "type": "array", "items": { "$ref": "#/definitions/nodeStatus" @@ -1913,40 +1915,62 @@ "x-go-package": "github.com/cockroachdb/cockroach/pkg/server" }, "rangeDescriptorInfo": { - "description": "rangeDescriptorInfo contains a subset of fields from roachpb.RangeDescriptor\nthat are safe to be returned from APIs.", + "description": "rangeDescriptorInfo contains a subset of fields from the Cockroach-internal\nrange descriptor that are safe to be returned from APIs.", "type": "object", "properties": { "end_key": { - "$ref": "#/definitions/RKey" + "description": "EndKey is the resolved Cockroach-internal key that denotes the end of\nthis range.", + "type": "array", + "items": { + "type": "integer", + "format": "uint8" + }, + "x-go-name": "EndKey" }, "queries_per_second": { + "description": "QueriesPerSecond is the number of queries per second this range is\nserving. Only set for hot ranges.", "type": "number", "format": "double", "x-go-name": "QueriesPerSecond" }, "range_id": { - "$ref": "#/definitions/RangeID" + "description": "RangeID is the integer id of this range.", + "type": "integer", + "format": "int64", + "x-go-name": "RangeID" }, "start_key": { - "$ref": "#/definitions/RKey" + "description": "StartKey is the resolved Cockroach-internal key that denotes the start of\nthis range.", + "type": "array", + "items": { + "type": "integer", + "format": "uint8" + }, + "x-go-name": "StartKey" }, "store_id": { - "$ref": "#/definitions/StoreID" + "description": "StoreID is the ID of the store this hot range is on. Only set for hot\nranges.", + "type": "integer", + "format": "int32", + "x-go-name": "StoreID" } }, "x-go-package": "github.com/cockroachdb/cockroach/pkg/server" }, "rangeInfo": { "type": "object", + "title": "Info related to a range.", "properties": { "desc": { "$ref": "#/definitions/rangeDescriptorInfo" }, "error_message": { + "description": "ErrorMessage is any error retrieved from the internal range info. For\ninternal use only.", "type": "string", "x-go-name": "ErrorMessage" }, "lease_history": { + "description": "LeaseHistory is for internal use only.", "type": "array", "items": { "$ref": "#/definitions/Lease" @@ -1957,14 +1981,21 @@ "$ref": "#/definitions/RangeProblems" }, "quiescent": { + "description": "Quiescent is for internal use only.", "type": "boolean", "x-go-name": "Quiescent" }, "source_node_id": { - "$ref": "#/definitions/NodeID" + "description": "SourceNodeID is the ID of the node where this range info was retrieved\nfrom.", + "type": "integer", + "format": "int32", + "x-go-name": "SourceNodeID" }, "source_store_id": { - "$ref": "#/definitions/StoreID" + "description": "SourceStoreID is the ID of the store on the node where this range info was\nretrieved from.", + "type": "integer", + "format": "int32", + "x-go-name": "SourceStoreID" }, "span": { "$ref": "#/definitions/PrettySpan" @@ -1973,6 +2004,7 @@ "$ref": "#/definitions/RangeStatistics" }, "ticking": { + "description": "Ticking is for internal use only.", "type": "boolean", "x-go-name": "Ticking" } diff --git a/pkg/ccl/changefeedccl/BUILD.bazel b/pkg/ccl/changefeedccl/BUILD.bazel index c47a872d58ac..cad807f7df70 100644 --- a/pkg/ccl/changefeedccl/BUILD.bazel +++ b/pkg/ccl/changefeedccl/BUILD.bazel @@ -195,5 +195,6 @@ go_test( "@com_github_shopify_sarama//:sarama", "@com_github_stretchr_testify//assert", "@com_github_stretchr_testify//require", + "@org_golang_x_text//collate", ], ) diff --git a/pkg/ccl/changefeedccl/avro.go b/pkg/ccl/changefeedccl/avro.go index 65253e84ba2d..a739f91c3858 100644 --- a/pkg/ccl/changefeedccl/avro.go +++ b/pkg/ccl/changefeedccl/avro.go @@ -296,6 +296,16 @@ func typeToAvroSchema(typ *types.T, reuseMap bool) (*avroSchemaField, error) { return tree.NewDString(x.(string)), nil }, ) + case types.CollatedStringFamily: + setNullable( + avroSchemaString, + func(d tree.Datum) (interface{}, error) { + return d.(*tree.DCollatedString).Contents, nil + }, + func(x interface{}) (tree.Datum, error) { + return tree.NewDCollatedString(x.(string), typ.Locale(), &tree.CollationEnvironment{}) + }, + ) case types.BytesFamily: setNullable( avroSchemaBytes, diff --git a/pkg/ccl/changefeedccl/avro_test.go b/pkg/ccl/changefeedccl/avro_test.go index 6b15c25b8ceb..4dcb9a99cab7 100644 --- a/pkg/ccl/changefeedccl/avro_test.go +++ b/pkg/ccl/changefeedccl/avro_test.go @@ -40,6 +40,7 @@ import ( "github.com/cockroachdb/cockroach/pkg/util/timeutil/pgdate" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" + "golang.org/x/text/collate" ) func parseTableDesc(createTableStmt string) (catalog.TableDescriptor, error) { @@ -247,8 +248,7 @@ func TestAvroSchema(t *testing.T) { case types.AnyFamily, types.OidFamily, types.TupleFamily: // These aren't expected to be needed for changefeeds. return true - case types.IntervalFamily, types.BitFamily, - types.CollatedStringFamily: + case types.IntervalFamily, types.BitFamily: // Implement these as customer demand dictates. return true case types.ArrayFamily: @@ -262,11 +262,24 @@ func TestAvroSchema(t *testing.T) { return !randgen.IsLegalColumnType(typ) } - // Generate a test for each column type with a random datum of that type. + typesToTest := make([]*types.T, 0, 256) + for _, typ := range types.OidToType { if skipType(typ) { continue } + typesToTest = append(typesToTest, typ) + switch typ.Family() { + case types.StringFamily: + collationTags := collate.Supported() + randCollationTag := collationTags[rand.Intn(len(collationTags))] + collatedType := types.MakeCollatedString(typ, randCollationTag.String()) + typesToTest = append(typesToTest, collatedType) + } + } + + // Generate a test for each column type with a random datum of that type. + for _, typ := range typesToTest { var datum tree.Datum datum, typ = overrideRandGen(typ) if datum == nil { @@ -352,27 +365,28 @@ func TestAvroSchema(t *testing.T) { // reference. t.Run("type_goldens", func(t *testing.T) { goldens := map[string]string{ - `BOOL`: `["null","boolean"]`, - `BOOL[]`: `["null",{"type":"array","items":["null","boolean"]}]`, - `BOX2D`: `["null","string"]`, - `BYTES`: `["null","bytes"]`, - `DATE`: `["null",{"type":"int","logicalType":"date"}]`, - `FLOAT8`: `["null","double"]`, - `GEOGRAPHY`: `["null","bytes"]`, - `GEOMETRY`: `["null","bytes"]`, - `INET`: `["null","string"]`, - `INT8`: `["null","long"]`, - `JSONB`: `["null","string"]`, - `STRING`: `["null","string"]`, - `TIME`: `["null",{"type":"long","logicalType":"time-micros"}]`, - `TIMETZ`: `["null","string"]`, - `TIMESTAMP`: `["null",{"type":"long","logicalType":"timestamp-micros"}]`, - `TIMESTAMPTZ`: `["null",{"type":"long","logicalType":"timestamp-micros"}]`, - `UUID`: `["null","string"]`, - `DECIMAL(3,2)`: `["null",{"type":"bytes","logicalType":"decimal","precision":3,"scale":2}]`, + `BOOL`: `["null","boolean"]`, + `BOOL[]`: `["null",{"type":"array","items":["null","boolean"]}]`, + `BOX2D`: `["null","string"]`, + `BYTES`: `["null","bytes"]`, + `DATE`: `["null",{"type":"int","logicalType":"date"}]`, + `FLOAT8`: `["null","double"]`, + `GEOGRAPHY`: `["null","bytes"]`, + `GEOMETRY`: `["null","bytes"]`, + `INET`: `["null","string"]`, + `INT8`: `["null","long"]`, + `JSONB`: `["null","string"]`, + `STRING`: `["null","string"]`, + `STRING COLLATE fr`: `["null","string"]`, + `TIME`: `["null",{"type":"long","logicalType":"time-micros"}]`, + `TIMETZ`: `["null","string"]`, + `TIMESTAMP`: `["null",{"type":"long","logicalType":"timestamp-micros"}]`, + `TIMESTAMPTZ`: `["null",{"type":"long","logicalType":"timestamp-micros"}]`, + `UUID`: `["null","string"]`, + `DECIMAL(3,2)`: `["null",{"type":"bytes","logicalType":"decimal","precision":3,"scale":2}]`, } - for _, typ := range append(types.Scalar, types.BoolArray) { + for _, typ := range append(types.Scalar, types.BoolArray, types.MakeCollatedString(types.String, `fr`)) { switch typ.Family() { case types.IntervalFamily, types.OidFamily, types.BitFamily: continue @@ -499,6 +513,9 @@ func TestAvroSchema(t *testing.T) { {sqlType: `BOOL[]`, sql: `'{true, true, false, null}'`, avro: `{"array":[{"boolean":true},{"boolean":true},{"boolean":false},null]}`}, + {sqlType: `VARCHAR COLLATE "fr"`, + sql: `'Bonjour' COLLATE "fr"`, + avro: `{"string":"Bonjour"}`}, } for _, test := range goldens { @@ -803,6 +820,12 @@ func BenchmarkEncodeString(b *testing.B) { benchmarkEncodeType(b, types.String, randEncDatumRow(types.String)) } +var collatedStringType *types.T = types.MakeCollatedString(types.String, `fr`) + +func BenchmarkEncodeCollatedString(b *testing.B) { + benchmarkEncodeType(b, collatedStringType, randEncDatumRow(collatedStringType)) +} + func BenchmarkEncodeDate(b *testing.B) { // RandDatum could return "interesting" dates (infinite past, etc). Alas, avro // doesn't support those yet, so override it to something we do support. diff --git a/pkg/ccl/changefeedccl/encoder_test.go b/pkg/ccl/changefeedccl/encoder_test.go index b7dd0c980d37..a0a251e06c97 100644 --- a/pkg/ccl/changefeedccl/encoder_test.go +++ b/pkg/ccl/changefeedccl/encoder_test.go @@ -347,6 +347,31 @@ func TestAvroArray(t *testing.T) { t.Run(`enterprise`, enterpriseTest(testFn)) } +func TestAvroCollatedString(t *testing.T) { + defer leaktest.AfterTest(t)() + defer log.Scope(t).Close(t) + + testFn := func(t *testing.T, db *gosql.DB, f cdctest.TestFeedFactory) { + reg := cdctest.MakeTestSchemaRegistry() + defer reg.Close() + + sqlDB := sqlutils.MakeSQLRunner(db) + sqlDB.Exec(t, `CREATE TABLE foo (a INT PRIMARY KEY, b string collate "fr-CA")`) + sqlDB.Exec(t, `INSERT INTO foo VALUES (1, 'désolée' collate "fr-CA")`) + + foo := feed(t, f, `CREATE CHANGEFEED FOR foo `+ + `WITH format=$1, confluent_schema_registry=$2`, + changefeedbase.OptFormatAvro, reg.URL()) + defer closeFeed(t, foo) + assertPayloadsAvro(t, reg, foo, []string{ + `foo: {"a":{"long":1}}->{"after":{"foo":{"a":{"long":1},"b":{"string":"désolée"}}}}`, + }) + } + + t.Run(`sinkless`, sinklessTest(testFn)) + t.Run(`enterprise`, enterpriseTest(testFn)) +} + func TestAvroSchemaNaming(t *testing.T) { defer leaktest.AfterTest(t)() defer log.Scope(t).Close(t) diff --git a/pkg/cmd/docgen/diagrams.go b/pkg/cmd/docgen/diagrams.go index 307d922eeca1..d14e247a9fd3 100644 --- a/pkg/cmd/docgen/diagrams.go +++ b/pkg/cmd/docgen/diagrams.go @@ -22,6 +22,7 @@ import ( "sort" "strings" "sync" + "time" "github.com/cockroachdb/cockroach/pkg/cmd/docgen/extract" "github.com/cockroachdb/cockroach/pkg/util/envutil" @@ -49,7 +50,8 @@ func init() { // BNF vars. var ( - addr string + addr string + bnfAPITimeout time.Duration ) cmdBNF := &cobra.Command{ @@ -58,7 +60,7 @@ func init() { Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { bnfDir := args[0] - bnf, err := runBNF(addr) + bnf, err := runBNF(addr, bnfAPITimeout) if err != nil { log.Fatal(err) } @@ -129,11 +131,14 @@ func init() { } cmdBNF.Flags().StringVar(&addr, "addr", "./pkg/sql/parser/sql.y", "Location of sql.y file. Can also specify an http address.") + cmdBNF.Flags().DurationVar(&bnfAPITimeout, "timeout", time.Second*120, "Timeout in seconds for bnf HTTP Api, "+ + "only relevant when the web api is used; default 120s.") // SVG vars. var ( - maxWorkers int - railroadJar string + maxWorkers int + railroadJar string + railroadAPITimeout time.Duration ) cmdSVG := &cobra.Command{ @@ -195,7 +200,7 @@ func init() { } defer f.Close() - rr, err := runRR(f, railroadJar) + rr, err := runRR(f, railroadJar, railroadAPITimeout) if err != nil { log.Fatalf("%s: %s\n", m, err) } @@ -246,6 +251,8 @@ func init() { cmdSVG.Flags().IntVar(&maxWorkers, "max-workers", 1, "maximum number of concurrent workers") cmdSVG.Flags().StringVar(&railroadJar, "railroad", "", "Location of Railroad.jar; empty to use website") + cmdSVG.Flags().DurationVar(&railroadAPITimeout, "timeout", time.Second*120, "Timeout in seconds for railroad HTTP Api, "+ + "only relevant when the web api is used; default 120s.") diagramCmd := &cobra.Command{ Use: "grammar", @@ -274,8 +281,8 @@ type stmtSpec struct { nosplit bool } -func runBNF(addr string) ([]byte, error) { - return extract.GenerateBNF(addr) +func runBNF(addr string, bnfAPITimeout time.Duration) ([]byte, error) { + return extract.GenerateBNF(addr, bnfAPITimeout) } func runParse( @@ -298,14 +305,14 @@ func runParse( return b, err } -func runRR(r io.Reader, railroadJar string) ([]byte, error) { +func runRR(r io.Reader, railroadJar string, railroadAPITimeout time.Duration) ([]byte, error) { b, err := ioutil.ReadAll(r) if err != nil { return nil, err } var html []byte if railroadJar == "" { - html, err = extract.GenerateRRNet(b) + html, err = extract.GenerateRRNet(b, railroadAPITimeout) } else { html, err = extract.GenerateRRJar(railroadJar, b) } diff --git a/pkg/cmd/docgen/extract/extract.go b/pkg/cmd/docgen/extract/extract.go index 791801120abb..dc8783dd8750 100644 --- a/pkg/cmd/docgen/extract/extract.go +++ b/pkg/cmd/docgen/extract/extract.go @@ -21,6 +21,7 @@ import ( "os/exec" "regexp" "strings" + "time" "unicode" "github.com/cockroachdb/cockroach/pkg/internal/rsg/yacc" @@ -63,7 +64,7 @@ func GenerateRRJar(jar string, bnf []byte) ([]byte, error) { } // GenerateRRNet generates the RR XHTML from a EBNF file. -func GenerateRRNet(bnf []byte) ([]byte, error) { +func GenerateRRNet(bnf []byte, railroadAPITimeout time.Duration) ([]byte, error) { rrLock.Lock() defer rrLock.Unlock() @@ -77,7 +78,8 @@ func GenerateRRNet(bnf []byte) ([]byte, error) { v.Add("options", "factoring") v.Add("options", "inline") - resp, err := httputil.Post(context.TODO(), rrAddr, "application/x-www-form-urlencoded", strings.NewReader(v.Encode())) + httpClient := httputil.NewClientWithTimeout(railroadAPITimeout) + resp, err := httpClient.Post(context.TODO(), rrAddr, "application/x-www-form-urlencoded", strings.NewReader(v.Encode())) if err != nil { return nil, err } @@ -95,10 +97,11 @@ func GenerateRRNet(bnf []byte) ([]byte, error) { // GenerateBNF Opens or downloads the .y file at addr and returns at as an EBNF // file. Unimplemented branches are removed. Resulting empty nodes and their // uses are further removed. Empty nodes are elided. -func GenerateBNF(addr string) (ebnf []byte, err error) { +func GenerateBNF(addr string, bnfAPITimeout time.Duration) (ebnf []byte, err error) { var b []byte if strings.HasPrefix(addr, "http") { - resp, err := httputil.Get(context.TODO(), addr) + httpClient := httputil.NewClientWithTimeout(bnfAPITimeout) + resp, err := httpClient.Get(context.TODO(), addr) if err != nil { return nil, err } diff --git a/pkg/server/api_v2_ranges.go b/pkg/server/api_v2_ranges.go index 8256b64a1db7..9794efe0f846 100644 --- a/pkg/server/api_v2_ranges.go +++ b/pkg/server/api_v2_ranges.go @@ -18,39 +18,54 @@ import ( "strconv" "strings" - "github.com/cockroachdb/cockroach/pkg/kv/kvserver/liveness/livenesspb" "github.com/cockroachdb/cockroach/pkg/roachpb" "github.com/cockroachdb/cockroach/pkg/server/serverpb" "github.com/cockroachdb/cockroach/pkg/util" "github.com/gorilla/mux" ) +// Status about a node. type nodeStatus struct { - // Fields that are a subset of NodeDescriptor. - NodeID roachpb.NodeID `json:"node_id"` - Address util.UnresolvedAddr `json:"address"` - Attrs roachpb.Attributes `json:"attrs"` - Locality roachpb.Locality `json:"locality"` - ServerVersion roachpb.Version `json:"ServerVersion"` - BuildTag string `json:"build_tag"` - StartedAt int64 `json:"started_at"` - ClusterName string `json:"cluster_name"` - SQLAddress util.UnresolvedAddr `json:"sql_address"` - - // Other fields that are a subset of roachpb.NodeStatus. - Metrics map[string]float64 `json:"metrics,omitempty"` - TotalSystemMemory int64 `json:"total_system_memory,omitempty"` - NumCpus int32 `json:"num_cpus,omitempty"` - UpdatedAt int64 `json:"updated_at,omitempty"` - - // Retrieved from the liveness status map. - LivenessStatus livenesspb.NodeLivenessStatus `json:"liveness_status"` + // NodeID is the integer ID of this node. + NodeID int32 `json:"node_id"` + // Address is the unresolved network listen address of this node. + Address util.UnresolvedAddr `json:"address"` + Attrs roachpb.Attributes `json:"attrs"` + Locality roachpb.Locality `json:"locality"` + // ServerVersion is the exact version of Cockroach this node is running. + ServerVersion roachpb.Version `json:"ServerVersion"` + // BuildTag is an internal build marker. + BuildTag string `json:"build_tag"` + // StartedAt is the time when this node was started, expressed as + // nanoseconds since Unix epoch. + StartedAt int64 `json:"started_at"` + // ClusterName is the string name of this cluster, if set. + ClusterName string `json:"cluster_name"` + // SQLAddress is the listen address to which SQL clients can connect. + SQLAddress util.UnresolvedAddr `json:"sql_address"` + + // Metrics contain the last sampled metrics for this node. + Metrics map[string]float64 `json:"metrics,omitempty"` + // TotalSystemMemory is the total amount of available system memory on this + // node (or cgroup), in bytes. + TotalSystemMemory int64 `json:"total_system_memory,omitempty"` + // NumCpus is the number of CPUs on this node. + NumCpus int32 `json:"num_cpus,omitempty"` + // UpdatedAt is the time at which the node status record was last updated, + // in nanoseconds since Unix epoch. + UpdatedAt int64 `json:"updated_at,omitempty"` + + // LivenessStatus is the status of the node from the perspective of the + // liveness subsystem. For internal use only. + LivenessStatus int32 `json:"liveness_status"` } // Response struct for listNodes. // // swagger:model nodesResponse type nodesResponse struct { + // Status of nodes. + // // swagger:allOf Nodes []nodeStatus `json:"nodes"` // Continuation offset for the next paginated call, if more values are present. @@ -101,7 +116,7 @@ func (a *apiV2Server) listNodes(w http.ResponseWriter, r *http.Request) { resp.Next = next for _, n := range nodes.Nodes { resp.Nodes = append(resp.Nodes, nodeStatus{ - NodeID: n.Desc.NodeID, + NodeID: int32(n.Desc.NodeID), Address: n.Desc.Address, Attrs: n.Desc.Attrs, Locality: n.Desc.Locality, @@ -114,7 +129,7 @@ func (a *apiV2Server) listNodes(w http.ResponseWriter, r *http.Request) { TotalSystemMemory: n.TotalSystemMemory, NumCpus: n.NumCpus, UpdatedAt: n.UpdatedAt, - LivenessStatus: nodes.LivenessByNodeID[n.Desc.NodeID], + LivenessStatus: int32(nodes.LivenessByNodeID[n.Desc.NodeID]), }) } writeJSONResponse(ctx, w, 200, resp) @@ -223,16 +238,24 @@ func (a *apiV2Server) listRange(w http.ResponseWriter, r *http.Request) { writeJSONResponse(ctx, w, 200, response) } -// rangeDescriptorInfo contains a subset of fields from roachpb.RangeDescriptor -// that are safe to be returned from APIs. +// rangeDescriptorInfo contains a subset of fields from the Cockroach-internal +// range descriptor that are safe to be returned from APIs. type rangeDescriptorInfo struct { - RangeID roachpb.RangeID `json:"range_id"` - StartKey roachpb.RKey `json:"start_key,omitempty"` - EndKey roachpb.RKey `json:"end_key,omitempty"` - - // Set for HotRanges. - StoreID roachpb.StoreID `json:"store_id"` - QueriesPerSecond float64 `json:"queries_per_second"` + // RangeID is the integer id of this range. + RangeID int64 `json:"range_id"` + // StartKey is the resolved Cockroach-internal key that denotes the start of + // this range. + StartKey []byte `json:"start_key,omitempty"` + // EndKey is the resolved Cockroach-internal key that denotes the end of + // this range. + EndKey []byte `json:"end_key,omitempty"` + + // StoreID is the ID of the store this hot range is on. Only set for hot + // ranges. + StoreID int32 `json:"store_id"` + // QueriesPerSecond is the number of queries per second this range is + // serving. Only set for hot ranges. + QueriesPerSecond float64 `json:"queries_per_second"` } func (r *rangeDescriptorInfo) init(rd *roachpb.RangeDescriptor) { @@ -241,33 +264,46 @@ func (r *rangeDescriptorInfo) init(rd *roachpb.RangeDescriptor) { return } *r = rangeDescriptorInfo{ - RangeID: rd.RangeID, + RangeID: int64(rd.RangeID), StartKey: rd.StartKey, EndKey: rd.EndKey, } } +// Info related to a range. type rangeInfo struct { // swagger:allOf Desc rangeDescriptorInfo `json:"desc"` - // Subset of fields copied from serverpb.RangeInfo - Span serverpb.PrettySpan `json:"span"` - SourceNodeID roachpb.NodeID `json:"source_node_id,omitempty"` - SourceStoreID roachpb.StoreID `json:"source_store_id,omitempty"` - ErrorMessage string `json:"error_message,omitempty"` - LeaseHistory []roachpb.Lease `json:"lease_history"` - Problems serverpb.RangeProblems `json:"problems"` - Stats serverpb.RangeStatistics `json:"stats"` - Quiescent bool `json:"quiescent,omitempty"` - Ticking bool `json:"ticking,omitempty"` + // Span is the pretty-ified start/end key span for this range. + Span serverpb.PrettySpan `json:"span"` + // SourceNodeID is the ID of the node where this range info was retrieved + // from. + SourceNodeID int32 `json:"source_node_id,omitempty"` + // SourceStoreID is the ID of the store on the node where this range info was + // retrieved from. + SourceStoreID int32 `json:"source_store_id,omitempty"` + // ErrorMessage is any error retrieved from the internal range info. For + // internal use only. + ErrorMessage string `json:"error_message,omitempty"` + // LeaseHistory is for internal use only. + LeaseHistory []roachpb.Lease `json:"lease_history"` + // Problems is a map of any issues reported by this range. For internal use + // only. + Problems serverpb.RangeProblems `json:"problems"` + // Stats is for internal use only. + Stats serverpb.RangeStatistics `json:"stats"` + // Quiescent is for internal use only. + Quiescent bool `json:"quiescent,omitempty"` + // Ticking is for internal use only. + Ticking bool `json:"ticking,omitempty"` } func (ri *rangeInfo) init(r serverpb.RangeInfo) { *ri = rangeInfo{ Span: r.Span, - SourceNodeID: r.SourceNodeID, - SourceStoreID: r.SourceStoreID, + SourceNodeID: int32(r.SourceNodeID), + SourceStoreID: int32(r.SourceStoreID), ErrorMessage: r.ErrorMessage, LeaseHistory: r.LeaseHistory, Problems: r.Problems, @@ -282,8 +318,10 @@ func (ri *rangeInfo) init(r serverpb.RangeInfo) { // // swagger:model nodeRangesResponse type nodeRangesResponse struct { + // Info about retrieved ranges. Ranges []rangeInfo `json:"ranges"` - Next int `json:"next,omitempty"` + // Continuation token for the next limited run. Use in the `offset` parameter. + Next int `json:"next,omitempty"` } // swagger:operation GET /nodes/{node_id}/ranges/ listNodeRanges @@ -455,7 +493,7 @@ func (a *apiV2Server) listHotRanges(w http.ResponseWriter, r *http.Request) { for _, hotRange := range store.HotRanges { var r rangeDescriptorInfo r.init(&hotRange.Desc) - r.StoreID = store.StoreID + r.StoreID = int32(store.StoreID) r.QueriesPerSecond = hotRange.QueriesPerSecond rangeDescriptorInfos = append(rangeDescriptorInfos, r) } diff --git a/pkg/server/api_v2_ranges_test.go b/pkg/server/api_v2_ranges_test.go index d1853f0ea5da..bfffae1c4d4f 100644 --- a/pkg/server/api_v2_ranges_test.go +++ b/pkg/server/api_v2_ranges_test.go @@ -99,8 +99,8 @@ func TestNodeRangesV2(t *testing.T) { t.Errorf("didn't get any ranges") } for _, ri := range nodeRangesResp.Ranges { - require.Equal(t, roachpb.NodeID(1), ri.SourceNodeID) - require.Equal(t, roachpb.StoreID(1), ri.SourceStoreID) + require.Equal(t, int32(1), ri.SourceNodeID) + require.Equal(t, int32(1), ri.SourceStoreID) require.GreaterOrEqual(t, len(ri.LeaseHistory), 1) require.NotEmpty(t, ri.Span.StartKey) require.NotEmpty(t, ri.Span.EndKey) diff --git a/pkg/server/serverpb/status.pb.go b/pkg/server/serverpb/status.pb.go index 88d4367167e1..cdadbde49887 100644 --- a/pkg/server/serverpb/status.pb.go +++ b/pkg/server/serverpb/status.pb.go @@ -644,6 +644,7 @@ func (m *RaftState_Progress) XXX_DiscardUnknown() { var xxx_messageInfo_RaftState_Progress proto.InternalMessageInfo +// RangeProblems describes issues reported by a range. For internal use only. type RangeProblems struct { Unavailable bool `protobuf:"varint,1,opt,name=unavailable,proto3" json:"unavailable,omitempty"` LeaderNotLeaseHolder bool `protobuf:"varint,2,opt,name=leader_not_lease_holder,json=leaderNotLeaseHolder,proto3" json:"leader_not_lease_holder,omitempty"` @@ -689,11 +690,16 @@ func (m *RangeProblems) XXX_DiscardUnknown() { var xxx_messageInfo_RangeProblems proto.InternalMessageInfo +// RangeStatistics describes statistics reported by a range. For internal use +// only. type RangeStatistics struct { + // Queries per second served by this range. + // // Note that queries per second will only be known by the leaseholder. // All other replicas will report it as 0. QueriesPerSecond float64 `protobuf:"fixed64,1,opt,name=queries_per_second,json=queriesPerSecond,proto3" json:"queries_per_second,omitempty"` - WritesPerSecond float64 `protobuf:"fixed64,2,opt,name=writes_per_second,json=writesPerSecond,proto3" json:"writes_per_second,omitempty"` + // Writes per second served by this range. + WritesPerSecond float64 `protobuf:"fixed64,2,opt,name=writes_per_second,json=writesPerSecond,proto3" json:"writes_per_second,omitempty"` } func (m *RangeStatistics) Reset() { *m = RangeStatistics{} } @@ -1932,8 +1938,10 @@ type ActiveQuery struct { // True if this query is distributed. IsDistributed bool `protobuf:"varint,4,opt,name=is_distributed,json=isDistributed,proto3" json:"is_distributed,omitempty"` // phase stores the current phase of execution for this query. - Phase ActiveQuery_Phase `protobuf:"varint,5,opt,name=phase,proto3,enum=cockroach.server.serverpb.ActiveQuery_Phase" json:"phase,omitempty"` - Progress float32 `protobuf:"fixed32,6,opt,name=progress,proto3" json:"progress,omitempty"` + Phase ActiveQuery_Phase `protobuf:"varint,5,opt,name=phase,proto3,enum=cockroach.server.serverpb.ActiveQuery_Phase" json:"phase,omitempty"` + // progress is an estimate of the fraction of this query that has been + // processed. + Progress float32 `protobuf:"fixed32,6,opt,name=progress,proto3" json:"progress,omitempty"` // The SQL statement fingerprint, compatible with StatementStatisticsKey. SqlAnon string `protobuf:"bytes,8,opt,name=sql_anon,json=sqlAnon,proto3" json:"sql_anon,omitempty"` } diff --git a/pkg/server/serverpb/status.proto b/pkg/server/serverpb/status.proto index 042a12a7b91e..ca7110d22523 100644 --- a/pkg/server/serverpb/status.proto +++ b/pkg/server/serverpb/status.proto @@ -159,6 +159,7 @@ message RaftState { uint64 lead_transferee = 7; } +// RangeProblems describes issues reported by a range. For internal use only. message RangeProblems { bool unavailable = 1; bool leader_not_lease_holder = 2; @@ -177,10 +178,15 @@ message RangeProblems { bool raft_log_too_large = 7; } +// RangeStatistics describes statistics reported by a range. For internal use +// only. message RangeStatistics { + // Queries per second served by this range. + // // Note that queries per second will only be known by the leaseholder. // All other replicas will report it as 0. double queries_per_second = 1; + // Writes per second served by this range. double writes_per_second = 2; } @@ -530,6 +536,8 @@ message ActiveQuery { // phase stores the current phase of execution for this query. Phase phase = 5; + // progress is an estimate of the fraction of this query that has been + // processed. float progress = 6; // The SQL statement fingerprint, compatible with StatementStatisticsKey.