From cd5d410a252cc7e4683ad2aa0e7ffe2263539e37 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Thu, 28 Dec 2023 00:20:31 -0500 Subject: [PATCH 1/3] Add grpc-gateway helper types (#102) Signed-off-by: Yuri Shkuro --- Makefile | 2 +- proto/api_v3/query_service.proto | 27 +++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 459f427..3624dcf 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ PROTOTOOL_VER=1.8.0 PROTOTOOL_IMAGE=uber/prototool:$(PROTOTOOL_VER) PROTOTOOL=docker run --rm -u ${shell id -u} -v "${PWD}:/go/src/${PROJECT_ROOT}" -w /go/src/${PROJECT_ROOT} $(PROTOTOOL_IMAGE) -PROTOC_VER=0.4.0 +PROTOC_VER=0.5.0 PROTOC_IMAGE=jaegertracing/protobuf:$(PROTOC_VER) PROTOC=docker run --rm -u ${shell id -u} -v "${PWD}:${PWD}" -w ${PWD} ${PROTOC_IMAGE} --proto_path=${PWD} diff --git a/proto/api_v3/query_service.proto b/proto/api_v3/query_service.proto index b809e94..8a3b0b4 100644 --- a/proto/api_v3/query_service.proto +++ b/proto/api_v3/query_service.proto @@ -130,3 +130,30 @@ service QueryService { // GetOperations returns operation names. rpc GetOperations(GetOperationsRequest) returns (GetOperationsResponse) {} } + +// Below are some helper types when using APIv3 via HTTP endpoints implemented via grpc-gateway. + +// GRPCGatewayError is the type returned when GRPC server returns an error. +// Note that for streaming responses it would be wrapped in GRPCGatewayWrapper below. +// Example: {"error":{"grpcCode":2,"httpCode":500,"message":"...","httpStatus":"text..."}}. +message GRPCGatewayError { + message GRPCGatewayErrorDetails { + int32 grpcCode = 1; + int32 httpCode = 2; + string message = 3; + string httpStatus = 4; + } + + GRPCGatewayErrorDetails error = 1; +} + +// GRPCGatewayWrapper is a type returned when GRPC service returns a stream. +// For some unknown reason grpc-gateway/v1 wraps chunk responses in {"result": {actual output}}. +// See https://github.com/grpc-ecosystem/grpc-gateway/issues/2189 +// TODO: it's not clear what happens when the server returns more than one chunk. +// The gateway will presumably combine then into a single HTTP response. +// Currently this is not possible because even though APIv3 GRPC Service is using output stream, +// its implementation reads all spans from QueryService at once and forms only a single chunk. +message GRPCGatewayWrapper { + SpansResponseChunk result = 1; +} From a97152c66d8f1562fcdcbbb85600d5f12c8aab9e Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Sun, 14 Jan 2024 14:11:06 -0500 Subject: [PATCH 2/3] Replace api_v3.SpansResponseChunk with opentelemetry.proto.trace.v1.TracesData (#103) See https://github.com/jaegertracing/jaeger/pull/5098 --- proto/api_v3/query_service.proto | 36 +++++++++++++------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/proto/api_v3/query_service.proto b/proto/api_v3/query_service.proto index 8a3b0b4..61feed0 100644 --- a/proto/api_v3/query_service.proto +++ b/proto/api_v3/query_service.proto @@ -38,17 +38,6 @@ message GetTraceRequest { ]; } -// Response object with spans. -message SpansResponseChunk { - // A list of OpenTelemetry ResourceSpans. - // In case of JSON format the ids (trace_id, span_id, parent_id) are encoded in base64 even though OpenTelemetry specification - // mandates to use hex encoding [2]. - // Base64 is chosen to keep compatibility with JSONPb codec. - // [1]: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto - // [2]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlphttp - repeated opentelemetry.proto.trace.v1.ResourceSpans resource_spans = 1; -} - // Query parameters to find traces. Except for num_traces, all fields should be treated // as forming a conjunction, e.g., "service_name='X' AND operation_name='Y' AND ...". // All fields are matched against individual spans, not at the trace level. @@ -118,11 +107,11 @@ service QueryService { // It means that the JSON response cannot be directly unmarshalled using JSONPb. // This can be fixed by first parsing into user-defined envelope with standard JSON library // or string manipulation to remove the envelope. Alternatively generate objects using OpenAPI. - rpc GetTrace(GetTraceRequest) returns (stream SpansResponseChunk) {} + rpc GetTrace(GetTraceRequest) returns (stream opentelemetry.proto.trace.v1.TracesData) {} // FindTraces searches for traces. // See GetTrace for JSON unmarshalling. - rpc FindTraces(FindTracesRequest) returns (stream SpansResponseChunk) {} + rpc FindTraces(FindTracesRequest) returns (stream opentelemetry.proto.trace.v1.TracesData) {} // GetServices returns service names. rpc GetServices(GetServicesRequest) returns (GetServicesResponse) {} @@ -131,10 +120,9 @@ service QueryService { rpc GetOperations(GetOperationsRequest) returns (GetOperationsResponse) {} } -// Below are some helper types when using APIv3 via HTTP endpoints implemented via grpc-gateway. +// Below are some helper types when using APIv3 via HTTP endpoints. // GRPCGatewayError is the type returned when GRPC server returns an error. -// Note that for streaming responses it would be wrapped in GRPCGatewayWrapper below. // Example: {"error":{"grpcCode":2,"httpCode":500,"message":"...","httpStatus":"text..."}}. message GRPCGatewayError { message GRPCGatewayErrorDetails { @@ -147,13 +135,17 @@ message GRPCGatewayError { GRPCGatewayErrorDetails error = 1; } -// GRPCGatewayWrapper is a type returned when GRPC service returns a stream. -// For some unknown reason grpc-gateway/v1 wraps chunk responses in {"result": {actual output}}. +// GRPCGatewayWrapper wraps streaming responses from GetTrace/FindTraces for HTTP. +// Today there is always only one response because internally the HTTP server gets +// data from QueryService that does not support multiple responses. But in the +// future the server may return multiple responeses using Transfer-Encoding: chunked. +// In case of errors, GRPCGatewayError above is used. +// +// Example: +// {"result": {"resourceSpans": ...}} +// // See https://github.com/grpc-ecosystem/grpc-gateway/issues/2189 -// TODO: it's not clear what happens when the server returns more than one chunk. -// The gateway will presumably combine then into a single HTTP response. -// Currently this is not possible because even though APIv3 GRPC Service is using output stream, -// its implementation reads all spans from QueryService at once and forms only a single chunk. +// message GRPCGatewayWrapper { - SpansResponseChunk result = 1; + opentelemetry.proto.trace.v1.TracesData result = 1; } From bdd5ab5514b3dbb8d3924cf6bc1b28e356df6ae3 Mon Sep 17 00:00:00 2001 From: Yuri Shkuro Date: Sun, 14 Jan 2024 14:30:09 -0500 Subject: [PATCH 3/3] Upgrade to opentelemetry-proto v1.1.0 (#104) * Upgrade to opentelemetry-proto v1.1.0 Signed-off-by: Yuri Shkuro * separate Signed-off-by: Yuri Shkuro * regen Signed-off-by: Yuri Shkuro --------- Signed-off-by: Yuri Shkuro --- Makefile | 12 +++++- opentelemetry-proto | 2 +- swagger/api_v3/query_service.swagger.json | 48 ++++++++++++++--------- 3 files changed, 40 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 3624dcf..f3a23a7 100644 --- a/Makefile +++ b/Makefile @@ -91,7 +91,11 @@ PROTOC_INTERNAL := $(PROTOC) \ --csharp_out=internal_access,base_namespace:${PROTO_GEN_CSHARP_DIR} \ --python_out=${PROTO_GEN_PYTHON_DIR} -proto: +.PHONY: proto +proto: proto-prepare proto-api-v2 proto-api-v3 + +.PHONY: proto-prepare +proto-prepare: mkdir -p ${PROTO_GEN_GO_DIR} \ ${PROTO_GEN_JAVA_DIR} \ ${PROTO_GEN_PYTHON_DIR} \ @@ -99,14 +103,18 @@ proto: ${PROTO_GEN_CPP_DIR} \ ${PROTO_GEN_CSHARP_DIR} +.PHONY: proto-api-v2 +proto-api-v2: $(PROTOC_WITHOUT_GRPC) \ proto/api_v2/model.proto - + $(PROTOC_WITH_GRPC) \ proto/api_v2/query.proto \ proto/api_v2/collector.proto \ proto/api_v2/sampling.proto +.PHONY: proto-api-v3 +proto-api-v3: # API v3 $(PROTOC_WITH_GRPC) \ proto/api_v3/query_service.proto diff --git a/opentelemetry-proto b/opentelemetry-proto index c4dfbc5..4ca4f03 160000 --- a/opentelemetry-proto +++ b/opentelemetry-proto @@ -1 +1 @@ -Subproject commit c4dfbc51f3cd4089778555a2ac5d9bc093ed2956 +Subproject commit 4ca4f0335c63cda7ab31ea7ed70d6553aee14dce diff --git a/swagger/api_v3/query_service.swagger.json b/swagger/api_v3/query_service.swagger.json index fc9de98..45a9c95 100644 --- a/swagger/api_v3/query_service.swagger.json +++ b/swagger/api_v3/query_service.swagger.json @@ -72,13 +72,13 @@ "type": "object", "properties": { "result": { - "$ref": "#/definitions/api_v3SpansResponseChunk" + "$ref": "#/definitions/v1TracesData" }, "error": { "$ref": "#/definitions/runtimeStreamError" } }, - "title": "Stream result of api_v3SpansResponseChunk" + "title": "Stream result of v1TracesData" } } }, @@ -150,13 +150,13 @@ "type": "object", "properties": { "result": { - "$ref": "#/definitions/api_v3SpansResponseChunk" + "$ref": "#/definitions/v1TracesData" }, "error": { "$ref": "#/definitions/runtimeStreamError" } }, - "title": "Stream result of api_v3SpansResponseChunk" + "title": "Stream result of v1TracesData" } } }, @@ -247,6 +247,11 @@ "type": "integer", "format": "int64", "description": "dropped_attributes_count is the number of dropped attributes. If the value is 0,\nthen no attributes were dropped." + }, + "flags": { + "type": "integer", + "format": "int64", + "description": "Flags, a bit field. 8 least significant bits are the trace\nflags as defined in W3C Trace Context specification. Readers\nMUST not assume that 24 most significant bits will be zero.\nWhen creating new spans, the most-significant 24-bits MUST be\nzero. To read the 8-bit W3C trace flag (use flags \u0026\nSPAN_FLAGS_TRACE_FLAGS_MASK). [Optional].\n\nSee https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions." } }, "description": "A pointer from the current span to another span in the same trace or in a\ndifferent trace. For example, this can be used in batching operations,\nwhere a single batch handler processes multiple requests from different\ntraces or when the handler receives a request from a different project." @@ -311,19 +316,6 @@ }, "description": "Operation encapsulates information about operation." }, - "api_v3SpansResponseChunk": { - "type": "object", - "properties": { - "resource_spans": { - "type": "array", - "items": { - "$ref": "#/definitions/v1ResourceSpans" - }, - "title": "A list of OpenTelemetry ResourceSpans.\nIn case of JSON format the ids (trace_id, span_id, parent_id) are encoded in base64 even though OpenTelemetry specification\nmandates to use hex encoding [2].\nBase64 is chosen to keep compatibility with JSONPb codec.\n[1]: https://github.com/open-telemetry/opentelemetry-proto/blob/main/opentelemetry/proto/trace/v1/trace.proto\n[2]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlphttp" - } - }, - "description": "Response object with spans." - }, "api_v3TraceQueryParameters": { "type": "object", "properties": { @@ -529,7 +521,7 @@ }, "schema_url": { "type": "string", - "description": "This schema_url applies to the data in the \"resource\" field. It does not apply\nto the data in the \"scope_spans\" field which have their own schema_url field." + "description": "The Schema URL, if known. This is the identifier of the Schema that the resource data\nis recorded in. To learn more about Schema URL see\nhttps://opentelemetry.io/docs/specs/otel/schemas/#schema-url\nThis schema_url applies to the data in the \"resource\" field. It does not apply\nto the data in the \"scope_spans\" field which have their own schema_url field." } }, "description": "A collection of ScopeSpans from a Resource." @@ -550,7 +542,7 @@ }, "schema_url": { "type": "string", - "description": "This schema_url applies to all spans and span events in the \"spans\" field." + "description": "The Schema URL, if known. This is the identifier of the Schema that the span data\nis recorded in. To learn more about Schema URL see\nhttps://opentelemetry.io/docs/specs/otel/schemas/#schema-url\nThis schema_url applies to all spans and span events in the \"spans\" field." } }, "description": "A collection of Spans produced by an InstrumentationScope." @@ -577,6 +569,11 @@ "format": "byte", "description": "The `span_id` of this span's parent span. If this is a root span, then this\nfield must be empty. The ID is an 8-byte array." }, + "flags": { + "type": "integer", + "format": "int64", + "description": "Flags, a bit field. 8 least significant bits are the trace\nflags as defined in W3C Trace Context specification. Readers\nMUST not assume that 24 most significant bits will be zero.\nTo read the 8-bit W3C trace flag, use `flags \u0026 SPAN_FLAGS_TRACE_FLAGS_MASK`.\n\nWhen creating span messages, if the message is logically forwarded from another source\nwith an equivalent flags fields (i.e., usually another OTLP span message), the field SHOULD\nbe copied as-is. If creating from a source that does not have an equivalent flags field\n(such as a runtime representation of an OpenTelemetry span), the high 24 bits MUST\nbe set to zero.\n\n[Optional].\n\nSee https://www.w3.org/TR/trace-context-2/#trace-flags for the flag definitions." + }, "name": { "type": "string", "description": "A description of the span's operation.\n\nFor example, the name can be a qualified method name or a file name\nand a line number where the operation is called. A best practice is to use\nthe same display name at the same call point in an application.\nThis makes it easier to correlate spans in different traces.\n\nThis field is semantically required to be set to non-empty string.\nEmpty value is equivalent to an unknown span name.\n\nThis field is required." @@ -652,6 +649,19 @@ } }, "description": "The Status type defines a logical error model that is suitable for different\nprogramming environments, including REST APIs and RPC APIs." + }, + "v1TracesData": { + "type": "object", + "properties": { + "resource_spans": { + "type": "array", + "items": { + "$ref": "#/definitions/v1ResourceSpans" + }, + "description": "An array of ResourceSpans.\nFor data coming from a single resource this array will typically contain\none element. Intermediary nodes that receive data from multiple origins\ntypically batch the data before forwarding further and in that case this\narray will contain multiple elements." + } + }, + "description": "TracesData represents the traces data that can be stored in a persistent storage,\nOR can be embedded by other protocols that transfer OTLP traces data but do\nnot implement the OTLP protocol.\n\nThe main difference between this message and collector protocol is that\nin this message there will not be any \"control\" or \"metadata\" specific to\nOTLP protocol.\n\nWhen new fields are added into this message, the OTLP request MUST be updated\nas well." } } }