From 326faeb6baa82ed2415186274ec5132067923cab Mon Sep 17 00:00:00 2001 From: YANGDB Date: Fri, 27 Jan 2023 16:14:41 -0800 Subject: [PATCH 1/4] Add Traces schema support for SSO which is OTEL compliant Signed-off-by: YANGDB --- schema/traces/README.md | 66 +++++++++++++++++ schema/traces/traces.mapping | 129 +++++++++++++++++++++++++++++++++ schema/traces/traces.schema | 134 +++++++++++++++++++++++++++++++++++ 3 files changed, 329 insertions(+) create mode 100644 schema/traces/README.md create mode 100644 schema/traces/traces.mapping create mode 100644 schema/traces/traces.schema diff --git a/schema/traces/README.md b/schema/traces/README.md new file mode 100644 index 000000000..58594fa4e --- /dev/null +++ b/schema/traces/README.md @@ -0,0 +1,66 @@ +# Traces Schema Support +The next section provides the Simple Schema for Observability support which conforms with the OTEL specification. + +- traces.mapping presents the template mapping for creating the Simple Schema for Observability index +- traces.schema presents the json schema validation for verification of a trace document conforms to the mapping structure + +## Traces +see [OTEL traces convention](https://github.com/open-telemetry/opentelemetry-specification/tree/main/semantic_conventions/trace) + +Traces in are defined implicitly by their Spans - In particular, a Trace can be thought of as a directed acyclic graph (DAG) of Spans, where the edges between Spans are defined as parent/child relationship. + +## Spans +A span represents an operation within a transaction. Each Span encapsulates the following state: + +* An operation name +* start and finish timestamp +* Attributes list of key-value pairs. +* Set of Events, each of which is itself a tuple (timestamp, name, Attributes) +* Parent's Span identifier. +* Links to causally-related Spans (via the SpanContext of those related Spans). +* SpanContext information required to reference a Span. + +### SpanContext +Represents all the information that identifies Span in the Trace and is propagated to child Spans and across process boundaries. +A **SpanContext** contains the tracing identifiers and the options that are propagated from parent to child Spans. + +* TraceId - It is worldwide unique with practically sufficient probability by being made as 16 randomly generated bytes - used to group all spans for a specific trace together across all processes. +* SpanId - It is the identifier for a span, globally unique with practically sufficient probability by being made as 8 randomly generated bytes. When passed to a child Span this identifier becomes the parent span id for the child Span. +* Tracestate - carries tracing-system specific context in a list of key value pairs. Tracestate allows different vendors propagate additional information and inter-operate with their legacy Id formats. For more details see this. + +Additional fields can be supported via the Attributes key/value store see [traces](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/README.md) + +### Structure +The default fields that are supported by the traces are +- **TraceId** : It is worldwide unique with practically sufficient probability by being made as 16 randomly generated bytes - used to group all spans for a specific trace together across all processes. +- **SpanId** : It is the identifier for a span, globally unique with practically sufficient probability by being made as 8 randomly generated bytes. When passed to a child Span this identifier becomes the parent span id for the child Span. +- **ParentId** : It is the identifier for a span's parent span. +- **TraceState** : carries tracing-system specific context in a list of key value pairs. Tracestate allows different vendors propagate additional information and inter-operate with their legacy Id formats. For more details see this. + +- **Name** : String representing the span's name +- **Kind** + - SpanKind.CLIENT + + - SpanKind.SERVER + + - SpanKind.CONSUMER + + - SpanKind.PRODUCER + + - SpanKind.INTERNAL + + +- **StartTime** : Start time of the event +- **EndTime** : End time of the event +- **Attributes** + - An Attribute is a key-value pair, which has the following structure [Attributes](https://github.com/open-telemetry/opentelemetry-specification/blob/b00980832b4b823155001df56dbf9203d4e53f98/specification/common/README.md#attribute) + +- **DroppedAttributesCount** : Integer counting the dropped attributes +- **Events** : A set of the next tuples (timestamp, name, Attributes) +- **DroppedEventsCount** : Integer counting the dropped events +- **Links** : links to causally-related Spans +- **DroppedLinksCount** : Integer counting the dropped links +- **Status** - + - UNSET= 0 -> The default status. + - OK= 1 -> The operation has been validated by an Application developer or Operator to have completed successfully. + - ERROR= 2 -> The operation contains an error. \ No newline at end of file diff --git a/schema/traces/traces.mapping b/schema/traces/traces.mapping new file mode 100644 index 000000000..bc1021815 --- /dev/null +++ b/schema/traces/traces.mapping @@ -0,0 +1,129 @@ +{ + "index_patterns": [ + "sso_traces-*-*" + ], + "mappings": { + "_meta": { + "version": "0.1.0-dev" + }, + "dynamic_templates": [ + { + "attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "attributes.*" + } + }, + { + "events_attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "events.attributes.*" + } + }, + { + "links_attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "links.attributes.*" + } + }, + { + "resources_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "resources.*" + } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "traceId": { + "ignore_above": 256, + "type": "keyword" + }, + "spanId": { + "ignore_above": 256, + "type": "keyword" + }, + "parentSpanId": { + "ignore_above": 256, + "type": "keyword" + }, + "traceState": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 128, + "type": "keyword" + }, + "startTime": { + "type": "date_nanos" + }, + "endTime": { + "type": "date_nanos" + }, + "droppedAttributesCount": { + "type": "long" + }, + "droppedEventsCount": { + "type": "long" + }, + "droppedLinksCount": { + "type": "long" + }, + "traceStatus": { + "properties": { + "code": { + "type": "array", + "items": { + "type": "int", + "enum": [0,1,2] + } + }, + "message": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "UNSET", + "OK", + "ERROR"] + } + } + } + }, + "events": { + "type": "nested", + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "type": "date_nanos" + } + } + }, + "links": { + "type": "nested" + } + } + } +} \ No newline at end of file diff --git a/schema/traces/traces.schema b/schema/traces/traces.schema new file mode 100644 index 000000000..e20c2d557 --- /dev/null +++ b/schema/traces/traces.schema @@ -0,0 +1,134 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "https://opensearch.org/schemas/SSO", + "definitions": { + "span": { + "type": "object", + "additionalProperties": false, + "properties": { + "traceId": { + "type": "string" + }, + "spanId": { + "type": "string" + }, + "status": { + "type": "object", + "$ref": "#/definitions/Status" + }, + "traceState": { + "type": "array", + "items": { + "type": "object" + } + }, + "parentSpanId": { + "type": "string" + }, + "name": { + "type": "string" + }, + "kind": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "CLIENT", + "SERVER", + "CONSUMER", + "PRODUCER", + "INTERNAL" + ] + } + }, + "startTime": { + "type": "string", + "format": "date-time" + }, + "endTime": { + "type": "string", + "format": "date-time" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/Event" + } + }, + "links": { + "type": "array", + "items": { + "type": "object" + } + }, + "droppedAttributesCount": { + "type": "integer" + }, + "droppedEventsCount": { + "type": "integer" + }, + "droppedLinksCount": { + "type": "integer" + } + }, + "required": [ + "traceId", + "spanId", + "startTime", + "endTime", + "kind", + "name", + "status" + ], + "title": "Spans" + }, + "Status": { + "type": "object", + "additionalProperties": false, + "properties": { + "code": { + "type": "integer" + }, + "message": { + "type": "string" + } + }, + "required": [ + "code", + "message" + ], + "title": "Status" + }, + "Event": { + "type": "object", + "additionalProperties": false, + "properties": { + "timestamp": { + "type": "string", + "format": "date-time" + }, + "name": { + "type": "string" + }, + "attributes": { + "$ref": "#/definitions/Attributes" + }, + "droppedAttributesCount": { + "type": "integer" + } + }, + "required": [ + "attributes", + "droppedAttributesCount", + "name", + "time" + ], + "title": "Event" + }, + "Attributes": { + "type": "object", + "additionalProperties": true, + "title": "Attributes" + } + } +} From 5d93e675b98a677847cbc6eec230cbd1c3f3972b Mon Sep 17 00:00:00 2001 From: YANGDB Date: Fri, 27 Jan 2023 16:14:41 -0800 Subject: [PATCH 2/4] add basic trace samples Signed-off-by: YANGDB Signed-off-by: YANGDB --- schema/traces/README.md | 61 ++++++++++ schema/traces/samples/traceA.json | 23 ++++ schema/traces/samples/traceB.json | 47 ++++++++ schema/traces/samples/traceC.json | 63 +++++++++++ schema/traces/traces.mapping | 139 +++++++++++++++++++++++ schema/traces/traces.schema | 177 ++++++++++++++++++++++++++++++ 6 files changed, 510 insertions(+) create mode 100644 schema/traces/README.md create mode 100644 schema/traces/samples/traceA.json create mode 100644 schema/traces/samples/traceB.json create mode 100644 schema/traces/samples/traceC.json create mode 100644 schema/traces/traces.mapping create mode 100644 schema/traces/traces.schema diff --git a/schema/traces/README.md b/schema/traces/README.md new file mode 100644 index 000000000..03e0c7de7 --- /dev/null +++ b/schema/traces/README.md @@ -0,0 +1,61 @@ +# Traces Schema Support +The next section provides the Simple Schema for Observability support which conforms with the OTEL specification. + +- traces.mapping presents the template mapping for creating the Simple Schema for Observability index +- traces.schema presents the json schema validation for verification of a trace document conforms to the mapping structure + +## Traces +see [OTEL traces convention](https://github.com/open-telemetry/opentelemetry-specification/tree/main/semantic_conventions/trace) + +Traces in are defined implicitly by their Spans - In particular, a Trace can be thought of as a directed acyclic graph (DAG) of Spans, where the edges between Spans are defined as parent/child relationship. + +## Spans +A span represents an operation within a transaction. Each Span encapsulates the following state: + +* An operation name +* start and finish timestamp +* Attributes list of key-value pairs. +* Set of Events, each of which is itself a tuple (timestamp, name, Attributes) +* Parent's Span identifier. +* Links to causally-related Spans (via the SpanContext of those related Spans). +* SpanContext information required to reference a Span. + +### SpanContext +Represents all the information that identifies Span in the Trace and is propagated to child Spans and across process boundaries. +A **SpanContext** contains the tracing identifiers and the options that are propagated from parent to child Spans. + +* TraceId - It is worldwide unique with practically sufficient probability by being made as 16 randomly generated bytes - used to group all spans for a specific trace together across all processes. +* SpanId - It is the identifier for a span, globally unique with practically sufficient probability by being made as 8 randomly generated bytes. When passed to a child Span this identifier becomes the parent span id for the child Span. +* Tracestate - carries tracing-system specific context in a list of key value pairs. Tracestate allows different vendors propagate additional information and inter-operate with their legacy Id formats. For more details see this. + +Additional fields can be supported via the Attributes key/value store see [traces](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/README.md) + +### Structure +The default fields that are supported by the traces are +- **TraceId** : It is worldwide unique with practically sufficient probability by being made as 16 randomly generated bytes - used to group all spans for a specific trace together across all processes. +- **SpanId** : It is the identifier for a span, globally unique with practically sufficient probability by being made as 8 randomly generated bytes. When passed to a child Span this identifier becomes the parent span id for the child Span. +- **ParentId** : It is the identifier for a span's parent span. +- **TraceState** : carries tracing-system specific context in a list of key value pairs. Tracestate allows different vendors propagate additional information and inter-operate with their legacy Id formats. For more details see this. + +- **Name** : String representing the span's name +- **Kind** + - SpanKind.CLIENT + - SpanKind.SERVER + - SpanKind.CONSUMER + - SpanKind.PRODUCER + - SpanKind.INTERNAL + +- **StartTime** : Start time of the event +- **EndTime** : End time of the event +- **Attributes** + - An Attribute is a key-value pair, which has the following structure [Attributes](https://github.com/open-telemetry/opentelemetry-specification/blob/b00980832b4b823155001df56dbf9203d4e53f98/specification/common/README.md#attribute) + +- **DroppedAttributesCount** : Integer counting the dropped attributes +- **Events** : A set of the next tuples (timestamp, name, Attributes) +- **DroppedEventsCount** : Integer counting the dropped events +- **Links** : links to causally-related Spans +- **DroppedLinksCount** : Integer counting the dropped links +- **Status** - + - UNSET= 0 -> The default status. + - OK= 1 -> The operation has been validated by an Application developer or Operator to have completed successfully. + - ERROR= 2 -> The operation contains an error. \ No newline at end of file diff --git a/schema/traces/samples/traceA.json b/schema/traces/samples/traceA.json new file mode 100644 index 000000000..e31f7a86d --- /dev/null +++ b/schema/traces/samples/traceA.json @@ -0,0 +1,23 @@ +{ + "traceId": "4fa04f117be100f476b175e41096e736", + "spanId": "e275ac9d21929e9b", + "traceState": "", + "parentSpanId": "", + "name": "client_checkout", + "kind": "INTERNAL", + "startTime": "2021-03-25T17:23:30.088478464Z", + "endTime": "2021-03-25T17:23:30.481628416Z", + "droppedAttributesCount": 0, + "droppedEventsCount": 0, + "droppedLinksCount": 0, + "resources":{ + "telemetry@sdk@name": "opentelemetry", + "telemetry@sdk@language": "python", + "telemetry@sdk@version": "0.14b0", + "service@name": "frontend-client", + "host@hostname": "ip-172-31-10-8.us-west-2.compute.internal" + }, + "status":{ + "code": 0 + } +} \ No newline at end of file diff --git a/schema/traces/samples/traceB.json b/schema/traces/samples/traceB.json new file mode 100644 index 000000000..d6cc2e4ea --- /dev/null +++ b/schema/traces/samples/traceB.json @@ -0,0 +1,47 @@ +{ + "traceId": "15d30e4d211d79e10fcaeab97015c90d", + "spanId": "5bcca8ba513bb54a", + "traceState": "", + "parentSpanId": "", + "name": "mysql", + "kind": "CLIENT", + "startTime": "2021-03-25T17:21:03.038618112Z", + "endTime": "2021-03-25T17:21:03.041571328Z", + "events": [ + { + "timestamp": "2021-03-25T17:21:03.041542912Z", + "name": "exception", + "attributes": { + "exception@message": "1050 %2842S01%29: Table %27User_Carts%27 already exists", + "exception@type": "ProgrammingError", + "exception@stacktrace": "Traceback %28most recent call last :File /usr/lib/python3.6/site-packages/opentelemetry/sdk/trace/__init__.py, line 804, in use_span yield spanFile /usr/lib/python3.6/site-packages/opentelemetry/instrumentation/dbapi/__init__.py, line 354, in traced_executionraise exFile /usr/lib/python3.6/site-packages/opentelemetry/instrumentation/dbapi/__init__.py, line 345, in traced_executionresult = query_method%28%2Aargs, %2A%2Akwargs%29File /usr/lib/python3.6/site-packages/mysql/connector/cursor.py" + }, + "droppedAttributesCount": 0 + } + ], + "links": [], + "droppedAttributesCount": 0, + "droppedEventsCount": 0, + "droppedLinksCount": 0, + "status": { + "status.message": "1050 %2842S01%29: Table %27User_Carts%27 already exists", + "status.code": 2 + }, + "attributes": { + "component": "mysql", + "db@user": "root", + "net@peer@name": "localhost", + "db@type": "sql", + "net@peer@port": 3306, + "db@instance": "", + "db@statement": "CREATE TABLE `User_Carts` %28 `ItemId` varchar%2816%29 NOT NULL, `TotalQty` int%2811%29 NOT NULL, PRIMARY KEY %28`ItemId`%29%29 ENGINE=InnoDB" + }, + "resources": { + "telemetry@sdk@language": "python", + "service@name": "database", + "telemetry@sdk@version": "0.14b0", + "service@instance@id": "140307275923408", + "telemetry@sdk@name": "opentelemetry", + "host@hostname": "ip-172-31-10-8.us-west-2.compute.internal" + } +} \ No newline at end of file diff --git a/schema/traces/samples/traceC.json b/schema/traces/samples/traceC.json new file mode 100644 index 000000000..a87b6571b --- /dev/null +++ b/schema/traces/samples/traceC.json @@ -0,0 +1,63 @@ +{ + "traceId": "c1d985bd02e1dbb85b444011f19a1ecc", + "spanId": "55a698828fe06a42", + "traceState": "", + "parentSpanId": "", + "name": "mysql", + "kind": "CLIENT", + "startTime": "2021-03-25T17:21:03.000792576Z", + "endTime": "2021-03-25T17:21:03.023983360Z", + "events": [ + { + "timestamp": "2021-03-25T17:21:03.023934720Z", + "name": "exception", + "attributes": { + "exception@message": "1050 %2842S01%29: Table Inventory_Items already exists", + "exception@type": "ProgrammingError", + "exception@stacktrace": "Traceback most recent call last" + }, + "droppedAttributesCount": 0 + } + ], + "links": [ + { + "traceId": "c1d985bd02e1dbb85b444011f19a1ecc", + "spanId": "55a698828fe06a42w2", + "traceState": "", + "attributes": { + "db@user": "root", + "net@peer@name": "localhost", + "component": "mysql", + "db@type": "sql", + "net@peer@port": 3306, + "db@instance": "", + "db@statement": "CREATE TABLE `Inventory_Items` %28 `ItemId` varchar%2816%29 NOT NULL, `TotalQty` int%2811%29 NOT NULL, PRIMARY KEY %28`ItemId`%29%29 ENGINE=InnoDB" + }, + "droppedAttributesCount": 0 + } + ], + "droppedAttributesCount": 0, + "droppedEventsCount": 0, + "droppedLinksCount": 0, + "resources": { + "telemetry@sdk@language": "python", + "telemetry@sdk@version": "0.14b0", + "service@instance@id": "140307275923408", + "service@name": "database", + "telemetry@sdk@name": "opentelemetry", + "host@hostname": "ip-172-31-10-8.us-west-2.compute.internal" + }, + "status": { + "code": 2, + "message": "1050 %2842S01%29: Table %27Inventory_Items%27 already exists" + }, + "attributes": { + "db@user": "root", + "net@peer@name": "localhost", + "component": "mysql", + "db@type": "sql", + "net@peer@port": 3306, + "db@instance": "", + "db@statement": "CREATE TABLE `Inventory_Items` %28 `ItemId` varchar%2816%29 NOT NULL, `TotalQty` int%2811%29 NOT NULL, PRIMARY KEY %28`ItemId`%29%29 ENGINE=InnoDB" + } +} diff --git a/schema/traces/traces.mapping b/schema/traces/traces.mapping new file mode 100644 index 000000000..e07cd149d --- /dev/null +++ b/schema/traces/traces.mapping @@ -0,0 +1,139 @@ +{ + "index_patterns": [ + "sso_traces-*-*" + ], + "mappings": { + "_meta": { + "version": "0.1.0-dev" + }, + "dynamic_templates": [ + { + "attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "attributes.*" + } + }, + { + "events_attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "events.attributes.*" + } + }, + { + "links_attributes_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "links.attributes.*" + } + }, + { + "resources_map": { + "mapping": { + "type": "keyword" + }, + "path_match": "resources.*" + } + } + ], + "_source": { + "enabled": true + }, + "properties": { + "traceId": { + "ignore_above": 256, + "type": "keyword" + }, + "spanId": { + "ignore_above": 256, + "type": "keyword" + }, + "traceState": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "parentSpanId": { + "ignore_above": 256, + "type": "keyword" + }, + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "kind": { + "ignore_above": 128, + "type": "keyword" + }, + "startTime": { + "type": "date_nanos" + }, + "endTime": { + "type": "date_nanos" + }, + "droppedAttributesCount": { + "type": "long" + }, + "droppedEventsCount": { + "type": "long" + }, + "droppedLinksCount": { + "type": "long" + }, + "status": { + "properties": { + "code": { + "ignore_above": 128, + "type": "keyword" + }, + "message": { + "ignore_above": 128, + "type": "keyword" + } + } + }, + "events": { + "type": "nested", + "properties": { + "name": { + "ignore_above": 1024, + "type": "keyword" + }, + "timestamp": { + "type": "date_nanos" + } + } + }, + "links": { + "type": "nested", + "properties": { + "traceId": { + "ignore_above": 256, + "type": "keyword" + }, + "spanId": { + "ignore_above": 256, + "type": "keyword" + }, + "traceState": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/schema/traces/traces.schema b/schema/traces/traces.schema new file mode 100644 index 000000000..625a2d1eb --- /dev/null +++ b/schema/traces/traces.schema @@ -0,0 +1,177 @@ +{ + "$schema": "http://json-schema.org/draft-06/schema#", + "$ref": "https://opensearch.org/schemas/SSO", + "definitions": { + "span": { + "type": "object", + "additionalProperties": false, + "properties": { + "traceId": { + "type": "string" + }, + "spanId": { + "type": "string" + }, + "traceState": { + "type": "array", + "items": { + "type": "object" + } + }, + "status": { + "type": "object", + "$ref": "#/definitions/Status" + }, + "parentSpanId": { + "type": "string" + }, + "name": { + "type": "string" + }, + "kind": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "CLIENT", + "SERVER", + "CONSUMER", + "PRODUCER", + "INTERNAL" + ] + } + }, + "startTime": { + "type": "string", + "format": "date-time" + }, + "endTime": { + "type": "string", + "format": "date-time" + }, + "events": { + "type": "array", + "items": { + "$ref": "#/definitions/Event" + } + }, + "links": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "traceId": { + "type": "string" + }, + "spanId": { + "type": "string" + }, + "traceState": { + "type": "array", + "items": { + "type": "object" + } + }, + "attributes": { + "$ref": "#/definitions/Attributes" + }, + "droppedAttributesCount": { + "type": "integer" + } + }, + "required": [ + "traceId", + "spanId", + "traceState" + ], + "title": "Links" + } + }, + "droppedAttributesCount": { + "type": "integer" + }, + "droppedEventsCount": { + "type": "integer" + }, + "droppedLinksCount": { + "type": "integer" + } + }, + "required": [ + "traceId", + "spanId", + "startTime", + "endTime", + "kind", + "name", + "status" + ], + "title": "Spans" + }, + "Status": { + "type": "object", + "additionalProperties": false, + "properties": { + "code": { + "type": "array", + "items": { + "type": "integer", + "enum": [ + 0, + 1, + 2 + ] + } + }, + "message": { + "type": "array", + "items": { + "type": "string", + "enum": [ + "UNSET", + "OK", + "ERROR" + ] + } + } + }, + "required": [ + "code", + "message" + ], + "title": "Status" + }, + "Event": { + "type": "object", + "additionalProperties": false, + "properties": { + "timestamp": { + "type": "string", + "format": "date-time" + }, + "name": { + "type": "string" + }, + "attributes": { + "$ref": "#/definitions/Attributes" + }, + "droppedAttributesCount": { + "type": "integer" + } + }, + "required": [ + "attributes", + "droppedAttributesCount", + "name", + "time" + ], + "title": "Event" + }, + "Attributes": { + "type": "object", + "additionalProperties": true, + "title": "Attributes" + } + } +} From 41e688ebad1e6e40e65115ff82f39bd1c65f83d1 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 8 Feb 2023 11:36:46 -0800 Subject: [PATCH 3/4] add specific info and support for data-stream API Signed-off-by: YANGDB --- schema/traces/README.md | 19 +++++++++++++++++++ schema/traces/samples/traceB.json | 2 +- schema/traces/samples/traceC.json | 2 +- schema/traces/traces.mapping | 2 +- schema/traces/traces.schema | 2 +- 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/schema/traces/README.md b/schema/traces/README.md index 03264309b..acb590002 100644 --- a/schema/traces/README.md +++ b/schema/traces/README.md @@ -18,6 +18,25 @@ The next section provides the Simple Schema for Observability support which conf - traces.mapping presents the template mapping for creating the Simple Schema for Observability index - traces.schema presents the json schema validation for verification of a trace document conforms to the mapping structure +### data-stream +[data-stream](https://opensearch.org/docs/latest/opensearch/data-streams/) Data streams simplify this process and enforce a setup that best suits time-series data, such as being designed primarily for append-only data and ensuring that each document has a timestamp field. +A data stream is internally composed of multiple backing indices. Search requests are routed to all the backing indices, while indexing requests are routed to the latest write index. + +As part of the Observability naming scheme, the value of the data stream fields combine to the name of the actual data stream : + +`{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. +This means the fields can only contain characters that are valid as part of names of data streams. + +- **type** conforms to one of the supported Observability signals (Traces, Logs, Metrics, Alerts) +- **dataset** user defined field that can mainly be utilized for describing the origin of the signal +- **namespace** user custom field that can be used to describe any customer domain specific classification + + +If nothing is stated in the namespace / dataset - the signal information would be routed into the default data-stream indices +- **Traces** - traces-default +- **Metrics** - metrics-default +- **Logs** - logs-default + ## Traces see [OTEL traces convention](https://github.com/open-telemetry/opentelemetry-specification/tree/main/semantic_conventions/trace) diff --git a/schema/traces/samples/traceB.json b/schema/traces/samples/traceB.json index 72bf9a68c..ac917ccde 100644 --- a/schema/traces/samples/traceB.json +++ b/schema/traces/samples/traceB.json @@ -28,7 +28,7 @@ "code": 2 }, "attributes": { - "dataflow": { + "data_stream": { "type": "span", "dataset": "mysql" }, diff --git a/schema/traces/samples/traceC.json b/schema/traces/samples/traceC.json index f057b0d9f..81ddad393 100644 --- a/schema/traces/samples/traceC.json +++ b/schema/traces/samples/traceC.json @@ -52,7 +52,7 @@ "message": "1050 %2842S01%29: Table %27Inventory_Items%27 already exists" }, "attributes": { - "dataflow": { + "data_stream": { "type": "span", "namespace": "exceptions", "dataset": "mysql" diff --git a/schema/traces/traces.mapping b/schema/traces/traces.mapping index 7bb3e59fa..c3d634334 100644 --- a/schema/traces/traces.mapping +++ b/schema/traces/traces.mapping @@ -112,7 +112,7 @@ "attributes": { "type": "object", "properties": { - "dataflow": { + "data_stream": { "properties": { "dataset": { "ignore_above": 128, diff --git a/schema/traces/traces.schema b/schema/traces/traces.schema index 3b2d232f4..a0e4ca454 100644 --- a/schema/traces/traces.schema +++ b/schema/traces/traces.schema @@ -184,7 +184,7 @@ "type": "object", "additionalProperties": true, "properties": { - "dataflow": { + "data_stream": { "$ref": "/schemas/Dataflow" } }, From 56d76dcfea4473d8053b3c8cf6dcf8d2c35ccea0 Mon Sep 17 00:00:00 2001 From: YANGDB Date: Wed, 8 Feb 2023 11:36:46 -0800 Subject: [PATCH 4/4] add specific info and support for data-stream API Signed-off-by: YANGDB --- schema/traces/README.md | 19 +++++++++++++++++++ schema/traces/samples/traceB.json | 2 +- schema/traces/samples/traceC.json | 2 +- schema/traces/traces.mapping | 2 +- schema/traces/traces.schema | 2 +- 5 files changed, 23 insertions(+), 4 deletions(-) diff --git a/schema/traces/README.md b/schema/traces/README.md index 03264309b..4b9014e64 100644 --- a/schema/traces/README.md +++ b/schema/traces/README.md @@ -18,6 +18,25 @@ The next section provides the Simple Schema for Observability support which conf - traces.mapping presents the template mapping for creating the Simple Schema for Observability index - traces.schema presents the json schema validation for verification of a trace document conforms to the mapping structure +### data-stream +[data-stream](https://opensearch.org/docs/latest/opensearch/data-streams/) Data streams simplify this process and enforce a setup that best suits time-series data, such as being designed primarily for append-only data and ensuring that each document has a timestamp field. +A data stream is internally composed of multiple backing indices. Search requests are routed to all the backing indices, while indexing requests are routed to the latest write index. + +As part of the Observability naming scheme, the value of the data stream fields combine to the name of the actual data stream : + +`{data_stream.type}-{data_stream.dataset}-{data_stream.namespace}`. +This means the fields can only contain characters that are valid as part of names of data streams. + +- **type** conforms to one of the supported Observability signals (Traces, Logs, Metrics, Alerts) +- **dataset** user defined field that can mainly be utilized for describing the origin of the signal +- **namespace** user custom field that can be used to describe any customer domain specific classification + + +If nothing is stated in the namespace / dataset - the signal information would be routed into the default data-stream indices +- **Traces** - traces-default-namespace +- **Metrics** - metrics-default-namespace +- **Logs** - logs-default-namespace + ## Traces see [OTEL traces convention](https://github.com/open-telemetry/opentelemetry-specification/tree/main/semantic_conventions/trace) diff --git a/schema/traces/samples/traceB.json b/schema/traces/samples/traceB.json index 72bf9a68c..ac917ccde 100644 --- a/schema/traces/samples/traceB.json +++ b/schema/traces/samples/traceB.json @@ -28,7 +28,7 @@ "code": 2 }, "attributes": { - "dataflow": { + "data_stream": { "type": "span", "dataset": "mysql" }, diff --git a/schema/traces/samples/traceC.json b/schema/traces/samples/traceC.json index f057b0d9f..81ddad393 100644 --- a/schema/traces/samples/traceC.json +++ b/schema/traces/samples/traceC.json @@ -52,7 +52,7 @@ "message": "1050 %2842S01%29: Table %27Inventory_Items%27 already exists" }, "attributes": { - "dataflow": { + "data_stream": { "type": "span", "namespace": "exceptions", "dataset": "mysql" diff --git a/schema/traces/traces.mapping b/schema/traces/traces.mapping index 7bb3e59fa..c3d634334 100644 --- a/schema/traces/traces.mapping +++ b/schema/traces/traces.mapping @@ -112,7 +112,7 @@ "attributes": { "type": "object", "properties": { - "dataflow": { + "data_stream": { "properties": { "dataset": { "ignore_above": 128, diff --git a/schema/traces/traces.schema b/schema/traces/traces.schema index 3b2d232f4..a0e4ca454 100644 --- a/schema/traces/traces.schema +++ b/schema/traces/traces.schema @@ -184,7 +184,7 @@ "type": "object", "additionalProperties": true, "properties": { - "dataflow": { + "data_stream": { "$ref": "/schemas/Dataflow" } },