From a42f9ed05197574140a3b751c50465c8682e966b Mon Sep 17 00:00:00 2001 From: Jack Berg Date: Fri, 30 Aug 2024 14:24:04 -0500 Subject: [PATCH 1/4] Prefer arrays of name value pairs over objects --- Makefile | 2 - README.md | 23 +++++++++++- examples/anchors.yaml | 3 +- examples/kitchen-sink.yaml | 27 ++++++++++++-- examples/sdk-config.yaml | 9 +++-- examples/sdk-migration-config.yaml | 9 +++-- schema/common.json | 29 ++++++++++----- schema/meter_provider.json | 5 ++- schema/resource.json | 59 +++++++++++++++++++++++++++--- 9 files changed, 133 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index d042e82..808aacd 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,6 @@ SCHEMA_FILES := $(shell find . -path './schema/*.json' -exec basename {} \; | so EXAMPLE_FILES := $(shell find . -path './examples/*.yaml' -exec basename {} \; | sort) $(shell mkdir -p out) -include validator/Makefile - .PHONY: all all: install-tools compile-schema validate-examples diff --git a/README.md b/README.md index 1324b21..d6588d7 100644 --- a/README.md +++ b/README.md @@ -15,8 +15,6 @@ The [examples](./examples) repository contains a variety of sample configuration - [sdk-migration-config.yaml](./examples/sdk-migration-config.yaml): Includes env var substitution references to all [standard env vars](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md) which map cleanly to file configuration (see notes in the example for the set of env vars which are not referenced). Note, SDKs parsing configuration files ignore all env vars besides those referenced via [env var substitution][]. This is a great starting point for transitioning from env var based configuration to file based configuration. - [sdk-config.yaml](./examples/sdk-config.yaml): Represents the typical default configuration. This is a good starting point if you are not using env var based configuration or wish to transition fully to file based configuration. Note, SDKs parsing configuration files ignore all env vars besides those referenced via [env var substitution][]. -[env var substitution]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/file-configuration.md#environment-variable-substitution - ## Code generation There are [several tools](https://json-schema.org/implementations.html) available to generate code from a JSON schema. The following shows an example for generating code from the JSON schema in Go: @@ -70,6 +68,24 @@ Properties should be modeled using the most appropriate data structures and type In instances where there is a type mismatch between the JSON schema and equivalent standard env var, an alternative version of the property may be provided to resolve the mismatch. For example, resource attributes are configured at `.resource.attributes`, but `.resource.attributes_list` is available with a format matching that of `OTEL_RESOURCE_ATTRIBUTES`. Alternative properties are reserved for cases where there is a demonstrated need for platforms to be able to participate in configuration and there is no reasonable alternative. +### Name-value pairs + +When a type requires a configurable list of name-value pairs (i.e. resource attributes, HTTP headers), model using an array of objects, each with `name` and `value` properties. While an arrays of name-value objects is slightly more verbose an object where each key-value is an entry, it is preferred because: + +* Avoids user input as keys, which ensures conformity with the [snake_case properties](#property-name-case) rule. +* Allows both the names and the values to be targets for [env var substitution]. For example: + + ```yaml + tracer_provider: + processors: + - batch: + exporter: + otlp: + headers: + - name: ${AUTHORIZATION_HEADER_NAME:-api-key} + value: ${AUTHORIZATION_HEADER_VALUE} + ``` + ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) @@ -89,3 +105,6 @@ Maintainers ([@open-telemetry/configuration-maintainers](https://github.com/orgs - [Tyler Yahn](https://github.com/tsloughter), Splunk *Find more about the maintainer role in [community repository](https://github.com/open-telemetry/community/blob/main/guides/contributor/membership.md#maintainer).* + +[env var substitution]: https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/file-configuration.md#environment-variable-substitution + diff --git a/examples/anchors.yaml b/examples/anchors.yaml index 5aed3bd..d956f41 100644 --- a/examples/anchors.yaml +++ b/examples/anchors.yaml @@ -8,7 +8,8 @@ exporters: client_key: /app/cert.pem client_certificate: /app/cert.pem headers: - api-key: !!str 1234 + - name: api-key + value: "1234" compression: gzip timeout: 10000 diff --git a/examples/kitchen-sink.yaml b/examples/kitchen-sink.yaml index fd56325..29263a5 100644 --- a/examples/kitchen-sink.yaml +++ b/examples/kitchen-sink.yaml @@ -78,7 +78,8 @@ logger_provider: # # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_LOGS_HEADERS headers: - api-key: "1234" + - name: api-key + value: "1234" # Configure compression. # # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_LOGS_COMPRESSION @@ -186,7 +187,8 @@ meter_provider: # # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_METRICS_HEADERS headers: - api-key: !!str 1234 + - name: api-key + value: "1234" # Configure compression. # # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_METRICS_COMPRESSION @@ -313,7 +315,8 @@ tracer_provider: # # Environment variable: OTEL_EXPORTER_OTLP_HEADERS, OTEL_EXPORTER_OTLP_TRACES_HEADERS headers: - api-key: !!str 1234 + - name: api-key + value: "1234" # Configure compression. # # Environment variable: OTEL_EXPORTER_OTLP_COMPRESSION, OTEL_EXPORTER_OTLP_TRACES_COMPRESSION @@ -416,7 +419,23 @@ resource: # Configure `service.name` resource attribute # # Environment variable: OTEL_SERVICE_NAME - service.name: !!str "unknown_service" + - name: service-name + string_value: "unknown_service" + # Configure other resource attributes with explicit types. + - name: bool_key + bool_value: true + - name: int_key + int_value: 1 + - name: double_key + double_value: 1.1 + - name: string_array_key + string_array_value: ["value1", "value2"] + - name: bool_array_key + bool_array_value: [ true, false ] + - name: int_array_key + int_array_value: [ 1, 2 ] + - name: double_array_key + double_array_value: [ 1.1, 2.2 ] # Configure resource attributes. Entries have lower priority than entries from .resource.attributes. # # The value is a list of comma separated key-value pairs, matching the format of OTEL_RESOURCE_ATTRIBUTES. See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md#general-sdk-configuration for details. diff --git a/examples/sdk-config.yaml b/examples/sdk-config.yaml index 9675500..a699ad1 100644 --- a/examples/sdk-config.yaml +++ b/examples/sdk-config.yaml @@ -16,7 +16,8 @@ resource: # Configure resource attributes. attributes: # Configure `service.name` resource attribute - service.name: unknown_service + - name: service.name + string_value: unknown_service # Configure general attribute limits. See also tracer_provider.limits, logger_provider.limits. attribute_limits: @@ -62,7 +63,7 @@ tracer_provider: # Configure max time (in milliseconds) to wait for each export. timeout: 10000 # Configure headers: - headers: {} + headers: [] # Configure span limits. See also attribute_limits. limits: # Configure max span attribute value size. Overrides attribute_limits.attribute_value_length_limit. @@ -131,7 +132,7 @@ meter_provider: # Configure max time (in milliseconds) to wait for each export. timeout: 10000 # Configure headers: - headers: {} + headers: [] # Configure temporality preference. temporality_preference: cumulative # Configure default histogram aggregation. @@ -170,7 +171,7 @@ logger_provider: # Configure max time (in milliseconds) to wait for each export. timeout: 10000 # Configure headers: - headers: {} + headers: [] # Configure log record limits. See also attribute_limits. limits: # Configure max log record attribute value size. Overrides attribute_limits.attribute_value_length_limit. diff --git a/examples/sdk-migration-config.yaml b/examples/sdk-migration-config.yaml index 26bdaac..aa9b5f3 100644 --- a/examples/sdk-migration-config.yaml +++ b/examples/sdk-migration-config.yaml @@ -47,7 +47,8 @@ resource: # Configure resource attributes. Entries have higher priority than entries from .resource.attributes_list. attributes: # Configure `service.name` resource attribute - service.name: ${OTEL_SERVICE_NAME:-unknown_service} + - name: service.name + string_value: ${OTEL_SERVICE_NAME:-unknown_service} # Configure resource attributes. Entries have lower priority than entries from .resource.attributes. # # The value is a list of comma separated key-value pairs, matching the format of OTEL_RESOURCE_ATTRIBUTES. See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md#general-sdk-configuration for details. @@ -97,7 +98,7 @@ tracer_provider: # Configure max time (in milliseconds) to wait for each export. timeout: ${OTEL_EXPORTER_OTLP_TRACES_TIMEOUT:-10000} # Configure headers: - headers: {} + headers: [] # Configure span limits. See also attribute_limits. limits: # Configure max span attribute value size. Overrides attribute_limits.attribute_value_length_limit. @@ -166,7 +167,7 @@ meter_provider: # Configure max time (in milliseconds) to wait for each export. timeout: ${OTEL_EXPORTER_OTLP_METRICS_TIMEOUT:-10000} # Configure headers: - headers: {} + headers: [] # Configure temporality preference. temporality_preference: ${OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE:-cumulative} # Configure default histogram aggregation. @@ -205,7 +206,7 @@ logger_provider: # Configure max time (in milliseconds) to wait for each export. timeout: ${OTEL_EXPORTER_OTLP_LOGS_TIMEOUT:-10000} # Configure headers: - headers: {} + headers: [] # Configure log record limits. See also attribute_limits. limits: # Configure max log record attribute value size. Overrides attribute_limits.attribute_value_length_limit. diff --git a/schema/common.json b/schema/common.json index 0332633..132bd0f 100644 --- a/schema/common.json +++ b/schema/common.json @@ -4,15 +4,6 @@ "title": "Common", "type": ["object", "null"], "$defs": { - "Headers": { - "type": ["object", "null"], - "title": "Headers", - "patternProperties": { - ".*": { - "type": ["string", "null"] - } - } - }, "IncludeExclude": { "type": "object", "additionalProperties": false, @@ -43,6 +34,21 @@ } } }, + "NameStringValuePair": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": ["string", "null"] + } + }, + "required": [ + "name", "value" + ] + }, "Otlp": { "type": ["object", "null"], "additionalProperties": false, @@ -64,7 +70,10 @@ "type": ["string", "null"] }, "headers": { - "$ref": "#/$defs/Headers" + "type": "array", + "items": { + "$ref": "common.json#/$defs/NameStringValuePair" + } }, "compression": { "type": ["string", "null"] diff --git a/schema/meter_provider.json b/schema/meter_provider.json index fdaad6f..41fa6ad 100644 --- a/schema/meter_provider.json +++ b/schema/meter_provider.json @@ -170,7 +170,10 @@ "type": ["string", "null"] }, "headers": { - "$ref": "common.json#/$defs/Headers" + "type": "array", + "items": { + "$ref": "common.json#/$defs/NameStringValuePair" + } }, "compression": { "type": ["string", "null"] diff --git a/schema/resource.json b/schema/resource.json index 1f0b9df..304e155 100644 --- a/schema/resource.json +++ b/schema/resource.json @@ -6,7 +6,10 @@ "additionalProperties": false, "properties": { "attributes": { - "$ref": "#/$defs/Attributes" + "type": "array", + "items": { + "$ref": "#/$defs/AttributeNameValue" + } }, "detectors": { "$ref": "#/$defs/Detectors" @@ -19,10 +22,56 @@ } }, "$defs": { - "Attributes": { - "title": "Attributes", - "type": ["object", "null"], - "additionalProperties": true + "AttributeNameValue": { + "title": "AttributeNameValue", + "type": "object", + "additionalProperties": false, + "minProperties": 2, + "maxProperties": 2, + "properties": { + "name": { + "type": "string" + }, + "string_value": { + "type": "string" + }, + "bool_value": { + "type": "boolean" + }, + "int_value": { + "type": "integer" + }, + "double_value": { + "type": "number" + }, + "string_array_value": { + "type": "array", + "items": { + "type": "string" + } + }, + "bool_array_value": { + "type": "array", + "items": { + "type": "boolean" + } + }, + "int_array_value": { + "type": "array", + "items": { + "type": "integer" + } + }, + "double_array_value": { + "type": "array", + "items": { + "type": "number" + } + } + }, + "required": [ + "name" + ] }, "Detectors": { "title": "Detectors", From 2849441ee24dae22f096cfbcfa7ec6c9949d6a8c Mon Sep 17 00:00:00 2001 From: Jack Berg Date: Tue, 3 Sep 2024 15:45:54 -0500 Subject: [PATCH 2/4] .resource.attributes are name, value, type --- examples/kitchen-sink.yaml | 28 ++++++++++---- examples/sdk-config.yaml | 2 +- examples/sdk-migration-config.yaml | 2 +- schema/resource.json | 60 ++++++++++++------------------ 4 files changed, 45 insertions(+), 47 deletions(-) diff --git a/examples/kitchen-sink.yaml b/examples/kitchen-sink.yaml index 29263a5..5611b94 100644 --- a/examples/kitchen-sink.yaml +++ b/examples/kitchen-sink.yaml @@ -414,28 +414,40 @@ tracer_provider: resource: # Configure resource attributes. Entries have higher priority than entries from .resource.attributes_list. # + # Each entry must contain .name and .value, and may optionally include .type, which defaults to "string" if not set. The value must match the type. Known types include: string, bool, int, double, string_array, bool_array, int_array, double_array. + # # Environment variable: OTEL_RESOURCE_ATTRIBUTES attributes: # Configure `service.name` resource attribute # # Environment variable: OTEL_SERVICE_NAME - name: service-name - string_value: "unknown_service" + value: unknown_service # Configure other resource attributes with explicit types. + - name: string_key + value: value + type: string - name: bool_key - bool_value: true + value: true + type: bool - name: int_key - int_value: 1 + value: 1 + type: int - name: double_key - double_value: 1.1 + value: 1.1 + type: double - name: string_array_key - string_array_value: ["value1", "value2"] + value: ["value1", "value2"] + type: string_array - name: bool_array_key - bool_array_value: [ true, false ] + value: [ true, false ] + type: bool_array - name: int_array_key - int_array_value: [ 1, 2 ] + value: [ 1, 2 ] + type: int_array - name: double_array_key - double_array_value: [ 1.1, 2.2 ] + value: [ 1.1, 2.2 ] + type: double_array # Configure resource attributes. Entries have lower priority than entries from .resource.attributes. # # The value is a list of comma separated key-value pairs, matching the format of OTEL_RESOURCE_ATTRIBUTES. See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md#general-sdk-configuration for details. diff --git a/examples/sdk-config.yaml b/examples/sdk-config.yaml index a699ad1..8a61ec9 100644 --- a/examples/sdk-config.yaml +++ b/examples/sdk-config.yaml @@ -17,7 +17,7 @@ resource: attributes: # Configure `service.name` resource attribute - name: service.name - string_value: unknown_service + value: unknown_service # Configure general attribute limits. See also tracer_provider.limits, logger_provider.limits. attribute_limits: diff --git a/examples/sdk-migration-config.yaml b/examples/sdk-migration-config.yaml index aa9b5f3..b735aa0 100644 --- a/examples/sdk-migration-config.yaml +++ b/examples/sdk-migration-config.yaml @@ -48,7 +48,7 @@ resource: attributes: # Configure `service.name` resource attribute - name: service.name - string_value: ${OTEL_SERVICE_NAME:-unknown_service} + value: ${OTEL_SERVICE_NAME:-unknown_service} # Configure resource attributes. Entries have lower priority than entries from .resource.attributes. # # The value is a list of comma separated key-value pairs, matching the format of OTEL_RESOURCE_ATTRIBUTES. See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/configuration/sdk-environment-variables.md#general-sdk-configuration for details. diff --git a/schema/resource.json b/schema/resource.json index 304e155..f059603 100644 --- a/schema/resource.json +++ b/schema/resource.json @@ -26,51 +26,37 @@ "title": "AttributeNameValue", "type": "object", "additionalProperties": false, - "minProperties": 2, - "maxProperties": 2, "properties": { "name": { "type": "string" }, - "string_value": { - "type": "string" - }, - "bool_value": { - "type": "boolean" - }, - "int_value": { - "type": "integer" - }, - "double_value": { - "type": "number" - }, - "string_array_value": { - "type": "array", - "items": { - "type": "string" - } + "value": { + "oneOf": [ + {"type": "string"}, + {"type": "number"}, + {"type": "boolean"}, + {"type": "null"}, + {"type": "array", "items": {"type": "string"}}, + {"type": "array", "items": {"type": "boolean"}}, + {"type": "array", "items": {"type": "number"}} + ] }, - "bool_array_value": { - "type": "array", - "items": { - "type": "boolean" - } - }, - "int_array_value": { - "type": "array", - "items": { - "type": "integer" - } - }, - "double_array_value": { - "type": "array", - "items": { - "type": "number" - } + "type": { + "enum": [ + null, + "string", + "bool", + "int", + "double", + "string_array", + "bool_array", + "int_array", + "double_array" + ] } }, "required": [ - "name" + "name", "value" ] }, "Detectors": { From 0fe5dbf6e730fb512c38e1bb6c16730a27bbe792 Mon Sep 17 00:00:00 2001 From: Jack Berg Date: Tue, 3 Sep 2024 16:54:49 -0500 Subject: [PATCH 3/4] Fix validator shell tests --- validator/shelltests/hex_integer.yaml | 3 ++- validator/shelltests/string_for_int.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/validator/shelltests/hex_integer.yaml b/validator/shelltests/hex_integer.yaml index b742f2c..b6504f9 100644 --- a/validator/shelltests/hex_integer.yaml +++ b/validator/shelltests/hex_integer.yaml @@ -9,7 +9,8 @@ resource: # Configure resource attributes. attributes: # Configure `service.name` resource attribute - service.name: "${OTEL_SERVICE_NAME:-unknown_service}" + - name: service.name + value: "${OTEL_SERVICE_NAME:-unknown_service}" attribute_limits: # Configure max attribute value size. diff --git a/validator/shelltests/string_for_int.yaml b/validator/shelltests/string_for_int.yaml index 8aec2c1..00818bb 100644 --- a/validator/shelltests/string_for_int.yaml +++ b/validator/shelltests/string_for_int.yaml @@ -9,7 +9,8 @@ resource: # Configure resource attributes. attributes: # Configure `service.name` resource attribute - service.name: "${OTEL_SERVICE_NAME:-unknown_service}" + - name: service.name + value: "${OTEL_SERVICE_NAME:-unknown_service}" attribute_limits: # Configure max attribute value size. From 5fb350ff66fac165b27f8bfc605667e2aa5210d8 Mon Sep 17 00:00:00 2001 From: jack-berg <34418638+jack-berg@users.noreply.github.com> Date: Wed, 18 Sep 2024 10:42:14 -0500 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Alex Boten <223565+codeboten@users.noreply.github.com> --- README.md | 2 +- examples/kitchen-sink.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d6588d7..8922968 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ In instances where there is a type mismatch between the JSON schema and equivale ### Name-value pairs -When a type requires a configurable list of name-value pairs (i.e. resource attributes, HTTP headers), model using an array of objects, each with `name` and `value` properties. While an arrays of name-value objects is slightly more verbose an object where each key-value is an entry, it is preferred because: +When a type requires a configurable list of name-value pairs (i.e. resource attributes, HTTP headers), model using an array of objects, each with `name` and `value` properties. While an array of name-value objects is slightly more verbose than an object where each key-value is an entry, the latter is preferred because: * Avoids user input as keys, which ensures conformity with the [snake_case properties](#property-name-case) rule. * Allows both the names and the values to be targets for [env var substitution]. For example: diff --git a/examples/kitchen-sink.yaml b/examples/kitchen-sink.yaml index 5611b94..732f657 100644 --- a/examples/kitchen-sink.yaml +++ b/examples/kitchen-sink.yaml @@ -421,7 +421,7 @@ resource: # Configure `service.name` resource attribute # # Environment variable: OTEL_SERVICE_NAME - - name: service-name + - name: service.name value: unknown_service # Configure other resource attributes with explicit types. - name: string_key