From 118e5f36458494943419f16c0339908780d5373e Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 29 Aug 2024 11:07:52 -0700 Subject: [PATCH 01/21] feat(multi-registry): Create first draft of the spec. --- .../examples/waf-vendor-registry/metrics.yaml | 44 ++++++ .../registry/waf_attributes.yaml | 26 ++++ docs/specs/multi-registry/multi_registry.md | 140 ++++++++++++++++++ 3 files changed, 210 insertions(+) create mode 100644 docs/specs/multi-registry/examples/waf-vendor-registry/metrics.yaml create mode 100644 docs/specs/multi-registry/examples/waf-vendor-registry/registry/waf_attributes.yaml create mode 100644 docs/specs/multi-registry/multi_registry.md diff --git a/docs/specs/multi-registry/examples/waf-vendor-registry/metrics.yaml b/docs/specs/multi-registry/examples/waf-vendor-registry/metrics.yaml new file mode 100644 index 00000000..9f7a837d --- /dev/null +++ b/docs/specs/multi-registry/examples/waf-vendor-registry/metrics.yaml @@ -0,0 +1,44 @@ +# New optional section to define external registries to import in the current registry. +# Note: Unused imports will be detected by Weaver and reported as warnings by default. +imports: + # Importing an external registry involves specifying the schema URL associated with that registry. + # The current schema file structure needs to be updated, either by adding a reference to a + # self-contained and fully resolved semantic convention or by directly integrating its content into the schema. + # This approach is also well aligned with the OpenTelemetry Protocol specification, + # which already specifies a schema URL at the resource and scope levels. + - schema_url: https://opentelemetry.io/schemas/1.27.0 + # A short and local name used to reference signals and attributes from the imported registry. + name: otel + +groups: + # The following declaration overrides the definition of the `otel:trace.http.server` span group and changes + # the requirement level of few attributes to `required`. + # This override does not affect the original definition in the imported registry. + # Note: Similar to attribute references, group references cannot change the group type. + - ref: otel:trace.http.server + attributes: + # References to attributes from the imported registry must always be prefixed with the imported name and a colon. + # Local references do not require any prefix. + - ref: otel:http.request.method + requirement_level: required + - ref: otel:client.address + requirement_level: required + - ref: otel:client.port + requirement_level: required + # Locally defined attributes can be added to the imported group. + - ref: waf.action.type + requirement_level: required + + # Locally defined metric group can both extend a locally defined attribute group and + # reference imported individual attributes. + - id: metric.waf.action.hit.count + type: metric + metric_name: waf.action.hit.count + brief: "The number of times the action was hit." + instrument: counter + unit: "{count}" + stability: stable + extends: registry.waf + attributes: + - ref: otel:http.request.method + requirement_level: required diff --git a/docs/specs/multi-registry/examples/waf-vendor-registry/registry/waf_attributes.yaml b/docs/specs/multi-registry/examples/waf-vendor-registry/registry/waf_attributes.yaml new file mode 100644 index 00000000..8e861529 --- /dev/null +++ b/docs/specs/multi-registry/examples/waf-vendor-registry/registry/waf_attributes.yaml @@ -0,0 +1,26 @@ +groups: + - id: registry.waf + type: attribute_group + display_name: WAF Attributes + brief: 'A set of specific WAF attributes.' + attributes: + - id: waf.action.type + stability: stable + type: + members: + - id: alarmed + value: "ALARMED" + brief: 'ALARMED action.' + stability: stable + - id: blocked + value: "BLOCKED" + brief: 'BLOCKED action.' + stability: stable + - id: legal + value: "LEGAL" + brief: 'LEGAL action.' + stability: stable + - id: dropped + value: "DROPPED" + brief: 'DROPPED action.' + stability: stable diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md new file mode 100644 index 00000000..32f0f683 --- /dev/null +++ b/docs/specs/multi-registry/multi_registry.md @@ -0,0 +1,140 @@ +# Multi-Registry Specification Proposal + +A series of changes are proposed to support multiple semantic convention registries in OpenTelemetry. + +## Semantic Convention Registry Changes + +- A semantic convention registry can be defined by anyone (e.g. a vendor, a community, an individual, an enterprise, + etc.). +- A semantic convention registry can import one or several semantic conventions from other published registries. +- A new optional section called `imports` will be added to semantic convention file defining groups. +- The `imports` section is a list of imported semantic conventions with their schema URL and alias. +- Aliases are only visible inside the file where they are defined. +- Aliases must be unique inside the file where they are defined. +- Schema URLs are used to fetch both OTEL schema and self-contained/resolved semantic convention registries. The way a + resolved registry is linked to an OTEL schema is TBD (could be a new URL pointing to the resolved registry or an + integration inside the schema file itself). +- Unused imported registries will be detected by Weaver and reported as warnings. +- A registry can only be imported as a self-contained/resolved semantic convention registry. +- A set of core policies will be enforced by Weaver for any registry OTEL or non-OTEL in order to ensure backward + compatibility and consistency across registries (list of core policies TBD). +- Any attribute or group of a registry is a referencable entity when the registry is imported. +- Group references are now supported to support the following use cases + - A registry can add new attributes to a group defined in another registry. + - A registry can override the attributes of a group defined in another registry (e.g. `requirement_level`). + - A registry can override a subset of group fields defined in another registry (list of fields TBD). +- Overrides defined in a registry are not propagated to the imported semantic conventions. +- Overrides defined in a registry are visible to registries importing the current registry. +- Group reference can't change the type of the group (similar to attribute reference). +- References to an imported group or attribute are always prefixed with the alias of the imported semantic convention + (e.g. `ref: otel:client.address`). The colon is used as a separator between the alias and the group or attribute name. +- References to entities (groups or attributes) defined in the local registry are never prefixed. +- A locally defined group can reference an imported group in its `extends` section. +- A locally defined group can reference an imported attribute in its `attributes` section. + +Note: A resolved semantic convention registry is self-contained and does not contain any complex constructs like +`imports`, `ref`, `extends`, etc. Their structure are less subject to change, making them good candidate for +publication, and making them easier to consume. + +Wonkiness to remove from the existing semantic convention schema: + +- Rename `metric_name` to `name` in the `metric` group for consistency with the other groups. +- Probably more TBD. + +Things we should avoid/minimize: + +- Name squatting: By relying on local aliases and URL schema, we are not relying on a naming convention approach based + on company names, etc. This should minimize the risk of name squatting. +- Name inconsistency: By enforcing core policies, we should minimize the risk of name inconsistency across registries. + +Alternatives: + +- We could make alias optional in the `imports` section. To do so, we would need to rely on Weaver to automatically + detect entity IDs which are defined both in the local semconv file and the imported registry. When a such conflict is + detected, Weaver will report an error and asl the user to define an alias for the imported registry. This approach + could be supported in the future if we see a need for it. + +Open Questions: +- Do we allow different versions of the same registry to be imported into different semantic convention files of the + same registry? +- Is there a relationship to define between the instrumentation scope name and version and the semantic convention + registry? + +## OpenTelemetry Schema Changes + +The OpenTelemetry schema file structure must be updated to either include the URL to a self-contained/resolved +semantic convention or to include the resolved registry itself. + +## Weaver Changes + +The following changes are proposed to Weaver: + +- Weaver must be able to support any operation on any semantic convention registry (check, resolve, generate, search, + ...). +- The command `weaver registry generate` must allow the generation of the referenced entities that belong to the + imported semantic convention registries or optionally the generation of all the entities of the imported registries. +- Extend the `--templates` parameter to allow git URL so OTEL templates (or community-based templates) can be reused + for any registry. +- Extend the `--policies` parameter to allow git URL so OTEL policies (or community-based policies) can be reused for + any registry. +- Add a step before the resolution process to build a deduplicated list of the imported registries. Download the + corresponding resolved registries and create a mapping url to resolved registry that will be used during the + resolution process to resolve the references to the imported registries (and to detect clashes between local and + imported IDs if aliases are not used). +- More TBD. + +Open Questions: +- Is a resolved registry contain any trace of the imported registries? + +## Protocol Changes + +No impact on OTLP and OTAP. + +A `schema_url` field is already present at the resource and scope levels. + +Ideally any component of the observability pipeline should be able to fetch the resolved semantic convention registry +just by knowing the schema URL of any resource or instrumentation scope. + +## OpenTelemetry SDKs Changes + +TBD + +Open Questions: + +- Can we enforce the presence of the schema URL in the resource and instrumentation scope? +- How do we convey the schema URL to the SDKs? Could that be part of the codegen done by Weaver? + +## Resolved Semantic Convention Registry Format + +The following properties are proposed for a resolved semantic convention registry: + +- Resolved semantic convention registry must be easy to consume and to publish + - Accessible via a URL. + - Self-contained, i.e. a single file. + - No `ref`, no `extends`, no `imports`, no alias, no other complex constructs. + - Yaml or JSON format so resolved registries can be easily consumed by any tool. +- Optional lineage section. + +The content of a resolved semantic convention registry depends on the: +- The semantic convention files composing the registry to resolve. +- The semantic convention registries imported. +- The configuration specified during the resolution process. + - Include all the entities of the imported registries + - Include only the referenced entities of the imported registries. + +Open Questions: + +- Do we keep track of the imported registries in the resolved registry? If yes, how? Lineage? +- Can we leverage the attribute deduplication mechanism to simplify the merging of imported registries? +- Can we extend the deduplication mechanism to the signals? +- Materialized resolved registry (what see the jq, template and policy engines) vs Published resolved registry. + - Materialized Resolved Registry: This is what the jq, template and policy engines see. In this format there are + no deduplication of declaration. This format is not meant to be published. + - Published Resolved Registry: In this format, the deduplication of declaration is automatically done by Weaver. + This format is meant to be published. + +## Priorities + +TBD + +- [Not Final] Start with attributes \ No newline at end of file From dba1a2c9389f1b3e4d33879d283337e98346d534 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 29 Aug 2024 14:45:17 -0700 Subject: [PATCH 02/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 61 ++++++++++++++++++- .../acme-http-server-lib/metric.yaml | 10 +++ .../registry/attributes.yaml | 25 ++++++++ .../acme-http-server-lib/trace.yaml | 12 ++++ .../multi-registry/registries/app/app.yaml | 28 +++++++++ .../registries/app/registry/attributes.yaml | 11 ++++ .../waf-vendor}/registry/waf_attributes.yaml | 0 .../waf-vendor/trace.yaml} | 16 +---- .../registries/waf-vendor/waf_action.yaml | 28 +++++++++ 9 files changed, 173 insertions(+), 18 deletions(-) create mode 100644 docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml create mode 100644 docs/specs/multi-registry/registries/acme-http-server-lib/registry/attributes.yaml create mode 100644 docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml create mode 100644 docs/specs/multi-registry/registries/app/app.yaml create mode 100644 docs/specs/multi-registry/registries/app/registry/attributes.yaml rename docs/specs/multi-registry/{examples/waf-vendor-registry => registries/waf-vendor}/registry/waf_attributes.yaml (100%) rename docs/specs/multi-registry/{examples/waf-vendor-registry/metrics.yaml => registries/waf-vendor/trace.yaml} (77%) create mode 100644 docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 32f0f683..fe06ebfa 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -1,11 +1,59 @@ # Multi-Registry Specification Proposal +Status: Work in Progress + +## Introduction + A series of changes are proposed to support multiple semantic convention registries in OpenTelemetry. +Note: This proposal tries to describe the overall changes needed to support the vision but that does not mean that we +can't introduce the changes incrementally. For example, we could start by only supporting attributes and then +progressively add support for metrics, events, spans, etc. + +## Use Case Example + +The following use case doesn't pretend to be exhaustive but it should give a good idea of the kind of multi-registry +use cases we'd like to support. + +[Diagram](TBD) + +Actors: +- OTEL: The OpenTelemetry project that publishes the OTEL semantic convention registry. So the community can discover + and use the signals defined by OTEL. +- WAF Vendor: A vendor of a Web Application Firewall (WAF) that wants to publish a semantic convention registry for + their product. So their customers can discover and use their signals. +- Cloud Vendor: A cloud provider that wants to publish a semantic convention registry for their services. +- OSS lib author: An author of an OSS library that wants to publish a semantic convention registry for his library. +- Enterprise App: An enterprise that wants to use the concept of semantic convention registry for their internal use. + +ToDo +- Actors +- Diagram +- Narrative +- Value Proposition for the different actors + +Benefits/Value Proposition per actor: +- OTEL: + - Can focus on the core signals and delegate the definition of more specific signals to the community. + - Can leverage the community to define more specific signals. + - Can leverage the community to define signals for specific domains. + - Can leverage the community to define signals for specific vendors. +- WAF Vendor: + - Reuse common signals defined by OTEL. So overall their customer experiences will be more consistent across + different vendors. + - Make it easier for their customers to discover and use their custom signals. +- Cloud Vendor: +- OSS lib author: +- App Developer: + ## Semantic Convention Registry Changes -- A semantic convention registry can be defined by anyone (e.g. a vendor, a community, an individual, an enterprise, - etc.). +- A semantic convention registry can be defined by anyone. For all the following examples, registry authors can + extend or amend the OTEL registry or create their own attributes and groups: + - A vendor publishes a semconv registry for their products, so their customers can discover and use their signals + - A community publishes a semconv registry for a specific domain that is too specific to be included in the OTEL registry + - An individual publishes a semconv registry for their own OSS library or project + - An enterprise creates internal semconv registries for their internal use - A semantic convention registry can import one or several semantic conventions from other published registries. - A new optional section called `imports` will be added to semantic convention file defining groups. - The `imports` section is a list of imported semantic conventions with their schema URL and alias. @@ -24,7 +72,8 @@ A series of changes are proposed to support multiple semantic convention registr - A registry can override the attributes of a group defined in another registry (e.g. `requirement_level`). - A registry can override a subset of group fields defined in another registry (list of fields TBD). - Overrides defined in a registry are not propagated to the imported semantic conventions. -- Overrides defined in a registry are visible to registries importing the current registry. +- Overrides defined in a registry are visible to registries importing the current registry. These attributes and groups + overrides are re-exported with some transformations by the local registry. - Group reference can't change the type of the group (similar to attribute reference). - References to an imported group or attribute are always prefixed with the alias of the imported semantic convention (e.g. `ref: otel:client.address`). The colon is used as a separator between the alias and the group or attribute name. @@ -122,6 +171,12 @@ The content of a resolved semantic convention registry depends on the: - Include all the entities of the imported registries - Include only the referenced entities of the imported registries. +More specifically, a resolved semantic convention registry contains: +- All the attributes registry specified locally in the semantic convention registry. +- All the groups specified locally in the semantic convention registry. +- All the attributes and groups imported but not re-exported locally are not included in the resolved registry. A + re-exported entity is an entity that is imported and referenced in the local registry with some overriding. + Open Questions: - Do we keep track of the imported registries in the resolved registry? If yes, how? Lineage? diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml new file mode 100644 index 00000000..ca475e17 --- /dev/null +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml @@ -0,0 +1,10 @@ +imports: + - schema_url: https://opentelemetry.io/schemas/1.27.0 + name: otel + +groups: + - ref: otel:http.server.request.duration + attributes: + - ref: otel:thread.id + requirement_level: required + - ref: route.name diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/registry/attributes.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/registry/attributes.yaml new file mode 100644 index 00000000..23dba82d --- /dev/null +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/registry/attributes.yaml @@ -0,0 +1,25 @@ +groups: + - id: registry.acme_http_server + type: attribute_group + display_name: Custom attributes for ACME HTTP Server + brief: 'A set of specific attributes for ACME HTTP Server.' + attributes: + - id: route.name + brief: 'The name of the route.' + stability: stable + requirement_level: required + type: string + - id: rate_limiter.type + brief: 'The type of rate limiter.' + stability: stable + requirement_level: required + type: + members: + - id: sliding + value: "SLIDING" + brief: 'SLIDING policy.' + stability: stable + - id: token + value: "TOKEN" + brief: 'TOKEN policy.' + stability: stable diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml new file mode 100644 index 00000000..8bbd282b --- /dev/null +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml @@ -0,0 +1,12 @@ +imports: + - schema_url: https://opentelemetry.io/schemas/1.27.0 + name: otel + +groups: + - ref: otel:trace.http.server + brief: 'Trace reported by ACME HTTP Server library.' + attributes: + - ref: otel:thread.id + requirement_level: required + - ref: route.name + - ref: rate_limiter.type \ No newline at end of file diff --git a/docs/specs/multi-registry/registries/app/app.yaml b/docs/specs/multi-registry/registries/app/app.yaml new file mode 100644 index 00000000..6ac70b3a --- /dev/null +++ b/docs/specs/multi-registry/registries/app/app.yaml @@ -0,0 +1,28 @@ +imports: + - schema_url: https://opentelemetry.io/schemas/1.27.0 + name: otel + - schema_url: https://github.com/acme-http-server-lib/schema_v0.7.1.yaml + name: acme + +groups: + - id: metric.endpoint.request.count + type: metric + metric_name: endpoint.request.count + brief: "The number of requests received by the endpoint." + instrument: counter + unit: "{count}" + stability: stable + attributes: + - ref: otel:user.roles + requirement_level: required + - ref: otel:client.address + requirement_level: required + - ref: otel:client.port + requirement_level: required + - ref: otel:http.request.method + requirement_level: required + - ref: otel:thread.id + requirement_level: required + - ref: acme:route.name + - ref: tenant.id + diff --git a/docs/specs/multi-registry/registries/app/registry/attributes.yaml b/docs/specs/multi-registry/registries/app/registry/attributes.yaml new file mode 100644 index 00000000..0d73d08e --- /dev/null +++ b/docs/specs/multi-registry/registries/app/registry/attributes.yaml @@ -0,0 +1,11 @@ +groups: + - id: registry.app + type: attribute_group + display_name: Enterprise Application Attributes + brief: 'A set of enterprise application attributes.' + attributes: + - id: tenant.id + brief: 'The customer tenant id.' + stability: stable + requirement_level: required + type: string diff --git a/docs/specs/multi-registry/examples/waf-vendor-registry/registry/waf_attributes.yaml b/docs/specs/multi-registry/registries/waf-vendor/registry/waf_attributes.yaml similarity index 100% rename from docs/specs/multi-registry/examples/waf-vendor-registry/registry/waf_attributes.yaml rename to docs/specs/multi-registry/registries/waf-vendor/registry/waf_attributes.yaml diff --git a/docs/specs/multi-registry/examples/waf-vendor-registry/metrics.yaml b/docs/specs/multi-registry/registries/waf-vendor/trace.yaml similarity index 77% rename from docs/specs/multi-registry/examples/waf-vendor-registry/metrics.yaml rename to docs/specs/multi-registry/registries/waf-vendor/trace.yaml index 9f7a837d..5fcf399b 100644 --- a/docs/specs/multi-registry/examples/waf-vendor-registry/metrics.yaml +++ b/docs/specs/multi-registry/registries/waf-vendor/trace.yaml @@ -27,18 +27,4 @@ groups: requirement_level: required # Locally defined attributes can be added to the imported group. - ref: waf.action.type - requirement_level: required - - # Locally defined metric group can both extend a locally defined attribute group and - # reference imported individual attributes. - - id: metric.waf.action.hit.count - type: metric - metric_name: waf.action.hit.count - brief: "The number of times the action was hit." - instrument: counter - unit: "{count}" - stability: stable - extends: registry.waf - attributes: - - ref: otel:http.request.method - requirement_level: required + requirement_level: required \ No newline at end of file diff --git a/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml b/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml new file mode 100644 index 00000000..311683e9 --- /dev/null +++ b/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml @@ -0,0 +1,28 @@ +# New optional section to define external registries to import in the current registry. +# Note: Unused imports will be detected by Weaver and reported as warnings by default. +imports: + # Importing an external registry involves specifying the schema URL associated with that registry. + # The current schema file structure needs to be updated, either by adding a reference to a + # self-contained and fully resolved semantic convention or by directly integrating its content into the schema. + # This approach is also well aligned with the OpenTelemetry Protocol specification, + # which already specifies a schema URL at the resource and scope levels. + - schema_url: https://opentelemetry.io/schemas/1.27.0 + # A short and local name used to reference signals and attributes from the imported registry. + # We use `my_otel` here just to show that it can be any name and that different aliases can be used for the same + # schema URL across different semconv files in the same registry. + name: my_otel + +groups: + # Locally defined metric group can both extend a locally defined attribute group and + # reference imported individual attributes. + - id: metric.waf.action.hit.count + type: metric + metric_name: waf.action.hit.count + brief: "The number of times the action was hit." + instrument: counter + unit: "{count}" + stability: stable + extends: registry.waf + attributes: + - ref: my_otel:http.request.method + requirement_level: required From 6d08430f82a65bfc9ebbeb8d1fd53884a2990352 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 29 Aug 2024 15:53:34 -0700 Subject: [PATCH 03/21] feat(multi-registry): Update draft of multi-registry spec. --- .../images/multi_registry_use_case.svg | 1 + docs/specs/multi-registry/multi_registry.md | 39 +++++++++++++++++-- .../acme-http-server-lib/metric.yaml | 6 +++ .../registry/attributes.yaml | 3 ++ .../acme-http-server-lib/trace.yaml | 11 ++++++ .../multi-registry/registries/app/app.yaml | 13 ++++--- .../waf-vendor/registry/waf_attributes.yaml | 4 ++ .../registries/waf-vendor/waf_action.yaml | 8 ++++ 8 files changed, 75 insertions(+), 10 deletions(-) create mode 100644 docs/specs/multi-registry/images/multi_registry_use_case.svg diff --git a/docs/specs/multi-registry/images/multi_registry_use_case.svg b/docs/specs/multi-registry/images/multi_registry_use_case.svg new file mode 100644 index 00000000..a460987c --- /dev/null +++ b/docs/specs/multi-registry/images/multi_registry_use_case.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index fe06ebfa..9019e5bf 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -1,4 +1,4 @@ -# Multi-Registry Specification Proposal +# Multi-Registry - Draft Proposal Status: Work in Progress @@ -15,7 +15,12 @@ progressively add support for metrics, events, spans, etc. The following use case doesn't pretend to be exhaustive but it should give a good idea of the kind of multi-registry use cases we'd like to support. -[Diagram](TBD) +The following diagram shows a small but concrete example of how multiple semantic convention registries could be used +together. + +![Multi-Registry Use Case](images/multi_registry_use_case.svg) + +The color-coding within the signal descriptions represents where the corresponding definition comes from. Actors: - OTEL: The OpenTelemetry project that publishes the OTEL semantic convention registry. So the community can discover @@ -45,11 +50,37 @@ Benefits/Value Proposition per actor: - Cloud Vendor: - OSS lib author: - App Developer: +- App SRE: + +## Design Principles + +- **Independent Ownership**: Any individual or organization should be able to create and maintain a semantic convention + registry independently, without requiring active coordination with the OTEL project. +- **Registry Accessibility**: Semantic convention registries can be either private or public, allowing flexibility + based on the needs of the owner. +- **Community Support Tools**: The OTEL project will provide and maintain tools to assist the community in creating, + validating, resolving, and publishing semantic convention registries (i.e. Weaver tool). +- **Core Policy Enforcement**: The OTEL project will establish and enforce a set of core policies that all semantic + convention registries must adhere to, ensuring consistency and reliability. +- **Cross-Registry References**: References between different semantic convention registries should be supported, + facilitating interoperability and integration across various registries. +- **Circular Reference Handling**: Circular references between semantic convention registries must be detected, + reported, and rejected to prevent conflicts and maintain the integrity of the system. + +## Design Principles + +- Anyone should be able to own a semantic convention registry without any active coordination mechanism with OTEL. +- A semantic convention registry can be private or public. +- The OTEL project will maintain a tool to help the community to create, validate, resolve, and publish semantic + convention registries. +- The OTEL project will maintain a set of core policies that must be enforced on any semantic convention registry. +- References between semantic conventions registries should be supported. +- Circular references between semantic conventions registries should be detected, reported, and rejected. ## Semantic Convention Registry Changes -- A semantic convention registry can be defined by anyone. For all the following examples, registry authors can - extend or amend the OTEL registry or create their own attributes and groups: +- A semantic convention registry can be defined by anyone and without any kind of coordination with OTEL. For all the + following examples, registry authors can extend or amend the OTEL registry or create their own attributes and groups: - A vendor publishes a semconv registry for their products, so their customers can discover and use their signals - A community publishes a semconv registry for a specific domain that is too specific to be included in the OTEL registry - An individual publishes a semconv registry for their own OSS library or project diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml index ca475e17..834b610a 100644 --- a/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml @@ -5,6 +5,12 @@ imports: groups: - ref: otel:http.server.request.duration attributes: + - ref: otel:http.request.method + requirement_level: required + - ref: otel:client.address + requirement_level: required + - ref: otel:client.port + requirement_level: required - ref: otel:thread.id requirement_level: required - ref: route.name diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/registry/attributes.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/registry/attributes.yaml index 23dba82d..4dceacf4 100644 --- a/docs/specs/multi-registry/registries/acme-http-server-lib/registry/attributes.yaml +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/registry/attributes.yaml @@ -4,6 +4,9 @@ groups: display_name: Custom attributes for ACME HTTP Server brief: 'A set of specific attributes for ACME HTTP Server.' attributes: + # The ACME HTTP server author does not use the standard `http.route` for an unknown reason. + # Unfortunately, we cannot avoid this. In the future, we could enhance Weaver to detect these + # types of similarities and report them as advice messages. - id: route.name brief: 'The name of the route.' stability: stable diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml index 8bbd282b..6d803459 100644 --- a/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml @@ -6,7 +6,18 @@ groups: - ref: otel:trace.http.server brief: 'Trace reported by ACME HTTP Server library.' attributes: + # The `requirement_level` of the following attributes is promoted to `required` + # Note: These 3 attributes are already attached to `trace.http.server` + - ref: otel:http.request.method + requirement_level: required + - ref: otel:client.address + requirement_level: required + - ref: otel:client.port + requirement_level: required + # A standard OTEL attribute is attached to the `trace.http.server` + # This attribute is not part of the OTEL definition of a `trace.http.server` - ref: otel:thread.id requirement_level: required + # Two locally defined attributes (see the attribute registry) - ref: route.name - ref: rate_limiter.type \ No newline at end of file diff --git a/docs/specs/multi-registry/registries/app/app.yaml b/docs/specs/multi-registry/registries/app/app.yaml index 6ac70b3a..c985e593 100644 --- a/docs/specs/multi-registry/registries/app/app.yaml +++ b/docs/specs/multi-registry/registries/app/app.yaml @@ -5,12 +5,10 @@ imports: name: acme groups: - - id: metric.endpoint.request.count - type: metric - metric_name: endpoint.request.count - brief: "The number of requests received by the endpoint." - instrument: counter - unit: "{count}" + - id: endpoint.error + type: event + name: endpoint.error + brief: "An endpoint error event." stability: stable attributes: - ref: otel:user.roles @@ -21,8 +19,11 @@ groups: requirement_level: required - ref: otel:http.request.method requirement_level: required + - ref: otel:error.type + requirement_level: required - ref: otel:thread.id requirement_level: required - ref: acme:route.name - ref: tenant.id + diff --git a/docs/specs/multi-registry/registries/waf-vendor/registry/waf_attributes.yaml b/docs/specs/multi-registry/registries/waf-vendor/registry/waf_attributes.yaml index 8e861529..93ac315e 100644 --- a/docs/specs/multi-registry/registries/waf-vendor/registry/waf_attributes.yaml +++ b/docs/specs/multi-registry/registries/waf-vendor/registry/waf_attributes.yaml @@ -4,7 +4,11 @@ groups: display_name: WAF Attributes brief: 'A set of specific WAF attributes.' attributes: + - id: waf.signature + brief: "The WAF signature" + type: string - id: waf.action.type + brief: "The WAF action type." stability: stable type: members: diff --git a/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml b/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml index 311683e9..7b8b0097 100644 --- a/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml +++ b/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml @@ -26,3 +26,11 @@ groups: attributes: - ref: my_otel:http.request.method requirement_level: required + - ref: my_otel:client.address + requirement_level: required + - ref: my_otel:client.port + requirement_level: required + - ref: waf.signature + requirement_level: required + - ref: waf.action.type + requirement_level: required From f075e431e2ed83bb5e90e82a50ec1548dd75c1f7 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 29 Aug 2024 15:55:49 -0700 Subject: [PATCH 04/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 9019e5bf..98fce6a0 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -27,17 +27,10 @@ Actors: and use the signals defined by OTEL. - WAF Vendor: A vendor of a Web Application Firewall (WAF) that wants to publish a semantic convention registry for their product. So their customers can discover and use their signals. -- Cloud Vendor: A cloud provider that wants to publish a semantic convention registry for their services. - OSS lib author: An author of an OSS library that wants to publish a semantic convention registry for his library. - Enterprise App: An enterprise that wants to use the concept of semantic convention registry for their internal use. -ToDo -- Actors -- Diagram -- Narrative -- Value Proposition for the different actors - -Benefits/Value Proposition per actor: +Benefits/Value Proposition per actor [TODO]: - OTEL: - Can focus on the core signals and delegate the definition of more specific signals to the community. - Can leverage the community to define more specific signals. @@ -47,7 +40,6 @@ Benefits/Value Proposition per actor: - Reuse common signals defined by OTEL. So overall their customer experiences will be more consistent across different vendors. - Make it easier for their customers to discover and use their custom signals. -- Cloud Vendor: - OSS lib author: - App Developer: - App SRE: @@ -67,16 +59,6 @@ Benefits/Value Proposition per actor: - **Circular Reference Handling**: Circular references between semantic convention registries must be detected, reported, and rejected to prevent conflicts and maintain the integrity of the system. -## Design Principles - -- Anyone should be able to own a semantic convention registry without any active coordination mechanism with OTEL. -- A semantic convention registry can be private or public. -- The OTEL project will maintain a tool to help the community to create, validate, resolve, and publish semantic - convention registries. -- The OTEL project will maintain a set of core policies that must be enforced on any semantic convention registry. -- References between semantic conventions registries should be supported. -- Circular references between semantic conventions registries should be detected, reported, and rejected. - ## Semantic Convention Registry Changes - A semantic convention registry can be defined by anyone and without any kind of coordination with OTEL. For all the From 7aa912958e30bfb1d49a283c6ba2e2fdff726ccb Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 29 Aug 2024 16:48:22 -0700 Subject: [PATCH 05/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 73 ++++++++++++--------- 1 file changed, 43 insertions(+), 30 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 98fce6a0..2341d751 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -6,43 +6,56 @@ Status: Work in Progress A series of changes are proposed to support multiple semantic convention registries in OpenTelemetry. -Note: This proposal tries to describe the overall changes needed to support the vision but that does not mean that we -can't introduce the changes incrementally. For example, we could start by only supporting attributes and then -progressively add support for metrics, events, spans, etc. +> Note: This proposal aims to describe the overall changes needed to realize this vision. However, +> this does not mean that we cannot introduce the changes incrementally. For example, we could start +> by supporting attributes only and then progressively add support for metrics, events, spans, etc. ## Use Case Example -The following use case doesn't pretend to be exhaustive but it should give a good idea of the kind of multi-registry -use cases we'd like to support. +The following use case is not intended to be exhaustive, but it should provide a good idea of the +types of multi-registry scenarios we aim to support. -The following diagram shows a small but concrete example of how multiple semantic convention registries could be used -together. +The diagram below illustrates a small but concrete example of how multiple semantic convention +registries could be used together. ![Multi-Registry Use Case](images/multi_registry_use_case.svg) -The color-coding within the signal descriptions represents where the corresponding definition comes from. - -Actors: -- OTEL: The OpenTelemetry project that publishes the OTEL semantic convention registry. So the community can discover - and use the signals defined by OTEL. -- WAF Vendor: A vendor of a Web Application Firewall (WAF) that wants to publish a semantic convention registry for - their product. So their customers can discover and use their signals. -- OSS lib author: An author of an OSS library that wants to publish a semantic convention registry for his library. -- Enterprise App: An enterprise that wants to use the concept of semantic convention registry for their internal use. - -Benefits/Value Proposition per actor [TODO]: -- OTEL: - - Can focus on the core signals and delegate the definition of more specific signals to the community. - - Can leverage the community to define more specific signals. - - Can leverage the community to define signals for specific domains. - - Can leverage the community to define signals for specific vendors. -- WAF Vendor: - - Reuse common signals defined by OTEL. So overall their customer experiences will be more consistent across - different vendors. - - Make it easier for their customers to discover and use their custom signals. -- OSS lib author: -- App Developer: -- App SRE: +The color-coding within the signal descriptions indicates the provenance of the corresponding definition. + +**Actors and Their Benefits** + +1. **OpenTelemetry:** + - **Value Proposition:** + - OTEL can focus on defining core signals while delegating the creation of more specific signals + to the community at scale. + - OTEL establishes the foundation for developing new uses and tools in the observability ecosystem. + +2. **Vendor:** + - **Value Proposition:** + - By publishing their own semantic convention registry, vendors make it easier for their customers + to discover and effectively use custom signals specific to their products. + - Vendors can reuse common signals defined by OTEL, ensuring consistency in customer experiences. This + strategy enhances the interoperability of their products within the larger observability ecosystem. + +3. **OSS Library Author:** + - **Value Proposition:** + - The OSS library author can reuse OTEL-defined attributes and signals, integrating them with custom + signals tailored to their library. + - Publishing a semantic convention registry for the library simplifies the integration process for + developers, making it easier to adopt and use the library in a consistent and standardized way. + - This approach also increases the attractiveness, visibility and usability of the library within + the community. + +4. **Enterprise Application:** + - **Value Proposition:** + - Enterprises can leverage the concept of semantic convention registries to import external registries + and simplify their observability integration. + - By creating internal registries, enterprises can define custom signals that align with their specific + needs and share these across teams and products, fostering internal consistency. + - This capability enhances collaboration and streamlines the observability practices within the organization. + +By supporting these actors and their specific use cases, the multi-registry approach enables a flexible, +community-driven model for defining and using semantic conventions across diverse domains and applications. ## Design Principles From e2bd4e04c9f617828385ad8f77eddf1bba8b3694 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 29 Aug 2024 17:11:42 -0700 Subject: [PATCH 06/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 84 +++++++++++---------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 2341d751..e7597bd1 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -65,8 +65,8 @@ community-driven model for defining and using semantic conventions across divers based on the needs of the owner. - **Community Support Tools**: The OTEL project will provide and maintain tools to assist the community in creating, validating, resolving, and publishing semantic convention registries (i.e. Weaver tool). -- **Core Policy Enforcement**: The OTEL project will establish and enforce a set of core policies that all semantic - convention registries must adhere to, ensuring consistency and reliability. +- **Core Policy Enforcement**: The OTEL project will establish and enforce a set of core policies (e.g. backward + compatibility policies) that all semantic convention registries must adhere to, ensuring consistency and reliability. - **Cross-Registry References**: References between different semantic convention registries should be supported, facilitating interoperability and integration across various registries. - **Circular Reference Handling**: Circular references between semantic convention registries must be detected, @@ -74,64 +74,70 @@ community-driven model for defining and using semantic conventions across divers ## Semantic Convention Registry Changes -- A semantic convention registry can be defined by anyone and without any kind of coordination with OTEL. For all the - following examples, registry authors can extend or amend the OTEL registry or create their own attributes and groups: - - A vendor publishes a semconv registry for their products, so their customers can discover and use their signals - - A community publishes a semconv registry for a specific domain that is too specific to be included in the OTEL registry - - An individual publishes a semconv registry for their own OSS library or project - - An enterprise creates internal semconv registries for their internal use +- A semantic convention registry can be defined by anyone, without requiring any active coordination with OTEL. + For all the following examples, registry authors can extend or amend the OTEL registry or create their own + attributes and groups (non exhaustive list): + - A vendor publishes a semantic convention registry for their products, allowing their customers to discover + and use their signals. + - A community publishes a semantic convention registry for a specific domain that is too specialized to be + included in the OTEL registry. + - An individual publishes a semantic convention registry for their own OSS library or project. + - An enterprise creates internal semantic convention registries for internal use. - A semantic convention registry can import one or several semantic conventions from other published registries. -- A new optional section called `imports` will be added to semantic convention file defining groups. -- The `imports` section is a list of imported semantic conventions with their schema URL and alias. -- Aliases are only visible inside the file where they are defined. -- Aliases must be unique inside the file where they are defined. -- Schema URLs are used to fetch both OTEL schema and self-contained/resolved semantic convention registries. The way a - resolved registry is linked to an OTEL schema is TBD (could be a new URL pointing to the resolved registry or an - integration inside the schema file itself). +- A new optional section called `imports` will be added to the semantic convention file defining groups. +- The `imports` section is a list of imported semantic conventions with their schema URLs and aliases. +- Aliases are only visible within the file where they are defined. +- Aliases must be unique within the file where they are defined. +- Schema URLs are used to fetch both OTEL schema and self-contained/resolved semantic convention registries. + The way a resolved registry is linked to an OTEL schema is TBD (it could be a new URL pointing to the resolved + registry or an integration within the schema file itself). - Unused imported registries will be detected by Weaver and reported as warnings. - A registry can only be imported as a self-contained/resolved semantic convention registry. -- A set of core policies will be enforced by Weaver for any registry OTEL or non-OTEL in order to ensure backward +- A set of core policies will be enforced by Weaver for any registry, OTEL or non-OTEL, to ensure backward compatibility and consistency across registries (list of core policies TBD). -- Any attribute or group of a registry is a referencable entity when the registry is imported. -- Group references are now supported to support the following use cases +- Any attribute or group in a registry is a referencable entity when the registry is imported. +- Group references are now supported to address the following use cases: - A registry can add new attributes to a group defined in another registry. - - A registry can override the attributes of a group defined in another registry (e.g. `requirement_level`). - - A registry can override a subset of group fields defined in another registry (list of fields TBD). + - A registry can override the attributes of a group defined in another registry (e.g., `requirement_level`). + - A registry can only override a subset of group fields defined in another registry (list of fields TBD). - Overrides defined in a registry are not propagated to the imported semantic conventions. -- Overrides defined in a registry are visible to registries importing the current registry. These attributes and groups - overrides are re-exported with some transformations by the local registry. -- Group reference can't change the type of the group (similar to attribute reference). -- References to an imported group or attribute are always prefixed with the alias of the imported semantic convention - (e.g. `ref: otel:client.address`). The colon is used as a separator between the alias and the group or attribute name. +- Overrides defined in a registry are visible to registries importing the current registry. These attribute + and group overrides are re-exported with some transformations by the local registry. +- A group reference cannot change the type of the imported group (similar to attribute references). +- References to an imported group or attribute are always prefixed with the alias of the imported semantic + convention (e.g., `ref: otel:client.address`). The colon is used as a separator between the alias and the + group or attribute name. - References to entities (groups or attributes) defined in the local registry are never prefixed. - A locally defined group can reference an imported group in its `extends` section. - A locally defined group can reference an imported attribute in its `attributes` section. -Note: A resolved semantic convention registry is self-contained and does not contain any complex constructs like -`imports`, `ref`, `extends`, etc. Their structure are less subject to change, making them good candidate for -publication, and making them easier to consume. +> Note: The JSON Schema for the semantic convention registry must be updated to reflect these changes. + +> Note: A resolved semantic convention registry is self-contained and does not include any complex constructs +> like `imports`, `ref`, `extends`, etc. Their **structure is less subject to change**, making them good +> candidates for publication and easier to consume. Wonkiness to remove from the existing semantic convention schema: -- Rename `metric_name` to `name` in the `metric` group for consistency with the other groups. +- Rename `metric_name` to `name` in the `metric` group for consistency with other groups. - Probably more TBD. -Things we should avoid/minimize: +Things to avoid/minimize: -- Name squatting: By relying on local aliases and URL schema, we are not relying on a naming convention approach based - on company names, etc. This should minimize the risk of name squatting. -- Name inconsistency: By enforcing core policies, we should minimize the risk of name inconsistency across registries. +- **Name squatting**: By relying on local aliases and URL schema, we reduce the risk of name squatting, as + the naming convention is not based on company names that are not necessarily unique and are complex to control. +- **Name inconsistency**: By enforcing core policies, we minimize the risk of name inconsistency across registries. Alternatives: -- We could make alias optional in the `imports` section. To do so, we would need to rely on Weaver to automatically - detect entity IDs which are defined both in the local semconv file and the imported registry. When a such conflict is - detected, Weaver will report an error and asl the user to define an alias for the imported registry. This approach - could be supported in the future if we see a need for it. +- We could make aliases optional in the imports section. To do so, we would need to rely on Weaver to automatically + detect entity IDs that are defined both in the local semantic convention file and the imported registry. When such + a conflict is detected, Weaver will report an error and ask the user to define an alias for the imported registry. + This approach could be supported in the future if the need arises. Open Questions: -- Do we allow different versions of the same registry to be imported into different semantic convention files of the - same registry? +- Do we allow different versions of the same registry to be imported into different semantic convention files of + the same registry? - Is there a relationship to define between the instrumentation scope name and version and the semantic convention registry? From 97ed99859e07ff40613d0277bc24d005d36af6b4 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Thu, 29 Aug 2024 17:22:02 -0700 Subject: [PATCH 07/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index e7597bd1..162f6f3b 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -22,6 +22,8 @@ registries could be used together. The color-coding within the signal descriptions indicates the provenance of the corresponding definition. +The YAML semantic convention files corresponding to the diagram are available in this [folder](registries). + **Actors and Their Benefits** 1. **OpenTelemetry:** @@ -136,6 +138,8 @@ Alternatives: This approach could be supported in the future if the need arises. Open Questions: +- How do we handle experimental entities in the context of multi-registry? Do we allow reference of experimental + entities across registries? - Do we allow different versions of the same registry to be imported into different semantic convention files of the same registry? - Is there a relationship to define between the instrumentation scope name and version and the semantic convention @@ -162,6 +166,7 @@ The following changes are proposed to Weaver: corresponding resolved registries and create a mapping url to resolved registry that will be used during the resolution process to resolve the references to the imported registries (and to detect clashes between local and imported IDs if aliases are not used). +- Weaver must be able to detect newer versions of imported registries and report them as warnings. - More TBD. Open Questions: From 2050cc7bbed1abd9dc3ff2838bfc69ec33d91cf0 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 30 Aug 2024 11:23:15 -0700 Subject: [PATCH 08/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 162f6f3b..a8224adc 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -71,20 +71,24 @@ community-driven model for defining and using semantic conventions across divers compatibility policies) that all semantic convention registries must adhere to, ensuring consistency and reliability. - **Cross-Registry References**: References between different semantic convention registries should be supported, facilitating interoperability and integration across various registries. +- **Conflict Avoidance through Scoping**: Without a scoping mechanism, we can't ensure that a signal or an attribute + name will never conflict with a future declaration in another registry. - **Circular Reference Handling**: Circular references between semantic convention registries must be detected, reported, and rejected to prevent conflicts and maintain the integrity of the system. ## Semantic Convention Registry Changes -- A semantic convention registry can be defined by anyone, without requiring any active coordination with OTEL. - For all the following examples, registry authors can extend or amend the OTEL registry or create their own - attributes and groups (non exhaustive list): +With the following changes, a semantic convention registry can be defined by anyone, without requiring any active +coordination with OTEL. For all the following examples, registry authors can extend or amend the OTEL registry or +create their own attributes and groups (non exhaustive list): - A vendor publishes a semantic convention registry for their products, allowing their customers to discover and use their signals. - A community publishes a semantic convention registry for a specific domain that is too specialized to be included in the OTEL registry. - An individual publishes a semantic convention registry for their own OSS library or project. - An enterprise creates internal semantic convention registries for internal use. + +Changes: - A semantic convention registry can import one or several semantic conventions from other published registries. - A new optional section called `imports` will be added to the semantic convention file defining groups. - The `imports` section is a list of imported semantic conventions with their schema URLs and aliases. @@ -103,12 +107,13 @@ community-driven model for defining and using semantic conventions across divers - A registry can override the attributes of a group defined in another registry (e.g., `requirement_level`). - A registry can only override a subset of group fields defined in another registry (list of fields TBD). - Overrides defined in a registry are not propagated to the imported semantic conventions. -- Overrides defined in a registry are visible to registries importing the current registry. These attribute - and group overrides are re-exported with some transformations by the local registry. +- Overrides defined in registry 'A' are visible to registries that import 'A'. These attribute and group overrides + are said to be exported and thus reusable by the registries that import 'A'. - A group reference cannot change the type of the imported group (similar to attribute references). - References to an imported group or attribute are always prefixed with the alias of the imported semantic convention (e.g., `ref: otel:client.address`). The colon is used as a separator between the alias and the - group or attribute name. + group or attribute name. If preferred, there is an alternative to make the alias optional, see the alternatives + section below. - References to entities (groups or attributes) defined in the local registry are never prefixed. - A locally defined group can reference an imported group in its `extends` section. - A locally defined group can reference an imported attribute in its `attributes` section. From 17f5b680999cc28b889d37ee98cb99216b1eb209 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 30 Aug 2024 15:48:18 -0700 Subject: [PATCH 09/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 243 ++++++++++++++------ 1 file changed, 169 insertions(+), 74 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index a8224adc..858949c9 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -1,4 +1,4 @@ -# Multi-Registry - Draft Proposal +# Multi-Registry - Proposal Status: Work in Progress @@ -6,17 +6,22 @@ Status: Work in Progress A series of changes are proposed to support multiple semantic convention registries in OpenTelemetry. -> Note: This proposal aims to describe the overall changes needed to realize this vision. However, -> this does not mean that we cannot introduce the changes incrementally. For example, we could start +> Note 1: This proposal aims to describe the overall changes needed to realize the multi-registry long term vision. +> However, this does not mean that we cannot introduce the changes incrementally. For example, we could start > by supporting attributes only and then progressively add support for metrics, events, spans, etc. +> Note 2: In this document, the term "semantic convention registry" refers to the collection of semantic +> conventions (attributes, groups, etc.) that define the semantics of the data model used in OpenTelemetry. +> The term "registry" is used interchangeably with "semantic convention registry". + +> Note 3: In this document, the term "OTEL" refers to the OpenTelemetry project. + ## Use Case Example The following use case is not intended to be exhaustive, but it should provide a good idea of the types of multi-registry scenarios we aim to support. -The diagram below illustrates a small but concrete example of how multiple semantic convention -registries could be used together. +The diagram below illustrates a small but concrete example of how multiple registries could be used together. ![Multi-Registry Use Case](images/multi_registry_use_case.svg) @@ -26,7 +31,7 @@ The YAML semantic convention files corresponding to the diagram are available in **Actors and Their Benefits** -1. **OpenTelemetry:** +1. **OTEL:** - **Value Proposition:** - OTEL can focus on defining core signals while delegating the creation of more specific signals to the community at scale. @@ -34,7 +39,7 @@ The YAML semantic convention files corresponding to the diagram are available in 2. **Vendor:** - **Value Proposition:** - - By publishing their own semantic convention registry, vendors make it easier for their customers + - By publishing their own registry, vendors make it easier for their customers to discover and effectively use custom signals specific to their products. - Vendors can reuse common signals defined by OTEL, ensuring consistency in customer experiences. This strategy enhances the interoperability of their products within the larger observability ecosystem. @@ -43,15 +48,15 @@ The YAML semantic convention files corresponding to the diagram are available in - **Value Proposition:** - The OSS library author can reuse OTEL-defined attributes and signals, integrating them with custom signals tailored to their library. - - Publishing a semantic convention registry for the library simplifies the integration process for + - Publishing a registry for the library simplifies the integration process for developers, making it easier to adopt and use the library in a consistent and standardized way. - This approach also increases the attractiveness, visibility and usability of the library within the community. 4. **Enterprise Application:** - **Value Proposition:** - - Enterprises can leverage the concept of semantic convention registries to import external registries - and simplify their observability integration. + - Enterprises can leverage the concept of registries to import external registries and simplify their observability + integration. - By creating internal registries, enterprises can define custom signals that align with their specific needs and share these across teams and products, fostering internal consistency. - This capability enhances collaboration and streamlines the observability practices within the organization. @@ -61,51 +66,54 @@ community-driven model for defining and using semantic conventions across divers ## Design Principles -- **Independent Ownership**: Any individual or organization should be able to create and maintain a semantic convention - registry independently, without requiring active coordination with the OTEL project. +- **Independent Ownership**: Any individual or organization should be able to create and maintain a registry + independently, without requiring human coordination with the OTEL project. - **Registry Accessibility**: Semantic convention registries can be either private or public, allowing flexibility based on the needs of the owner. - **Community Support Tools**: The OTEL project will provide and maintain tools to assist the community in creating, - validating, resolving, and publishing semantic convention registries (i.e. Weaver tool). + validating, resolving, and publishing registries (i.e. Weaver tool). - **Core Policy Enforcement**: The OTEL project will establish and enforce a set of core policies (e.g. backward - compatibility policies) that all semantic convention registries must adhere to, ensuring consistency and reliability. -- **Cross-Registry References**: References between different semantic convention registries should be supported, - facilitating interoperability and integration across various registries. + compatibility policies) that all registries must adhere to, ensuring consistency and reliability. +- **Cross-Registry References**: References between different registries should be supported, facilitating + interoperability and integration across various registries. - **Conflict Avoidance through Scoping**: Without a scoping mechanism, we can't ensure that a signal or an attribute name will never conflict with a future declaration in another registry. -- **Circular Reference Handling**: Circular references between semantic convention registries must be detected, - reported, and rejected to prevent conflicts and maintain the integrity of the system. +- **Circular Reference Handling**: Circular references between registries must be detected, reported, and rejected to + prevent conflicts and maintain the integrity of the system. ## Semantic Convention Registry Changes -With the following changes, a semantic convention registry can be defined by anyone, without requiring any active -coordination with OTEL. For all the following examples, registry authors can extend or amend the OTEL registry or -create their own attributes and groups (non exhaustive list): - - A vendor publishes a semantic convention registry for their products, allowing their customers to discover - and use their signals. - - A community publishes a semantic convention registry for a specific domain that is too specialized to be - included in the OTEL registry. - - An individual publishes a semantic convention registry for their own OSS library or project. - - An enterprise creates internal semantic convention registries for internal use. +With the following changes, a registry can be defined by anyone, without requiring any active coordination with OTEL. +For all the following examples, registry authors can extend or amend the OTEL registry or create their own attributes +and groups (non-exhaustive list): + - A vendor publishes a registry for their products, allowing their customers to discover and use their signals. + - A community publishes a registry for a specific domain that is too specialized to be included in the OTEL registry. + - An individual publishes a registry for their own OSS library or project. + - An enterprise creates internal registries for internal use. Changes: -- A semantic convention registry can import one or several semantic conventions from other published registries. +- A registry can import one or several semantic conventions from other published registries. - A new optional section called `imports` will be added to the semantic convention file defining groups. - The `imports` section is a list of imported semantic conventions with their schema URLs and aliases. -- Aliases are only visible within the file where they are defined. -- Aliases must be unique within the file where they are defined. -- Schema URLs are used to fetch both OTEL schema and self-contained/resolved semantic convention registries. +- Aliases defined in a file with one `imports` section and no `groups` are global aliases and are visible in all the + files of the registry otherwise aliases are local to a file (see discussion + [here](#do-we-allow-an-alias-defined-in-one-semconv-file-to-be-visible-in-another-semconv-file)). +- Only one alias is allowed for a specific `schema_url` across all the files of the registry (see discussion + [here](#do-we-allow-importing-the-same-external-registry-with-different-aliases-within-the-scope-of-a-registry-)). +- Only one version of a specific imported registry is allowed across all the files of the registry (see discussion + [here](#do-we-allow-importing-the-same-registry-with-different-versions)). +- Schema URLs are used to fetch both OTEL schema and self-contained/resolved registries. The way a resolved registry is linked to an OTEL schema is TBD (it could be a new URL pointing to the resolved registry or an integration within the schema file itself). -- Unused imported registries will be detected by Weaver and reported as warnings. -- A registry can only be imported as a self-contained/resolved semantic convention registry. +- Unused imported registries will be detected by Weaver and reported as errors. +- A registry can only be imported as a self-contained/resolved registry. - A set of core policies will be enforced by Weaver for any registry, OTEL or non-OTEL, to ensure backward - compatibility and consistency across registries (list of core policies TBD). -- Any attribute or group in a registry is a referencable entity when the registry is imported. -- Group references are now supported to address the following use cases: - - A registry can add new attributes to a group defined in another registry. - - A registry can override the attributes of a group defined in another registry (e.g., `requirement_level`). - - A registry can only override a subset of group fields defined in another registry (list of fields TBD). + compatibility and consistency across registries (see section [Core Policies](#core-policies)). +- Any stable attribute or group in a registry is a referencable entity when the registry is imported. +- Experimental entities in a registry can not be referenced into other registries (see discussion + [here](#do-we-allow-reference-of-experimental-entities-across-registries)). +- Group references are now supported to extend or overrides group fields and attributes of an imported group. (see + discussion [here](#do-we-allow-references-to-groups-or-attributes-defined-in-an-imported-registry)). - Overrides defined in a registry are not propagated to the imported semantic conventions. - Overrides defined in registry 'A' are visible to registries that import 'A'. These attribute and group overrides are said to be exported and thus reusable by the registries that import 'A'. @@ -118,9 +126,9 @@ Changes: - A locally defined group can reference an imported group in its `extends` section. - A locally defined group can reference an imported attribute in its `attributes` section. -> Note: The JSON Schema for the semantic convention registry must be updated to reflect these changes. +> Note: The JSON Schema for the registry must be updated to reflect these changes. -> Note: A resolved semantic convention registry is self-contained and does not include any complex constructs +> Note: A resolved registry is self-contained and does not include any complex constructs > like `imports`, `ref`, `extends`, etc. Their **structure is less subject to change**, making them good > candidates for publication and easier to consume. @@ -142,40 +150,128 @@ Alternatives: a conflict is detected, Weaver will report an error and ask the user to define an alias for the imported registry. This approach could be supported in the future if the need arises. -Open Questions: -- How do we handle experimental entities in the context of multi-registry? Do we allow reference of experimental - entities across registries? -- Do we allow different versions of the same registry to be imported into different semantic convention files of - the same registry? -- Is there a relationship to define between the instrumentation scope name and version and the semantic convention - registry? - -## OpenTelemetry Schema Changes - -The OpenTelemetry schema file structure must be updated to either include the URL to a self-contained/resolved -semantic convention or to include the resolved registry itself. +### Discussions + +#### Do we allow importing the same external registry with different aliases within the scope of a registry? +- Short answer: No for consistency and readability. +- Long answer: Although technically it is not a problem, it is a good practice to use the same alias for the same + registry inside the same file or across different files of the same registry. This practice makes the overall + registry more readable and maintainable. Weaver will report an error if different aliases are used for the same + registry in the scope of a single registry. + +#### Do we allow an alias defined in one semconv file to be visible in another semconv file? +- Short answer: Yes, under certain conditions. +- Long answer: An alias defined in a file containing only an imports section is visible in all the files of the + registry. Such an alias is called a global alias. A global alias is useful when the same imported registry is + used in multiple files within the same registry. In this situation, defining a global alias simplifies the + management of dependencies, as updating to a new registry version will only require a single change in the + registry file containing the global aliases. Weaver will report an error if an alias not defined locally or as + a global alias is used in a semconv file. + +#### Do we allow importing the same registry with different versions? +- Short answer: No for consistency. +- Long answer: Although technically it is possible to support multiple versions of an imported registry, it is a + good practice to use align the versions inside the same registry. This practice make the overall registry more + consistent and easy to reason about. Weaver will report an error if different versions are used for the same + registry in the scope of a single registry. + +#### Do we allow reference of experimental entities across registries? +- Short answer: No for backward compatibility. +- Long answer: Experimental entities are not meant to be used in production. Experimental entities are subject to + change or removal without any notice in the next version of the registry. The type of an experimental entity can + be changed. Weaver will detect and report an error if an experimental entity is referenced across registries. + +#### Do we allow circular references between registries? +- Short answer: No. +- Long answer: Circular references are already prohibited inside a registry, so circular references between + registries can lead to the kind of problems (e.g. unresolvable references). Weaver will detect and report + an error if a circular reference is detected between registries. + +#### Do we allow references to groups or attributes defined in an imported registry? +- Short answer: Yes. +- Long answer: References to groups or attributes defined in an imported registry are allowed. This feature + allows a registry to extend or override the attributes of an imported group. Weaver will detect and report an + error if a reference is made to an entity that is not defined in the imported registry. + +### Open Questions: +- Can we override any field of a group defined in an imported registry? No for `type` and `attributes` fields, what + about the other fields? +- Is there a relationship to define between the instrumentation scope name and version and the registry? + + +## Core Policies + +These core policies aim to ensure consistency and backward compatibility across registries. + +### Group Policies +- No group ID collisions +- Only attribute groups are allowed in the attribute registry +- Ref attributes are not allowed in the attribute registry +- ID attributes are not allowed in groups outside of the attribute registry + +### Attribute Policies +- No attribute name collisions +- Attributes cannot be removed +- Attributes cannot "degrade" in stability (e.g., stable -> experimental) +- Stable attributes cannot change type +- All attributes must be defined inside a semantic convention registry +- Attribute names must match the following regular expression: `^[a-z][a-z0-9]*([._][a-z0-9]+)*$` +- Attributes with prefixes are not allowed; attributes must be fully qualified + +### Enum Member Policies +- Stable members cannot change stability +- Values cannot change +- IDs cannot be removed +- Enum IDs must match the following regular expression: `^[a-z][a-z0-9]*([._][a-z0-9]+)*$` + +### Metrics +- No metric name collisions +- Metrics cannot be removed +- Metrics cannot "degrade" in stability (e.g., stable -> experimental) +- Stable metric units cannot change +- Stable metric instruments cannot change +- The set of required/recommended attributes must remain the same +- Metric names must match the following regular expression: `^[a-z][a-z0-9]*([._][a-z0-9]+)*$` + +### Events +- No event name collisions +- Events cannot be removed +- Events cannot "degrade" in stability (e.g., stable -> experimental) +- The set of required/recommended attributes must remain the same +- Event names must match the following regular expression: `^[a-z][a-z0-9]*([._][a-z0-9]+)*$` + +### Spans +- No span name collisions +- Spans cannot be removed +- Spans cannot "degrade" in stability (e.g., stable -> experimental) +- The set of required/recommended attributes must remain the same +- Span names must match the following regular expression: `^[a-z][a-z0-9]*([._][a-z0-9]+)*$` + +## OTEL Schema Changes + +The OTEL schema file structure must be updated to either include the URL to a self-contained/resolved +registry or to include the resolved registry itself. ## Weaver Changes The following changes are proposed to Weaver: -- Weaver must be able to support any operation on any semantic convention registry (check, resolve, generate, search, +- Weaver must be able to support any operation on any registry (check, resolve, generate, search, ...). - The command `weaver registry generate` must allow the generation of the referenced entities that belong to the - imported semantic convention registries or optionally the generation of all the entities of the imported registries. + imported registries or optionally the generation of all the entities of the imported registries. - Extend the `--templates` parameter to allow git URL so OTEL templates (or community-based templates) can be reused for any registry. - Extend the `--policies` parameter to allow git URL so OTEL policies (or community-based policies) can be reused for any registry. -- Add a step before the resolution process to build a deduplicated list of the imported registries. Download the - corresponding resolved registries and create a mapping url to resolved registry that will be used during the - resolution process to resolve the references to the imported registries (and to detect clashes between local and - imported IDs if aliases are not used). -- Weaver must be able to detect newer versions of imported registries and report them as warnings. +- Maintain a list of global and local aliases and detect any conflict between them. Detect when the same schema URL + is imported with different aliases. Detect when an alias (global or local) is not used in the registry. - More TBD. Open Questions: - Is a resolved registry contain any trace of the imported registries? +- It will be nice to detect when a new registry version is available. How can we detect the presence of a new registry + version? A file containing of the schema URL per version for a registry? Some kind of naming conventions for the URL? ## Protocol Changes @@ -183,10 +279,10 @@ No impact on OTLP and OTAP. A `schema_url` field is already present at the resource and scope levels. -Ideally any component of the observability pipeline should be able to fetch the resolved semantic convention registry -just by knowing the schema URL of any resource or instrumentation scope. +Ideally any component of the observability pipeline should be able to fetch the resolved registry just by knowing the +schema URL of any resource or instrumentation scope. -## OpenTelemetry SDKs Changes +## OTEL SDKs Changes TBD @@ -197,32 +293,33 @@ Open Questions: ## Resolved Semantic Convention Registry Format -The following properties are proposed for a resolved semantic convention registry: +The following properties are proposed for a resolved registry: -- Resolved semantic convention registry must be easy to consume and to publish +- Resolved registry must be easy to consume and to publish - Accessible via a URL. - Self-contained, i.e. a single file. - No `ref`, no `extends`, no `imports`, no alias, no other complex constructs. - Yaml or JSON format so resolved registries can be easily consumed by any tool. - Optional lineage section. -The content of a resolved semantic convention registry depends on the: +The content of a resolved registry depends on the: - The semantic convention files composing the registry to resolve. -- The semantic convention registries imported. +- The registries imported. - The configuration specified during the resolution process. - Include all the entities of the imported registries - Include only the referenced entities of the imported registries. -More specifically, a resolved semantic convention registry contains: -- All the attributes registry specified locally in the semantic convention registry. -- All the groups specified locally in the semantic convention registry. +More specifically, a resolved registry contains: +- All the attributes registry specified locally in the registry. +- All the groups specified locally in the registry. - All the attributes and groups imported but not re-exported locally are not included in the resolved registry. A re-exported entity is an entity that is imported and referenced in the local registry with some overriding. Open Questions: - Do we keep track of the imported registries in the resolved registry? If yes, how? Lineage? -- Can we leverage the attribute deduplication mechanism to simplify the merging of imported registries? +- Can we leverage the attribute deduplication mechanism to simplify the merging of imported registries? ToDo -> Explain + attribute deduplication mechanism. - Can we extend the deduplication mechanism to the signals? - Materialized resolved registry (what see the jq, template and policy engines) vs Published resolved registry. - Materialized Resolved Registry: This is what the jq, template and policy engines see. In this format there are @@ -232,6 +329,4 @@ Open Questions: ## Priorities -TBD - -- [Not Final] Start with attributes \ No newline at end of file +TBD \ No newline at end of file From 5cef1e2faca800cfbc9cf3330c2a3eb257465def Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 30 Aug 2024 16:52:20 -0700 Subject: [PATCH 10/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 52 ++++++++++++--------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 858949c9..3a745461 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -21,21 +21,23 @@ A series of changes are proposed to support multiple semantic convention registr The following use case is not intended to be exhaustive, but it should provide a good idea of the types of multi-registry scenarios we aim to support. -The diagram below illustrates a small but concrete example of how multiple registries could be used together. +The diagram below illustrates a small but realistic example of how multiple registries could be used together. +For more details, the YAML semantic convention files corresponding to the diagram are available in this +[folder](registries). ![Multi-Registry Use Case](images/multi_registry_use_case.svg) The color-coding within the signal descriptions indicates the provenance of the corresponding definition. -The YAML semantic convention files corresponding to the diagram are available in this [folder](registries). - **Actors and Their Benefits** 1. **OTEL:** - **Value Proposition:** - OTEL can focus on defining core signals while delegating the creation of more specific signals to the community at scale. - - OTEL establishes the foundation for developing new uses and tools in the observability ecosystem. + - OTEL establishes the foundation for developing new uses and tools in the observability ecosystem (e.g. data + catalog integration, dashboard generation, database schema generation, metadata to improve AI assisted Query + engines, etc.). 2. **Vendor:** - **Value Proposition:** @@ -56,10 +58,10 @@ The YAML semantic convention files corresponding to the diagram are available in 4. **Enterprise Application:** - **Value Proposition:** - Enterprises can leverage the concept of registries to import external registries and simplify their observability - integration. + integration for complex systems. - By creating internal registries, enterprises can define custom signals that align with their specific - needs and share these across teams and products, fostering internal consistency. - - This capability enhances collaboration and streamlines the observability practices within the organization. + needs and share these across teams and products, fostering internal consistency. This capability enhances + collaboration and streamlines the observability practices within the organization. By supporting these actors and their specific use cases, the multi-registry approach enables a flexible, community-driven model for defining and using semantic conventions across diverse domains and applications. @@ -68,28 +70,29 @@ community-driven model for defining and using semantic conventions across divers - **Independent Ownership**: Any individual or organization should be able to create and maintain a registry independently, without requiring human coordination with the OTEL project. -- **Registry Accessibility**: Semantic convention registries can be either private or public, allowing flexibility - based on the needs of the owner. -- **Community Support Tools**: The OTEL project will provide and maintain tools to assist the community in creating, - validating, resolving, and publishing registries (i.e. Weaver tool). +- **Registry Accessibility**: Registries can be either private or public, allowing flexibility based on the needs + of the owner. +- **Common Format**: Published registries will adhere to a common format, making them easy to consume and integrate + with other registries. This format is known as "resolved registry format". +- **Community Support Tools**: The OTEL project will provide and maintain an extensible tool (Weaver) to assist the + community in creating, validating, resolving, and publishing registries. Weaver will also support documentation + generation and code generation for any registry. - **Core Policy Enforcement**: The OTEL project will establish and enforce a set of core policies (e.g. backward - compatibility policies) that all registries must adhere to, ensuring consistency and reliability. + compatibility policies) that all registries must adhere to, ensuring consistency and reliability. These policies + will be enforced by Weaver. - **Cross-Registry References**: References between different registries should be supported, facilitating interoperability and integration across various registries. - **Conflict Avoidance through Scoping**: Without a scoping mechanism, we can't ensure that a signal or an attribute - name will never conflict with a future declaration in another registry. + name will never conflict with a future declaration in another registry. Every imported registry will be scoped with + a unique alias to avoid conflicts. - **Circular Reference Handling**: Circular references between registries must be detected, reported, and rejected to prevent conflicts and maintain the integrity of the system. ## Semantic Convention Registry Changes -With the following changes, a registry can be defined by anyone, without requiring any active coordination with OTEL. -For all the following examples, registry authors can extend or amend the OTEL registry or create their own attributes -and groups (non-exhaustive list): - - A vendor publishes a registry for their products, allowing their customers to discover and use their signals. - - A community publishes a registry for a specific domain that is too specialized to be included in the OTEL registry. - - An individual publishes a registry for their own OSS library or project. - - An enterprise creates internal registries for internal use. +With the following changes, a registry can be defined by anyone, without requiring any human coordination with OTEL. +Registry authors can extend or amend entities defined in other registries or create their own attributes +and groups. Changes: - A registry can import one or several semantic conventions from other published registries. @@ -329,4 +332,11 @@ Open Questions: ## Priorities -TBD \ No newline at end of file +TBD + +## General Open Questions + +- What about introducing a new type of semconv file that will let end-users define global overrides and global redact + directives? For example, the requirement level of attributes such as `client.address` or `server.address` will be + better defined by the end-user than by the library author, or a vendor. A similar approach could be used for redact + directives in order to address privacy concerns. \ No newline at end of file From e20011d082369511a1a48368ad38208d5c112318 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 30 Aug 2024 17:08:45 -0700 Subject: [PATCH 11/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 58 ++++++++++++--------- 1 file changed, 33 insertions(+), 25 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 3a745461..8eb57d44 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -96,30 +96,31 @@ and groups. Changes: - A registry can import one or several semantic conventions from other published registries. -- A new optional section called `imports` will be added to the semantic convention file defining groups. -- The `imports` section is a list of imported semantic conventions with their schema URLs and aliases. -- Aliases defined in a file with one `imports` section and no `groups` are global aliases and are visible in all the +- A published registry is a resolved registry (see resolved registry format below) accessible via a URL. +- A new optional section called `imports` will be added to the semantic convention file defining groups. The `imports` + section is a list of imported semantic conventions with their schema URLs and aliases. +- Aliases defined in a file with `global_imports` section are global aliases and are visible in all the files of the registry otherwise aliases are local to a file (see discussion - [here](#do-we-allow-an-alias-defined-in-one-semconv-file-to-be-visible-in-another-semconv-file)). -- Only one alias is allowed for a specific `schema_url` across all the files of the registry (see discussion + [here](#do-we-allow-an-alias-defined-in-one-semconv-file-to-be-visible-in-another-semconv-file)). Note that these + files can not contain any group definitions. +- Only one alias is allowed for a specific `schema_url` across all the files of a registry (see discussion [here](#do-we-allow-importing-the-same-external-registry-with-different-aliases-within-the-scope-of-a-registry-)). -- Only one version of a specific imported registry is allowed across all the files of the registry (see discussion +- Only one version of a specific imported registry is allowed across all the files of a registry (see discussion [here](#do-we-allow-importing-the-same-registry-with-different-versions)). - Schema URLs are used to fetch both OTEL schema and self-contained/resolved registries. The way a resolved registry is linked to an OTEL schema is TBD (it could be a new URL pointing to the resolved registry or an integration within the schema file itself). -- Unused imported registries will be detected by Weaver and reported as errors. +- Import a registry and not using it is considered an error. - A registry can only be imported as a self-contained/resolved registry. -- A set of core policies will be enforced by Weaver for any registry, OTEL or non-OTEL, to ensure backward - compatibility and consistency across registries (see section [Core Policies](#core-policies)). +- Any registry must comply with the OTEL core policies to ensure backward compatibility and consistency across + registries (see section [Core Policies](#core-policies)). - Any stable attribute or group in a registry is a referencable entity when the registry is imported. - Experimental entities in a registry can not be referenced into other registries (see discussion [here](#do-we-allow-reference-of-experimental-entities-across-registries)). - Group references are now supported to extend or overrides group fields and attributes of an imported group. (see discussion [here](#do-we-allow-references-to-groups-or-attributes-defined-in-an-imported-registry)). -- Overrides defined in a registry are not propagated to the imported semantic conventions. -- Overrides defined in registry 'A' are visible to registries that import 'A'. These attribute and group overrides - are said to be exported and thus reusable by the registries that import 'A'. +- A registry 'A' can import semantic conventions from a registry 'B'. 'A' can override some entities of 'B'. These + overrides are not propagates to 'B'. 'B' can not import the registry 'A' (circular dependencies are not allowed). - A group reference cannot change the type of the imported group (similar to attribute references). - References to an imported group or attribute are always prefixed with the alias of the imported semantic convention (e.g., `ref: otel:client.address`). The colon is used as a separator between the alias and the @@ -129,16 +130,10 @@ Changes: - A locally defined group can reference an imported group in its `extends` section. - A locally defined group can reference an imported attribute in its `attributes` section. -> Note: The JSON Schema for the registry must be updated to reflect these changes. - -> Note: A resolved registry is self-contained and does not include any complex constructs -> like `imports`, `ref`, `extends`, etc. Their **structure is less subject to change**, making them good -> candidates for publication and easier to consume. - -Wonkiness to remove from the existing semantic convention schema: - -- Rename `metric_name` to `name` in the `metric` group for consistency with other groups. -- Probably more TBD. +> Wonkiness to remove from the existing semantic convention schema: +> +> - Rename `metric_name` to `name` in the `metric` group for consistency with other groups. +> - Probably more TBD. Things to avoid/minimize: @@ -174,7 +169,7 @@ Alternatives: #### Do we allow importing the same registry with different versions? - Short answer: No for consistency. - Long answer: Although technically it is possible to support multiple versions of an imported registry, it is a - good practice to use align the versions inside the same registry. This practice make the overall registry more + good practice to align the versions inside the same registry. This practice make the overall registry more consistent and easy to reason about. Weaver will report an error if different versions are used for the same registry in the scope of a single registry. @@ -197,8 +192,8 @@ Alternatives: error if a reference is made to an entity that is not defined in the imported registry. ### Open Questions: -- Can we override any field of a group defined in an imported registry? No for `type` and `attributes` fields, what - about the other fields? +- Can we override any field of a group defined in an imported registry? No for `type` field, what about the other + fields? - Is there a relationship to define between the instrumentation scope name and version and the registry? @@ -250,6 +245,10 @@ These core policies aim to ensure consistency and backward compatibility across - The set of required/recommended attributes must remain the same - Span names must match the following regular expression: `^[a-z][a-z0-9]*([._][a-z0-9]+)*$` +## OTEL Semantic Convention JSON Schema Changes + +[TODO] The JSON Schema for the registry must be updated to reflect these changes. + ## OTEL Schema Changes The OTEL schema file structure must be updated to either include the URL to a self-contained/resolved @@ -296,6 +295,10 @@ Open Questions: ## Resolved Semantic Convention Registry Format +> Note: A resolved registry is self-contained and does not include any complex constructs +> like `imports`, `ref`, `extends`, etc. Their **structure is less subject to change**, making them good +> candidates for publication and easier to consume. + The following properties are proposed for a resolved registry: - Resolved registry must be easy to consume and to publish @@ -305,6 +308,11 @@ The following properties are proposed for a resolved registry: - Yaml or JSON format so resolved registries can be easily consumed by any tool. - Optional lineage section. +The structure of resolved registries is less subject to change. We can always add advanced support for embedding, +more complex inheritance mechanisms, etc. The resolved registry format will not change because all these complex +constructs will be removed during the resolution process. This property makes resolved registries good candidates +for publication and easier to consume. + The content of a resolved registry depends on the: - The semantic convention files composing the registry to resolve. - The registries imported. From e963180020306942a0d03ca447884dd29a974f7c Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 30 Aug 2024 17:09:57 -0700 Subject: [PATCH 12/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 8eb57d44..4a556185 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -159,7 +159,7 @@ Alternatives: #### Do we allow an alias defined in one semconv file to be visible in another semconv file? - Short answer: Yes, under certain conditions. -- Long answer: An alias defined in a file containing only an imports section is visible in all the files of the +- Long answer: An alias defined in a file containing `global_imports` section is visible in all the files of the registry. Such an alias is called a global alias. A global alias is useful when the same imported registry is used in multiple files within the same registry. In this situation, defining a global alias simplifies the management of dependencies, as updating to a new registry version will only require a single change in the From 02e58ebfcb72b2216f6c9ffa4b822ab28ea0ce68 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 30 Aug 2024 23:58:26 -0700 Subject: [PATCH 13/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 4a556185..71d0f960 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -71,7 +71,7 @@ community-driven model for defining and using semantic conventions across divers - **Independent Ownership**: Any individual or organization should be able to create and maintain a registry independently, without requiring human coordination with the OTEL project. - **Registry Accessibility**: Registries can be either private or public, allowing flexibility based on the needs - of the owner. + of the owner. Registries are accessible via a URL. - **Common Format**: Published registries will adhere to a common format, making them easy to consume and integrate with other registries. This format is known as "resolved registry format". - **Community Support Tools**: The OTEL project will provide and maintain an extensible tool (Weaver) to assist the From 0a698de841eb40ecc9c4c11f8579b8a2708fa1bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Laurent=20Qu=C3=A9rel?= Date: Sat, 31 Aug 2024 13:02:09 -0700 Subject: [PATCH 14/21] Update multi_registry.md --- docs/specs/multi-registry/multi_registry.md | 37 +++++++++++---------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 71d0f960..a9e6a71a 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -68,25 +68,26 @@ community-driven model for defining and using semantic conventions across divers ## Design Principles -- **Independent Ownership**: Any individual or organization should be able to create and maintain a registry - independently, without requiring human coordination with the OTEL project. -- **Registry Accessibility**: Registries can be either private or public, allowing flexibility based on the needs - of the owner. Registries are accessible via a URL. -- **Common Format**: Published registries will adhere to a common format, making them easy to consume and integrate - with other registries. This format is known as "resolved registry format". -- **Community Support Tools**: The OTEL project will provide and maintain an extensible tool (Weaver) to assist the - community in creating, validating, resolving, and publishing registries. Weaver will also support documentation - generation and code generation for any registry. -- **Core Policy Enforcement**: The OTEL project will establish and enforce a set of core policies (e.g. backward - compatibility policies) that all registries must adhere to, ensuring consistency and reliability. These policies - will be enforced by Weaver. +- **Independent Ownership**: Individuals or organizations should be able to create and maintain a registry + independently, without requiring coordination with the OTEL project. +- **Registry Accessibility**: Registries should be either private or public, allowing flexibility based on + the needs of the owner. Registries should be accessible via a URL. +- **Common Format**: A published registry should adhere to a common format, making it easy to consume and + integrate with other registries. +- **Self-Contained**: A published registry should be self-contained and stored in a single file. +- **Community Support Tools**: The OTEL project should provide and maintain an extensible tool, Weaver, to + assist the community in creating, validating, resolving, and publishing registries. Weaver should also + support documentation generation and code generation for any registry. +- **Core Policy Enforcement**: The OTEL project should establish and enforce a set of core policies (e.g., + backward compatibility policies) that all registries must adhere to, ensuring consistency and reliability. + These policies should be enforced by Weaver. - **Cross-Registry References**: References between different registries should be supported, facilitating interoperability and integration across various registries. -- **Conflict Avoidance through Scoping**: Without a scoping mechanism, we can't ensure that a signal or an attribute - name will never conflict with a future declaration in another registry. Every imported registry will be scoped with - a unique alias to avoid conflicts. -- **Circular Reference Handling**: Circular references between registries must be detected, reported, and rejected to - prevent conflicts and maintain the integrity of the system. +- **Conflict Avoidance through Scoping**: A scoping mechanism should be implemented to ensure that a signal + or an attribute name does not conflict with future declarations in other registries. Every imported + registry should be scoped with a unique alias to avoid conflicts. +- **Circular Reference Handling**: Circular references between registries should be detected, reported, and + rejected to prevent conflicts and maintain the integrity of the system. ## Semantic Convention Registry Changes @@ -347,4 +348,4 @@ TBD - What about introducing a new type of semconv file that will let end-users define global overrides and global redact directives? For example, the requirement level of attributes such as `client.address` or `server.address` will be better defined by the end-user than by the library author, or a vendor. A similar approach could be used for redact - directives in order to address privacy concerns. \ No newline at end of file + directives in order to address privacy concerns. From fe45217941877a157d12d1f3c83e0c4d75c8f138 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Tue, 3 Sep 2024 11:06:52 -0700 Subject: [PATCH 15/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 71d0f960..9c384e82 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -4,7 +4,20 @@ Status: Work in Progress ## Introduction -A series of changes are proposed to support multiple semantic convention registries in OpenTelemetry. +This document outlines the proposal for supporting multiple semantic convention registries in OpenTelemetry. The goal is +to enable the community to define and publish their own semantic conventions, which can be used independently of the +core OpenTelemetry Semantic Conventions. This approach allows the community to define custom signals and attributes +specific to their domain or use case while still leveraging the core signals and attributes defined by OpenTelemetry. + +Multi-registries have the potential to significantly to develop the OpenTelemetry ecosystem and enhance the flexibility +and extensibility of OpenTelemetry, enabling a wide range of use cases and scenarios. For example, vendors can publish +their own registries containing signals and attributes specific to their products, while open-source library authors can +define custom signals tailored to their libraries. Enterprises can also create internal registries to define custom +signals that align with their specific needs and share these across teams and products. + +The proposal outlines the changes needed to support multiple semantic convention registries in OpenTelemetry, including +the core policies that all registries must adhere to, the format of resolved registries, and the modifications required +in Weaver to support the creation, validation, generation, packaging, and publication of registries. > Note 1: This proposal aims to describe the overall changes needed to realize the multi-registry long term vision. > However, this does not mean that we cannot introduce the changes incrementally. For example, we could start From 5562cefb9b6cd4312f547f3a5c90937b8bef6d8d Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Tue, 3 Sep 2024 11:13:15 -0700 Subject: [PATCH 16/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index a890778c..b406115e 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -85,20 +85,22 @@ community-driven model for defining and using semantic conventions across divers independently, without requiring coordination with the OTEL project. - **Registry Accessibility**: Registries should be either private or public, allowing flexibility based on the needs of the owner. Registries should be accessible via a URL. -- **Common Format**: A published registry should adhere to a common format, making it easy to consume and - integrate with other registries. +- **Common Format**: A published registry should adhere to a common packaging and format, making it easy to + consume and integrate with other registries. - **Self-Contained**: A published registry should be self-contained and stored in a single file. +- **Version Discoverability**: The various versions of a registry should be discoverable and accessible via a + URL. - **Community Support Tools**: The OTEL project should provide and maintain an extensible tool, Weaver, to - assist the community in creating, validating, resolving, and publishing registries. Weaver should also - support documentation generation and code generation for any registry. + assist the community in creating, validating, resolving, packaging, and publishing registries. Weaver should + also support documentation generation and code generation for any registry. - **Core Policy Enforcement**: The OTEL project should establish and enforce a set of core policies (e.g., backward compatibility policies) that all registries must adhere to, ensuring consistency and reliability. These policies should be enforced by Weaver. - **Cross-Registry References**: References between different registries should be supported, facilitating interoperability and integration across various registries. - **Conflict Avoidance through Scoping**: A scoping mechanism should be implemented to ensure that a signal - or an attribute name does not conflict with future declarations in other registries. Every imported - registry should be scoped with a unique alias to avoid conflicts. + or attribute name does not conflict with future declarations in other registries. Every imported registry + should be scoped with a unique name or alias to avoid conflicts. - **Circular Reference Handling**: Circular references between registries should be detected, reported, and rejected to prevent conflicts and maintain the integrity of the system. From 706f7dc22f42ad1976987023f8cdfe9d00a530fe Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 6 Sep 2024 20:10:19 -0700 Subject: [PATCH 17/21] feat(multi-registry): Update draft of multi-registry spec. --- .../images/otel-semconv-open-ecosystem.svg | 1 + docs/specs/multi-registry/multi_registry.md | 630 +++++++++++------- 2 files changed, 381 insertions(+), 250 deletions(-) create mode 100644 docs/specs/multi-registry/images/otel-semconv-open-ecosystem.svg diff --git a/docs/specs/multi-registry/images/otel-semconv-open-ecosystem.svg b/docs/specs/multi-registry/images/otel-semconv-open-ecosystem.svg new file mode 100644 index 00000000..21070b71 --- /dev/null +++ b/docs/specs/multi-registry/images/otel-semconv-open-ecosystem.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index b406115e..db7b02bc 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -1,38 +1,41 @@ -# Multi-Registry - Proposal +# OpenTelemetry Semantic Convention Multi-Registry - Proposal Status: Work in Progress ## Introduction -This document outlines the proposal for supporting multiple semantic convention registries in OpenTelemetry. The goal is +This document outlines the proposal to support multiple semantic convention registries in OpenTelemetry. The goal is to enable the community to define and publish their own semantic conventions, which can be used independently of the core OpenTelemetry Semantic Conventions. This approach allows the community to define custom signals and attributes -specific to their domain or use case while still leveraging the core signals and attributes defined by OpenTelemetry. +specific to their domains or use cases while still leveraging the core signals and attributes defined by OpenTelemetry. -Multi-registries have the potential to significantly to develop the OpenTelemetry ecosystem and enhance the flexibility -and extensibility of OpenTelemetry, enabling a wide range of use cases and scenarios. For example, vendors can publish -their own registries containing signals and attributes specific to their products, while open-source library authors can -define custom signals tailored to their libraries. Enterprises can also create internal registries to define custom +Multi-registries have the potential to significantly expand the OpenTelemetry ecosystem, enhancing flexibility +and extensibility. This will enable a wide range of use cases and scenarios. For example, vendors could publish +their own registries containing signals and attributes specific to their products, while open-source library authors could +define custom signals tailored to their libraries. Enterprises could also create internal registries to define custom signals that align with their specific needs and share these across teams and products. -The proposal outlines the changes needed to support multiple semantic convention registries in OpenTelemetry, including -the core policies that all registries must adhere to, the format of resolved registries, and the modifications required -in Weaver to support the creation, validation, generation, packaging, and publication of registries. +![Semantic Convention Registries + Weaver = Open Ecosystem](images/otel-semconv-open-ecosystem.svg) -> Note 1: This proposal aims to describe the overall changes needed to realize the multi-registry long term vision. -> However, this does not mean that we cannot introduce the changes incrementally. For example, we could start -> by supporting attributes only and then progressively add support for metrics, events, spans, etc. +This proposal outlines the changes required to support multiple semantic convention registries in OpenTelemetry, including +the core policies that all registries must adhere to, the format of published registries, and the modifications needed +in Weaver to support the creation, validation, generation, packaging, and publication of registries. -> Note 2: In this document, the term "semantic convention registry" refers to the collection of semantic -> conventions (attributes, groups, etc.) that define the semantics of the data model used in OpenTelemetry. -> The term "registry" is used interchangeably with "semantic convention registry". +> [!NOTE] +> This proposal describes the overall changes required to achieve the multi-registry vision long term. However, these +> changes can be introduced incrementally. For example, we could start by supporting attributes only and gradually add +> support for metrics, events, spans, etc. -> Note 3: In this document, the term "OTEL" refers to the OpenTelemetry project. +> [!NOTE] +> In this document, “semantic convention registry” refers to a collection of semantic convention entities (attributes, +> groups, signals, etc.) that define the semantics of the data model used in OpenTelemetry. The terms “registry” and +> “semantic convention registry” are used interchangeably. The term “entity” refers to any semantic convention +> attribute, group, or signal. The abbreviation “OTEL” refers to the OpenTelemetry project. ## Use Case Example -The following use case is not intended to be exhaustive, but it should provide a good idea of the -types of multi-registry scenarios we aim to support. +The following use case is not exhaustive but provides an example of the types of multi-registry scenarios we aim to +support. The diagram below illustrates a small but realistic example of how multiple registries could be used together. For more details, the YAML semantic convention files corresponding to the diagram are available in this @@ -44,12 +47,14 @@ The color-coding within the signal descriptions indicates the provenance of the **Actors and Their Benefits** +The multi-registry approach supports various actors, each benefiting differently from its flexibility and scalability. + 1. **OTEL:** - **Value Proposition:** - OTEL can focus on defining core signals while delegating the creation of more specific signals to the community at scale. - OTEL establishes the foundation for developing new uses and tools in the observability ecosystem (e.g. data - catalog integration, dashboard generation, database schema generation, metadata to improve AI assisted Query + catalog integration, dashboard generation, database schema generation, metadata to improve AI assisted query engines, etc.). 2. **Vendor:** @@ -61,7 +66,7 @@ The color-coding within the signal descriptions indicates the provenance of the 3. **OSS Library Author:** - **Value Proposition:** - - The OSS library author can reuse OTEL-defined attributes and signals, integrating them with custom + - OSS library authors can reuse OTEL-defined attributes and signals, integrating them with custom signals tailored to their library. - Publishing a registry for the library simplifies the integration process for developers, making it easier to adopt and use the library in a consistent and standardized way. @@ -81,143 +86,359 @@ community-driven model for defining and using semantic conventions across divers ## Design Principles +- **Complexity on the system, not the user**: The system should handle the complexity of managing multiple registries, + ensuring that users can easily create, validate, and publish registries without needing to understand the + intricacies of the resolution process. - **Independent Ownership**: Individuals or organizations should be able to create and maintain a registry independently, without requiring coordination with the OTEL project. -- **Registry Accessibility**: Registries should be either private or public, allowing flexibility based on - the needs of the owner. Registries should be accessible via a URL. -- **Common Format**: A published registry should adhere to a common packaging and format, making it easy to - consume and integrate with other registries. -- **Self-Contained**: A published registry should be self-contained and stored in a single file. -- **Version Discoverability**: The various versions of a registry should be discoverable and accessible via a - URL. +- **Core Policy Enforcement**: The OTEL project should establish and enforce core policies (e.g., backward + compatibility) that all published registries must adhere to, ensuring consistency and reliability. +- **Registry Accessibility**: Registries should be either private or public, depending on the owner’s needs, + and accessible via a URL. +- **Cross-Registry References**: References between different registries should be supported to facilitate + interoperability and integration across various registries. +- **Clear Rules to Avoid Conflicts**: A well-defined set of rules should be established and enforced to prevent + conflicts among imported attributes and signals. Users are responsible for resolving any conflicts that cannot + be automatically resolved by the tooling. +- **Circular Reference Handling**: Circular references between registries should be automatically detected, + reported, and rejected to maintain system integrity and prevent conflicts. +- **Publication Format**: A published registry must adhere to a format and packaging designed to ensure stability, + ease of exchange, and consumption by other registries or tools. +- **Self-Contained**: A published registry should be self-contained. +- **Version Discoverability**: Versions of a published registry should be easily discoverable and accessible via a URL. +- **Transparency for Telemetry Consumers**: Downstream consumers of telemetry data (e.g., backends and dashboards) + should never be exposed to conflicts within or between registries. They should see attributes and signals as + defined, without scope or conflict resolution directives, ensuring a consistent and reliable data experience. - **Community Support Tools**: The OTEL project should provide and maintain an extensible tool, Weaver, to assist the community in creating, validating, resolving, packaging, and publishing registries. Weaver should - also support documentation generation and code generation for any registry. -- **Core Policy Enforcement**: The OTEL project should establish and enforce a set of core policies (e.g., - backward compatibility policies) that all registries must adhere to, ensuring consistency and reliability. - These policies should be enforced by Weaver. -- **Cross-Registry References**: References between different registries should be supported, facilitating - interoperability and integration across various registries. -- **Conflict Avoidance through Scoping**: A scoping mechanism should be implemented to ensure that a signal - or attribute name does not conflict with future declarations in other registries. Every imported registry - should be scoped with a unique name or alias to avoid conflicts. -- **Circular Reference Handling**: Circular references between registries should be detected, reported, and - rejected to prevent conflicts and maintain the integrity of the system. + also support documentation and code generation for any published registry. +- **Clear, Actionable, and Educational Error Messages**: Weaver should detect and report any errors or inconsistencies + in a registry, providing users with clear and actionable feedback. ## Semantic Convention Registry Changes -With the following changes, a registry can be defined by anyone, without requiring any human coordination with OTEL. -Registry authors can extend or amend entities defined in other registries or create their own attributes -and groups. - -Changes: -- A registry can import one or several semantic conventions from other published registries. -- A published registry is a resolved registry (see resolved registry format below) accessible via a URL. -- A new optional section called `imports` will be added to the semantic convention file defining groups. The `imports` - section is a list of imported semantic conventions with their schema URLs and aliases. -- Aliases defined in a file with `global_imports` section are global aliases and are visible in all the - files of the registry otherwise aliases are local to a file (see discussion - [here](#do-we-allow-an-alias-defined-in-one-semconv-file-to-be-visible-in-another-semconv-file)). Note that these - files can not contain any group definitions. -- Only one alias is allowed for a specific `schema_url` across all the files of a registry (see discussion - [here](#do-we-allow-importing-the-same-external-registry-with-different-aliases-within-the-scope-of-a-registry-)). -- Only one version of a specific imported registry is allowed across all the files of a registry (see discussion - [here](#do-we-allow-importing-the-same-registry-with-different-versions)). -- Schema URLs are used to fetch both OTEL schema and self-contained/resolved registries. - The way a resolved registry is linked to an OTEL schema is TBD (it could be a new URL pointing to the resolved - registry or an integration within the schema file itself). -- Import a registry and not using it is considered an error. -- A registry can only be imported as a self-contained/resolved registry. -- Any registry must comply with the OTEL core policies to ensure backward compatibility and consistency across - registries (see section [Core Policies](#core-policies)). -- Any stable attribute or group in a registry is a referencable entity when the registry is imported. -- Experimental entities in a registry can not be referenced into other registries (see discussion - [here](#do-we-allow-reference-of-experimental-entities-across-registries)). -- Group references are now supported to extend or overrides group fields and attributes of an imported group. (see - discussion [here](#do-we-allow-references-to-groups-or-attributes-defined-in-an-imported-registry)). -- A registry 'A' can import semantic conventions from a registry 'B'. 'A' can override some entities of 'B'. These - overrides are not propagates to 'B'. 'B' can not import the registry 'A' (circular dependencies are not allowed). -- A group reference cannot change the type of the imported group (similar to attribute references). -- References to an imported group or attribute are always prefixed with the alias of the imported semantic - convention (e.g., `ref: otel:client.address`). The colon is used as a separator between the alias and the - group or attribute name. If preferred, there is an alternative to make the alias optional, see the alternatives - section below. -- References to entities (groups or attributes) defined in the local registry are never prefixed. -- A locally defined group can reference an imported group in its `extends` section. -- A locally defined group can reference an imported attribute in its `attributes` section. - -> Wonkiness to remove from the existing semantic convention schema: -> -> - Rename `metric_name` to `name` in the `metric` group for consistency with other groups. -> - Probably more TBD. - -Things to avoid/minimize: - -- **Name squatting**: By relying on local aliases and URL schema, we reduce the risk of name squatting, as - the naming convention is not based on company names that are not necessarily unique and are complex to control. -- **Name inconsistency**: By enforcing core policies, we minimize the risk of name inconsistency across registries. - -Alternatives: - -- We could make aliases optional in the imports section. To do so, we would need to rely on Weaver to automatically - detect entity IDs that are defined both in the local semantic convention file and the imported registry. When such - a conflict is detected, Weaver will report an error and ask the user to define an alias for the imported registry. - This approach could be supported in the future if the need arises. - -### Discussions - -#### Do we allow importing the same external registry with different aliases within the scope of a registry? -- Short answer: No for consistency and readability. -- Long answer: Although technically it is not a problem, it is a good practice to use the same alias for the same - registry inside the same file or across different files of the same registry. This practice makes the overall - registry more readable and maintainable. Weaver will report an error if different aliases are used for the same - registry in the scope of a single registry. - -#### Do we allow an alias defined in one semconv file to be visible in another semconv file? -- Short answer: Yes, under certain conditions. -- Long answer: An alias defined in a file containing `global_imports` section is visible in all the files of the - registry. Such an alias is called a global alias. A global alias is useful when the same imported registry is - used in multiple files within the same registry. In this situation, defining a global alias simplifies the - management of dependencies, as updating to a new registry version will only require a single change in the - registry file containing the global aliases. Weaver will report an error if an alias not defined locally or as - a global alias is used in a semconv file. - -#### Do we allow importing the same registry with different versions? -- Short answer: No for consistency. -- Long answer: Although technically it is possible to support multiple versions of an imported registry, it is a - good practice to align the versions inside the same registry. This practice make the overall registry more - consistent and easy to reason about. Weaver will report an error if different versions are used for the same - registry in the scope of a single registry. - -#### Do we allow reference of experimental entities across registries? -- Short answer: No for backward compatibility. -- Long answer: Experimental entities are not meant to be used in production. Experimental entities are subject to - change or removal without any notice in the next version of the registry. The type of an experimental entity can - be changed. Weaver will detect and report an error if an experimental entity is referenced across registries. - -#### Do we allow circular references between registries? -- Short answer: No. -- Long answer: Circular references are already prohibited inside a registry, so circular references between - registries can lead to the kind of problems (e.g. unresolvable references). Weaver will detect and report - an error if a circular reference is detected between registries. - -#### Do we allow references to groups or attributes defined in an imported registry? -- Short answer: Yes. -- Long answer: References to groups or attributes defined in an imported registry are allowed. This feature - allows a registry to extend or override the attributes of an imported group. Weaver will detect and report an - error if a reference is made to an entity that is not defined in the imported registry. - -### Open Questions: -- Can we override any field of a group defined in an imported registry? No for `type` field, what about the other - fields? -- Is there a relationship to define between the instrumentation scope name and version and the registry? - - -## Core Policies +The following sections describe the changes needed in the semantic convention registry to support multiple registries. + +### Directory Structure + +The directory structure of a registry remains mostly unchanged, with the addition of a new file called +`weaver_registry.yaml`. This file contains the registry metadata, such as the name, version, and description. +It is required for published registries or registries with dependencies on other registries. + +```plaintext +registry_root/ + domain_1 + domain_2 + ... + registry + weaver_registry.yaml +``` + +### New `weaver_registry.yaml` File + +This file is used to describe the registry and its dependencies. It is optional for standalone registries +but required for published registries or registries with dependencies on other registries. + +```yaml +name: +description: +version: +schema_url: +owner: + name: + url: +maintainers: + - name: + email: + url: +dependencies: + - name: + version: + repository: + alias: +configuration: + allow_experimental_ref: # default value to be discussed +``` + +A registry can import one or more semantic conventions from other published registries. These dependencies +are declared in the `dependencies` section. The `alias` field is optional but must be specified +if multiple imported registries share the same `name`. This design allows Weaver to automatically update +dependencies to the latest version when a new version becomes available (see the [Registry Update](#registry-update) +section). It also allows registry authors to use registry names as preferred, without concerns about +conflicts. + +Sections such as [Registry Resolution](#registry-resolution), [Registry Packaging](#registry-packaging), and +[Registry Publication](#registry-publication) provide more details on how the `weaver_registry.yaml` file +is used in these processes. + +Open Questions: +- Should we follow SemVer 2 for registry versions? It seems advisable, as Weaver can detect breaking changes. +- How do we retrieve a published registry URL from a schema URL? Perhaps a new field in the schema URL file could + point to the published registry URL, possibly named `registry_url`. + +### Semantic Convention File Structure + +The current [JSON Schema](https://github.com/open-telemetry/weaver/blob/main/schemas/semconv.schema.json) that +describes the files of the semantic convention registry remains mostly unchanged. However, a few minor updates are +required: + +- The ability to reference a group must be supported, allowing a registry to extend or override fields of an + imported group. +- The structure of a reference must be enhanced to disambiguate attribute or signal references from other registries + when necessary (more details in the [Registry Resolution](#registry-resolution) section). + +Currently, a reference follows dot notation (e.g., `ref: client.address`). When there is no ambiguity or conflict +detected by Weaver, this notation is sufficient. However, when a conflict occurs, the user must disambiguate the +reference by prefixing it with the registry name (e.g., `ref: otel:client.address`). The prefix corresponds +to the registry name as defined in the `weaver_registry.yaml` file under the `dependencies` section. A colon is used +as a separator between the name and the attribute or group name. References to groups or attributes defined in the +local registry are never prefixed. + +In a group, the `extends` section can now reference an imported group using this new reference syntax. Similarly, the +`attributes` section of a group can reference an imported attribute. + +Referencing a group is currently unsupported. There are several options for implementing this feature, depending on +the uniqueness of the group ID agreed upon: + +- If a group ID is unique per group type, the following syntax could reference a group: + +```yaml +groups: + - metric_ref: + - event_ref: + - span_ref: + - ... +``` + +- If a group ID is unique across group types, the following syntax could reference a group: + +```yaml +groups: + ref: +``` + +> [!IMPORTANT] +> A decision on the uniqueness of group IDs across group types is required to determine the appropriate referencing +> syntax. + +Similar to attribute references, group references cannot override the type, stability, or deprecation status of the +imported group. + +References to experimental entities across registries are allowed under certain conditions: + +- An entity referencing an experimental entity must also be marked as experimental. +- The flag `allow_experimental_ref` in the `weaver_registry.yaml` file must be set to `true`. + +### Registry Resolution + +The registry resolution process needs to handle more complex scenarios, especially when dealing with multiple +registries. It begins by parsing the `weaver_registry.yaml` file of the registry being resolved. All registries listed +under `dependencies` are loaded, parsed, and passed to the conflict resolution stage (assuming that any published +registries have already completed their resolution process). The dependencies of registries form a directed acyclic +graph. Each registry may be maintained by different owners and have different release schedules, which can introduce +conflicts, particularly when dependencies import different versions of a common ancestor or independently override +entities defined in the same ancestor registry. + +The resolution process is designed to automatically detect and resolve most conflicts without user intervention. +However, when a conflict cannot be resolved consistently and predictably, the user must disambiguate by explicitly +choosing the imported registry to resolve the conflict. + +The process hinges on the concept of mergeable entities (attributes or groups). For two entities to be considered +mergeable, they must meet the following basic criteria: +1. Both entities must be of the same type (e.g., attribute, metric, event, span, etc.). +2. They must share the same ID. +3. They must have a common ancestor (i.e., both were originally defined in the same registry). + +Since entities are complex objects consisting of multiple fields, and override mechanisms are allowed, one of the +following additional conditions must be met: +- **Structural equivalence**: If the two entities have identical fields (i.e., both field names and values match), they + are considered structurally equivalent and will be automatically deduplicated. +- **Version compatibility**: If the two entities have not been overridden in any intermediate registry but come from + different versions of the same registry, the entity from the most recent version is used. +- **Disjoint overrides**: If the two entities have been overridden in intermediate registries but only in disjoint + fields (i.e., no overlapping fields are overridden), the entities are automatically merged. + +If none of these conditions are met, the user must resolve the conflict by explicitly prefixing the reference with the +appropriate registry name. + +A formal definition of this resolution process is being developed, possibly based on Quint. + +Two entities with the same type (e.g., attribute) and ID, defined in separate dependencies, +are not considered mergeable even if they are structurally equivalent. This is because these entities originate from +different registries without coordination, meaning their similarity is likely coincidental. While it would be +technically possible to merge them automatically, this is not currently allowed for safety. +If automatic merging becomes a common request, a specific configuration option in the +`weaver_registry.yaml` file could enable this behavior. + +The detection mechanism for circular dependencies must be expanded to identify circular dependencies +between registries. + +Finally, the resolution process must be capable of generating an OTEL schema file that includes the URL of the published +registry, along with all transformation rules, both local and derived from imported registries. This allows consumers of +the published registry to navigate across different versions of the registry, accommodating components of the +observability stack that may not be aligned in terms of versioning. + +> [!NOTE] +> The resolution process outlined above relies on specific requirements for published registries. The self-contained +> format of these registries must include the necessary provenance information to determine: 1) the common ancestor of +> two entities, and 2) versioning details. + +> [!IMPORTANT] +> Telemetry consumers (i.e., anyone or anything downstream from signal production, such as backends, dashboards, users, etc.) +> will observe the signals and attributes by their name, not by their optional prefix used for disambiguation. Other +> metadata and provenance information can also be consulted. + +### Registry Check + +The Weaver registry check command will be extended to apply the following mandatory policies when used in the context +of registry packaging: + +- Having a dependency on a published registry and not using it is considered an error at the time of packaging. +- Any packaged registry must comply with the OTEL core policies to ensure backward compatibility and consistency + across registries (see [Core Policies](#core-policies)). + +[Probably more TBD here] + +### Registry Packaging + +Before publication, a registry must be packaged into a self-contained file that is easy to reference, download, and +consume. The packaging process is responsible for: +- Running the resolution process to ensure the registry is resolvable. +- Running OTEL core policies to ensure compatibility and consistency. +- Running user-defined policies to ensure compliance with custom rules. +- Serializing the resolved registry into a single file (exact format TBD). This file is self-contained and does not + contain complex constructs like `ref`, `extends`, etc. However, the provenance of the entities/fields is included + to support the multi-registry resolution process described earlier. +- Creating a compressed file containing the resolved registry and the `weaver_registry.yaml` file. The registry package + is named `-.gz`. + +The provenance information must include the name, version, and URL of the registry where the entity is +defined. + +Open Questions: +- Should the OTEL Schema URL corresponding to the packaged registry be included in the package? + +### Registry Publication + +A published registry is a self-contained, resolved registry accessible via a URL. The URL of a published registry is +`/-.gz`. The repository is the URL where registry versions are published. The latest version +of a registry is also accessible via `/-latest.gz`. + +Weaver can maintain a directory containing all published versions of a registry. In that case, the `-.gz` +file and the `-latest.gz` file are automatically stored in the directory. + +### Registry Update + +For any published registry, Weaver can discover the latest version by fetching the file located at +`/-latest.gz` and parsing the `weaver_registry.yaml` file. This mechanism allows Weaver to +automatically suggest the latest version of dependencies. + +## Resolved Semantic Convention Registry Format + +The purpose of the resolved registry format is to provide a self-contained, easy-to-consume file containing all the +entities of a registry. This format excludes complex constructs such as `ref` or `extends` and is defined in a single +file. These properties make it easy for any tool — not just Weaver — to publish and consume it. This shift is key +to fostering a rich ecosystem around Semantic Conventions and OpenTelemetry. By removing complex constructs, +the format becomes more stable and less prone to changes. Any new mechanisms introduced in the semantic convention +format (e.g., embedding or more complex inheritance) will not affect the resolved registry format as long as the +resolution process can transform the unresolved registry into a resolved one. + +The file format for the resolved registry is still under discussion but will likely be in YAML or JSON to +ensure compatibility with a wide range of tools. + +A resolved registry consists of the following components: +- An internal, self-contained catalog of deduplicated attribute definitions, ensuring attributes are not defined + multiple times in the registry. +- An internal catalog of registry identifiers, represented as triples (registry name, version, URL). +- A list of resolved groups with their attributes, using internal references to the attribute catalog. +- Provenance information for each entity, linking it to the corresponding entry in the registry identifier catalog, + where the entity is either defined or overridden. + +This format is optimized to minimize the in-memory size of the resolved registry, as many attributes are expected to be +shared across multiple entities and registries. + +Weaver provides an additional mechanism to transform the resolved registry format into a materialized version. This +materialized format is less optimized for size (i.e., no internal catalogs, with all attribute definitions fully +materialized in each entity) but more optimized for consumption by other tools. Weaver currently uses this process +before invoking the jq, template, and policy engines. + +### Weaver Changes + +Weaver currently supports the following workflows: + +- **check**: Check or validate a registry for consistency and compliance with the core policies. +- **resolve**: Resolve a registry and generate a resolved registry YAML file. +- **generate**: Generate code or documentation from a registry. + +To support multiple registries, the following new workflows are proposed: + +- **search**: Search for a specific entity (attribute, group, etc.) in a registry or across multiple registries. +- **update**: Detect and update the latest versions of a registry’s dependencies. +- **package**: Package a registry or a set of registries into a single file. +- **publish**: Publish a registry to a folder. + +The command `weaver registry generate` must allow the generation of referenced entities from +imported registries and optionally generate entities defined in these registries but not referenced in the +local registry. The `--templates` parameter must support git URLs so community-based templates (including the official +OTEL templates) can be reused for any registry. + +Similarly, the `--policies` parameter must support git URLs so community-based policies (including those defined by +OTEL) can be applied to any registry. + +### OTEL Semantic Convention JSON Schema Changes + +The JSON Schema for specifying semantic convention groups must be updated to support group references. Additionally, +the reference format should be extended to optionally allow a reference to be prefixed by a registry name (e.g., +`otel:client.address`). + +### OTEL Schema Changes + +The OTEL schema file structure must be updated to either include the URL to a published registry or to include the +published registry itself. ([Decision TBD] I have a reference for the first option). + +## Protocol Changes + +No impact on OTLP and OTAP. + +A `schema_url` field is already present at the resource and scope levels. + +Ideally, any component of the observability pipeline should be able to fetch the published registry just by knowing the +schema URL of any resource or instrumentation scope. + +## OTEL SDKs Changes + +To leverage the multi-registry feature, users should be able to generate code from their custom registry and use it as a +replacement for the default OTEL registry. + +A more advanced feature would be generating type-safe client SDKs exposing all the entities defined in the custom +registry. A proof of concept is available in Weaver for the Rust language. + +Open Questions: + +- Can we enforce the presence of the schema URL at the resource and instrumentation scope levels? +- How do we convey the schema URL to the SDKs? Could this be part of the codegen done by Weaver? + +## Future Work + +In addition to implementing the multi-registry feature in Weaver, the following future work is planned: + +- Efforts to formalize the resolution process are ongoing, with the aim of ensuring the process is consistent and fully + addresses all targeted use cases. +- Privacy and security concerns have not been fully addressed in this proposal. The fact that certain sensitive + attributes are not imported into your application’s registry does not guarantee they won’t appear in your + application’s telemetry, as they could be defined and generated by linked libraries. Additionally, the override + process only affects downstream instrumentation. Introducing a new type of semantic convention file that allows + end-users to define global overrides and redact directives could help address these concerns. This new file format + could be used by client SDKs or intermediary proxies to filter or redact sensitive attributes. + +# Appendices + +## Appendix A - Core Policies These core policies aim to ensure consistency and backward compatibility across registries. ### Group Policies + - No group ID collisions - Only attribute groups are allowed in the attribute registry - Ref attributes are not allowed in the attribute registry @@ -261,106 +482,15 @@ These core policies aim to ensure consistency and backward compatibility across - The set of required/recommended attributes must remain the same - Span names must match the following regular expression: `^[a-z][a-z0-9]*([._][a-z0-9]+)*$` -## OTEL Semantic Convention JSON Schema Changes - -[TODO] The JSON Schema for the registry must be updated to reflect these changes. - -## OTEL Schema Changes - -The OTEL schema file structure must be updated to either include the URL to a self-contained/resolved -registry or to include the resolved registry itself. - -## Weaver Changes - -The following changes are proposed to Weaver: - -- Weaver must be able to support any operation on any registry (check, resolve, generate, search, - ...). -- The command `weaver registry generate` must allow the generation of the referenced entities that belong to the - imported registries or optionally the generation of all the entities of the imported registries. -- Extend the `--templates` parameter to allow git URL so OTEL templates (or community-based templates) can be reused - for any registry. -- Extend the `--policies` parameter to allow git URL so OTEL policies (or community-based policies) can be reused for - any registry. -- Maintain a list of global and local aliases and detect any conflict between them. Detect when the same schema URL - is imported with different aliases. Detect when an alias (global or local) is not used in the registry. -- More TBD. - -Open Questions: -- Is a resolved registry contain any trace of the imported registries? -- It will be nice to detect when a new registry version is available. How can we detect the presence of a new registry - version? A file containing of the schema URL per version for a registry? Some kind of naming conventions for the URL? - -## Protocol Changes - -No impact on OTLP and OTAP. - -A `schema_url` field is already present at the resource and scope levels. - -Ideally any component of the observability pipeline should be able to fetch the resolved registry just by knowing the -schema URL of any resource or instrumentation scope. - -## OTEL SDKs Changes - -TBD - -Open Questions: - -- Can we enforce the presence of the schema URL in the resource and instrumentation scope? -- How do we convey the schema URL to the SDKs? Could that be part of the codegen done by Weaver? - -## Resolved Semantic Convention Registry Format - -> Note: A resolved registry is self-contained and does not include any complex constructs -> like `imports`, `ref`, `extends`, etc. Their **structure is less subject to change**, making them good -> candidates for publication and easier to consume. - -The following properties are proposed for a resolved registry: - -- Resolved registry must be easy to consume and to publish - - Accessible via a URL. - - Self-contained, i.e. a single file. - - No `ref`, no `extends`, no `imports`, no alias, no other complex constructs. - - Yaml or JSON format so resolved registries can be easily consumed by any tool. -- Optional lineage section. - -The structure of resolved registries is less subject to change. We can always add advanced support for embedding, -more complex inheritance mechanisms, etc. The resolved registry format will not change because all these complex -constructs will be removed during the resolution process. This property makes resolved registries good candidates -for publication and easier to consume. - -The content of a resolved registry depends on the: -- The semantic convention files composing the registry to resolve. -- The registries imported. -- The configuration specified during the resolution process. - - Include all the entities of the imported registries - - Include only the referenced entities of the imported registries. - -More specifically, a resolved registry contains: -- All the attributes registry specified locally in the registry. -- All the groups specified locally in the registry. -- All the attributes and groups imported but not re-exported locally are not included in the resolved registry. A - re-exported entity is an entity that is imported and referenced in the local registry with some overriding. - -Open Questions: - -- Do we keep track of the imported registries in the resolved registry? If yes, how? Lineage? -- Can we leverage the attribute deduplication mechanism to simplify the merging of imported registries? ToDo -> Explain - attribute deduplication mechanism. -- Can we extend the deduplication mechanism to the signals? -- Materialized resolved registry (what see the jq, template and policy engines) vs Published resolved registry. - - Materialized Resolved Registry: This is what the jq, template and policy engines see. In this format there are - no deduplication of declaration. This format is not meant to be published. - - Published Resolved Registry: In this format, the deduplication of declaration is automatically done by Weaver. - This format is meant to be published. - -## Priorities - -TBD - -## General Open Questions - -- What about introducing a new type of semconv file that will let end-users define global overrides and global redact - directives? For example, the requirement level of attributes such as `client.address` or `server.address` will be - better defined by the end-user than by the library author, or a vendor. A similar approach could be used for redact - directives in order to address privacy concerns. +Liudmila's feedback on the core policies (need to be discussed): +``` +I think we need to come up with more nuanced rules. E.g. +- it's ok to change recommended attribute to required. +- it's ok to remove recommended (if you have good reasons not to populate recommended, this is your way to tell that + you don't populate it) +- it's ok to change conditionally required to recommended or required (if condition is always true) +- it might be ok to promote opt-in to recommended or even required + +maybe those rules are complex enough that we just want to start with a simple one that all required attributes should +stay required +``` \ No newline at end of file From 422022fb6da5d418c0c63e2522fd1fdd8d6555b3 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Fri, 6 Sep 2024 21:24:15 -0700 Subject: [PATCH 18/21] feat(multi-registry): Update draft of multi-registry spec. --- docs/specs/multi-registry/multi_registry.md | 47 +++++++++++-------- .../acme-http-server-lib/metric.yaml | 4 -- .../acme-http-server-lib/trace.yaml | 4 -- .../acme-http-server-lib/weaver_registry.yaml | 12 +++++ .../multi-registry/registries/app/app.yaml | 6 --- .../registries/app/weaver_registry.yaml | 15 ++++++ .../registries/waf-vendor/trace.yaml | 19 -------- .../registries/waf-vendor/waf_action.yaml | 16 ------- .../waf-vendor/weaver_registry.yaml | 12 +++++ 9 files changed, 67 insertions(+), 68 deletions(-) create mode 100644 docs/specs/multi-registry/registries/acme-http-server-lib/weaver_registry.yaml create mode 100644 docs/specs/multi-registry/registries/app/weaver_registry.yaml create mode 100644 docs/specs/multi-registry/registries/waf-vendor/weaver_registry.yaml diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index db7b02bc..1a601116 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -1,11 +1,11 @@ -# OpenTelemetry Semantic Convention Multi-Registry - Proposal +# OpenTelemetry Semantic Convention - Multi-Registry Proposal Status: Work in Progress ## Introduction This document outlines the proposal to support multiple semantic convention registries in OpenTelemetry. The goal is -to enable the community to define and publish their own semantic conventions, which can be used independently of the +to enable the community to define and publish their own semantic conventions, which can be operated independently of the core OpenTelemetry Semantic Conventions. This approach allows the community to define custom signals and attributes specific to their domains or use cases while still leveraging the core signals and attributes defined by OpenTelemetry. @@ -15,6 +15,8 @@ their own registries containing signals and attributes specific to their product define custom signals tailored to their libraries. Enterprises could also create internal registries to define custom signals that align with their specific needs and share these across teams and products. +The effect of combining semantic conventions, multiple registries, and Weaver is illustrated in the diagram below. + ![Semantic Convention Registries + Weaver = Open Ecosystem](images/otel-semconv-open-ecosystem.svg) This proposal outlines the changes required to support multiple semantic convention registries in OpenTelemetry, including @@ -32,6 +34,10 @@ in Weaver to support the creation, validation, generation, packaging, and public > “semantic convention registry” are used interchangeably. The term “entity” refers to any semantic convention > attribute, group, or signal. The abbreviation “OTEL” refers to the OpenTelemetry project. +> [!NOTE] +> A very close concept to the multi-registry proposal was discussed in the past under the name of "Application +> Telemetry Schema". See the [Application Telemetry Schema Proposal](https://github.com/open-telemetry/oteps/blob/main/text/0243-app-telemetry-schema-vision-roadmap.md). + ## Use Case Example The following use case is not exhaustive but provides an example of the types of multi-registry scenarios we aim to @@ -102,10 +108,11 @@ community-driven model for defining and using semantic conventions across divers be automatically resolved by the tooling. - **Circular Reference Handling**: Circular references between registries should be automatically detected, reported, and rejected to maintain system integrity and prevent conflicts. -- **Publication Format**: A published registry must adhere to a format and packaging designed to ensure stability, - ease of exchange, and consumption by other registries or tools. -- **Self-Contained**: A published registry should be self-contained. -- **Version Discoverability**: Versions of a published registry should be easily discoverable and accessible via a URL. +- **Open Publication Format**: A published registry must adhere to a open format and packaging designed to ensure + stability, ease of exchange, and consumption by other registries or tools. +- **Self-Contained**: A published registry should be self-contained to make the overall system more robust and + resilient to changes in dependencies. +- **Version Discoverability**: Available versions of a published registry should be easily discoverable. - **Transparency for Telemetry Consumers**: Downstream consumers of telemetry data (e.g., backends and dashboards) should never be exposed to conflicts within or between registries. They should see attributes and signals as defined, without scope or conflict resolution directives, ensuring a consistent and reliable data experience. @@ -127,14 +134,14 @@ It is required for published registries or registries with dependencies on other ```plaintext registry_root/ - domain_1 - domain_2 + domain_1/ + domain_2/ ... - registry + resources/ weaver_registry.yaml ``` -### New `weaver_registry.yaml` File +### `weaver_registry.yaml` File This file is used to describe the registry and its dependencies. It is optional for standalone registries but required for published registries or registries with dependencies on other registries. @@ -162,17 +169,19 @@ configuration: A registry can import one or more semantic conventions from other published registries. These dependencies are declared in the `dependencies` section. The `alias` field is optional but must be specified -if multiple imported registries share the same `name`. This design allows Weaver to automatically update -dependencies to the latest version when a new version becomes available (see the [Registry Update](#registry-update) -section). It also allows registry authors to use registry names as preferred, without concerns about -conflicts. +if multiple imported registries share the same `name`. + +Based on the `dependencies` section, Weaver can automatically update dependencies to the latest version when a new +version becomes available (see the [Registry Update](#registry-update) section). It also allows registry authors to use +registry names as preferred, without concerns about conflicts. Sections such as [Registry Resolution](#registry-resolution), [Registry Packaging](#registry-packaging), and [Registry Publication](#registry-publication) provide more details on how the `weaver_registry.yaml` file is used in these processes. Open Questions: -- Should we follow SemVer 2 for registry versions? It seems advisable, as Weaver can detect breaking changes. +- Should we follow SemVer 2 for registry versions? It seems advisable, as Weaver can detect breaking changes. However, + the resolution process doesn't rely on SemVer 2 in an important way. - How do we retrieve a published registry URL from a schema URL? Perhaps a new field in the schema URL file could point to the published registry URL, possibly named `registry_url`. @@ -197,10 +206,10 @@ local registry are never prefixed. In a group, the `extends` section can now reference an imported group using this new reference syntax. Similarly, the `attributes` section of a group can reference an imported attribute. -Referencing a group is currently unsupported. There are several options for implementing this feature, depending on -the uniqueness of the group ID agreed upon: +In the current semantic conventions specification, referencing a group is currently unsupported. There are several +options for implementing this feature, depending on the uniqueness of the group ID agreed upon: -- If a group ID is unique per group type, the following syntax could reference a group: +- If a group ID is unique per group type, the following syntax could reference a group without ambiguity: ```yaml groups: @@ -244,7 +253,7 @@ However, when a conflict cannot be resolved consistently and predictably, the us choosing the imported registry to resolve the conflict. The process hinges on the concept of mergeable entities (attributes or groups). For two entities to be considered -mergeable, they must meet the following basic criteria: +mergeable, they must, at least, meet the following basic criteria: 1. Both entities must be of the same type (e.g., attribute, metric, event, span, etc.). 2. They must share the same ID. 3. They must have a common ancestor (i.e., both were originally defined in the same registry). diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml index 834b610a..ebd6469c 100644 --- a/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/metric.yaml @@ -1,7 +1,3 @@ -imports: - - schema_url: https://opentelemetry.io/schemas/1.27.0 - name: otel - groups: - ref: otel:http.server.request.duration attributes: diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml index 6d803459..8fc5b03c 100644 --- a/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/trace.yaml @@ -1,7 +1,3 @@ -imports: - - schema_url: https://opentelemetry.io/schemas/1.27.0 - name: otel - groups: - ref: otel:trace.http.server brief: 'Trace reported by ACME HTTP Server library.' diff --git a/docs/specs/multi-registry/registries/acme-http-server-lib/weaver_registry.yaml b/docs/specs/multi-registry/registries/acme-http-server-lib/weaver_registry.yaml new file mode 100644 index 00000000..ab7e9536 --- /dev/null +++ b/docs/specs/multi-registry/registries/acme-http-server-lib/weaver_registry.yaml @@ -0,0 +1,12 @@ +name: acme_http_server_lib +version: 1.0.0 +schema_url: https://acme_server.com/schemas/1.0.0 +owner: + name: ACME HTTP Server Library + contact: contact@acme.com +maintainers: + - name: Sarah Connor +dependencies: + - name: otel + version: 1.27.0 + repository: https://opentelemetry.io/registries/ \ No newline at end of file diff --git a/docs/specs/multi-registry/registries/app/app.yaml b/docs/specs/multi-registry/registries/app/app.yaml index c985e593..6255339d 100644 --- a/docs/specs/multi-registry/registries/app/app.yaml +++ b/docs/specs/multi-registry/registries/app/app.yaml @@ -1,9 +1,3 @@ -imports: - - schema_url: https://opentelemetry.io/schemas/1.27.0 - name: otel - - schema_url: https://github.com/acme-http-server-lib/schema_v0.7.1.yaml - name: acme - groups: - id: endpoint.error type: event diff --git a/docs/specs/multi-registry/registries/app/weaver_registry.yaml b/docs/specs/multi-registry/registries/app/weaver_registry.yaml new file mode 100644 index 00000000..fc127c40 --- /dev/null +++ b/docs/specs/multi-registry/registries/app/weaver_registry.yaml @@ -0,0 +1,15 @@ +name: my_app +version: 1.0.0 +schema_url: https://mycompany.com/schemas/my_app/1.0.0 +owner: + name: MyCompany + contact: registry@mycompany.com +maintainers: + - name: John Doe +dependencies: + - name: otel + version: 1.27.0 + repository: https://opentelemetry.io/registries/ + - name: acme + version: 0.7.1 + repository: https://github.com/acme-http-server-lib/registries/ diff --git a/docs/specs/multi-registry/registries/waf-vendor/trace.yaml b/docs/specs/multi-registry/registries/waf-vendor/trace.yaml index 5fcf399b..7fffda50 100644 --- a/docs/specs/multi-registry/registries/waf-vendor/trace.yaml +++ b/docs/specs/multi-registry/registries/waf-vendor/trace.yaml @@ -1,30 +1,11 @@ -# New optional section to define external registries to import in the current registry. -# Note: Unused imports will be detected by Weaver and reported as warnings by default. -imports: - # Importing an external registry involves specifying the schema URL associated with that registry. - # The current schema file structure needs to be updated, either by adding a reference to a - # self-contained and fully resolved semantic convention or by directly integrating its content into the schema. - # This approach is also well aligned with the OpenTelemetry Protocol specification, - # which already specifies a schema URL at the resource and scope levels. - - schema_url: https://opentelemetry.io/schemas/1.27.0 - # A short and local name used to reference signals and attributes from the imported registry. - name: otel - groups: - # The following declaration overrides the definition of the `otel:trace.http.server` span group and changes - # the requirement level of few attributes to `required`. - # This override does not affect the original definition in the imported registry. - # Note: Similar to attribute references, group references cannot change the group type. - ref: otel:trace.http.server attributes: - # References to attributes from the imported registry must always be prefixed with the imported name and a colon. - # Local references do not require any prefix. - ref: otel:http.request.method requirement_level: required - ref: otel:client.address requirement_level: required - ref: otel:client.port requirement_level: required - # Locally defined attributes can be added to the imported group. - ref: waf.action.type requirement_level: required \ No newline at end of file diff --git a/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml b/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml index 7b8b0097..a4799c02 100644 --- a/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml +++ b/docs/specs/multi-registry/registries/waf-vendor/waf_action.yaml @@ -1,20 +1,4 @@ -# New optional section to define external registries to import in the current registry. -# Note: Unused imports will be detected by Weaver and reported as warnings by default. -imports: - # Importing an external registry involves specifying the schema URL associated with that registry. - # The current schema file structure needs to be updated, either by adding a reference to a - # self-contained and fully resolved semantic convention or by directly integrating its content into the schema. - # This approach is also well aligned with the OpenTelemetry Protocol specification, - # which already specifies a schema URL at the resource and scope levels. - - schema_url: https://opentelemetry.io/schemas/1.27.0 - # A short and local name used to reference signals and attributes from the imported registry. - # We use `my_otel` here just to show that it can be any name and that different aliases can be used for the same - # schema URL across different semconv files in the same registry. - name: my_otel - groups: - # Locally defined metric group can both extend a locally defined attribute group and - # reference imported individual attributes. - id: metric.waf.action.hit.count type: metric metric_name: waf.action.hit.count diff --git a/docs/specs/multi-registry/registries/waf-vendor/weaver_registry.yaml b/docs/specs/multi-registry/registries/waf-vendor/weaver_registry.yaml new file mode 100644 index 00000000..db71d3b7 --- /dev/null +++ b/docs/specs/multi-registry/registries/waf-vendor/weaver_registry.yaml @@ -0,0 +1,12 @@ +name: waf_vendor +version: 1.5.1 +schema_url: https://waf_vendor.com/schemas/1.5.1 +owner: + name: WAF Vendor + contact: contact@waf_vendor.com +maintainers: + - name: Neo +dependencies: + - name: otel + version: 1.27.0 + repository: https://opentelemetry.io/registries/ \ No newline at end of file From 5d68d7dfb79f88bb90c2318d8e760f69ade5b31c Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Sun, 8 Sep 2024 21:45:35 -0700 Subject: [PATCH 19/21] feat(multi-registry): uniqueness within groups is scoped by the type of group --- docs/specs/multi-registry/multi_registry.md | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index 1a601116..cc9befee 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -206,10 +206,9 @@ local registry are never prefixed. In a group, the `extends` section can now reference an imported group using this new reference syntax. Similarly, the `attributes` section of a group can reference an imported attribute. -In the current semantic conventions specification, referencing a group is currently unsupported. There are several -options for implementing this feature, depending on the uniqueness of the group ID agreed upon: - -- If a group ID is unique per group type, the following syntax could reference a group without ambiguity: +In the current semantic conventions specification, referencing a group is currently unsupported. Uniqueness within +groups is scoped by the type of group. It is entirely possible to have an event and a metric identified by the same ID. +Therefore, the following approach is proposed to reference a group: ```yaml groups: @@ -219,17 +218,6 @@ groups: - ... ``` -- If a group ID is unique across group types, the following syntax could reference a group: - -```yaml -groups: - ref: -``` - -> [!IMPORTANT] -> A decision on the uniqueness of group IDs across group types is required to determine the appropriate referencing -> syntax. - Similar to attribute references, group references cannot override the type, stability, or deprecation status of the imported group. From 3d625ad6bae62d9521992379276c1d97636700c3 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Sun, 8 Sep 2024 22:21:56 -0700 Subject: [PATCH 20/21] feat(multi-registry): dependency version can be a version number or latest --- docs/specs/multi-registry/multi_registry.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index cc9befee..f05c46b8 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -160,7 +160,7 @@ maintainers: url: dependencies: - name: - version: + version: repository: alias: configuration: From 58d4f9d774b86b7be31157a2be80412d94731dd0 Mon Sep 17 00:00:00 2001 From: Laurent Querel Date: Wed, 11 Sep 2024 15:03:32 -0700 Subject: [PATCH 21/21] chore(spec): Explain how Weaver will help resolving conflicts --- docs/specs/multi-registry/multi_registry.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/specs/multi-registry/multi_registry.md b/docs/specs/multi-registry/multi_registry.md index f05c46b8..e794f3fb 100644 --- a/docs/specs/multi-registry/multi_registry.md +++ b/docs/specs/multi-registry/multi_registry.md @@ -270,6 +270,14 @@ If automatic merging becomes a common request, a specific configuration option i The detection mechanism for circular dependencies must be expanded to identify circular dependencies between registries. +Weaver must be able to detect and report conflicts, circular dependencies, and other related issues with clear and +actionable error messages. These messages will heavily rely on the provenance information maintained by Weaver +throughout the resolution process of each registry. This ensures the user gains a clear understanding of the origin of +the conflict and the steps needed to resolve it. + +Additionally, Weaver must support a flag to enable, on-demand, a verbose mode that explains the resolution process. This +will allow users to understand when and why entities are deduplicated, merged, or overridden. + Finally, the resolution process must be capable of generating an OTEL schema file that includes the URL of the published registry, along with all transformation rules, both local and derived from imported registries. This allows consumers of the published registry to navigate across different versions of the registry, accommodating components of the