From 8e33392f972bff7e2b097e0a7e82997113da4195 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Mon, 8 Jan 2024 08:41:02 -0800 Subject: [PATCH 1/7] Support codegen by namespace (#243) Co-authored-by: Armin Ruech <7052238+arminru@users.noreply.github.com> Co-authored-by: Alexander Wert --- .github/workflows/semconvgen.yml | 12 +- semantic-conventions/CHANGELOG.md | 4 + semantic-conventions/README.md | 59 ++++++- .../src/opentelemetry/semconv/main.py | 9 +- .../semconv/model/semantic_attribute.py | 6 + .../semconv/model/semantic_convention.py | 3 + .../opentelemetry/semconv/templating/code.py | 150 ++++++++++++++++-- .../data/jinja/attribute_templates/template | 19 +-- .../all/FirstAttributes.java | 18 +++ .../all/SecondAttributes.java | 8 + .../all/ThirdAttributes.java | 13 ++ .../group_by_root_namespace/attributes.yml | 56 +++++++ .../attributes_and_metrics/First.java | 19 +++ .../attributes_and_metrics/SecondGroup.java | 19 +++ .../attributes_and_metrics/semconv.yml | 30 ++++ .../attributes_and_metrics/template | 76 +++++++++ .../no_group_prefix/FooAttributes.java | 13 ++ .../no_group_prefix/OtherAttributes.java | 8 + .../attributes_no_group_prefix.yml | 20 +++ .../single_file/All.java | 34 ++++ .../single_file/semconv.yml | 51 ++++++ .../single_file/template_single_file | 74 +++++++++ .../stable/ThirdAttributesStable.java | 8 + .../stable/template_only_stable | 49 ++++++ .../group_by_root_namespace/template_all | 45 ++++++ .../src/tests/semconv/templating/test_code.py | 146 +++++++++++++++-- 26 files changed, 903 insertions(+), 46 deletions(-) create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FirstAttributes.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/SecondAttributes.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/ThirdAttributes.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/First.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/SecondGroup.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/semconv.yml create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/template create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/FooAttributes.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/OtherAttributes.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/attributes_no_group_prefix.yml create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/All.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/semconv.yml create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/template_single_file create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/ThirdAttributesStable.java create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/template_only_stable create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/template_all diff --git a/.github/workflows/semconvgen.yml b/.github/workflows/semconvgen.yml index fcc3834d..f34296a9 100644 --- a/.github/workflows/semconvgen.yml +++ b/.github/workflows/semconvgen.yml @@ -13,7 +13,7 @@ jobs: tests: runs-on: ubuntu-latest defaults: - run: + run: working-directory: semantic-conventions/ steps: - uses: actions/checkout@v4 @@ -69,3 +69,13 @@ jobs: else tag_and_push "${GITHUB_REF#"refs/tags/"}" fi + - name: Push the Dev Docker image + if: startsWith(github.ref, 'refs/heads/feature/') + run: | + echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + function tag_and_push { + docker tag semconvgen "otel/semconvgen:${1}" && docker push "otel/semconvgen:${1}" + } + TAG="${GITHUB_REF#"refs/heads/"}" + TAG="${TAG/"/"/"-"}" + tag_and_push "${TAG}" diff --git a/semantic-conventions/CHANGELOG.md b/semantic-conventions/CHANGELOG.md index 786f0f40..068fdd35 100644 --- a/semantic-conventions/CHANGELOG.md +++ b/semantic-conventions/CHANGELOG.md @@ -16,6 +16,10 @@ Please update the changelog as part of any significant pull request. ([#271](https://github.com/open-telemetry/build-tools/pull/271)) - Add link to requirement levels definition from Markdown table title. ([#222](https://github.com/open-telemetry/build-tools/pull/222)) +- Added code-generation mode that groups attributes by the root namespace and ability to write each group into individual file. + [BREAKING] The `--file-per-group ` that used to create multiple directories (like `output//file`) now generates + multiple files (`output/file`) instead. + ([#243](https://github.com/open-telemetry/build-tools/pull/243)) ## v0.23.0 diff --git a/semantic-conventions/README.md b/semantic-conventions/README.md index e52a12a2..cd8f17e4 100644 --- a/semantic-conventions/README.md +++ b/semantic-conventions/README.md @@ -107,13 +107,14 @@ This way, multiple files are generated. The value of `pattern` can be one of the - `semconv_id`: The id of the semantic convention. - `prefix`: The prefix with which all attributes starts with. - `extends`: The id of the parent semantic convention. +- `root_namespace`: The root namespace of attribute to group by. Finally, additional value can be passed to the template in form of `key=value` pairs separated by comma using the `--parameters [{key=value},]+` or `-D` flag. ### Customizing Jinja's Whitespace Control -The image also supports customising +The image also supports customizing [Whitespace Control in Jinja templates](https://jinja.palletsprojects.com/en/3.1.x/templates/#whitespace-control) via the additional flag `--trim-whitespace`. Providing the flag will enable both `lstrip_blocks` and `trim_blocks`. @@ -153,3 +154,59 @@ Following checks are performed so it's considered non-critical and it's possible to suppress it with `--ignore-warnings` +### Accessing Semantic Conventions in the template + +When template is processed, it has access to a set of variables that depends on the `--file-per-group` value (or lack of it). +You can access properties of these variables and call Jinja or Python functions defined on them. + +#### Single file (no `--file-per-group` pattern is provided) + +Processes all parsed semantic conventions + +- `semconvs` - the dictionary containing parsed `BaseSemanticConvention` instances with semconv `id` as a key +- `attributes_and_templates` - the dictionary containing all attributes (including template ones) grouped by their root namespace. + Each element in the dictionary is a list of attributes that share the same root namespace. Attributes that don't have a namespace + appear under `""` key. +- `attributes` - the list of all attributes from all parsed semantic conventions. Does not include template attributes. +- `attribute_templates` - the list of all attribute templates from all parsed semantic conventions. + +#### The `root_namespace` pattern + +Processes a single namespace and is called for each namespace detected. + +- `attributes_and_templates` - the list containing all attributes (including template ones) in the given root namespace. +- `root_namespace` - the root namespace being processed. + +#### Other patterns + +Processes a single pattern value and is called for each distinct value. + +- `semconv` - the instance of parsed `BaseSemanticConvention` being processed. + +### Filtering and mapping + +Jinja templates has a notion of [filters](https://jinja.palletsprojects.com/en/2.11.x/templates/#list-of-builtin-filters) allowing to transform objects or filter lists. + +Semconvgen supports following additional filters to simplify common operations in templates. + +#### `SemanticAttribute` operations + +1. `is_definition` - Checks if the attribute is the original definition of the attribute and not a reference. +2. `is_deprecated` - Checks if the attribute is deprecated. The same check can also be done with `(attribute.stability | string()) == "StabilityLevel.DEPRECATED"` +3. `is_experimental` - Checks if the attribute is experimental. The same check can also be done with `(attribute.stability | string()) == "StabilityLevel.EXPERIMENTAL"` +4. `is_stable` - Checks if the attribute is experimental. The same check can also be done with `(attribute.stability | string()) == "StabilityLevel.STABLE"` +5. `is_template` - Checks if the attribute is a template attribute. + +#### String operations + +1. `first_up` - Upper-cases the first character in the string. Does not modify anything else +2. `regex_replace(text, pattern, replace)` - Makes regex-based replace in `text` string using `pattern`` +3. `to_camelcase` - Converts a string to camel case (using `.` and `_` as words delimiter in the original string). + The first character of every word is upper-cased, other characters are lower-cased. E.g. `foo.bAR_baz` becomes `fooBarBaz` +4. `to_const_name` - Converts a string to Python or Java constant name (SNAKE_CASE) replacing `.` or `-` with `_`. E.g. + `foo.bAR-baz` becomes `FOO_BAR_BAZ`. +5. `to_doc_brief` - Trims whitespace and removes dot at the end. E.g. ` Hello world.\t` becomes `Hello world` + +#### `BaseSemanticConvention` operations + +1. `is_metric` - Checks if semantic convention describes a metric. diff --git a/semantic-conventions/src/opentelemetry/semconv/main.py b/semantic-conventions/src/opentelemetry/semconv/main.py index 461d0c28..e14f7b8f 100644 --- a/semantic-conventions/src/opentelemetry/semconv/main.py +++ b/semantic-conventions/src/opentelemetry/semconv/main.py @@ -155,7 +155,8 @@ def add_code_parser(subparsers): parser.add_argument( "--output", "-o", - help="Specify the output file for the code generation.", + help="Specify the output file name for the code generation. " + "See also `--file-per-group` on how to generate multiple files.", type=str, required=True, ) @@ -169,8 +170,10 @@ def add_code_parser(subparsers): parser.add_argument( "--file-per-group", dest="pattern", - help="Each Semantic Convention is processed by the template and store in a different file. PATTERN is expected " - "to be the name of a SemanticConvention field and is prepended as a prefix to the output argument", + help="Semantic conventions are processed by the template and stored in a different file. " + "File names start with a 'pattern' and end with the name specified in the 'output' argument. " + "The 'pattern' can either match 'root_namespace' to group attributes by the root namespace or " + "match a name of Semantic Convention property which value will be used as a file name prefix.", type=str, ) parser.add_argument( diff --git a/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py b/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py index 32996f9d..9a53ed7e 100644 --- a/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py @@ -59,6 +59,7 @@ class SemanticAttribute: sampling_relevant: bool note: str position: List[int] + root_namespace: str inherited: bool = False imported: bool = False @@ -211,6 +212,10 @@ def parse( fqn = fqn.strip() parsed_brief = TextWithLinks(brief.strip() if brief else "") parsed_note = TextWithLinks(note.strip()) + + namespaces = fqn.split(".") + root_namespace = namespaces[0] if len(namespaces) > 1 else "" + attr = SemanticAttribute( fqn=fqn, attr_id=attr_id, @@ -226,6 +231,7 @@ def parse( sampling_relevant=sampling_relevant, note=parsed_note, position=position, + root_namespace=root_namespace, ) if attr.fqn in attributes: position = position_data[list(attribute)[0]] diff --git a/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py b/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py index 932cd578..2f320e25 100644 --- a/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py @@ -266,6 +266,9 @@ def __init__(self, group, strict_validation=True): self.metric_name = group.get("metric_name") self.unit = group.get("unit") self.instrument = group.get("instrument") + + namespaces = self.metric_name.split(".") + self.root_namespace = namespaces[0] if len(namespaces) > 1 else "" self.validate() def validate(self): diff --git a/semantic-conventions/src/opentelemetry/semconv/templating/code.py b/semantic-conventions/src/opentelemetry/semconv/templating/code.py index b8865291..aeb29d6e 100644 --- a/semantic-conventions/src/opentelemetry/semconv/templating/code.py +++ b/semantic-conventions/src/opentelemetry/semconv/templating/code.py @@ -21,12 +21,15 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape from opentelemetry.semconv.model.semantic_attribute import ( + AttributeType, RequirementLevel, SemanticAttribute, + StabilityLevel, TextWithLinks, ) from opentelemetry.semconv.model.semantic_convention import ( BaseSemanticConvention, + MetricSemanticConvention, SemanticConventionSet, ) from opentelemetry.semconv.model.utils import ID_RE @@ -160,10 +163,36 @@ def to_camelcase(name: str, first_upper=False) -> str: return first + "".join(word.capitalize() for word in rest) +def first_up(name: str) -> str: + return name[0].upper() + name[1:] + + +def is_stable(obj: typing.Union[SemanticAttribute, BaseSemanticConvention]) -> bool: + return obj.stability == StabilityLevel.STABLE + + def is_deprecated(obj: typing.Union[SemanticAttribute, BaseSemanticConvention]) -> bool: return obj.deprecated is not None +def is_experimental( + obj: typing.Union[SemanticAttribute, BaseSemanticConvention] +) -> bool: + return obj.stability is None or obj.stability == StabilityLevel.EXPERIMENTAL + + +def is_definition(attribute: SemanticAttribute) -> bool: + return attribute.is_local and attribute.ref is None + + +def is_template(attribute: SemanticAttribute) -> bool: + return AttributeType.is_template_type(str(attribute.attr_type)) + + +def is_metric(semconv: BaseSemanticConvention) -> bool: + return isinstance(semconv, MetricSemanticConvention) + + class CodeRenderer: pattern = f"{{{ID_RE.pattern}}}" @@ -196,6 +225,7 @@ def get_data_single_file( "semconvs": semconvset.models, "attributes": semconvset.attributes(), "attribute_templates": semconvset.attribute_templates(), + "attributes_and_templates": self._grouped_attribute_definitions(semconvset), } data.update(self.parameters) return data @@ -214,20 +244,30 @@ def setup_environment(env: Environment, trim_whitespace: bool): env.filters["to_const_name"] = to_const_name env.filters["merge"] = merge env.filters["to_camelcase"] = to_camelcase + env.filters["first_up"] = first_up env.filters["to_html_links"] = to_html_links env.filters["regex_replace"] = regex_replace env.filters["render_markdown"] = render_markdown env.filters["is_deprecated"] = is_deprecated + env.filters["is_definition"] = is_definition + env.filters["is_stable"] = is_stable + env.filters["is_experimental"] = is_experimental + env.filters["is_template"] = is_template + env.filters["is_metric"] = is_metric + env.tests["is_stable"] = is_stable + env.tests["is_experimental"] = is_experimental env.tests["is_deprecated"] = is_deprecated + env.tests["is_definition"] = is_definition + env.tests["is_template"] = is_template + env.tests["is_metric"] = is_metric env.trim_blocks = trim_whitespace env.lstrip_blocks = trim_whitespace @staticmethod - def prefix_output_file(file_name, pattern, semconv): + def prefix_output_file(file_name, prefix): basename = os.path.basename(file_name) dirname = os.path.dirname(file_name) - value = getattr(semconv, pattern) - return os.path.join(dirname, to_camelcase(value, True), basename) + return os.path.join(dirname, to_camelcase(prefix, True) + basename) def render( self, @@ -243,19 +283,97 @@ def render( autoescape=select_autoescape([""]), ) self.setup_environment(env, self.trim_whitespace) - if pattern: - for semconv in semconvset.models.values(): - output_name = self.prefix_output_file(output_file, pattern, semconv) - data = self.get_data_multiple_files(semconv, template_path) - template = env.get_template(file_name, globals=data) - template.globals["now"] = datetime.datetime.utcnow() - template.globals["version"] = os.environ.get("ARTIFACT_VERSION", "dev") - template.globals["RequirementLevel"] = RequirementLevel - template.stream(data).dump(output_name) + if pattern == "root_namespace": + self._render_group_by_root_namespace( + semconvset, template_path, file_name, output_file, env + ) + elif pattern is not None: + self._render_by_pattern( + semconvset, template_path, file_name, output_file, pattern, env + ) else: data = self.get_data_single_file(semconvset, template_path) template = env.get_template(file_name, globals=data) - template.globals["now"] = datetime.datetime.utcnow() - template.globals["version"] = os.environ.get("ARTIFACT_VERSION", "dev") - template.globals["RequirementLevel"] = RequirementLevel - template.stream(data).dump(output_file) + self._write_template_to_file(template, data, output_file) + + def _render_by_pattern( + self, + semconvset: SemanticConventionSet, + template_path: str, + file_name: str, + output_file: str, + pattern: str, + env: Environment, + ): + for semconv in semconvset.models.values(): + prefix = getattr(semconv, pattern) + output_name = self.prefix_output_file(output_file, prefix) + data = self.get_data_multiple_files(semconv, template_path) + template = env.get_template(file_name, globals=data) + self._write_template_to_file(template, data, output_name) + + def _render_group_by_root_namespace( + self, + semconvset: SemanticConventionSet, + template_path: str, + file_name: str, + output_file: str, + env: Environment, + ): + attribute_and_templates = self._grouped_attribute_definitions(semconvset) + metrics = self._grouped_metric_definitions(semconvset) + for ns, attribute_and_templates in attribute_and_templates.items(): + sanitized_ns = ns if ns != "" else "other" + output_name = self.prefix_output_file(output_file, sanitized_ns) + + data = { + "template": template_path, + "attributes_and_templates": attribute_and_templates, + "metrics": metrics.get(ns) or [], + "root_namespace": sanitized_ns, + } + data.update(self.parameters) + + template = env.get_template(file_name, globals=data) + self._write_template_to_file(template, data, output_name) + + def _grouped_attribute_definitions(self, semconvset): + grouped_attributes = {} + for semconv in semconvset.models.values(): + for attr in semconv.attributes_and_templates: + if not is_definition(attr): # skip references + continue + if attr.root_namespace not in grouped_attributes: + grouped_attributes[attr.root_namespace] = [] + grouped_attributes[attr.root_namespace].append(attr) + + for ns in grouped_attributes: + grouped_attributes[ns] = sorted(grouped_attributes[ns], key=lambda a: a.fqn) + return grouped_attributes + + def _grouped_metric_definitions(self, semconvset): + grouped_metrics = {} + for semconv in semconvset.models.values(): + if not is_metric(semconv): + continue + + if semconv.root_namespace not in grouped_metrics: + grouped_metrics[semconv.root_namespace] = [] + + grouped_metrics[semconv.root_namespace].append(semconv) + + for ns in grouped_metrics: + grouped_metrics[ns] = sorted( + grouped_metrics[ns], key=lambda a: a.metric_name + ) + return grouped_metrics + + def _write_template_to_file(self, template, data, output_name): + template.globals["now"] = datetime.datetime.utcnow() + template.globals["version"] = os.environ.get("ARTIFACT_VERSION", "dev") + template.globals["RequirementLevel"] = RequirementLevel + + content = template.render(data) + if content != "": + with open(output_name, "w", encoding="utf-8") as f: + f.write(content) diff --git a/semantic-conventions/src/tests/data/jinja/attribute_templates/template b/semantic-conventions/src/tests/data/jinja/attribute_templates/template index e32d00a3..c6b139df 100644 --- a/semantic-conventions/src/tests/data/jinja/attribute_templates/template +++ b/semantic-conventions/src/tests/data/jinja/attribute_templates/template @@ -25,22 +25,13 @@ {%- elif type == "double" -%} doubleKey {%- else -%} - {{lowerFirst(type)}}Key + {{ type | to_camelcase(False)}}Key {%- endif -%} {%- endmacro %} -{%- macro print_value(type, value) -%} - {{ "\"" if type == "String"}}{{value}}{{ "\"" if type == "String"}} -{%- endmacro %} -{%- macro upFirst(text) -%} - {{ text[0]|upper}}{{text[1:] }} -{%- endmacro %} -{%- macro lowerFirst(text) -%} - {{ text[0]|lower}}{{text[1:] }} -{%- endmacro %} package io.opentelemetry.instrumentation.api.attributetemplates; class AttributesTemplate { -{%- for attribute_template in attribute_templates if attribute_template.is_local and not attribute_template.ref %} +{%- for attribute_template in attribute_templates | select("is_definition") %} /** * {{attribute_template.brief | render_markdown(code="{{@code {0}}}", paragraph="{0}")}} @@ -57,9 +48,9 @@ class AttributesTemplate { {%- if attribute_template | is_deprecated %} @Deprecated {%- endif %} - public static final AttributeKey<{{upFirst(to_java_return_type(attribute_template.instantiated_type | string))}}> {{attribute_template.fqn | to_const_name}} = {{to_java_key_type(attribute_template.instantiated_type | string)}}("{{attribute_template.fqn}}"); + public static final AttributeKey<{{ to_java_return_type(attribute_template.instantiated_type | string) | to_camelcase(True)}}> {{attribute_template.fqn | to_const_name}} = {{to_java_key_type(attribute_template.instantiated_type | string)}}("{{attribute_template.fqn}}"); {%- endfor %} -{%- for attribute in attributes if attribute.is_local and not attribute.ref %} +{%- for attribute in attributes | select("is_definition") %} /** * {{attribute.brief | render_markdown(code="{{@code {0}}}", paragraph="{0}")}} @@ -76,6 +67,6 @@ class AttributesTemplate { {%- if attribute | is_deprecated %} @Deprecated {%- endif %} - public static final AttributeKey<{{upFirst(to_java_return_type(attribute.instantiated_type | string))}}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + public static final AttributeKey<{{ to_java_return_type(attribute.instantiated_type | string) | to_camelcase(True)}}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); {%- endfor %} } diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FirstAttributes.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FirstAttributes.java new file mode 100644 index 00000000..9281a5f2 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FirstAttributes.java @@ -0,0 +1,18 @@ +package io.opentelemetry.instrumentation.api.semconv; + +class FirstAttributes { + /** + * short description of attr_one + */ + public static final AttributeKey FIRST_ATTR_ONE = booleanKey("first.attr_one"); + + /** + * short description of attr_one_a + */ + public static final AttributeKey FIRST_ATTR_ONE_A = longKey("first.attr_one_a"); + + /** + * this is the description of attribute template + */ + public static final AttributeKeyTemplate FIRST_ATTR_TEMPLATE_ONE = stringKey("first.attr_template_one"); +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/SecondAttributes.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/SecondAttributes.java new file mode 100644 index 00000000..abc59440 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/SecondAttributes.java @@ -0,0 +1,8 @@ +package io.opentelemetry.instrumentation.api.semconv; + +class SecondAttributes { + /** + * short description of attr_two + */ + public static final AttributeKey SECOND_ATTR_TWO = stringKey("second.attr_two"); +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/ThirdAttributes.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/ThirdAttributes.java new file mode 100644 index 00000000..e4a9bca1 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/ThirdAttributes.java @@ -0,0 +1,13 @@ +package io.opentelemetry.instrumentation.api.semconv; + +class ThirdAttributes { + /** + * this is the description of attribute template + */ + public static final AttributeKeyTemplate THIRD_ATTR_TEMPLATE_THREE = stringKey("third.attr_template_three"); + + /** + * short description of attr_three + */ + public static final AttributeKey THIRD_ATTR_THREE = stringKey("third.attr_three"); +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml new file mode 100644 index 00000000..f4a1a6cc --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml @@ -0,0 +1,56 @@ +groups: + - id: first_group_id + type: attribute_group + brief: first description + prefix: first + attributes: + - id: attr_one + type: boolean + brief: short description of attr_one + - id: attr_template_one + type: template[string] + brief: > + this is the description of attribute template + examples: 'example' + + - id: second_group_id + brief: second description + prefix: second + span_kind: client + extends: first_group_id + attributes: + - id: attr_two + type: string + brief: short description of attr_two + examples: ['example_one', 'example_two'] + - id: first_group_part_two + type: resource + brief: first_a description + prefix: first + attributes: + - id: attr_one_a + type: int + brief: short description of attr_one_a + - ref: second.attr_two + - ref: third.attr_template_three + - id: third_group_id + brief: third description + prefix: third + attributes: + - id: attr_three + type: string + brief: short description of attr_three + examples: "3" + stability: stable + - id: attr_template_three + type: template[string] + brief: > + this is the description of attribute template + examples: 'example' + - id: forth_group_id + brief: forth description + attributes: + - id: attr_four + type: string + brief: short description of attr_four + examples: "4" diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/First.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/First.java new file mode 100644 index 00000000..7ec15b57 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/First.java @@ -0,0 +1,19 @@ +package io.opentelemetry.instrumentation.api.semconv; + +import io.opentelemetry.api.metrics.Meter; + +class First { + /** + * short description of attr_one + */ + public static final AttributeKey FIRST_ATTR_ONE = booleanKey("first.attr_one"); + /** + * first metric description + * Experimental: False + */ + public static final LongCounterBuilder createFirstMetric(Meter meter) { + return meter.counterBuilder("first.metric") + .setDescription("first metric description") + .setUnit("{one}"); + } +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/SecondGroup.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/SecondGroup.java new file mode 100644 index 00000000..20fbc0ed --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/SecondGroup.java @@ -0,0 +1,19 @@ +package io.opentelemetry.instrumentation.api.semconv; + +import io.opentelemetry.api.metrics.Meter; + +class SecondGroup { + /** + * short description of attr_two + */ + public static final AttributeKey SECOND_GROUP_ATTR_TWO = longKey("second_group.attr_two"); + /** + * second metric description + * Experimental: True + */ + public static final DoubleHistogramBuilder createSecondGroupMetric(Meter meter) { + return meter.histogramBuilder("second_group.metric") + .setDescription("second metric description") + .setUnit("s"); + } +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/semconv.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/semconv.yml new file mode 100644 index 00000000..3d2c670b --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/semconv.yml @@ -0,0 +1,30 @@ +groups: + - id: first_group_id + type: attribute_group + brief: first description + prefix: first + attributes: + - id: attr_one + type: boolean + brief: short description of attr_one + + - id: first_metric_id + brief: first metric description + metric_name: first.metric + instrument: counter + type: metric + unit: "{one}" + stability: stable + extends: first_group_id + + - id: second_group_id + brief: second metric description + metric_name: second_group.metric + type: metric + instrument: histogram + unit: "s" + prefix: second_group + attributes: + - id: attr_two + type: int + brief: short description of attr_two diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/template b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/template new file mode 100644 index 00000000..f1cacd96 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/template @@ -0,0 +1,76 @@ +{%- macro to_java_return_type(type) -%} + {%- if type == "string" -%} + String + {%- elif type == "string[]" -%} + List + {%- elif type == "boolean" -%} + boolean + {%- elif type == "int" -%} + long + {%- elif type == "double" -%} + double + {%- else -%} + {{type}} + {%- endif -%} +{%- endmacro %} +{%- macro to_java_key_type(type) -%} + {%- if type == "string" -%} + stringKey + {%- elif type == "string[]" -%} + stringArrayKey + {%- elif type == "boolean" -%} + booleanKey + {%- elif type == "int" -%} + longKey + {%- elif type == "double" -%} + doubleKey + {%- else -%} + {{ type | to_camelcase(False)}}Key + {%- endif -%} +{%- endmacro %} +{%- macro to_java_instrument_builder_factory(instrument) -%} + {%- if instrument == "counter" -%} + counterBuilder + {%- elif instrument == "histogram" -%} + histogramBuilder + {%- elif instrument == "updowncounter" -%} + upDownCounterBuilder + {%- elif instrument == "gauge" -%} + gaugeBuilder + {%- endif -%} +{%- endmacro %} +{%- macro to_java_instrument_builder_type(instrument) -%} + {%- if instrument == "counter" -%} + LongCounterBuilder + {%- elif instrument == "histogram" -%} + DoubleHistogramBuilder + {%- elif instrument == "updowncounter" -%} + LongUpDownCounterBuilder + {%- elif instrument == "gauge" -%} + DoubleGaugeBuilder + {%- endif -%} +{%- endmacro %} +package io.opentelemetry.instrumentation.api.semconv; + +import io.opentelemetry.api.metrics.Meter; + +class {{ root_namespace | to_camelcase(True) }} { +{%- for attribute in attributes_and_templates %} + + /** + * {{attribute.brief | render_markdown(code="{{@code {0}}}", paragraph="{0}")}} + */ + public static final AttributeKey<{{ to_java_return_type(attribute.instantiated_type | string) | first_up }}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); +{% endfor %} +{%- for metric in metrics %} + /** + * {{metric.brief | to_doc_brief}} + * Experimental: {{ metric | is_experimental }} + */ + public static final {{ to_java_instrument_builder_type(metric.instrument) }} create{{metric.metric_name | to_camelcase(True)}}(Meter meter) { + return meter.{{to_java_instrument_builder_factory(metric.instrument)}}("{{ metric.metric_name }}") + .setDescription("{{ metric.brief }}") + .setUnit("{{ metric.unit }}"); + } +{% endfor %} +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/FooAttributes.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/FooAttributes.java new file mode 100644 index 00000000..b09c7dc5 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/FooAttributes.java @@ -0,0 +1,13 @@ +package io.opentelemetry.instrumentation.api.semconv; + +class FooAttributes { + /** + * short description of attr_one + */ + public static final AttributeKey FOO_ATTR_ONE = booleanKey("foo.attr_one"); + + /** + * short description of foo.attr_two + */ + public static final AttributeKey FOO_ATTR_TWO = stringKey("foo.attr_two"); +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/OtherAttributes.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/OtherAttributes.java new file mode 100644 index 00000000..9a7b38b2 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/OtherAttributes.java @@ -0,0 +1,8 @@ +package io.opentelemetry.instrumentation.api.semconv; + +class OtherAttributes { + /** + * short description of bar_attr + */ + public static final AttributeKey BAR_ATTR = stringKey("bar_attr"); +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/attributes_no_group_prefix.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/attributes_no_group_prefix.yml new file mode 100644 index 00000000..d2dcb72e --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/attributes_no_group_prefix.yml @@ -0,0 +1,20 @@ +groups: + - id: group_with_prefix + type: attribute_group + brief: description + prefix: foo + attributes: + - id: attr_one + type: boolean + brief: short description of attr_one + - id: group_with_no_prefix + brief: description + attributes: + - id: foo.attr_two + type: string + brief: short description of foo.attr_two + examples: "foo" + - id: bar_attr + type: string + brief: short description of bar_attr + examples: "bar" \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/All.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/All.java new file mode 100644 index 00000000..cd1cabeb --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/All.java @@ -0,0 +1,34 @@ +package io.opentelemetry.instrumentation.api.semconv; + +class AllAttributes { + class FirstAttributes { + /** + * short description of attr_one + */ + public static final AttributeKey FIRST_ATTR_ONE = booleanKey("first.attr_one"); + + /** + * short description of attr_one_a + */ + public static final AttributeKey FIRST_ATTR_ONE_A = longKey("first.attr_one_a"); + + /** + * this is the description of attribute template + */ + public static final AttributeKeyTemplate FIRST_ATTR_TEMPLATE_ONE = stringKey("first.attr_template_one"); + } + class SecondAttributes { + /** + * short description of attr_two + */ + public static final AttributeKey SECOND_ATTR_TWO = stringKey("second.attr_two"); + } + /** + * short description of attr_four + */ + public static final AttributeKey ATTR_FOUR = stringKey("attr_four"); + /** + * first metric description + */ + public static final String FIRST_METRIC_NAME = "first.metric.name"; +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/semconv.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/semconv.yml new file mode 100644 index 00000000..37d33fc2 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/semconv.yml @@ -0,0 +1,51 @@ +groups: + - id: first_group_id + type: attribute_group + brief: first description + prefix: first + attributes: + - id: attr_one + type: boolean + brief: short description of attr_one + - id: attr_template_one + type: template[string] + brief: > + this is the description of attribute template + examples: 'example' + + - id: second_group_id + brief: second description + prefix: second + span_kind: client + extends: first_group_id + attributes: + - id: attr_two + type: string + brief: short description of attr_two + examples: ['example_one', 'example_two'] + + - id: first_group_part_two + type: resource + brief: first_a description + prefix: first + attributes: + - id: attr_one_a + type: int + brief: short description of attr_one_a + - ref: second.attr_two + + - id: forth_group_id + brief: forth description + attributes: + - id: attr_four + type: string + brief: short description of attr_four + examples: "4" + + - id: first_metric_id + brief: first metric description + metric_name: first.metric.name + instrument: counter + type: metric + unit: "{one}" + extends: first_group_id \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/template_single_file b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/template_single_file new file mode 100644 index 00000000..986ec3d2 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/template_single_file @@ -0,0 +1,74 @@ +{%- macro to_java_return_type(type) -%} + {%- if type == "string" -%} + String + {%- elif type == "string[]" -%} + List + {%- elif type == "boolean" -%} + boolean + {%- elif type == "int" -%} + long + {%- elif type == "double" -%} + double + {%- else -%} + {{type}} + {%- endif -%} +{%- endmacro %} +{%- macro to_java_key_type(type) -%} + {%- if type == "string" -%} + stringKey + {%- elif type == "string[]" -%} + stringArrayKey + {%- elif type == "boolean" -%} + booleanKey + {%- elif type == "int" -%} + longKey + {%- elif type == "double" -%} + doubleKey + {%- else -%} + {{ type | to_camelcase(False)}}Key + {%- endif -%} +{%- endmacro %} +package io.opentelemetry.instrumentation.api.semconv; + +class AllAttributes { +{%- for root_ns in attributes_and_templates %} + + {% if root_ns != "" %} + class {{root_ns | first_up}}Attributes { + {%- for attribute in attributes_and_templates[root_ns] %} + + /** + * {{attribute.brief | render_markdown(code="{{@code {0}}}", paragraph="{0}")}} + */ + {% if attribute | is_template %} + public static final AttributeKeyTemplate<{{ to_java_return_type(attribute.instantiated_type | string) | first_up}}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + {% else %} + public static final AttributeKey<{{ to_java_return_type(attribute.instantiated_type | string) | first_up }}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + {% endif %} + + {%- endfor %} + } + {%- endif %} +{%- endfor %} +{# non-namespaced attributes #} +{%- for attribute in attributes_and_templates[""] %} + /** + * {{attribute.brief | render_markdown(code="{{@code {0}}}", paragraph="{0}")}} + */ + {% if attribute | is_template %} + public static final AttributeKeyTemplate<{{ to_java_return_type(attribute.instantiated_type | string) | first_up}}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + {% else %} + public static final AttributeKey<{{ to_java_return_type(attribute.instantiated_type | string) | first_up }}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + {% endif %} + +{%- endfor %} +{%- for id in semconvs %} +{%- if semconvs[id] | is_metric %} +{% set metric = semconvs[id] %} + /** + * {{metric.brief | to_doc_brief}} + */ + public static final String {{metric.metric_name | to_const_name}} = "{{metric.metric_name}}"; +{% endif %} +{% endfor %} +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/ThirdAttributesStable.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/ThirdAttributesStable.java new file mode 100644 index 00000000..71263bf3 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/ThirdAttributesStable.java @@ -0,0 +1,8 @@ +package io.opentelemetry.instrumentation.api.semconv; + +class ThirdAttributes { + /** + * short description of attr_three + */ + public static final AttributeKey THIRD_ATTR_THREE = stringKey("third.attr_three"); +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/template_only_stable b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/template_only_stable new file mode 100644 index 00000000..deb83073 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/template_only_stable @@ -0,0 +1,49 @@ +{%- macro to_java_return_type(type) -%} + {%- if type == "string" -%} + String + {%- elif type == "string[]" -%} + List + {%- elif type == "boolean" -%} + boolean + {%- elif type == "int" -%} + long + {%- elif type == "double" -%} + double + {%- else -%} + {{type}} + {%- endif -%} +{%- endmacro %} +{%- macro to_java_key_type(type) -%} + {%- if type == "string" -%} + stringKey + {%- elif type == "string[]" -%} + stringArrayKey + {%- elif type == "boolean" -%} + booleanKey + {%- elif type == "int" -%} + longKey + {%- elif type == "double" -%} + doubleKey + {%- else -%} + {{ type | to_camelcase(False)}}Key + {%- endif -%} +{%- endmacro %} +{%- set stable_attributes_and_templates = attributes_and_templates | select("is_stable") | list %} + +{%- if stable_attributes_and_templates | count > 0 %} +package io.opentelemetry.instrumentation.api.semconv; + +class {{ root_namespace | to_camelcase(True) }}Attributes { +{%- for attribute in stable_attributes_and_templates %} + + /** + * {{attribute.brief | render_markdown(code="{{@code {0}}}", paragraph="{0}")}} + */ + {% if attribute | is_template %} + public static final AttributeKeyTemplate<{{ to_java_return_type(attribute.instantiated_type | string) | first_up}}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + {% else %} + public static final AttributeKey<{{ to_java_return_type(attribute.instantiated_type | string) | first_up }}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + {% endif %} +{% endfor %} +} +{%- endif %} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/template_all b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/template_all new file mode 100644 index 00000000..2b26c2a7 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/template_all @@ -0,0 +1,45 @@ +{%- macro to_java_return_type(type) -%} + {%- if type == "string" -%} + String + {%- elif type == "string[]" -%} + List + {%- elif type == "boolean" -%} + boolean + {%- elif type == "int" -%} + long + {%- elif type == "double" -%} + double + {%- else -%} + {{type}} + {%- endif -%} +{%- endmacro %} +{%- macro to_java_key_type(type) -%} + {%- if type == "string" -%} + stringKey + {%- elif type == "string[]" -%} + stringArrayKey + {%- elif type == "boolean" -%} + booleanKey + {%- elif type == "int" -%} + longKey + {%- elif type == "double" -%} + doubleKey + {%- else -%} + {{ type | to_camelcase(False)}}Key + {%- endif -%} +{%- endmacro %} +package io.opentelemetry.instrumentation.api.semconv; + +class {{ root_namespace | to_camelcase(True) }}Attributes { +{%- for attribute in attributes_and_templates %} + + /** + * {{attribute.brief | render_markdown(code="{{@code {0}}}", paragraph="{0}")}} + */ + {% if attribute | is_template %} + public static final AttributeKeyTemplate<{{ to_java_return_type(attribute.instantiated_type | string) | first_up}}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + {% else %} + public static final AttributeKey<{{ to_java_return_type(attribute.instantiated_type | string) | first_up }}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); + {% endif %} +{% endfor %} +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/semconv/templating/test_code.py b/semantic-conventions/src/tests/semconv/templating/test_code.py index 643505c9..e60eb927 100644 --- a/semantic-conventions/src/tests/semconv/templating/test_code.py +++ b/semantic-conventions/src/tests/semconv/templating/test_code.py @@ -1,4 +1,5 @@ -import io +import os +import tempfile from opentelemetry.semconv.model.semantic_convention import SemanticConventionSet from opentelemetry.semconv.templating.code import CodeRenderer @@ -12,9 +13,10 @@ def test_codegen_units(test_file_path, read_test_file): template_path = test_file_path("jinja", "metrics", "units_template") renderer = CodeRenderer({}, trim_whitespace=False) - output = io.StringIO() - renderer.render(semconv, template_path, output, None) - result = output.getvalue() + filename = os.path.join(tempfile.mkdtemp(), "Attributes.java") + renderer.render(semconv, template_path, filename, None) + with open(filename, "r", encoding="utf-8") as f: + result = f.read() expected = read_test_file("jinja", "metrics", "expected.java") @@ -32,9 +34,10 @@ def test_strip_blocks_enabled(test_file_path, read_test_file): ) renderer = CodeRenderer({}, trim_whitespace=True) - output = io.StringIO() - renderer.render(semconv, template_path, output, None) - result = output.getvalue() + filename = os.path.join(tempfile.mkdtemp(), "Attributes.java") + renderer.render(semconv, template_path, filename, None) + with open(filename, "r", encoding="utf-8") as f: + result = f.read() expected = read_test_file( "jinja", "metrics", "expected_trim_whitespace_enabled.java" @@ -53,10 +56,131 @@ def test_codegen_attribute_templates(test_file_path, read_test_file): template_path = test_file_path("jinja", "attribute_templates", "template") renderer = CodeRenderer({}, trim_whitespace=False) - output = io.StringIO() - renderer.render(semconv, template_path, output, None) - result = output.getvalue() - + filename = os.path.join(tempfile.mkdtemp(), "Attributes.java") + renderer.render(semconv, template_path, filename, None) + with open(filename, "r", encoding="utf-8") as f: + result = f.read() expected = read_test_file("jinja", "attribute_templates", "expected.java") assert result == expected + + +def test_codegen_attribute_root_ns(test_file_path, read_test_file): + semconv = SemanticConventionSet(debug=False) + + semconv.parse(test_file_path("jinja", "group_by_root_namespace", "attributes.yml")) + semconv.finish() + + template_path = test_file_path("jinja", "group_by_root_namespace", "template_all") + renderer = CodeRenderer({}, trim_whitespace=True) + + test_path = os.path.join("group_by_root_namespace", "all") + tmppath = tempfile.mkdtemp() + renderer.render( + semconv, + template_path, + os.path.join(tmppath, "Attributes.java"), + "root_namespace", + ) + + first = read_test_file("jinja", test_path, "FirstAttributes.java") + check_file(tmppath, "FirstAttributes.java", first) + + second = read_test_file("jinja", test_path, "SecondAttributes.java") + check_file(tmppath, "SecondAttributes.java", second) + + third = read_test_file("jinja", test_path, "ThirdAttributes.java") + check_file(tmppath, "ThirdAttributes.java", third) + + +def test_codegen_attribute_root_ns_stable(test_file_path, read_test_file): + semconv = SemanticConventionSet(debug=False) + semconv.parse(test_file_path("jinja", "group_by_root_namespace", "attributes.yml")) + semconv.finish() + + test_path = os.path.join("group_by_root_namespace", "stable") + template_path = test_file_path("jinja", test_path, "template_only_stable") + renderer = CodeRenderer({}, trim_whitespace=True) + + tmppath = tempfile.mkdtemp() + renderer.render( + semconv, + template_path, + os.path.join(tmppath, "Attributes.java"), + "root_namespace", + ) + + thirdStable = read_test_file("jinja", test_path, "ThirdAttributesStable.java") + check_file(tmppath, "ThirdAttributes.java", thirdStable) + assert not os.path.isfile(os.path.join(tmppath, "FirstAttributes.java")) + assert not os.path.isfile(os.path.join(tmppath, "SecondAttributes.java")) + + +def test_codegen_attribute_root_ns_no_group_prefix(test_file_path, read_test_file): + semconv = SemanticConventionSet(debug=False) + + test_path = os.path.join("group_by_root_namespace", "no_group_prefix") + semconv.parse(test_file_path("jinja", test_path, "attributes_no_group_prefix.yml")) + semconv.finish() + + template_path = test_file_path("jinja", "group_by_root_namespace", "template_all") + renderer = CodeRenderer({}, trim_whitespace=True) + + tmppath = tempfile.mkdtemp() + renderer.render( + semconv, + template_path, + os.path.join(tmppath, "Attributes.java"), + "root_namespace", + ) + + res = read_test_file("jinja", test_path, "FooAttributes.java") + check_file(tmppath, "FooAttributes.java", res) + + other = read_test_file("jinja", test_path, "OtherAttributes.java") + check_file(tmppath, "OtherAttributes.java", other) + + +def test_codegen_attribute_root_ns_single_file(test_file_path, read_test_file): + semconv = SemanticConventionSet(debug=False) + + test_path = os.path.join("group_by_root_namespace", "single_file") + semconv.parse(test_file_path("jinja", test_path, "semconv.yml")) + semconv.finish() + + template_path = test_file_path("jinja", test_path, "template_single_file") + renderer = CodeRenderer({}, trim_whitespace=True) + + tmppath = tempfile.mkdtemp() + renderer.render(semconv, template_path, os.path.join(tmppath, "All.java"), None) + + result = read_test_file("jinja", test_path, "All.java") + check_file(tmppath, "All.java", result) + + +def test_codegen_attribute_root_ns_metrics(test_file_path, read_test_file): + semconv = SemanticConventionSet(debug=False) + + test_path = os.path.join("group_by_root_namespace", "attributes_and_metrics") + semconv.parse(test_file_path("jinja", test_path, "semconv.yml")) + semconv.finish() + + template_path = test_file_path("jinja", test_path, "template") + renderer = CodeRenderer({}, trim_whitespace=True) + + tmppath = tempfile.mkdtemp() + renderer.render( + semconv, template_path, os.path.join(tmppath, ".java"), "root_namespace" + ) + + first = read_test_file("jinja", test_path, "First.java") + check_file(tmppath, "First.java", first) + + second = read_test_file("jinja", test_path, "SecondGroup.java") + check_file(tmppath, "SecondGroup.java", second) + + +def check_file(tmppath, actual_filename, expected_content): + with open(os.path.join(tmppath, actual_filename), "r", encoding="utf-8") as f: + actual = f.read() + assert actual == expected_content From aa970ac58f4189943fb55d4546d60ffdc57a7235 Mon Sep 17 00:00:00 2001 From: Armin Ruech <7052238+arminru@users.noreply.github.com> Date: Mon, 12 Feb 2024 18:19:54 +0100 Subject: [PATCH 2/7] Update semconvgen.yml workflow to run on feature/* branches (#256) --- .github/workflows/semconvgen.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/semconvgen.yml b/.github/workflows/semconvgen.yml index f34296a9..29d38234 100644 --- a/.github/workflows/semconvgen.yml +++ b/.github/workflows/semconvgen.yml @@ -2,9 +2,13 @@ name: Semantic Convention Generator on: push: tags: [ '**' ] - branches: [ main ] + branches: + - main + - 'feature/**' pull_request: - branches: [ main ] + branches: + - main + - 'feature/**' paths: - .github/workflows/semconvgen.yml - 'semantic-conventions/**' From 4bc84308ddada576646e95086755ce851e83fa79 Mon Sep 17 00:00:00 2001 From: Josh Suereth Date: Thu, 15 Feb 2024 08:59:44 -0500 Subject: [PATCH 3/7] Allow --output to be templatized when generating multiple files. (#263) --- semantic-conventions/README.md | 25 +++++++++++- .../opentelemetry/semconv/templating/code.py | 31 ++++++++++++--- .../src/tests/semconv/templating/test_code.py | 39 +++++++++++++++++-- 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/semantic-conventions/README.md b/semantic-conventions/README.md index cd8f17e4..d671ad10 100644 --- a/semantic-conventions/README.md +++ b/semantic-conventions/README.md @@ -101,7 +101,7 @@ By default, all models are fed into the specified template at once, i.e. only a This is helpful to generate constants for the semantic attributes, [example from opentelemetry-java](https://github.com/open-telemetry/semantic-conventions-java#generating-semantic-conventions). If the parameter `--file-per-group {pattern}` is set, a single yaml model is fed into the template -and the value of `pattern` is resolved from the model and attached as prefix to the output argument. +and the value of `pattern` is resolved from the model and may be used in the output argument. This way, multiple files are generated. The value of `pattern` can be one of the following: - `semconv_id`: The id of the semantic convention. @@ -109,6 +109,29 @@ This way, multiple files are generated. The value of `pattern` can be one of the - `extends`: The id of the parent semantic convention. - `root_namespace`: The root namespace of attribute to group by. +The `--output` parameter, when `--file-per-group` is used is evaluated as a template. The following variables are provided to output: + +- `prefix`: A prefix name for files, determined from the grouping. e.g. `http`, `database`, `user-agent`. +- `pascal_prefix`: A Pascal-case prefix name for files. e.g. `Http`, `Database`, `UserAgent`. +- `camel_prefix`: A camel-case prefix name for files. e.g. `http`, `database`, `userAgent`. +- `snake_prefix`: A snake-case prefix name for files. e.g. `http`, `database`, `user_agent`. + +For example, you could do the following: + +```bash +docker run --rm \ + -v ${SCRIPT_DIR}/opentelemetry-specification/semantic_conventions/trace:/source \ + -v ${SCRIPT_DIR}/templates:/templates \ + -v ${ROOT_DIR}/semconv/src/main/java/io/opentelemetry/semconv/trace/attributes/:/output \ + otel/semconvgen:$GENERATOR_VERSION \ + --yaml-root /source \ + code \ + --template /templates/SemanticAttributes.java.j2 \ + --file-per-group root_namespace \ + --output "/output/{{pascal_prefix}}Attributes.java" \ + ...other parameters... +``` + Finally, additional value can be passed to the template in form of `key=value` pairs separated by comma using the `--parameters [{key=value},]+` or `-D` flag. diff --git a/semantic-conventions/src/opentelemetry/semconv/templating/code.py b/semantic-conventions/src/opentelemetry/semconv/templating/code.py index aeb29d6e..6b828fcb 100644 --- a/semantic-conventions/src/opentelemetry/semconv/templating/code.py +++ b/semantic-conventions/src/opentelemetry/semconv/templating/code.py @@ -163,6 +163,13 @@ def to_camelcase(name: str, first_upper=False) -> str: return first + "".join(word.capitalize() for word in rest) +def to_snake_case(name): + name = re.sub("(.)([A-Z][a-z]+)", r"\1_\2", name) + name = re.sub("__([A-Z])", r"_\1", name) + name = re.sub("([a-z0-9])([A-Z])", r"\1_\2", name) + return name.lower() + + def first_up(name: str) -> str: return name[0].upper() + name[1:] @@ -264,10 +271,22 @@ def setup_environment(env: Environment, trim_whitespace: bool): env.lstrip_blocks = trim_whitespace @staticmethod - def prefix_output_file(file_name, prefix): - basename = os.path.basename(file_name) - dirname = os.path.dirname(file_name) - return os.path.join(dirname, to_camelcase(prefix, True) + basename) + def prefix_output_file(env, file_name, prefix): + # We treat incoming file names as a pattern. + # We allow will give them access to the same jinja model as file creation + # and we'll make sure a few things are available there, specifically: + # pascal case, camel case and snake case + data = { + "prefix": prefix, + "pascal_prefix": to_camelcase(prefix, True), + "camel_prefix": to_camelcase(prefix, False), + "snake_prefix": to_snake_case(prefix), + } + template = env.from_string(file_name) + full_name = template.render(data) + dirname = os.path.dirname(full_name) + basename = os.path.basename(full_name) + return os.path.join(dirname, basename) def render( self, @@ -307,7 +326,7 @@ def _render_by_pattern( ): for semconv in semconvset.models.values(): prefix = getattr(semconv, pattern) - output_name = self.prefix_output_file(output_file, prefix) + output_name = self.prefix_output_file(env, output_file, prefix) data = self.get_data_multiple_files(semconv, template_path) template = env.get_template(file_name, globals=data) self._write_template_to_file(template, data, output_name) @@ -324,7 +343,7 @@ def _render_group_by_root_namespace( metrics = self._grouped_metric_definitions(semconvset) for ns, attribute_and_templates in attribute_and_templates.items(): sanitized_ns = ns if ns != "" else "other" - output_name = self.prefix_output_file(output_file, sanitized_ns) + output_name = self.prefix_output_file(env, output_file, sanitized_ns) data = { "template": template_path, diff --git a/semantic-conventions/src/tests/semconv/templating/test_code.py b/semantic-conventions/src/tests/semconv/templating/test_code.py index e60eb927..e85a81de 100644 --- a/semantic-conventions/src/tests/semconv/templating/test_code.py +++ b/semantic-conventions/src/tests/semconv/templating/test_code.py @@ -79,7 +79,7 @@ def test_codegen_attribute_root_ns(test_file_path, read_test_file): renderer.render( semconv, template_path, - os.path.join(tmppath, "Attributes.java"), + os.path.join(tmppath, "{{pascal_prefix}}Attributes.java"), "root_namespace", ) @@ -93,6 +93,34 @@ def test_codegen_attribute_root_ns(test_file_path, read_test_file): check_file(tmppath, "ThirdAttributes.java", third) +def test_codegen_attribute_root_ns_snake_case_file(test_file_path, read_test_file): + semconv = SemanticConventionSet(debug=False) + + semconv.parse(test_file_path("jinja", "group_by_root_namespace", "attributes.yml")) + semconv.finish() + + template_path = test_file_path("jinja", "group_by_root_namespace", "template_all") + renderer = CodeRenderer({}, trim_whitespace=True) + + test_path = os.path.join("group_by_root_namespace", "all") + tmppath = tempfile.mkdtemp() + renderer.render( + semconv, + template_path, + os.path.join(tmppath, "{{snake_prefix}}_attributes.java"), + "root_namespace", + ) + + first = read_test_file("jinja", test_path, "FirstAttributes.java") + check_file(tmppath, "first_attributes.java", first) + + second = read_test_file("jinja", test_path, "SecondAttributes.java") + check_file(tmppath, "second_attributes.java", second) + + third = read_test_file("jinja", test_path, "ThirdAttributes.java") + check_file(tmppath, "third_attributes.java", third) + + def test_codegen_attribute_root_ns_stable(test_file_path, read_test_file): semconv = SemanticConventionSet(debug=False) semconv.parse(test_file_path("jinja", "group_by_root_namespace", "attributes.yml")) @@ -106,7 +134,7 @@ def test_codegen_attribute_root_ns_stable(test_file_path, read_test_file): renderer.render( semconv, template_path, - os.path.join(tmppath, "Attributes.java"), + os.path.join(tmppath, "{{pascal_prefix}}Attributes.java"), "root_namespace", ) @@ -130,7 +158,7 @@ def test_codegen_attribute_root_ns_no_group_prefix(test_file_path, read_test_fil renderer.render( semconv, template_path, - os.path.join(tmppath, "Attributes.java"), + os.path.join(tmppath, "{{pascal_prefix}}Attributes.java"), "root_namespace", ) @@ -170,7 +198,10 @@ def test_codegen_attribute_root_ns_metrics(test_file_path, read_test_file): tmppath = tempfile.mkdtemp() renderer.render( - semconv, template_path, os.path.join(tmppath, ".java"), "root_namespace" + semconv, + template_path, + os.path.join(tmppath, "{{pascal_prefix}}.java"), + "root_namespace", ) first = read_test_file("jinja", test_path, "First.java") From eccadce704203603b7bf5177db881343398d0017 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Tue, 20 Feb 2024 12:20:11 -0800 Subject: [PATCH 4/7] Add metrics to the context of non-scoped codegen (#270) --- semantic-conventions/README.md | 3 +- .../opentelemetry/semconv/templating/code.py | 9 ++++++ .../data/jinja/metrics/expected_metrics.java | 16 ++++++++++ .../tests/data/jinja/metrics/metrics_template | 11 +++++++ .../src/tests/data/yaml/metrics/metrics.yaml | 30 +++++++++++++++++++ .../src/tests/semconv/templating/test_code.py | 18 +++++++++++ 6 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 semantic-conventions/src/tests/data/jinja/metrics/expected_metrics.java create mode 100644 semantic-conventions/src/tests/data/jinja/metrics/metrics_template create mode 100644 semantic-conventions/src/tests/data/yaml/metrics/metrics.yaml diff --git a/semantic-conventions/README.md b/semantic-conventions/README.md index d671ad10..33876b6c 100644 --- a/semantic-conventions/README.md +++ b/semantic-conventions/README.md @@ -189,9 +189,10 @@ Processes all parsed semantic conventions - `semconvs` - the dictionary containing parsed `BaseSemanticConvention` instances with semconv `id` as a key - `attributes_and_templates` - the dictionary containing all attributes (including template ones) grouped by their root namespace. Each element in the dictionary is a list of attributes that share the same root namespace. Attributes that don't have a namespace - appear under `""` key. + appear under `""` key. Attributes and templates are sorted by attribute name. - `attributes` - the list of all attributes from all parsed semantic conventions. Does not include template attributes. - `attribute_templates` - the list of all attribute templates from all parsed semantic conventions. +- `metrics` - the list of all metric semantic conventions sorted by metric name. #### The `root_namespace` pattern diff --git a/semantic-conventions/src/opentelemetry/semconv/templating/code.py b/semantic-conventions/src/opentelemetry/semconv/templating/code.py index 6b828fcb..e1a554c8 100644 --- a/semantic-conventions/src/opentelemetry/semconv/templating/code.py +++ b/semantic-conventions/src/opentelemetry/semconv/templating/code.py @@ -233,6 +233,7 @@ def get_data_single_file( "attributes": semconvset.attributes(), "attribute_templates": semconvset.attribute_templates(), "attributes_and_templates": self._grouped_attribute_definitions(semconvset), + "metrics": self._all_metrics_definitions(semconvset), } data.update(self.parameters) return data @@ -387,6 +388,14 @@ def _grouped_metric_definitions(self, semconvset): ) return grouped_metrics + def _all_metrics_definitions(self, semconvset): + all_metrics = [] + for semconv in semconvset.models.values(): + if is_metric(semconv): + all_metrics.append(semconv) + + return sorted(all_metrics, key=lambda a: a.metric_name) + def _write_template_to_file(self, template, data, output_name): template.globals["now"] = datetime.datetime.utcnow() template.globals["version"] = os.environ.get("ARTIFACT_VERSION", "dev") diff --git a/semantic-conventions/src/tests/data/jinja/metrics/expected_metrics.java b/semantic-conventions/src/tests/data/jinja/metrics/expected_metrics.java new file mode 100644 index 00000000..8dccdd34 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/metrics/expected_metrics.java @@ -0,0 +1,16 @@ +class AllMetrics { + /** + * first metric description + * Unit: {one} + * Instrument: counter + * Experimental: False + */ + public static final String FIRST_METRIC = "first.metric"; + /** + * second metric description + * Unit: s + * Instrument: histogram + * Experimental: True + */ + public static final String SECOND_GROUP_METRIC = "second_group.metric"; +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/metrics/metrics_template b/semantic-conventions/src/tests/data/jinja/metrics/metrics_template new file mode 100644 index 00000000..c49a3803 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/metrics/metrics_template @@ -0,0 +1,11 @@ +class AllMetrics { +{%- for metric in metrics %} + /** + * {{metric.brief | to_doc_brief}} + * Unit: {{ metric.unit }} + * Instrument: {{ metric.instrument }} + * Experimental: {{ metric | is_experimental }} + */ + public static final String {{ metric.metric_name | to_const_name }} = "{{metric.metric_name}}"; +{%- endfor %} +} diff --git a/semantic-conventions/src/tests/data/yaml/metrics/metrics.yaml b/semantic-conventions/src/tests/data/yaml/metrics/metrics.yaml new file mode 100644 index 00000000..3d2c670b --- /dev/null +++ b/semantic-conventions/src/tests/data/yaml/metrics/metrics.yaml @@ -0,0 +1,30 @@ +groups: + - id: first_group_id + type: attribute_group + brief: first description + prefix: first + attributes: + - id: attr_one + type: boolean + brief: short description of attr_one + + - id: first_metric_id + brief: first metric description + metric_name: first.metric + instrument: counter + type: metric + unit: "{one}" + stability: stable + extends: first_group_id + + - id: second_group_id + brief: second metric description + metric_name: second_group.metric + type: metric + instrument: histogram + unit: "s" + prefix: second_group + attributes: + - id: attr_two + type: int + brief: short description of attr_two diff --git a/semantic-conventions/src/tests/semconv/templating/test_code.py b/semantic-conventions/src/tests/semconv/templating/test_code.py index e85a81de..5bc45f05 100644 --- a/semantic-conventions/src/tests/semconv/templating/test_code.py +++ b/semantic-conventions/src/tests/semconv/templating/test_code.py @@ -23,6 +23,24 @@ def test_codegen_units(test_file_path, read_test_file): assert result == expected +def test_codegen_metrics_all(test_file_path, read_test_file): + semconv = SemanticConventionSet(debug=False) + semconv.parse(test_file_path("yaml", "metrics", "metrics.yaml")) + semconv.finish() + + template_path = test_file_path("jinja", "metrics", "metrics_template") + renderer = CodeRenderer({}, trim_whitespace=False) + + filename = os.path.join(tempfile.mkdtemp(), "AllMetrics.java") + renderer.render(semconv, template_path, filename, None) + with open(filename, "r", encoding="utf-8") as f: + result = f.read() + + expected = read_test_file("jinja", "metrics", "expected_metrics.java") + + assert result == expected + + def test_strip_blocks_enabled(test_file_path, read_test_file): """Tests that the Jinja whitespace control params are fed to the Jinja environment""" semconv = SemanticConventionSet(debug=False) From 441547d4f1e06201be0aa9729b09598e45587c37 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Wed, 21 Feb 2024 05:43:41 -0800 Subject: [PATCH 5/7] Add `enum_attributes` to the context, also adds print_member_value helper (#266) Co-authored-by: Josh Suereth --- .gitignore | 3 + semantic-conventions/README.md | 219 +++++++++++++++++- .../opentelemetry/semconv/templating/code.py | 13 ++ .../all/FifthAttributes.java | 27 +++ .../all/FirstAttributes.java | 10 +- .../group_by_root_namespace/attributes.yml | 30 ++- .../group_by_root_namespace/template_all | 14 +- .../src/tests/semconv/templating/test_code.py | 6 + 8 files changed, 313 insertions(+), 9 deletions(-) create mode 100644 semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FifthAttributes.java diff --git a/.gitignore b/.gitignore index 21e871b4..ff67d5cf 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,6 @@ # Vim .swp + +# Python +*.whl diff --git a/semantic-conventions/README.md b/semantic-conventions/README.md index 33876b6c..864c656c 100644 --- a/semantic-conventions/README.md +++ b/semantic-conventions/README.md @@ -198,7 +198,8 @@ Processes all parsed semantic conventions Processes a single namespace and is called for each namespace detected. -- `attributes_and_templates` - the list containing all attributes (including template ones) in the given root namespace. +- `attributes_and_templates` - the list containing all attributes (including template ones) in the given root namespace. Attributes are sorted by their name. +- `enum_attributes` - the list containing all enum attributes in the given root namespace. Attributes are sorted by their name. - `root_namespace` - the root namespace being processed. #### Other patterns @@ -220,6 +221,7 @@ Semconvgen supports following additional filters to simplify common operations i 3. `is_experimental` - Checks if the attribute is experimental. The same check can also be done with `(attribute.stability | string()) == "StabilityLevel.EXPERIMENTAL"` 4. `is_stable` - Checks if the attribute is experimental. The same check can also be done with `(attribute.stability | string()) == "StabilityLevel.STABLE"` 5. `is_template` - Checks if the attribute is a template attribute. +6. `attribute | print_member_value(member)` - Applies to enum attributes only and takes `EnumMember` as a parameter. Prints value of a given enum member as a constant - strings are quoted, integers are printed as is. #### String operations @@ -234,3 +236,218 @@ Semconvgen supports following additional filters to simplify common operations i #### `BaseSemanticConvention` operations 1. `is_metric` - Checks if semantic convention describes a metric. + + +### Examples + +#### Generate all attributes in files grouped by root namespace + +First, we should iterate over all attributes. +```jinja +{%- for attribute in attributes_and_templates %} +... +{%- endfor %} +``` + +Now, for each attribute we want to generate constant declaration like + +```python +SERVER_ADDRESS = "server.address" +""" +Server domain name if available without reverse DNS lookup; otherwise, IP address or Unix domain socket name. +Note: When observed from the client side, and when communicating through an intermediary, `server.address` SHOULD represent the server address behind any intermediaries, for example proxies, if it's available. +""" + +we can achieve it with the following template: + +```jinja +{{attribute.fqn | to_const_name}} = "{{attribute.fqn}}" +""" +{{attribute.brief | to_doc_brief}}. +{%- if attribute.note %} +Note: {{attribute.note | to_doc_brief | indent}}. +{%- endif %} +""" +``` + +We should also annotate deprecated attributes and potentially generate template attributes differently. +Here's a full example: + +```jinja +{%- for attribute in attributes_and_templates %} + +{% if attribute | is_template %} +{{attribute.fqn | to_const_name}}_TEMPLATE = "{{attribute.fqn}}" +{%- else %} +{{attribute.fqn | to_const_name}} = "{{attribute.fqn}}" +{%- endif %} +""" +{{attribute.brief | to_doc_brief}}. +{%- if attribute.note %} +Note: {{attribute.note | to_doc_brief | indent}}. +{%- endif %} + +{%- if attribute | is_deprecated %} +Deprecated: {{attribute.deprecated | to_doc_brief}}. +{%- endif %} +""" + +{%- endfor %} +``` + +#### Filter attributes based on stability + +It's possible to split attributes into stable and unstable for example to ship them in different artifacts or namespaces. + +You can achieve it by running code generation twice with different filters and output destinations. + +Here's an example of how to keep one template file for both: + +```jinja +{%- set filtered_attributes = attributes_and_templates | select(filter) | list %} +{%- for attribute in attributes_and_templates %} +... +{%- endfor %} +``` + +Here we apply a Jinja test named `filter` which we can define in the generation script: + +```bash +docker run --rm \ + -v ${SCRIPT_DIR}/semantic-conventions/model:/source \ + -v ${SCRIPT_DIR}/templates:/templates \ + -v ${ROOT_DIR}/opentelemetry-semantic-conventions/src/opentelemetry/semconv/:/output \ + otel/semconvgen:$OTEL_SEMCONV_GEN_IMG_VERSION \ + -f /source code \ + --template /templates/semantic_attributes.j2 \ + --output /output/{{snake_prefix}}_attributes.py \ + --file-per-group root_namespace \ + -Dfilter=is_stable +``` + +Here we run the generation with `filter` variable set to `is_stable`, which resolves to `attributes_and_templates | select("is_stable")` expression. +It will apply `is_stable` custom function to each attribute and collect only stable ones. + +We can also generate experimental attributes by changing the destination path and filter value: + +```bash +docker run --rm \ + -v ${SCRIPT_DIR}/semantic-conventions/model:/source \ + -v ${SCRIPT_DIR}/templates:/templates \ + -v ${ROOT_DIR}/opentelemetry-semantic-conventions/src/opentelemetry/semconv/:/output \ + otel/semconvgen:$OTEL_SEMCONV_GEN_IMG_VERSION \ + -f /source code \ + --template /templates/semantic_attributes.j2 \ + --output /output/experimental/{{snake_prefix}}_attributes.py \ + --file-per-group root_namespace \ + -Dfilter=is_experimental +``` + +#### Generate enum definitions + +Enum attribute members could be generated in the following way: + +```jinja +{%- for attribute in enum_attributes %} + +{%- set class_name = attribute.fqn | to_camelcase(True) ~ "Values" %} +{%- set type = attribute.attr_type.enum_type %} +class {{class_name}}(Enum): + {%- for member in attribute.attr_type.members %} + {{ member.member_id | to_const_name }} = {{ attribute | print_member_value(member) }} + """{{member.brief | to_doc_brief}}.""" + + {% endfor %} + +{% endfor %} +``` + +resulting in en enum like this: + +```python +class NetworkTransportValues(Enum): + TCP = "tcp" + """TCP.""" + + UDP = "udp" + """UDP.""" + + PIPE = "pipe" + """Named or anonymous pipe.""" + + UNIX = "unix" + """Unix domain socket.""" +``` + +#### Exclude certain namespaces + +In some cases you might want to skip certain namespaces. For example, JVM attribute and metric definitions might not be very useful in Python application. + +You can create a list of excluded namespaces and pass it over to the template as parameter (or hardcode it): + +```jinja +{%- if root_namespace not in ("jvm", "dotnet") %} +... +{%- endif %} +``` + +If result of the rendering is empty string, code generator does not store it. + +#### Generate metric definitions + +You can generate metric names as constants, but could also generate method definitions that create instruments and populate name, description, and unit: + +```python +""" +Duration of HTTP client requests +""" +@staticmethod +def create_http_client_request_duration(meter: Meter) -> Histogram: + return meter.create_histogram( + name="http.client.request.duration", + description="Duration of HTTP client requests.", + unit="s", + ) +``` + +Since metric types (like `Histogram`) and factory methods (like `create_histogram`) depend on the language, it's necessary to define mappings in the template. + +For example, this is a macro rendering Python instrument type name based on the semantic convention type: + +```jinja +{%- macro to_python_instrument_type(instrument) -%} + {%- if instrument == "counter" -%} + Counter + {%- elif instrument == "histogram" -%} + Histogram + {%- elif instrument == "updowncounter" -%} + UpDownCounter + {%- elif instrument == "gauge" -%} + ObservableGauge + {%- endif -%} +{%- endmacro %} +``` + +We'd need a very similar one for factory method. + +This is the template that generates above metric definition: + +```java + """ + {{metric.brief | to_doc_brief}} + """ + @staticmethod + {%- if metric.instrument == "gauge" %} + def create_{{ metric.metric_name | replace(".", "_") }}(meter: Meter, callback: Sequence[Callable]) -> {{to_python_instrument_type(metric.instrument)}}: + {%- else %} + def create_{{ metric.metric_name | replace(".", "_") }}(meter: Meter) -> {{to_python_instrument_type(metric.instrument)}}: + {%- endif %} + return meter.create_{{to_python_instrument_factory(metric.instrument)}}( + name="{{ metric.metric_name }}", + {%- if metric.instrument == "gauge" %} + callback=callback, + {%- endif %} + description="{{ metric.brief }}", + unit="{{ metric.unit }}", + ) +``` \ No newline at end of file diff --git a/semantic-conventions/src/opentelemetry/semconv/templating/code.py b/semantic-conventions/src/opentelemetry/semconv/templating/code.py index e1a554c8..721a8462 100644 --- a/semantic-conventions/src/opentelemetry/semconv/templating/code.py +++ b/semantic-conventions/src/opentelemetry/semconv/templating/code.py @@ -22,6 +22,8 @@ from opentelemetry.semconv.model.semantic_attribute import ( AttributeType, + EnumAttributeType, + EnumMember, RequirementLevel, SemanticAttribute, StabilityLevel, @@ -125,6 +127,15 @@ def to_doc_brief(doc_string: typing.Optional[str]) -> str: return doc_string +def print_member_value(attr: SemanticAttribute, member: EnumMember) -> str: + if ( + isinstance(attr.attr_type, EnumAttributeType) + and attr.attr_type.enum_type == "string" + ): + return f'"{member.value}"' + return str(member.value) + + def to_html_links(doc_string: typing.Optional[typing.Union[str, TextWithLinks]]) -> str: if doc_string is None: return "" @@ -256,6 +267,7 @@ def setup_environment(env: Environment, trim_whitespace: bool): env.filters["to_html_links"] = to_html_links env.filters["regex_replace"] = regex_replace env.filters["render_markdown"] = render_markdown + env.filters["print_member_value"] = print_member_value env.filters["is_deprecated"] = is_deprecated env.filters["is_definition"] = is_definition env.filters["is_stable"] = is_stable @@ -349,6 +361,7 @@ def _render_group_by_root_namespace( data = { "template": template_path, "attributes_and_templates": attribute_and_templates, + "enum_attributes": [a for a in attribute_and_templates if a.is_enum], "metrics": metrics.get(ns) or [], "root_namespace": sanitized_ns, } diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FifthAttributes.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FifthAttributes.java new file mode 100644 index 00000000..490a7aa3 --- /dev/null +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FifthAttributes.java @@ -0,0 +1,27 @@ +package io.opentelemetry.instrumentation.api.semconv; + +class FifthAttributes { + /** + * short description of attr_five_int + */ + public static final AttributeKey FIFTH_ATTR_FIVE_INT = enumKey("fifth.attr_five_int"); + + /** + * short description of attr_five_string + */ + public static final AttributeKey FIFTH_ATTR_FIVE_STRING = enumKey("fifth.attr_five_string"); + public static final class FifthAttrFiveIntValues { + /** First enum2 value.*/ + public static final long ENUM2_ONE = 1; + /** Second enum2 value.*/ + public static final long ENUM2_TWO = 2; + } + + public static final class FifthAttrFiveStringValues { + /** First enum1 value.*/ + public static final String ENUM1_ONE = "one"; + /** Second enum1 value.*/ + public static final String ENUM1_TWO = "two"; + } + +} \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FirstAttributes.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FirstAttributes.java index 9281a5f2..0d15179f 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FirstAttributes.java +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/all/FirstAttributes.java @@ -1,11 +1,6 @@ package io.opentelemetry.instrumentation.api.semconv; class FirstAttributes { - /** - * short description of attr_one - */ - public static final AttributeKey FIRST_ATTR_ONE = booleanKey("first.attr_one"); - /** * short description of attr_one_a */ @@ -15,4 +10,9 @@ class FirstAttributes { * this is the description of attribute template */ public static final AttributeKeyTemplate FIRST_ATTR_TEMPLATE_ONE = stringKey("first.attr_template_one"); + + /** + * short description of last_attr + */ + public static final AttributeKey FIRST_LAST_ATTR = booleanKey("first.last_attr"); } \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml index f4a1a6cc..fecaf66f 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml @@ -4,9 +4,9 @@ groups: brief: first description prefix: first attributes: - - id: attr_one + - id: last_attr # intentionally out of alphabetical order to test sorting type: boolean - brief: short description of attr_one + brief: short description of last_attr - id: attr_template_one type: template[string] brief: > @@ -54,3 +54,29 @@ groups: type: string brief: short description of attr_four examples: "4" + - id: fifth_group_id + brief: fifth description + prefix: fifth + attributes: + - id: attr_five_string + type: + members: + - id: enum1_one + value: "one" + brief: "First enum1 value" + - id: enum1_two + value: "two" + brief: "Second enum1 value" + brief: short description of attr_five_string + examples: one + - id: attr_five_int + type: + members: + - id: enum2_one + value: 1 + brief: "First enum2 value" + - id: enum2_two + value: 2 + brief: "Second enum2 value" + brief: short description of attr_five_int + examples: 1 diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/template_all b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/template_all index 2b26c2a7..adc735eb 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/template_all +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/template_all @@ -42,4 +42,16 @@ class {{ root_namespace | to_camelcase(True) }}Attributes { public static final AttributeKey<{{ to_java_return_type(attribute.instantiated_type | string) | first_up }}> {{attribute.fqn | to_const_name}} = {{to_java_key_type(attribute.instantiated_type | string)}}("{{attribute.fqn}}"); {% endif %} {% endfor %} -} \ No newline at end of file + +{%- for attribute in enum_attributes %} + {% set type = to_java_return_type(attribute.attr_type.enum_type) %} + public static final class {{attribute.fqn | to_camelcase(True) ~ "Values"}} { + {% for member in attribute.attr_type.members %} + /** {{member.brief | to_doc_brief}}.*/ + {% set value = attribute | print_member_value(member) %} + public static final {{ type }} {{ member.member_id | to_const_name }} = {{ value }}; + {% endfor %} + } + +{% endfor %} +} diff --git a/semantic-conventions/src/tests/semconv/templating/test_code.py b/semantic-conventions/src/tests/semconv/templating/test_code.py index 5bc45f05..034d2b95 100644 --- a/semantic-conventions/src/tests/semconv/templating/test_code.py +++ b/semantic-conventions/src/tests/semconv/templating/test_code.py @@ -110,6 +110,9 @@ def test_codegen_attribute_root_ns(test_file_path, read_test_file): third = read_test_file("jinja", test_path, "ThirdAttributes.java") check_file(tmppath, "ThirdAttributes.java", third) + fifth = read_test_file("jinja", test_path, "FifthAttributes.java") + check_file(tmppath, "FifthAttributes.java", fifth) + def test_codegen_attribute_root_ns_snake_case_file(test_file_path, read_test_file): semconv = SemanticConventionSet(debug=False) @@ -138,6 +141,9 @@ def test_codegen_attribute_root_ns_snake_case_file(test_file_path, read_test_fil third = read_test_file("jinja", test_path, "ThirdAttributes.java") check_file(tmppath, "third_attributes.java", third) + fifth = read_test_file("jinja", test_path, "FifthAttributes.java") + check_file(tmppath, "fifth_attributes.java", fifth) + def test_codegen_attribute_root_ns_stable(test_file_path, read_test_file): semconv = SemanticConventionSet(debug=False) From 66e445867e5feae164e9e74951f86fa526034e0b Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Thu, 29 Feb 2024 09:51:55 -0800 Subject: [PATCH 6/7] fix tests and rebase --- semantic-conventions/CHANGELOG.md | 8 ++ semantic-conventions/README.md | 79 +++++++++++-------- .../src/opentelemetry/semconv/main.py | 19 ++++- .../vnext.yaml | 2 +- .../compat/enum_member_removed/vnext.yaml | 2 +- .../compat/enum_member_removed/vprev.yaml | 2 +- .../data/compat/removed_attribute/vprev.yaml | 2 +- .../group_by_root_namespace/attributes.yml | 12 +++ .../attributes_and_metrics/semconv.yml | 3 + .../attributes_no_group_prefix.yml | 3 + .../single_file/semconv.yml | 8 +- .../single_file/template_single_file | 2 +- .../stable/ThirdAttributesStable.java | 2 +- .../stable/template_only_stable | 2 +- .../data/jinja/metrics/expected_metrics.java | 2 +- .../tests/data/jinja/metrics/metrics_template | 1 + .../markdown/stability/badges_expected.md | 10 +++ .../markdown/stability/labels_expected.md | 10 +++ .../src/tests/data/yaml/metrics/metrics.yaml | 4 + 19 files changed, 126 insertions(+), 47 deletions(-) create mode 100644 semantic-conventions/src/tests/data/markdown/stability/badges_expected.md create mode 100644 semantic-conventions/src/tests/data/markdown/stability/labels_expected.md diff --git a/semantic-conventions/CHANGELOG.md b/semantic-conventions/CHANGELOG.md index 068fdd35..c393c5e3 100644 --- a/semantic-conventions/CHANGELOG.md +++ b/semantic-conventions/CHANGELOG.md @@ -20,6 +20,14 @@ Please update the changelog as part of any significant pull request. [BREAKING] The `--file-per-group ` that used to create multiple directories (like `output//file`) now generates multiple files (`output/file`) instead. ([#243](https://github.com/open-telemetry/build-tools/pull/243)) +- Update `semconvgen.yml` workflow to run on feature/* branches. + ([#256](https://github.com/open-telemetry/build-tools/pull/256)) +- Allow --output to be templatized when generating multiple files. + ([#263](https://github.com/open-telemetry/build-tools/pull/263)) +- Add `metrics` to the context of non-scoped code generation + ([#270](https://github.com/open-telemetry/build-tools/pull/270)) +- Add `enum_attributes` to the context, adds `print_member_value` helper + ([#266](https://github.com/open-telemetry/build-tools/pull/266)) ## v0.23.0 diff --git a/semantic-conventions/README.md b/semantic-conventions/README.md index 864c656c..008da103 100644 --- a/semantic-conventions/README.md +++ b/semantic-conventions/README.md @@ -88,6 +88,38 @@ semantic conventions that have the tag `network`. `` will print a table describing a single metric `http.server.active_requests`. +## Version compatibility check + +You can check compatibility between the local one specified with `--yaml-root` and sepcific OpenTelemetry semantic convention version using the following command: + +```bash +docker run --rm otel/semconvgen --yaml-root {yaml_folder} compatibility --previous-version {semconv version} +``` + +The `{semconv version}` (e.g. `1.24.0`) is the previously released version of semantic conventions. + +Following checks are performed + +- On all attributes and metrics (experimental and stable): + - attributes and metrics must not be removed + - enum attribute members must not be removed + +- On stable attributes and attribute templates: + - stability must not be changed + - the type of attribute must not be changed + - enum attribute: type of value must not be changed + +- On stable enum attribute members: + - stability must not be changed + - `id` and `value` must not be changed + +- On stable metrics: + - stability must not be changed + - instrument and unit must not be changed + - new attributes should not be added. + This check does not take into account opt-in attributes. Adding new attributes to metric is not always breaking, + so it's considered non-critical and it's possible to suppress it with `--ignore-warnings` + ## Code Generator The image supports [Jinja](https://jinja.palletsprojects.com/en/2.11.x/) templates to generate code from the models. @@ -135,6 +167,18 @@ docker run --rm \ Finally, additional value can be passed to the template in form of `key=value` pairs separated by comma using the `--parameters [{key=value},]+` or `-D` flag. +Generating code from older versions of semantic conventions with new tooling is, in general, not supported. +However in some cases minor incompatibilities in semantic conventions can be ignored by setting `--strict-validation` flag to `false` + +```bash +docker run --rm \ + otel/semconvgen:$GENERATOR_VERSION \ + --yaml-root /source \ + `--strict-validation false` + code \ + ...other parameters... +``` + ### Customizing Jinja's Whitespace Control The image also supports customizing @@ -144,39 +188,6 @@ via the additional flag `--trim-whitespace`. Providing the flag will enable both ### Enabling/disabling support for colored diffs in error messages The `COLORED_DIFF` environment variable is set in the `semantic-conventions` `Dockerfile`. When this environment varibale is set, errors related to reformatting tables will show a "colored diff" using standard ANSI control characters. While this should be supported natively in any modern terminal environment, you may unset this variable if issues arise. Doing so will enable a "fall back" of non-colored inline diffs showing what was "added" and what was "removed", followed by the exact tokens added/removed encased in single quotes. -## Version compatibility check - -You can check compatibility between the local one specified with `--yaml-root` and sepcific OpenTelemetry semantic convention version using the following command: - -```bash -docker run --rm otel/semconvgen --yaml-root {yaml_folder} compatibility --previous-version {semconv version} -``` - -The `{semconv version}` (e.g. `1.24.0`) is the previously released version of semantic conventions. - -Following checks are performed - -- On all attributes and metrics (experimental and stable): - - attributes and metrics must not be removed - - enum attribute members must not be removed - -- On stable attributes and attribute templates: - - stability must not be changed - - the type of attribute must not be changed - - enum attribute: type of value must not be changed - -- On stable enum attribute members: - - stability must not be changed - - `id` and `value` must not be changed - -- On stable metrics: - - stability must not be changed - - instrument and unit must not be changed - - new attributes should not be added. - This check does not take into account opt-in attributes. Adding new attributes to metric is not always breaking, - so it's considered non-critical and it's possible to suppress it with `--ignore-warnings` - - ### Accessing Semantic Conventions in the template When template is processed, it has access to a set of variables that depends on the `--file-per-group` value (or lack of it). @@ -450,4 +461,4 @@ This is the template that generates above metric definition: description="{{ metric.brief }}", unit="{{ metric.unit }}", ) -``` \ No newline at end of file +``` diff --git a/semantic-conventions/src/opentelemetry/semconv/main.py b/semantic-conventions/src/opentelemetry/semconv/main.py index e14f7b8f..e9662695 100644 --- a/semantic-conventions/src/opentelemetry/semconv/main.py +++ b/semantic-conventions/src/opentelemetry/semconv/main.py @@ -35,14 +35,14 @@ def parse_semconv( - yaml_root: str, exclude: str, debug: bool, parser + yaml_root: str, exclude: str, debug: bool, strict_validation: bool, parser ) -> SemanticConventionSet: semconv = SemanticConventionSet(debug) files = find_yaml(yaml_root, exclude) for file in sorted(files): if not file.endswith(".yaml") and not file.endswith(".yml"): parser.error(f"{file} is not a yaml file.") - semconv.parse(file, False) + semconv.parse(file, strict_validation) semconv.finish() if semconv.has_error(): sys.exit(1) @@ -72,7 +72,9 @@ def main(): parser = setup_parser() args = parser.parse_args() check_args(args, parser) - semconv = parse_semconv(args.yaml_root, args.exclude, args.debug, parser) + semconv = parse_semconv( + args.yaml_root, args.exclude, args.debug, args.strict_validation, parser + ) semconv_filter = parse_only_filter(args.only, parser) filter_semconv(semconv, semconv_filter) if len(semconv.models) == 0: @@ -104,7 +106,9 @@ def process_markdown(semconv, args): def check_compatibility(semconv, args, parser): prev_semconv_path = download_previous_version(args.previous_version) - prev_semconv = parse_semconv(prev_semconv_path, args.exclude, args.debug, parser) + prev_semconv = parse_semconv( + prev_semconv_path, args.exclude, args.debug, args.strict_validation, parser + ) compatibility_checker = CompatibilityChecker(semconv, prev_semconv) problems = compatibility_checker.check() @@ -307,6 +311,13 @@ def setup_parser(): nargs="*", help="YAML file containing a Semantic Convention", ) + parser.add_argument( + "--strict-validation", + help="Fail on non-critical yaml validation issues.", + required=False, + default=True, + action="store_false", + ) subparsers = parser.add_subparsers(dest="flavor") add_code_parser(subparsers) add_md_parser(subparsers) diff --git a/semantic-conventions/src/tests/data/compat/attribute_stable_to_experimental/vnext.yaml b/semantic-conventions/src/tests/data/compat/attribute_stable_to_experimental/vnext.yaml index d83717a1..de6b05df 100644 --- a/semantic-conventions/src/tests/data/compat/attribute_stable_to_experimental/vnext.yaml +++ b/semantic-conventions/src/tests/data/compat/attribute_stable_to_experimental/vnext.yaml @@ -22,4 +22,4 @@ groups: brief: "Request headers." note: "Request headers note." examples: '`first.fifth_attr.bar=["foo"]`' - stability: experimental \ No newline at end of file + stability: experimental diff --git a/semantic-conventions/src/tests/data/compat/enum_member_removed/vnext.yaml b/semantic-conventions/src/tests/data/compat/enum_member_removed/vnext.yaml index 2c6eb19d..3b07b235 100644 --- a/semantic-conventions/src/tests/data/compat/enum_member_removed/vnext.yaml +++ b/semantic-conventions/src/tests/data/compat/enum_member_removed/vnext.yaml @@ -15,4 +15,4 @@ groups: brief: "third attribute" note: "third attribute note" examples: ["two"] - stability: experimental \ No newline at end of file + stability: experimental diff --git a/semantic-conventions/src/tests/data/compat/enum_member_removed/vprev.yaml b/semantic-conventions/src/tests/data/compat/enum_member_removed/vprev.yaml index 9ec933d4..db094829 100644 --- a/semantic-conventions/src/tests/data/compat/enum_member_removed/vprev.yaml +++ b/semantic-conventions/src/tests/data/compat/enum_member_removed/vprev.yaml @@ -15,4 +15,4 @@ groups: brief: "third attribute" note: "third attribute note" examples: ["one"] - stability: experimental \ No newline at end of file + stability: experimental diff --git a/semantic-conventions/src/tests/data/compat/removed_attribute/vprev.yaml b/semantic-conventions/src/tests/data/compat/removed_attribute/vprev.yaml index 5d3189d3..c78f2a4f 100644 --- a/semantic-conventions/src/tests/data/compat/removed_attribute/vprev.yaml +++ b/semantic-conventions/src/tests/data/compat/removed_attribute/vprev.yaml @@ -21,4 +21,4 @@ groups: type: template[string[]] brief: "request headers" examples: '`first.fifth_attr.foo=["bar"]`' - stability: experimental \ No newline at end of file + stability: experimental diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml index fecaf66f..72415d1a 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes.yml @@ -6,9 +6,11 @@ groups: attributes: - id: last_attr # intentionally out of alphabetical order to test sorting type: boolean + stability: experimental brief: short description of last_attr - id: attr_template_one type: template[string] + stability: experimental brief: > this is the description of attribute template examples: 'example' @@ -21,6 +23,7 @@ groups: attributes: - id: attr_two type: string + stability: experimental brief: short description of attr_two examples: ['example_one', 'example_two'] - id: first_group_part_two @@ -30,6 +33,7 @@ groups: attributes: - id: attr_one_a type: int + stability: experimental brief: short description of attr_one_a - ref: second.attr_two - ref: third.attr_template_three @@ -44,6 +48,7 @@ groups: stability: stable - id: attr_template_three type: template[string] + stability: experimental brief: > this is the description of attribute template examples: 'example' @@ -52,6 +57,7 @@ groups: attributes: - id: attr_four type: string + stability: experimental brief: short description of attr_four examples: "4" - id: fifth_group_id @@ -59,24 +65,30 @@ groups: prefix: fifth attributes: - id: attr_five_string + stability: experimental type: members: - id: enum1_one value: "one" + stability: experimental brief: "First enum1 value" - id: enum1_two value: "two" + stability: experimental brief: "Second enum1 value" brief: short description of attr_five_string examples: one - id: attr_five_int + stability: experimental type: members: - id: enum2_one value: 1 + stability: experimental brief: "First enum2 value" - id: enum2_two value: 2 + stability: experimental brief: "Second enum2 value" brief: short description of attr_five_int examples: 1 diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/semconv.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/semconv.yml index 3d2c670b..16f47c9e 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/semconv.yml +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/attributes_and_metrics/semconv.yml @@ -6,6 +6,7 @@ groups: attributes: - id: attr_one type: boolean + stability: experimental brief: short description of attr_one - id: first_metric_id @@ -20,6 +21,7 @@ groups: - id: second_group_id brief: second metric description metric_name: second_group.metric + stability: experimental type: metric instrument: histogram unit: "s" @@ -27,4 +29,5 @@ groups: attributes: - id: attr_two type: int + stability: experimental brief: short description of attr_two diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/attributes_no_group_prefix.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/attributes_no_group_prefix.yml index d2dcb72e..eb58fd47 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/attributes_no_group_prefix.yml +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/no_group_prefix/attributes_no_group_prefix.yml @@ -6,15 +6,18 @@ groups: attributes: - id: attr_one type: boolean + stability: experimental brief: short description of attr_one - id: group_with_no_prefix brief: description attributes: - id: foo.attr_two type: string + stability: experimental brief: short description of foo.attr_two examples: "foo" - id: bar_attr type: string + stability: experimental brief: short description of bar_attr examples: "bar" \ No newline at end of file diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/semconv.yml b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/semconv.yml index 37d33fc2..cdd432bc 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/semconv.yml +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/semconv.yml @@ -6,9 +6,11 @@ groups: attributes: - id: attr_one type: boolean + stability: experimental brief: short description of attr_one - id: attr_template_one type: template[string] + stability: experimental brief: > this is the description of attribute template examples: 'example' @@ -21,6 +23,7 @@ groups: attributes: - id: attr_two type: string + stability: experimental brief: short description of attr_two examples: ['example_one', 'example_two'] @@ -31,6 +34,7 @@ groups: attributes: - id: attr_one_a type: int + stability: experimental brief: short description of attr_one_a - ref: second.attr_two @@ -39,13 +43,15 @@ groups: attributes: - id: attr_four type: string + stability: experimental brief: short description of attr_four examples: "4" - id: first_metric_id brief: first metric description metric_name: first.metric.name + stability: experimental instrument: counter type: metric unit: "{one}" - extends: first_group_id \ No newline at end of file + extends: first_group_id diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/template_single_file b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/template_single_file index 986ec3d2..98ff6d26 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/template_single_file +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/single_file/template_single_file @@ -71,4 +71,4 @@ class AllAttributes { public static final String {{metric.metric_name | to_const_name}} = "{{metric.metric_name}}"; {% endif %} {% endfor %} -} \ No newline at end of file +} diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/ThirdAttributesStable.java b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/ThirdAttributesStable.java index 71263bf3..ece8c736 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/ThirdAttributesStable.java +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/ThirdAttributesStable.java @@ -5,4 +5,4 @@ class ThirdAttributes { * short description of attr_three */ public static final AttributeKey THIRD_ATTR_THREE = stringKey("third.attr_three"); -} \ No newline at end of file +} diff --git a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/template_only_stable b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/template_only_stable index deb83073..60d75caa 100644 --- a/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/template_only_stable +++ b/semantic-conventions/src/tests/data/jinja/group_by_root_namespace/stable/template_only_stable @@ -46,4 +46,4 @@ class {{ root_namespace | to_camelcase(True) }}Attributes { {% endif %} {% endfor %} } -{%- endif %} \ No newline at end of file +{% endif %} diff --git a/semantic-conventions/src/tests/data/jinja/metrics/expected_metrics.java b/semantic-conventions/src/tests/data/jinja/metrics/expected_metrics.java index 8dccdd34..86727325 100644 --- a/semantic-conventions/src/tests/data/jinja/metrics/expected_metrics.java +++ b/semantic-conventions/src/tests/data/jinja/metrics/expected_metrics.java @@ -13,4 +13,4 @@ class AllMetrics { * Experimental: True */ public static final String SECOND_GROUP_METRIC = "second_group.metric"; -} \ No newline at end of file +} diff --git a/semantic-conventions/src/tests/data/jinja/metrics/metrics_template b/semantic-conventions/src/tests/data/jinja/metrics/metrics_template index c49a3803..9941bb9f 100644 --- a/semantic-conventions/src/tests/data/jinja/metrics/metrics_template +++ b/semantic-conventions/src/tests/data/jinja/metrics/metrics_template @@ -9,3 +9,4 @@ class AllMetrics { public static final String {{ metric.metric_name | to_const_name }} = "{{metric.metric_name}}"; {%- endfor %} } + diff --git a/semantic-conventions/src/tests/data/markdown/stability/badges_expected.md b/semantic-conventions/src/tests/data/markdown/stability/badges_expected.md new file mode 100644 index 00000000..de1439da --- /dev/null +++ b/semantic-conventions/src/tests/data/markdown/stability/badges_expected.md @@ -0,0 +1,10 @@ +# Common Attributes + + +| Attribute | Type | Description | Examples | Requirement Level | +|---|---|---|---|---| +| [`test.def_stability`](labels_expected.md) | boolean | | | Required | +| [`test.deprecated_attr`](labels_expected.md) | boolean | ![Deprecated](https://img.shields.io/badge/-deprecated-red)
| | Required | +| [`test.exp_attr`](labels_expected.md) | boolean | | | Required | +| [`test.stable_attr`](labels_expected.md) | boolean | ![Stable](https://img.shields.io/badge/-stable-lightgreen)
| | Required | + diff --git a/semantic-conventions/src/tests/data/markdown/stability/labels_expected.md b/semantic-conventions/src/tests/data/markdown/stability/labels_expected.md new file mode 100644 index 00000000..ab7f6194 --- /dev/null +++ b/semantic-conventions/src/tests/data/markdown/stability/labels_expected.md @@ -0,0 +1,10 @@ +# Common Attributes + + +| Attribute | Type | Description | Examples | Requirement Level | +|---|---|---|---|---| +| [`test.def_stability`](labels_expected.md) | boolean | | | Required | +| [`test.deprecated_attr`](labels_expected.md) | boolean | **Deprecated: Removed.**
| | Required | +| [`test.exp_attr`](labels_expected.md) | boolean | | | Required | +| [`test.stable_attr`](labels_expected.md) | boolean | | | Required | + diff --git a/semantic-conventions/src/tests/data/yaml/metrics/metrics.yaml b/semantic-conventions/src/tests/data/yaml/metrics/metrics.yaml index 3d2c670b..6a02479b 100644 --- a/semantic-conventions/src/tests/data/yaml/metrics/metrics.yaml +++ b/semantic-conventions/src/tests/data/yaml/metrics/metrics.yaml @@ -1,11 +1,13 @@ groups: - id: first_group_id type: attribute_group + stability: experimental brief: first description prefix: first attributes: - id: attr_one type: boolean + stability: experimental brief: short description of attr_one - id: first_metric_id @@ -20,6 +22,7 @@ groups: - id: second_group_id brief: second metric description metric_name: second_group.metric + stability: experimental type: metric instrument: histogram unit: "s" @@ -27,4 +30,5 @@ groups: attributes: - id: attr_two type: int + stability: experimental brief: short description of attr_two From 76b82030f7ababc85f84d3daf65da9c86e2c2ec2 Mon Sep 17 00:00:00 2001 From: Liudmila Molkova Date: Fri, 15 Mar 2024 08:48:05 -0700 Subject: [PATCH 7/7] Apply suggestions from code review Co-authored-by: Joao Grassi <5938087+joaopgrassi@users.noreply.github.com> --- semantic-conventions/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/semantic-conventions/README.md b/semantic-conventions/README.md index 8c6ef9d0..4be014a2 100644 --- a/semantic-conventions/README.md +++ b/semantic-conventions/README.md @@ -93,7 +93,7 @@ semantic conventions that have the tag `network`. ## Version compatibility check -You can check compatibility between the local one specified with `--yaml-root` and sepcific OpenTelemetry semantic convention version using the following command: +You can check compatibility between the local one specified with `--yaml-root` and specific OpenTelemetry semantic convention version using the following command: ```bash docker run --rm otel/semconvgen --yaml-root {yaml_folder} compatibility --previous-version {semconv version} @@ -101,7 +101,7 @@ docker run --rm otel/semconvgen --yaml-root {yaml_folder} compatibility --previo The `{semconv version}` (e.g. `1.24.0`) is the previously released version of semantic conventions. -Following checks are performed +The following checks are performed: - On all attributes and metrics (experimental and stable): - attributes and metrics must not be removed @@ -193,7 +193,7 @@ The `COLORED_DIFF` environment variable is set in the `semantic-conventions` `Do ### Accessing Semantic Conventions in the template -When template is processed, it has access to a set of variables that depends on the `--file-per-group` value (or lack of it). +When the template is processed, it has access to a set of variables that depends on the `--file-per-group` value (or lack of it). You can access properties of these variables and call Jinja or Python functions defined on them. #### Single file (no `--file-per-group` pattern is provided) @@ -226,7 +226,7 @@ Processes a single pattern value and is called for each distinct value. Jinja templates has a notion of [filters](https://jinja.palletsprojects.com/en/2.11.x/templates/#list-of-builtin-filters) allowing to transform objects or filter lists. -Semconvgen supports following additional filters to simplify common operations in templates. +Semconvgen supports the following additional filters to simplify common operations in templates. #### `SemanticAttribute` operations @@ -251,7 +251,6 @@ Semconvgen supports following additional filters to simplify common operations i 1. `is_metric` - Checks if semantic convention describes a metric. - ### Examples #### Generate all attributes in files grouped by root namespace