From 0c6d95011e0277b6977c630b19363adfc339e14d Mon Sep 17 00:00:00 2001 From: Alexander Wert Date: Wed, 19 Jul 2023 12:15:06 +0200 Subject: [PATCH] Refactored template attributes generation Signed-off-by: Alexander Wert --- .../semconv/model/semantic_attribute.py | 314 +++++++----------- .../semconv/model/semantic_convention.py | 9 +- .../src/opentelemetry/semconv/model/utils.py | 17 + .../semconv/templating/markdown/__init__.py | 25 +- .../attribute_templates.yaml | 14 +- .../tests/data/yaml/attribute_templates.yml | 10 +- .../semconv/model/test_semantic_attribute.py | 30 +- 7 files changed, 176 insertions(+), 243 deletions(-) diff --git a/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py b/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py index a2847ed8..564db50b 100644 --- a/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py @@ -23,6 +23,8 @@ from opentelemetry.semconv.model.exceptions import ValidationError from opentelemetry.semconv.model.utils import ( check_no_missing_keys, + is_template_attribute, + validate_attribute_id, validate_id, validate_values, ) @@ -67,6 +69,7 @@ class SemanticAttribute: position: List[int] inherited: bool = False imported: bool = False + is_template: bool = False def import_attribute(self): return replace(self, imported=True) @@ -84,7 +87,7 @@ def is_enum(self): @staticmethod def parse( - prefix, semconv_stability, yaml_attributes + prefix, semconv_stability, yaml_attributes, parse_template_attributes=False ) -> "Dict[str, SemanticAttribute]": """This method parses the yaml representation for semantic attributes creating the respective SemanticAttribute objects. @@ -107,138 +110,136 @@ def parse( return attributes for attribute in yaml_attributes: - attr = SemanticAttribute.parse_semantic_attribute( - prefix, attribute, semconv_stability, allowed_keys - ) - if attr.fqn in attributes: - position_data = attribute.lc.data - position = position_data[list(attribute)[0]] - msg = ( - "Attribute id " - + attr.fqn - + " is already present at line " - + str(attributes[attr.fqn].position[0] + 1) - ) - raise ValidationError.from_yaml_pos(position, msg) - attributes[attr.fqn] = attr - return attributes - @staticmethod - def parse_semantic_attribute( - prefix, attribute, semconv_stability, allowed_keys - ) -> "SemanticAttribute": - """This method parses the yaml representation for a single semantic attribute - creating the respective SemanticAttribute object. - """ + validate_values(attribute, allowed_keys) + attr_id = attribute.get("id") + ref = attribute.get("ref") + position_data = attribute.lc.data + position = position_data[next(iter(attribute))] + if attr_id is None and ref is None: + msg = "At least one of id or ref is required." + raise ValidationError.from_yaml_pos(position, msg) + if attr_id is not None: + validate_attribute_id(attr_id, position_data["id"]) + attr_type, brief, examples = SemanticAttribute.parse_id(attribute) + if prefix: + fqn = f"{prefix}.{attr_id}" + else: + fqn = attr_id + else: + # Ref + attr_type = None + if "type" in attribute: + msg = f"Ref attribute '{ref}' must not declare a type" + raise ValidationError.from_yaml_pos(position, msg) + brief = attribute.get("brief") + examples = attribute.get("examples") + ref = ref.strip() + fqn = ref + + is_template = is_template_attribute(fqn) + if (is_template and not parse_template_attributes) or ( + not is_template and parse_template_attributes + ): + continue + + required_value_map = { + "required": RequirementLevel.REQUIRED, + "conditionally_required": RequirementLevel.CONDITIONALLY_REQUIRED, + "recommended": RequirementLevel.RECOMMENDED, + "opt_in": RequirementLevel.OPT_IN, + } + requirement_level_msg = "" + requirement_level_val = attribute.get("requirement_level", "") + requirement_level: Optional[RequirementLevel] + if isinstance(requirement_level_val, CommentedMap): + + if len(requirement_level_val) != 1: + position = position_data["requirement_level"] + msg = "Multiple requirement_level values are not allowed!" + raise ValidationError.from_yaml_pos(position, msg) - validate_values(attribute, allowed_keys) - attr_id = attribute.get("id") - ref = attribute.get("ref") - position_data = attribute.lc.data - position = position_data[next(iter(attribute))] - if attr_id is None and ref is None: - msg = "At least one of id or ref is required." - raise ValidationError.from_yaml_pos(position, msg) - if attr_id is not None: - validate_id(attr_id, position_data["id"]) - attr_type, brief, examples = SemanticAttribute.parse_id(attribute) - if prefix: - fqn = f"{prefix}.{attr_id}" + recommended_msg = requirement_level_val.get("recommended", None) + condition_msg = requirement_level_val.get( + "conditionally_required", None + ) + if condition_msg is not None: + requirement_level = RequirementLevel.CONDITIONALLY_REQUIRED + requirement_level_msg = condition_msg + elif recommended_msg is not None: + requirement_level = RequirementLevel.RECOMMENDED + requirement_level_msg = recommended_msg else: - fqn = attr_id - else: - # Ref - attr_type = None - if "type" in attribute: - msg = f"Ref attribute '{ref}' must not declare a type" - raise ValidationError.from_yaml_pos(position, msg) - brief = attribute.get("brief") - examples = attribute.get("examples") - ref = ref.strip() - fqn = ref - - required_value_map = { - "required": RequirementLevel.REQUIRED, - "conditionally_required": RequirementLevel.CONDITIONALLY_REQUIRED, - "recommended": RequirementLevel.RECOMMENDED, - "opt_in": RequirementLevel.OPT_IN, - } - requirement_level_msg = "" - requirement_level_val = attribute.get("requirement_level", "") - requirement_level: Optional[RequirementLevel] - if isinstance(requirement_level_val, CommentedMap): + requirement_level = required_value_map.get(requirement_level_val) - if len(requirement_level_val) != 1: + if requirement_level_val and requirement_level is None: position = position_data["requirement_level"] - msg = "Multiple requirement_level values are not allowed!" + msg = ( + f"Value '{requirement_level_val}' for required field is not allowed" + ) raise ValidationError.from_yaml_pos(position, msg) - recommended_msg = requirement_level_val.get("recommended", None) - condition_msg = requirement_level_val.get("conditionally_required", None) - if condition_msg is not None: - requirement_level = RequirementLevel.CONDITIONALLY_REQUIRED - requirement_level_msg = condition_msg - elif recommended_msg is not None: - requirement_level = RequirementLevel.RECOMMENDED - requirement_level_msg = recommended_msg - else: - requirement_level = required_value_map.get(requirement_level_val) - - if requirement_level_val and requirement_level is None: - position = position_data["requirement_level"] - msg = f"Value '{requirement_level_val}' for required field is not allowed" - raise ValidationError.from_yaml_pos(position, msg) - - if ( - requirement_level == RequirementLevel.CONDITIONALLY_REQUIRED - and not requirement_level_msg - ): - position = position_data["requirement_level"] - msg = "Missing message for conditionally required field!" - raise ValidationError.from_yaml_pos(position, msg) + if ( + requirement_level == RequirementLevel.CONDITIONALLY_REQUIRED + and not requirement_level_msg + ): + position = position_data["requirement_level"] + msg = "Missing message for conditionally required field!" + raise ValidationError.from_yaml_pos(position, msg) - tag = attribute.get("tag", "").strip() - stability, deprecated = SemanticAttribute.parse_stability_deprecated( - attribute.get("stability"), attribute.get("deprecated"), position_data - ) - if ( - semconv_stability == StabilityLevel.DEPRECATED - and stability is not StabilityLevel.DEPRECATED - ): - position = ( - position_data["stability"] - if "stability" in position_data - else position_data["deprecated"] + tag = attribute.get("tag", "").strip() + stability, deprecated = SemanticAttribute.parse_stability_deprecated( + attribute.get("stability"), attribute.get("deprecated"), position_data ) - msg = f"Semantic convention stability set to deprecated but attribute '{attr_id}' is {stability}" - raise ValidationError.from_yaml_pos(position, msg) - stability = stability or semconv_stability or StabilityLevel.STABLE - sampling_relevant = ( - AttributeType.to_bool("sampling_relevant", attribute) - if attribute.get("sampling_relevant") - else False - ) - note = attribute.get("note", "") - fqn = fqn.strip() - parsed_brief = TextWithLinks(brief.strip() if brief else "") - parsed_note = TextWithLinks(note.strip()) - attr = SemanticAttribute( - fqn=fqn, - attr_id=attr_id, - ref=ref, - attr_type=attr_type, - brief=parsed_brief, - examples=examples, - tag=tag, - deprecated=deprecated, - stability=stability, - requirement_level=requirement_level, - requirement_level_msg=str(requirement_level_msg).strip(), - sampling_relevant=sampling_relevant, - note=parsed_note, - position=position, - ) - return attr + if ( + semconv_stability == StabilityLevel.DEPRECATED + and stability is not StabilityLevel.DEPRECATED + ): + position = ( + position_data["stability"] + if "stability" in position_data + else position_data["deprecated"] + ) + msg = f"Semantic convention stability set to deprecated but attribute '{attr_id}' is {stability}" + raise ValidationError.from_yaml_pos(position, msg) + stability = stability or semconv_stability or StabilityLevel.STABLE + sampling_relevant = ( + AttributeType.to_bool("sampling_relevant", attribute) + if attribute.get("sampling_relevant") + else False + ) + note = attribute.get("note", "") + fqn = fqn.strip() + parsed_brief = TextWithLinks(brief.strip() if brief else "") + parsed_note = TextWithLinks(note.strip()) + attr = SemanticAttribute( + fqn=fqn, + attr_id=attr_id, + ref=ref, + attr_type=attr_type, + brief=parsed_brief, + examples=examples, + tag=tag, + deprecated=deprecated, + stability=stability, + requirement_level=requirement_level, + requirement_level_msg=str(requirement_level_msg).strip(), + sampling_relevant=sampling_relevant, + note=parsed_note, + position=position, + is_template=is_template, + ) + if attr.fqn in attributes: + position = position_data[list(attribute)[0]] + msg = ( + "Attribute id " + + fqn + + " is already present at line " + + str(attributes[fqn].position[0] + 1) + ) + raise ValidationError.from_yaml_pos(position, msg) + attributes[fqn] = attr + return attributes @staticmethod def parse_id(attribute): @@ -340,71 +341,6 @@ def equivalent_to(self, other: "SemanticAttribute"): return False -@dataclass -class AttributeTemplate(SemanticAttribute): - parameter_name: str = "key" - - @staticmethod - def parse_attribute_templates( - prefix, semconv_stability, yaml_attributes - ) -> "Dict[str, AttributeTemplate]": - """This method parses the yaml representation for attribute templates - creating the respective AttributeTemplate objects. - """ - attributes = {} # type: Dict[str, AttributeTemplate] - allowed_keys = ( - "id", - "type", - "brief", - "examples", - "ref", - "tag", - "deprecated", - "stability", - "requirement_level", - "sampling_relevant", - "note", - "parameter_name", - ) - if not yaml_attributes: - return attributes - - for attribute in yaml_attributes: - attr = SemanticAttribute.parse_semantic_attribute( - prefix, attribute, semconv_stability, allowed_keys - ) - if attr.fqn in attributes: - position_data = attribute.lc.data - position = position_data[list(attribute)[0]] - msg = ( - "Attribute id " - + attr.fqn - + " is already present at line " - + str(attributes[attr.fqn].position[0] + 1) - ) - raise ValidationError.from_yaml_pos(position, msg) - parameter_name = attribute.get("parameter_name") - attr_template = AttributeTemplate( - fqn=attr.fqn, - attr_id=attr.attr_id, - ref=attr.ref, - attr_type=attr.attr_type, - brief=attr.brief, - examples=attr.examples, - tag=attr.tag, - deprecated=attr.deprecated, - stability=attr.stability, - requirement_level=attr.requirement_level, - requirement_level_msg=attr.requirement_level_msg, - sampling_relevant=attr.sampling_relevant, - note=attr.note, - position=attr.position, - parameter_name=parameter_name if parameter_name is not None else "key", - ) - attributes[attr_template.fqn] = attr_template - return attributes - - class AttributeType: # https://yaml.org/type/bool.html diff --git a/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py b/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py index c5ddbb9e..0dd19a34 100644 --- a/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py @@ -23,7 +23,6 @@ from opentelemetry.semconv.model.constraints import AnyOf, Include, parse_constraints from opentelemetry.semconv.model.exceptions import ValidationError from opentelemetry.semconv.model.semantic_attribute import ( - AttributeTemplate, RequirementLevel, SemanticAttribute, unique_attributes, @@ -125,6 +124,10 @@ def attribute_templates(self): return list(self.attr_templates_by_name.values()) + @property + def attributes_and_templates(self): + return self.attributes + self.attribute_templates + def __init__(self, group): super().__init__(group) @@ -143,8 +146,8 @@ def __init__(self, group): self.attrs_by_name = SemanticAttribute.parse( self.prefix, self.stability, group.get("attributes") ) - self.attr_templates_by_name = AttributeTemplate.parse_attribute_templates( - self.prefix, self.stability, group.get("attribute_templates") + self.attr_templates_by_name = SemanticAttribute.parse( + self.prefix, self.stability, group.get("attributes"), True ) def contains_attribute(self, attr: "SemanticAttribute"): diff --git a/semantic-conventions/src/opentelemetry/semconv/model/utils.py b/semantic-conventions/src/opentelemetry/semconv/model/utils.py index 8f114c3c..ee91fc8f 100644 --- a/semantic-conventions/src/opentelemetry/semconv/model/utils.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/utils.py @@ -18,6 +18,7 @@ from opentelemetry.semconv.model.exceptions import ValidationError ID_RE = re.compile("([a-z](\\.?[a-z0-9_-]+)+)") +ID_ATTRIBUTE_RE = re.compile("([a-z](\\.?[a-z0-9_-]+)+(\\.<[a-z0-9_-]+>)?)") """Identifiers must start with a lowercase ASCII letter and contain only lowercase, digits 0-9, underscore, dash (not recommended) and dots. Each dot must be followed by at least one allowed non-dot character.""" @@ -31,6 +32,22 @@ def validate_id(semconv_id, position): ) +def validate_attribute_id(semconv_attribute_id, position): + if not ID_ATTRIBUTE_RE.fullmatch(semconv_attribute_id): + raise ValidationError.from_yaml_pos( + position, + f"Invalid attribute id {semconv_attribute_id}. " + "Semantic Convention attribute id MUST match {ID_ATTRIBUTE_RE.pattern}", + ) + + +def is_template_attribute(semconv_attribute_id): + return ( + not ID_RE.fullmatch(semconv_attribute_id) + and ID_ATTRIBUTE_RE.fullmatch(semconv_attribute_id) is not None + ) + + def validate_values(yaml, keys, mandatory=()): """This method checks only valid keywords and value types are used""" unwanted = [k for k in yaml.keys() if k not in keys] diff --git a/semantic-conventions/src/opentelemetry/semconv/templating/markdown/__init__.py b/semantic-conventions/src/opentelemetry/semconv/templating/markdown/__init__.py index 22c321cd..b868e1c1 100644 --- a/semantic-conventions/src/opentelemetry/semconv/templating/markdown/__init__.py +++ b/semantic-conventions/src/opentelemetry/semconv/templating/markdown/__init__.py @@ -22,7 +22,6 @@ from opentelemetry.semconv.model.constraints import AnyOf, Include from opentelemetry.semconv.model.semantic_attribute import ( - AttributeTemplate, EnumAttributeType, EnumMember, RequirementLevel, @@ -98,12 +97,7 @@ def to_markdown_attr( """ This method renders attributes as markdown table entry """ - attribute_name = ( - attribute.fqn + ".<" + attribute.parameter_name + ">" - if isinstance(attribute, AttributeTemplate) - else attribute.fqn - ) - name = self.render_attribute_id(attribute_name) + name = self.render_attribute_id(attribute.fqn) attr_type = ( "enum" if isinstance(attribute.attr_type, EnumAttributeType) @@ -190,7 +184,8 @@ def to_markdown_attribute_table( ): attr_to_print = [] for attr in sorted( - semconv.attributes, key=lambda a: "" if a.ref is None else a.ref + semconv.attributes_and_templates, + key=lambda a: "" if a.ref is None else a.ref, ): if self.render_ctx.group_key is not None: if attr.tag == self.render_ctx.group_key: @@ -199,16 +194,6 @@ def to_markdown_attribute_table( if self.render_ctx.is_full or attr.is_local: attr_to_print.append(attr) - for attr_template in sorted( - semconv.attribute_templates, key=lambda a: "" if a.ref is None else a.ref - ): - if self.render_ctx.group_key is not None: - if attr_template.tag == self.render_ctx.group_key: - attr_to_print.append(attr_template) - continue - if self.render_ctx.is_full or attr_template.is_local: - attr_to_print.append(attr_template) - if self.render_ctx.group_key is not None and not attr_to_print: raise ValueError( f"No attributes retained for '{semconv.semconv_id}' filtering by '{self.render_ctx.group_key}'" @@ -407,7 +392,9 @@ def _create_attribute_location_dict(self): ) a: SemanticAttribute valid_attr = ( - a for a in semconv.attributes if a.is_local and not a.ref + a + for a in semconv.attributes_and_templates + if a.is_local and not a.ref ) for attr in valid_attr: m[attr.fqn] = md diff --git a/semantic-conventions/src/tests/data/markdown/attribute_templates/attribute_templates.yaml b/semantic-conventions/src/tests/data/markdown/attribute_templates/attribute_templates.yaml index 0fd5eadb..26fbf97a 100644 --- a/semantic-conventions/src/tests/data/markdown/attribute_templates/attribute_templates.yaml +++ b/semantic-conventions/src/tests/data/markdown/attribute_templates/attribute_templates.yaml @@ -6,15 +6,13 @@ groups: note: > These conventions can be used for http and https schemes and various HTTP versions like 1.1, 2 and SPDY. - attribute_templates: - - id: request.header - type: string - brief: > - HTTP request headers, `` being the normalized HTTP Header name - (lowercase, with - characters replaced by _), the value being the header values. - examples: '`http.request.header.content_type=["application/json"]`' - parameter_name: key attributes: + - id: request.header. + type: string + brief: > + HTTP request headers, `` being the normalized HTTP Header name + (lowercase, with - characters replaced by _), the value being the header values. + examples: '`http.request.header.content_type=["application/json"]`' - id: request.method type: string requirement_level: required diff --git a/semantic-conventions/src/tests/data/yaml/attribute_templates.yml b/semantic-conventions/src/tests/data/yaml/attribute_templates.yml index 44132832..39c936c1 100644 --- a/semantic-conventions/src/tests/data/yaml/attribute_templates.yml +++ b/semantic-conventions/src/tests/data/yaml/attribute_templates.yml @@ -1,21 +1,19 @@ -attribute_templates: - - id: attribute_template_one +attributes: + - id: attribute_template_one. tag: tag-one type: string brief: > this is the description of the first attribute template examples: 'This is a good example of the first attribute template' - parameter_name: key - - id: attribute_template_two + - id: attribute_template_two. tag: tag-two type: int brief: > this is the description of the second attribute template. It's a number. examples: [1000, 10, 1] - parameter_name: other_key - - id: attribute_template_three + - id: attribute_three tag: tag-three type: boolean brief: > diff --git a/semantic-conventions/src/tests/semconv/model/test_semantic_attribute.py b/semantic-conventions/src/tests/semconv/model/test_semantic_attribute.py index 7012172a..26a52dda 100644 --- a/semantic-conventions/src/tests/semconv/model/test_semantic_attribute.py +++ b/semantic-conventions/src/tests/semconv/model/test_semantic_attribute.py @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from opentelemetry.semconv.model.semantic_attribute import ( - AttributeTemplate, - SemanticAttribute, -) +from opentelemetry.semconv.model.semantic_attribute import SemanticAttribute def test_parse(load_yaml): @@ -56,25 +53,24 @@ def test_parse_deprecated(load_yaml): def test_parse_attribute_templates(load_yaml): yaml = load_yaml("attribute_templates.yml") - attribute_templates = AttributeTemplate.parse_attribute_templates( - "prefix", "", yaml.get("attribute_templates") + attribute_templates = SemanticAttribute.parse( + "prefix", "", yaml.get("attributes"), True ) - assert len(attribute_templates) == 3 + assert len(attribute_templates) == 2 expected_keys = sorted( [ - "prefix.attribute_template_one", - "prefix.attribute_template_two", - "prefix.attribute_template_three", + "prefix.attribute_template_one.", + "prefix.attribute_template_two.", ] ) actual_keys = sorted(list(attribute_templates.keys())) assert actual_keys == expected_keys - first_attribute = attribute_templates["prefix.attribute_template_one"] - assert first_attribute.attr_id == "attribute_template_one" + first_attribute = attribute_templates["prefix.attribute_template_one."] + assert first_attribute.fqn == "prefix.attribute_template_one." assert first_attribute.tag == "tag-one" assert first_attribute.ref is None assert first_attribute.attr_type == "string" @@ -85,10 +81,8 @@ def test_parse_attribute_templates(load_yaml): assert first_attribute.examples == [ "This is a good example of the first attribute template" ] - assert first_attribute.parameter_name == "key" + assert first_attribute.is_template is True - second_attribute = attribute_templates["prefix.attribute_template_two"] - assert second_attribute.parameter_name == "other_key" - - third_attribute = attribute_templates["prefix.attribute_template_three"] - assert third_attribute.parameter_name == "key" + second_attribute = attribute_templates["prefix.attribute_template_two."] + assert second_attribute.fqn == "prefix.attribute_template_two." + assert second_attribute.is_template is True