Update semconv tool with stability fields and MD render. (#43)

* Update semconv tool with stability fields and MD render.

* Bump version

semantic-conventions/CHANGELOG.md

@@ -2,6 +2,11 @@
 
 Please update the changelog as part of any significant pull request.
 
+## v0.4.0
+
+- Add stability fields. (#35)
+- Add Markdown render for code generation.
+
 ## v0.3.1
 
 - Fix markdown generator for int enums. (#36)

semantic-conventions/requirements.txt

@@ -7,6 +7,6 @@ ruamel.yaml.clib==0.2.0
 typed-ast==1.4.1
 typing-extensions==3.7.4.2
 Jinja2==2.11.2
-
 pytest==6.1.1
-ipdb==0.13.4
+ipdb==0.13.4
+mistune==2.0.0a6

semantic-conventions/src/opentelemetry/semconv/main.py

@@ -23,6 +23,7 @@
 from opentelemetry.semconv.templating.code import CodeRenderer
 
 from opentelemetry.semconv.templating.markdown import MarkdownRenderer
+from opentelemetry.semconv.templating.markdown.options import MarkdownOptions
 
 
 def parse_semconv(args, parser) -> SemanticConventionSet:
@@ -73,10 +74,16 @@ def main():
 
 
 def process_markdown(semconv, args):
-    exclude = exclude_file_list(args.markdown_root, args.exclude)
-    md_renderer = MarkdownRenderer(
-        args.markdown_root, semconv, exclude, args.md_break_conditional, args.md_check
+    options = MarkdownOptions(
+        check_only=args.md_check,
+        enable_stable=args.md_stable,
+        enable_experimental=args.md_experimental,
+        enable_deprecated=args.md_enable_deprecated,
+        use_badge=args.md_use_badges,
+        break_count=args.md_break_conditional,
+        exclude_files=exclude_file_list(args.markdown_root, args.exclude),
     )
+    md_renderer = MarkdownRenderer(args.markdown_root, semconv, options)
     md_renderer.render_md()
 
 
@@ -155,6 +162,32 @@ def add_md_parser(subparsers):
         required=False,
         action="store_true",
     )
+    parser.add_argument(
+        "--md-use-badges",
+        help="Use stability badges instead of labels for attributes.",
+        required=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "--md-stable",
+        help="Add labels to attributes marked as stable.",
+        required=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "--md-experimental",
+        help="Add labels to attributes marked as experimental.",
+        required=False,
+        action="store_true",
+    )
+    parser.add_argument(
+        "--md-disable-deprecated",
+        help="Removes deprecated notes of deprecated attributes.",
+        required=False,
+        default=True,
+        dest="md_enable_deprecated",
+        action="store_false",
+    )
 
 
 def setup_parser():

semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py

@@ -35,9 +35,17 @@ class Required(Enum):
     NO = 3
 
 
+class StabilityLevel(Enum):
+    STABLE = 1
+    EXPERIMENTAL = 2
+    DEPRECATED = 3
+
+
 class HasAttributes:
-    def _set_attributes(self, prefix, node):
-        self.attrs_by_name = SemanticAttribute.parse(prefix, node.get("attributes"))
+    def _set_attributes(self, prefix, stability, node):
+        self.attrs_by_name = SemanticAttribute.parse(
+            prefix, stability, node.get("attributes")
+        )
 
     @property
     def attributes(self):
@@ -64,6 +72,7 @@ class SemanticAttribute:
     brief: str
     examples: List[Union[str, int, bool]]
     tag: str
+    stability: StabilityLevel
     deprecated: str
     required: Required
     required_msg: str
@@ -88,7 +97,7 @@ def is_enum(self):
         return isinstance(self.attr_type, EnumAttributeType)
 
     @staticmethod
-    def parse(prefix, yaml_attributes):
+    def parse(prefix, semconv_stability, yaml_attributes):
         """ This method parses the yaml representation for semantic attributes
             creating the respective SemanticAttribute objects.
         """
@@ -101,6 +110,7 @@ def parse(prefix, yaml_attributes):
             "ref",
             "tag",
             "deprecated",
+            "stability",
             "required",
             "sampling_relevant",
             "note",
@@ -112,20 +122,20 @@ def parse(prefix, yaml_attributes):
             validate_values(attribute, allowed_keys)
             attr_id = attribute.get("id")
             ref = attribute.get("ref")
-            position = attribute.lc.data[list(attribute)[0]]
+            position_data = attribute.lc.data
+            position = position_data[next(iter(attribute))]
             if attr_id is None and ref is None:
                 msg = "At least one of id or ref is required."
                 raise ValidationError.from_yaml_pos(position, msg)
             if attr_id is not None:
-                validate_id(attr_id, attribute.lc.data["id"])
+                validate_id(attr_id, position_data["id"])
                 attr_type, brief, examples = SemanticAttribute.parse_id(attribute)
                 fqn = "{}.{}".format(prefix, attr_id)
                 attr_id = attr_id.strip()
             else:
                 # Ref
                 attr_type = None
                 if "type" in attribute:
-                    position = attribute.lc.data[list(attribute)[0]]
                     msg = "Ref attribute '{}' must not declare a type".format(ref)
                     raise ValidationError.from_yaml_pos(position, msg)
                 brief = attribute.get("brief")
@@ -144,56 +154,66 @@ def parse(prefix, yaml_attributes):
                 required = Required.CONDITIONAL
                 required_msg = required_val.get("conditional", None)
                 if required_msg is None:
-                    position = attribute.lc.data["required"]
+                    position = position_data["required"]
                     msg = "Missing message for conditional required field!"
                     raise ValidationError.from_yaml_pos(position, msg)
             else:
                 required = required_value_map.get(required_val)
                 if required == Required.CONDITIONAL:
-                    position = attribute.lc.data["required"]
+                    position = position_data["required"]
                     msg = "Missing message for conditional required field!"
                     raise ValidationError.from_yaml_pos(position, msg)
             if required is None:
-                position = attribute.lc.data["required"]
+                position = position_data["required"]
                 msg = "Value '{}' for required field is not allowed".format(
                     required_val
                 )
                 raise ValidationError.from_yaml_pos(position, msg)
             tag = attribute.get("tag", "").strip()
-            deprecated = attribute.get("deprecated")
-            if deprecated is not None:
-                if AttributeType.get_type(deprecated) != "string" or deprecated == "":
-                    position = attribute.lc.data["deprecated"]
-                    msg = (
-                        "Deprecated field expects a string that specify why the attribute is deprecated and/or what"
-                        " to use instead! "
-                    )
-                    raise ValidationError.from_yaml_pos(position, msg)
-                deprecated = deprecated.strip()
+            stability, deprecated = SemanticAttribute.parse_stability_deprecated(
+                attribute.get("stability"), attribute.get("deprecated"), position_data
+            )
+            if (
+                semconv_stability == StabilityLevel.DEPRECATED
+                and stability is not StabilityLevel.DEPRECATED
+            ):
+                position = (
+                    position_data["stability"]
+                    if "stability" in position_data
+                    else position_data["deprecated"]
+                )
+                msg = "Semantic convention stability set to deprecated but attribute '{}' is {}".format(
+                    attr_id, stability
+                )
+                raise ValidationError.from_yaml_pos(position, msg)
+            stability = stability or semconv_stability or StabilityLevel.STABLE
             sampling_relevant = (
                 AttributeType.to_bool("sampling_relevant", attribute)
                 if attribute.get("sampling_relevant")
                 else False
             )
             note = attribute.get("note", "")
             fqn = fqn.strip()
+            parsed_brief = TextWithLinks(brief.strip() if brief else "")
+            parsed_note = TextWithLinks(note.strip())
             attr = SemanticAttribute(
                 fqn=fqn,
                 attr_id=attr_id,
                 ref=ref,
                 attr_type=attr_type,
-                brief=brief.strip() if brief else "",
+                brief=parsed_brief,
                 examples=examples,
                 tag=tag,
                 deprecated=deprecated,
+                stability=stability,
                 required=required,
                 required_msg=str(required_msg).strip(),
                 sampling_relevant=sampling_relevant,
-                note=note.strip(),
+                note=parsed_note,
                 position=position,
             )
             if attr.fqn in attributes:
-                position = attribute.lc.data[list(attribute)[0]]
+                position = position_data[list(attribute)[0]]
                 msg = (
                     "Attribute id "
                     + fqn
@@ -253,6 +273,48 @@ def parse_id(attribute):
             AttributeType.check_examples_type(attr_type, examples, zlass)
         return attr_type, str(brief), examples
 
+    @staticmethod
+    def parse_stability_deprecated(stability, deprecated, position_data):
+        if deprecated is not None and stability is None:
+            stability = "deprecated"
+        if deprecated is not None:
+            if stability is not None and stability != "deprecated":
+                position = position_data["deprecated"]
+                msg = "There is a deprecation message but the stability is set to '{}'".format(
+                    stability
+                )
+                raise ValidationError.from_yaml_pos(position, msg)
+            if AttributeType.get_type(deprecated) != "string" or deprecated == "":
+                position = position_data["deprecated"]
+                msg = (
+                    "Deprecated field expects a string that specifies why the attribute is deprecated and/or what"
+                    " to use instead! "
+                )
+                raise ValidationError.from_yaml_pos(position, msg)
+            deprecated = deprecated.strip()
+        if stability is not None:
+            stability = SemanticAttribute.check_stability(
+                stability,
+                position_data["stability"]
+                if "stability" in position_data
+                else position_data["deprecated"],
+            )
+        return stability, deprecated
+
+    @staticmethod
+    def check_stability(stability_value, position):
+
+        stability_value_map = {
+            "deprecated": StabilityLevel.DEPRECATED,
+            "experimental": StabilityLevel.EXPERIMENTAL,
+            "stable": StabilityLevel.STABLE,
+        }
+        val = stability_value_map.get(stability_value)
+        if val is not None:
+            return val
+        msg = "Value '{}' is not allowed as a stability marker".format(stability_value)
+        raise ValidationError.from_yaml_pos(position, msg)
+
     def equivalent_to(self, other: "SemanticAttribute"):
         if self.attr_id is not None:
             if self.fqn == other.fqn:
@@ -395,7 +457,9 @@ def parse(attribute_type):
             for member in attribute_type["members"]:
                 validate_values(member, allowed_keys, mandatory_keys)
                 if not EnumAttributeType.is_valid_enum_value(member["value"]):
-                    raise ValidationError(0, 0, "Invalid value used in enum: <{}>".format(member["value"]))
+                    raise ValidationError(
+                        0, 0, "Invalid value used in enum: <{}>".format(member["value"])
+                    )
                 members.append(
                     EnumMember(
                         member_id=member["id"],
@@ -419,3 +483,43 @@ class EnumMember:
     value: str
     brief: str
     note: str
+
+
+class MdLink:
+    text: str
+    url: str
+
+    def __init__(self, text, url):
+        self.text = text
+        self.url = url
+
+    def __str__(self):
+        return "[{}]({})".format(self.text, self.url)
+
+
+class TextWithLinks(str):
+    parts: List[Union[str, MdLink]]
+    raw_text: str
+    md_link = re.compile("\[([^\[\]]+)\]\(([^)]+)")
+
+    def __init__(self, text):
+        super().__init__()
+        self.raw_text = text
+        self.parts = []
+        last_position = 0
+        for match in self.md_link.finditer(text):
+            prev_text = text[last_position : match.start()]
+            link = MdLink(match.group(1), match.group(2))
+            if prev_text:
+                self.parts.append(prev_text)
+            self.parts.append(link)
+            last_position = match.end() + 1
+        last_part = text[last_position:]
+        if last_part:
+            self.parts.append(last_part)
+
+    def __str__(self):
+        str_list = []
+        for elm in self.parts:
+            str_list.append(elm.__str__())
+        return "".join(str_list)

semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py

@@ -25,6 +25,7 @@
 from opentelemetry.semconv.model.semantic_attribute import (
     HasAttributes,
     SemanticAttribute,
+    StabilityLevel,
     Required,
     unique_attributes,
 )
@@ -116,6 +117,12 @@ def __init__(self, group):
         self.semconv_id = self.id
         self.note = group.get("note", "").strip()
         self.prefix = group.get("prefix", "").strip()
+        stability = group.get("stability")
+        deprecated = group.get("deprecated")
+        position_data = group.lc.data
+        self.stability, self.deprecated = SemanticAttribute.parse_stability_deprecated(
+            stability, deprecated, position_data
+        )
         self.extends = group.get("extends", "").strip()
         self.constraints = parse_constraints(group.get("constraints", ()))
 
@@ -187,14 +194,15 @@ class ResourceSemanticConvention(HasAttributes, BaseSemanticConvention):
         "brief",
         "note",
         "prefix",
+        "stability",
         "extends",
         "attributes",
         "constraints",
     )
 
     def __init__(self, group):
         super().__init__(group)
-        self._set_attributes(self.prefix, group)
+        self._set_attributes(self.prefix, self.stability, group)
 
 
 class SpanSemanticConvention(HasAttributes, BaseSemanticConvention):
@@ -206,6 +214,7 @@ class SpanSemanticConvention(HasAttributes, BaseSemanticConvention):
         "brief",
         "note",
         "prefix",
+        "stability",
         "extends",
         "span_kind",
         "attributes",
@@ -214,7 +223,7 @@ class SpanSemanticConvention(HasAttributes, BaseSemanticConvention):
 
     def __init__(self, group):
         super().__init__(group)
-        self._set_attributes(self.prefix, group)
+        self._set_attributes(self.prefix, self.stability, group)
         self.span_kind = SpanKind.parse(group.get("span_kind"))
         if self.span_kind is None:
             position = group.lc.data["span_kind"]