From 15b014dc072eccfca00ef44bffe7670016e0f9e1 Mon Sep 17 00:00:00 2001 From: Giovanni Liva Date: Mon, 10 Aug 2020 08:12:25 +0200 Subject: [PATCH 1/4] First cut of SemConvGen --- semantic-conventions/Dockerfile | 14 + semantic-conventions/README.md | 98 ++++ semantic-conventions/requirements.txt | 9 + semantic-conventions/setup.cfg | 36 ++ semantic-conventions/setup.py | 19 + .../src/dynatrace/__init__.py | 0 .../src/dynatrace/semconv/__init__.py | 0 .../src/dynatrace/semconv/main.py | 180 +++++++ .../src/dynatrace/semconv/model/__init__.py | 0 .../dynatrace/semconv/model/constraints.py | 58 +++ .../src/dynatrace/semconv/model/exceptions.py | 37 ++ .../semconv/model/semantic_attribute.py | 384 +++++++++++++++ .../semconv/model/semantic_convention.py | 457 ++++++++++++++++++ .../src/dynatrace/semconv/model/utils.py | 50 ++ .../dynatrace/semconv/templating/__init__.py | 0 .../src/dynatrace/semconv/templating/code.py | 131 +++++ .../dynatrace/semconv/templating/markdown.py | 404 ++++++++++++++++ .../src/dynatrace/semconv/version.py | 15 + 18 files changed, 1892 insertions(+) create mode 100644 semantic-conventions/Dockerfile create mode 100644 semantic-conventions/README.md create mode 100644 semantic-conventions/requirements.txt create mode 100644 semantic-conventions/setup.cfg create mode 100644 semantic-conventions/setup.py create mode 100644 semantic-conventions/src/dynatrace/__init__.py create mode 100644 semantic-conventions/src/dynatrace/semconv/__init__.py create mode 100644 semantic-conventions/src/dynatrace/semconv/main.py create mode 100644 semantic-conventions/src/dynatrace/semconv/model/__init__.py create mode 100644 semantic-conventions/src/dynatrace/semconv/model/constraints.py create mode 100644 semantic-conventions/src/dynatrace/semconv/model/exceptions.py create mode 100644 semantic-conventions/src/dynatrace/semconv/model/semantic_attribute.py create mode 100644 semantic-conventions/src/dynatrace/semconv/model/semantic_convention.py create mode 100644 semantic-conventions/src/dynatrace/semconv/model/utils.py create mode 100644 semantic-conventions/src/dynatrace/semconv/templating/__init__.py create mode 100644 semantic-conventions/src/dynatrace/semconv/templating/code.py create mode 100644 semantic-conventions/src/dynatrace/semconv/templating/markdown.py create mode 100644 semantic-conventions/src/dynatrace/semconv/version.py diff --git a/semantic-conventions/Dockerfile b/semantic-conventions/Dockerfile new file mode 100644 index 00000000..a78a5096 --- /dev/null +++ b/semantic-conventions/Dockerfile @@ -0,0 +1,14 @@ +ARG ALPINE_VERSION=3.12 +ARG PYTHON_VERSION=3.8.5 + +FROM python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} +LABEL maintainer="The OpenTelemetry Authors" +ADD . /semconvgen/ +WORKDIR semconvgen +RUN rm -f README.md +RUN apk --update add --virtual build-dependencies build-base \ + && pip install -r requirements.txt \ + && apk del build-dependencies + +RUN python setup.py install +ENTRYPOINT ["gen-semconv"] \ No newline at end of file diff --git a/semantic-conventions/README.md b/semantic-conventions/README.md new file mode 100644 index 00000000..56bcf831 --- /dev/null +++ b/semantic-conventions/README.md @@ -0,0 +1,98 @@ +# Semantic Convention generator + Docker + +A docker image to process Semantic Convention YAML models. + + +# Usage + +The image can be used to generate Markdown tables or code. + +```bash +docker run --rm -v: -v: otel/semconvgen [OPTION] +``` + +For help try: + +```bash +docker run --rm otel/semconvgen -h +``` + +## Markdown Tables + +Tables can be generated using the command: + +```bash +docker run --rm otel/semconvgen --yaml-root {yaml_folder} markdown --markdown-root {markdown_folder} +``` + +Where `{yaml_folder}` is the absolute path to the directory containing the yaml files and +`{markdown_folder}` the absolute path to the directory containing the markdown definitions +(`specification` for [opentelemetry-specification](https://github.com/open-telemetry/opentelemetry-specification/tree/master/)). + +The tool will automatically replace the tables with the up to date definition of the semantic conventions. +To do so, the tool looks for special tags in the markdown. + +``` + + +``` + +Everything between these two tags will be replaced with the table definition. +The `{semantic_convention_id}` MUST be the `id` field in the yaml files of the semantic convention +for which we want to generate the table. +After `{semantic_convention_id}`, optional parameters enclosed in parentheses can be added to customize the output: + +- `tag={tag}`: prints only the attributes that have `{tag}` as a tag; +- `full`: prints attributes and constraints inherited from the parent semantic conventions or from included ones; +- `ref`: prints attributes that are referenced from another semantic convention; +- `remove_constraint`: does not print additional constraints of the semantic convention. + +### Examples + +These examples assume that a semantic convention with the id `http.server` extends another semantic convention with the id `http`. + +`` will print only the attributes and constraints of the `http.server` semantic +convention. + +`` will print the attributes and constraints of the `http` semantic +convention and also the attributes and constraints of the `http.server` semantic convention. + +`` is equivalent to ``. + +`` will print the constraints and attributes of the `http.server` semantic +convention that have the tag `network`. + +`` will print the constraints and attributes of both `http` and `http.server` +semantic conventions that have the tag `network`. + +## Code Generator + +The image supports [Jinja](https://jinja.palletsprojects.com/en/2.11.x/) templates to generate code from the models. + +For example, the following template is used by the [opentelemetry-java-instrumentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/master/library-api/src/main/java/io/opentelemetry/instrumentation/api/typedspan) to generate Java classes. +[Template Link.](https://gist.github.com/thisthat/7e34742f4a7f1b5df57118f859a19c3b) + +The image can generate code with the following command: + +```bash +docker run --rm otel/semconvgen --yaml-root {yaml_folder} code --template {jinja-file} --output {output-file} +``` + +By default, all models are fed into the specified template at once, i.e. only a single file is generated. +This is helpful to generate constants for the semantic attributes, [example from opentelemetry-java](https://github.com/open-telemetry/opentelemetry-java/blob/master/api/src/main/java/io/opentelemetry/trace/attributes/SemanticAttributes.java). + +If the parameter `--file-per-group {pattern}` is set, a single yaml model is fed into the template +and the value of `pattern` is resolved from the model and attached as prefix to the output argument. +This way, multiple files are generated. The value of `pattern` can be one of the following: +- `semconv_id`: The id of the semantic convention. +- `prefix`: The prefix with which all attributes starts with. +- `extends`: The id of the parent semantic convention. + +Finally, additional value can be passed to the template in form of `key=value` pairs separated by +comma using the `--parameters [{key=value},]+` flag. + +For example, to generate the typed spans used by the [opentelemetry-java-instrumentation](https://github.com/open-telemetry/opentelemetry-java-instrumentation/tree/master/library-api/src/main/java/io/opentelemetry/instrumentation/api/typedspan), the following command can be used: + +```bash +docker run --rm otel/semconvgen --yaml-root ${yamls} code --template typed_span_class.java.j2 --file-per-group semconv_id -o ${output}/Span.java +``` \ No newline at end of file diff --git a/semantic-conventions/requirements.txt b/semantic-conventions/requirements.txt new file mode 100644 index 00000000..dce5c694 --- /dev/null +++ b/semantic-conventions/requirements.txt @@ -0,0 +1,9 @@ +black==19.10b0 +dataclasses==0.6 +mypy==0.770 +mypy-extensions==0.4.3 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.0 +typed-ast==1.4.1 +typing-extensions==3.7.4.2 +Jinja2==2.11.2 \ No newline at end of file diff --git a/semantic-conventions/setup.cfg b/semantic-conventions/setup.cfg new file mode 100644 index 00000000..c8387d02 --- /dev/null +++ b/semantic-conventions/setup.cfg @@ -0,0 +1,36 @@ +# Copyright 2019, Dynatrace + +[metadata] +name = semconvgen +description = Dynatrace Semantic Convention utility +author = Dynatrace +author_email = giovanni.liva@dynatrace.com +url = https://www.dynatrace.com/ +platforms = any +classifiers = + Development Status :: 1 - Alpha + Intended Audience :: Developers + Programming Language :: Python :: 3.6 + +[options] +python_requires = >=3.6 +package_dir= + =src +packages=find_namespace: +install_requires = + black==19.10b0 + dataclasses==0.6 + mypy==0.770 + mypy-extensions==0.4.3 + ruamel.yaml==0.16.10 + ruamel.yaml.clib==0.2.0 + typed-ast==1.4.1 + typing-extensions==3.7.4.2 + Jinja2==2.11.2 + +[options.packages.find] +where = src + +[options.entry_points] +console_scripts = + gen-semconv = dynatrace.semconv.main:main \ No newline at end of file diff --git a/semantic-conventions/setup.py b/semantic-conventions/setup.py new file mode 100644 index 00000000..4d614926 --- /dev/null +++ b/semantic-conventions/setup.py @@ -0,0 +1,19 @@ +import os + +import setuptools + +BASE_DIR = os.path.dirname(__file__) +VERSION_FILENAME = os.path.join(BASE_DIR, "src", "dynatrace", "semconv", "version.py") +PACKAGE_INFO = {} +with open(VERSION_FILENAME) as f: + exec(f.read(), PACKAGE_INFO) + + +VERSION_SUFFIX = os.environ.get("SEMCONGEN_VERSION_SUFFIX") +PUBLIC_VERSION = PACKAGE_INFO["__version__"] + +setuptools.setup( + version=PUBLIC_VERSION + if not VERSION_SUFFIX + else PUBLIC_VERSION + "+" + VERSION_SUFFIX +) diff --git a/semantic-conventions/src/dynatrace/__init__.py b/semantic-conventions/src/dynatrace/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/semantic-conventions/src/dynatrace/semconv/__init__.py b/semantic-conventions/src/dynatrace/semconv/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/semantic-conventions/src/dynatrace/semconv/main.py b/semantic-conventions/src/dynatrace/semconv/main.py new file mode 100644 index 00000000..2b028627 --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/main.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 + +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import glob +import sys + +from dynatrace.semconv.model.semantic_convention import SemanticConventionSet +from dynatrace.semconv.templating.code import CodeRenderer + +from dynatrace.semconv.templating.markdown import MarkdownRenderer + + +def parse_semconv(args, parser) -> SemanticConventionSet: + semconv = SemanticConventionSet(args.debug) + find_yaml(args) + for file in sorted(args.files): + if not file.endswith(".yaml"): + parser.error("{} is not a yaml file.".format(file)) + semconv.parse(file) + semconv.finish() + if semconv.has_error(): + sys.exit(1) + return semconv + + +def exclude_file_list(folder: str, pattern: str) -> list: + if not pattern: + return [] + sep = "/" + if folder.endswith("/"): + sep = "" + file_names = glob.glob(folder + sep + pattern, recursive=True) + return file_names + + +def main(): + parser = setup_parser() + args = parser.parse_args() + check_args(args, parser) + semconv = parse_semconv(args, parser) + + if args.flavor == "code": + renderer = CodeRenderer.from_commandline_params(args.parameters) + renderer.render(semconv, args.template, args.output, args.pattern) + elif args.flavor == "markdown": + process_markdown(semconv, args) + + +def process_markdown(semconv, args): + exclude = exclude_file_list(args.markdown_root, args.exclude) + md_renderer = MarkdownRenderer( + args.markdown_root, semconv, exclude, args.md_break_conditional, args.md_check + ) + md_renderer.render_md() + + +def find_yaml(args): + if args.yaml_root is not None: + exclude = set( + exclude_file_list(args.yaml_root if args.yaml_root else "", args.exclude) + ) + file_names = ( + set(glob.glob("{}/**/*.yaml".format(args.yaml_root), recursive=True)) + - exclude + ) + args.files.extend(sorted(file_names)) + + +def check_args(arguments, parser): + files = arguments.yaml_root is None and len(arguments.files) == 0 + if files: + parser.error("Either --yaml-root or YAML_FILE must be present") + + +def add_code_parser(subparsers): + parser = subparsers.add_parser("code") + parser.add_argument( + "--output", + "-o", + help="Specify the output file for the code generation.", + type=str, + required=True, + ) + parser.add_argument( + "--template", + "-t", + help="Specify the template to use for code generation", + type=str, + required=True, + ) + parser.add_argument( + "--file-per-group", + dest="pattern", + help="Each Semantic Convention is processed by the template and store in a different file. PATTERN is expected " + "to be the name of a SemanticConvention field and is prepended as a prefix to the output argument", + type=str, + ) + parser.add_argument( + "--parameters", + "-D", + dest="parameters", + action="append", + help="List of key=value pairs separated by comma. These values are fed into the template as is.", + type=str, + ) + + +def add_md_parser(subparsers): + parser = subparsers.add_parser("markdown") + parser.add_argument( + "--markdown-root", + "-md", + help="Specify folder of the markdown files", + type=str, + required=True, + ) + parser.add_argument( + "--md-break-conditional", + "-bc", + help="Set the number of chars before moving conditional causes of attributes to footnotes", + type=str, + required=False, + default=MarkdownRenderer.default_break_conditional_labels, + ) + parser.add_argument( + "--md-check", + help="Don't write the files back, just return the status. Return code 0 means nothing would change. Return " + "code 1 means some files would change.", + required=False, + action="store_true", + ) + + +def setup_parser(): + parser = argparse.ArgumentParser( + description="Process Semantic Conventions yaml files." + ) + parser.add_argument( + "--debug", "-d", help="Enable debug output", action="store_true" + ) + parser.add_argument( + "--yaml-root", + "-f", + metavar="folder", + help="Read all YAML files from a folder", + type=str, + ) + parser.add_argument( + "--exclude", "-e", help="Exclude the matching files using GLOB syntax", type=str + ) + parser.add_argument( + "files", + metavar="YAML_FILE", + type=str, + nargs="*", + help="YAML file containing a Semantic Convention", + ) + subparsers = parser.add_subparsers(dest="flavor") + add_code_parser(subparsers) + add_md_parser(subparsers) + + return parser + + +if __name__ == "__main__": + main() diff --git a/semantic-conventions/src/dynatrace/semconv/model/__init__.py b/semantic-conventions/src/dynatrace/semconv/model/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/semantic-conventions/src/dynatrace/semconv/model/constraints.py b/semantic-conventions/src/dynatrace/semconv/model/constraints.py new file mode 100644 index 00000000..3c7a3455 --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/model/constraints.py @@ -0,0 +1,58 @@ +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass, field, replace +from typing import List, Tuple, Set + +from dynatrace.semconv.model.semantic_attribute import SemanticAttribute + + +# We cannot frozen due to later evaluation of the attributes +@dataclass +class AnyOf: + """Defines a constraint where at least one of the list of attributes must be set. + The implementation of this class is evaluated in two times. At parsing time, the choice_list_ids field is + populated. After all yaml files are parsed, the choice_list_attributes field is populated with the object + representation of the attribute ids of choice_list_ids. + + Attributes: + choice_list_ids Contains the lists of attributes ids that must be set. + inherited True if it is inherited by another semantic convention, i.e. by include or extends. + choice_list_attributes Contains the list of attributes objects. This list contains the same lists of + attributes of choice_list_ids but instead of the ids, it contains the respective + objects representations. + """ + + choice_list_ids: Tuple[Tuple[str, ...]] + inherited: bool = False + choice_list_attributes: Tuple[Tuple[SemanticAttribute, ...]] = () + + def __eq__(self, other): + if not isinstance(other, AnyOf): + return False + return self.choice_list_ids == other.choice_list_ids + + def __hash__(self): + return hash(self.choice_list_ids) + + def add_attributes(self, attr: List[SemanticAttribute]): + self.choice_list_attributes += (attr,) + + def inherit_anyof(self): + return replace(self, inherited=True) + + +@dataclass(frozen=True) +class Include: + semconv_id: str diff --git a/semantic-conventions/src/dynatrace/semconv/model/exceptions.py b/semantic-conventions/src/dynatrace/semconv/model/exceptions.py new file mode 100644 index 00000000..918609ad --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/model/exceptions.py @@ -0,0 +1,37 @@ +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class ValidationError(Exception): + """ Exception raised if validation errors occur + Attributes: + line -- line in the file where the error occurred + column -- column in the file where the error occurred + message -- reason of the error + """ + + @classmethod + def from_yaml_pos(cls, pos, msg): + # the yaml parser starts counting from 0 + # while in document is usually reported starting from 1 + return cls(pos[0] + 1, pos[1] + 1, msg) + + def __init__(self, line, column, message): + super(ValidationError, self).__init__(line, column, message) + self.message = message + self.line = line + self.column = column + + def __str__(self): + return "{} - @{}:{}".format(self.message, self.line, self.column) diff --git a/semantic-conventions/src/dynatrace/semconv/model/semantic_attribute.py b/semantic-conventions/src/dynatrace/semconv/model/semantic_attribute.py new file mode 100644 index 00000000..34be27c0 --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/model/semantic_attribute.py @@ -0,0 +1,384 @@ +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numbers +import re +from collections.abc import Iterable +from dataclasses import dataclass, replace +from enum import Enum +from typing import List, Union + +from ruamel.yaml.comments import CommentedMap, CommentedSeq + +from dynatrace.semconv.model.exceptions import ValidationError +from dynatrace.semconv.model.utils import ( + validate_values, + validate_id, + check_no_missing_keys, +) + + +class Required(Enum): + ALWAYS = 1 + CONDITIONAL = 2 + NO = 3 + + +@dataclass +class SemanticAttribute: + fqn: str + attr_id: str + ref: str + attr_type: Union[str, "EnumAttributeType"] + brief: str + examples: List[Union[str, int, bool]] + tag: str + deprecated: str + required: Required + required_msg: str + sampling_relevant: bool + note: str + position: List[int] + inherited: bool = False + imported: bool = False + + def import_attribute(self): + return replace(self, imported=True) + + def inherit_attribute(self): + return replace(self, inherited=True) + + @property + def is_local(self): + return not self.imported and not self.inherited + + @property + def is_enum(self): + return isinstance(self.attr_type, EnumAttributeType) + + @staticmethod + def parse(prefix, yaml_attributes): + """ This method parses the yaml representation for semantic attributes + creating the respective SemanticAttribute objects. + """ + attributes = {} + allowed_keys = ( + "id", + "type", + "brief", + "examples", + "ref", + "tag", + "deprecated", + "required", + "sampling_relevant", + "note", + ) + for attribute in yaml_attributes: + validate_values(attribute, allowed_keys) + attr_id = attribute.get("id") + ref = attribute.get("ref") + position = attribute.lc.data[list(attribute)[0]] + if attr_id is None and ref is None: + msg = "At least one of id or ref is required." + raise ValidationError.from_yaml_pos(position, msg) + if attr_id is not None: + validate_id(attr_id, attribute.lc.data["id"]) + attr_type, brief, examples = SemanticAttribute.parse_id(attribute) + fqn = "{}.{}".format(prefix, attr_id) + attr_id = attr_id.strip() + else: + # Ref + attr_type = None + if "type" in attribute: + position = attribute.lc.data[list(attribute)[0]] + msg = "Ref attribute '{}' must not declare a type".format(ref) + raise ValidationError.from_yaml_pos(position, msg) + brief = attribute.get("brief") + examples = attribute.get("examples") + ref = ref.strip() + fqn = ref + + required_value_map = { + "always": Required.ALWAYS, + "conditional": Required.CONDITIONAL, + "": Required.NO, + } + required_msg = "" + required_val = attribute.get("required", "") + if isinstance(required_val, CommentedMap): + required = Required.CONDITIONAL + required_msg = required_val.get("conditional", None) + if required_msg is None: + position = attribute.lc.data["required"] + msg = "Missing message for conditional required field!" + raise ValidationError.from_yaml_pos(position, msg) + else: + required = required_value_map.get(required_val) + if required == Required.CONDITIONAL: + position = attribute.lc.data["required"] + msg = "Missing message for conditional required field!" + raise ValidationError.from_yaml_pos(position, msg) + if required is None: + position = attribute.lc.data["required"] + msg = "Value '{}' for required field is not allowed".format( + required_val + ) + raise ValidationError.from_yaml_pos(position, msg) + tag = attribute.get("tag", "").strip() + deprecated = attribute.get("deprecated") + if deprecated is not None: + if AttributeType.get_type(deprecated) != "string" or deprecated == "": + position = attribute.lc.data["deprecated"] + msg = ( + "Deprecated field expects a string that specify why the attribute is deprecated and/or what" + " to use instead! " + ) + raise ValidationError.from_yaml_pos(position, msg) + deprecated = deprecated.strip() + sampling_relevant = ( + AttributeType.to_bool("sampling_relevant", attribute) + if attribute.get("sampling_relevant") + else False + ) + note = attribute.get("note", "") + fqn = fqn.strip() + attr = SemanticAttribute( + fqn=fqn, + attr_id=attr_id, + ref=ref, + attr_type=attr_type, + brief=brief.strip() if brief else "", + examples=examples, + tag=tag, + deprecated=deprecated, + required=required, + required_msg=str(required_msg).strip(), + sampling_relevant=sampling_relevant, + note=note.strip(), + position=position, + ) + if attr.fqn in attributes: + position = attribute.lc.data[list(attribute)[0]] + msg = ( + "Attribute id " + + fqn + + " is already present at line " + + str(attributes.get(fqn).position[0] + 1) + ) + raise ValidationError.from_yaml_pos(position, msg) + attributes[fqn] = attr + return attributes + + @staticmethod + def parse_id(attribute): + check_no_missing_keys(attribute, ["type", "brief"]) + attr_val = attribute["type"] + try: + attr_type = EnumAttributeType.parse(attr_val) + except ValidationError as e: + position = attribute.lc.data["type"] + raise ValidationError.from_yaml_pos(position, e.message) from e + brief = attribute["brief"] + zlass = ( + AttributeType.type_mapper(attr_type) + if isinstance(attr_type, str) + else "enum" + ) + + examples = attribute.get("examples") + is_simple_type = AttributeType.is_simple_type(attr_type) + # if we are an array, examples must already be an array + if ( + is_simple_type + and attr_type.endswith("[]") + and not isinstance(examples, CommentedSeq) + ): + position = attribute.lc.data[list(attribute)[0]] + msg = "Non array examples for {} are not allowed".format(attr_type) + raise ValidationError.from_yaml_pos(position, msg) + if ( + (is_simple_type or isinstance(attr_type, EnumAttributeType)) + and not isinstance(examples, CommentedSeq) + and examples is not None + ): + examples = [examples] + if is_simple_type and attr_type not in ["boolean", "boolean[]"]: + if examples is None or (len(examples) == 0): + position = attribute.lc.data[list(attribute)[0]] + msg = "Empty examples for {} are not allowed".format(attr_type) + raise ValidationError.from_yaml_pos(position, msg) + if is_simple_type and attr_type not in ["boolean", "boolean[]"]: + AttributeType.check_examples_type(attr_type, examples, zlass) + return attr_type, str(brief), examples + + def equivalent_to(self, other: "SemanticAttribute"): + if self.attr_id is not None: + if self.fqn == other.fqn: + return True + elif self == other: + return True + return False + + +class AttributeType: + + # https://yaml.org/type/bool.html + bool_type = re.compile( + "y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF" + ) + + bool_type_true = re.compile("y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON") + bool_type_false = re.compile("n|N|no|No|NO|false|False|FALSE|off|Off|OFF") + + @staticmethod + def get_type(t): + if isinstance(t, numbers.Number): + return "number" + if AttributeType.bool_type.fullmatch(t): + return "boolean" + return "string" + + @staticmethod + def is_simple_type(attr_type: str): + return attr_type in ( + "string", + "string[]", + "number", + "number[]", + "boolean", + "boolean[]", + ) + + @staticmethod + def type_mapper(attr_type: str): + type_mapper = { + "number": int, + "number[]": int, + "string": str, + "string[]": str, + "boolean": bool, + "boolean[]": bool, + } + return type_mapper.get(attr_type) + + @staticmethod + def check_examples_type(attr_type, examples, zlass): + """ This method checks example are correctly typed + """ + index = -1 + for example in examples: + index += 1 + if attr_type.endswith("[]") and isinstance(example, Iterable): + # Multi array example + for element in example: + if not isinstance(element, zlass): + position = examples.lc.data[index] + msg = "Example with wrong type. Expected {} examples but is was {}.".format( + attr_type, type(element) + ) + raise ValidationError.from_yaml_pos(position, msg) + else: # Single value example or array example with a single example array + if not isinstance(example, zlass): + position = examples.lc.data[index] + msg = "Example with wrong type. Expected {} examples but is was {}.".format( + attr_type, type(example) + ) + raise ValidationError.from_yaml_pos(position, msg) + + @staticmethod + def to_bool(key, parent_object): + """ This method translate yaml boolean values to python boolean values + """ + yaml_value = parent_object.get(key) + if isinstance(yaml_value, bool): + return yaml_value + if isinstance(yaml_value, str): + if AttributeType.bool_type_true.fullmatch(yaml_value): + return True + elif AttributeType.bool_type_false.fullmatch(yaml_value): + return False + position = parent_object.lc.data[key] + msg = "Value '{}' for {} field is not allowed".format(yaml_value, key) + raise ValidationError.from_yaml_pos(position, msg) + + +@dataclass +class EnumAttributeType: + custom_values: bool + members: "List[EnumMember]" + enum_type: str + + def __init__(self, custom_values, members, enum_type): + self.custom_values = custom_values + self.members = members + self.enum_type = enum_type + + def __str__(self): + return self.enum_type + + @staticmethod + def parse(attribute_type): + """ This method parses the yaml representation for semantic attribute types. + If the type is an enumeration, it generated the EnumAttributeType object, + otherwise it returns the basic type as string. + """ + if isinstance(attribute_type, str): + if AttributeType.is_simple_type(attribute_type): + return attribute_type + else: # Wrong type used - rise the exception and fill the missing data in the parent + raise ValidationError( + 0, 0, "Invalid type: {} is not allowed".format(attribute_type) + ) + else: + allowed_keys = ["allow_custom_values", "members"] + mandatory_keys = ["members"] + validate_values(attribute_type, allowed_keys, mandatory_keys) + custom_values = ( + bool(attribute_type.get("allow_custom_values")) + if "allow_custom_values" in attribute_type + else False + ) + members = [] + if attribute_type["members"] is None or len(attribute_type["members"]) < 1: + # Missing members - rise the exception and fill the missing data in the parent + raise ValidationError(0, 0, "Enumeration without values!") + + allowed_keys = ["id", "value", "brief", "note"] + mandatory_keys = ["id", "value"] + for member in attribute_type["members"]: + validate_values(member, allowed_keys, mandatory_keys) + members.append( + EnumMember( + member_id=member["id"], + value=member["value"], + brief=member.get("brief") + if "brief" in member + else member["id"], + note=member.get("note") if "note" in member else "", + ) + ) + enum_type = AttributeType.get_type(members[0].value) + for m in members: + if enum_type != AttributeType.get_type(m.value): + raise ValidationError(0, 0, "Enumeration type inconsistent!") + return EnumAttributeType(custom_values, members, enum_type) + + +@dataclass +class EnumMember: + member_id: str + value: str + brief: str + note: str diff --git a/semantic-conventions/src/dynatrace/semconv/model/semantic_convention.py b/semantic-conventions/src/dynatrace/semconv/model/semantic_convention.py new file mode 100644 index 00000000..d1483545 --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/model/semantic_convention.py @@ -0,0 +1,457 @@ +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from dataclasses import dataclass, field +from enum import Enum +from typing import List, Union, Iterable, Tuple + +import typing +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedSeq + +from dynatrace.semconv.model.constraints import Include, AnyOf +from dynatrace.semconv.model.exceptions import ValidationError +from dynatrace.semconv.model.semantic_attribute import SemanticAttribute, Required +from dynatrace.semconv.model.utils import validate_values, validate_id + + +class SpanKind(Enum): + EMPTY = 1 + CLIENT = 2 + SERVER = 3 + CONSUMER = 4 + PRODUCER = 5 + INTERNAL = 6 + + @staticmethod + def parse(span_kind_value): + if span_kind_value is None: + return SpanKind.EMPTY + kind_map = { + "client": SpanKind.CLIENT, + "server": SpanKind.SERVER, + "producer": SpanKind.PRODUCER, + "consumer": SpanKind.CONSUMER, + "internal": SpanKind.INTERNAL, + } + return kind_map.get(span_kind_value) + + +@dataclass +class SemanticConvention: + """ Contains the model extracted from a yaml file + """ + + semconv_id: str + brief: str + note: str + prefix: str + extends: str + span_kind: SpanKind + attrs_by_name: "Dict[str, SemanticAttribute]" + constraints: "Set[Union[Include, AnyOf]]" + _position: List[int] + + @property + def attributes(self): + return list(self.attrs_by_name.values()) + + @staticmethod + def parse(yaml_file): + yaml = YAML().load(yaml_file) + models = [] + available_keys = ( + "id", + "brief", + "note", + "prefix", + "extends", + "span_kind", + "attributes", + "constraints", + ) + mandatory_keys = ("id", "brief") + for group in yaml["groups"]: + validate_values(group, available_keys, mandatory_keys) + validate_id(group["id"], group.lc.data["id"]) + span_kind = SpanKind.parse(group.get("span_kind")) + if span_kind is None: + position = group.lc.data["span_kind"] + msg = "Invalid value for span_kind: {}".format(group.get("span_kind")) + raise ValidationError.from_yaml_pos(position, msg) + prefix = group.get("prefix", "") + if prefix != "": + validate_id(prefix, group.lc.data["prefix"]) + position = group.lc.data["id"] + model = SemanticConvention( + semconv_id=group["id"].strip(), + brief=str(group["brief"]).strip(), + note=group.get("note", "").strip(), + prefix=prefix.strip(), + extends=group.get("extends", "").strip(), + span_kind=span_kind, + attrs_by_name=SemanticAttribute.parse(prefix, group.get("attributes")) + if "attributes" in group + else {}, + constraints=SemanticConvention.parse_constraint( + group.get("constraints", ()) + ), + _position=position, + ) + models.append(model) + return models + + @staticmethod + def parse_constraint(yaml_constraints): + """ This method parses the yaml representation for semantic convention attributes + creating a list of Constraint objects. + """ + constraints = () + allowed_keys = ("include", "any_of") + for constraint in yaml_constraints: + validate_values(constraint, allowed_keys) + if len(constraint.keys()) > 1: + position = constraint.lc.data[list(constraint)[1]] + msg = "Invalid entry in constraint array - multiple top-level keys in entry." + raise ValidationError.from_yaml_pos(position, msg) + if "include" in constraint: + constraints += (Include(constraint.get("include")),) + elif "any_of" in constraint: + choice_sets = () + for constraint_list in constraint.get("any_of"): + inner_id_list = () + if isinstance(constraint_list, CommentedSeq): + inner_id_list = tuple( + attr_constraint for attr_constraint in constraint_list + ) + else: + inner_id_list += (constraint_list,) + choice_sets += (inner_id_list,) + constraints += (AnyOf(choice_sets),) + return constraints + + def contains_attribute(self, attr: "SemanticAttribute"): + for local_attr in self.attributes: + if local_attr.attr_id is not None: + if local_attr.fqn == attr.fqn: + return True + if local_attr == attr: + return True + return False + + def all_attributes(self): + attr: SemanticAttribute + return SemanticConvention.unique_attr( + [attr for attr in self.attributes] + self.conditional_attributes() + ) + + def sampling_attributes(self): + attr: SemanticAttribute + return SemanticConvention.unique_attr( + [attr for attr in self.attributes if attr.sampling_relevant] + ) + + def required_attributes(self): + attr: SemanticAttribute + return SemanticConvention.unique_attr( + [attr for attr in self.attributes if attr.required == Required.ALWAYS] + ) + + def conditional_attributes(self): + attr: SemanticAttribute + return SemanticConvention.unique_attr( + [attr for attr in self.attributes if attr.required == Required.CONDITIONAL] + ) + + @staticmethod + def unique_attr(l: list) -> list: + output = [] + for x in l: + if x.fqn not in [attr.fqn for attr in output]: + output.append(x) + return output + + def any_of(self): + result = [] + for constraint in self.constraints: + if isinstance(constraint, AnyOf): + result.append(constraint) + return result + + def has_attribute_constraint(self, attr): + return any( + attribute.equivalent_to(attr) + for constraint in self.constraints + if isinstance(constraint, AnyOf) + for attr_list in constraint.choice_list_attributes + for attribute in attr_list + ) + + +@dataclass +class SemanticConventionSet: + """ Contains the list of models. + From this structure we will generate md/constants/etc with a pretty print of the structure. + """ + + debug: bool + models: typing.Dict[str, SemanticConvention] = field(default_factory=dict) + errors: bool = False + + def parse(self, file): + with open(file, "r", encoding="utf-8") as yaml_file: + try: + semconv_models = SemanticConvention.parse(yaml_file) + for model in semconv_models: + if model.semconv_id in self.models: + self.errors = True + print("Error parsing {}\n".format(file), file=sys.stderr) + print( + "Semantic convention '{}' is already defined.".format( + model.semconv_id + ), + file=sys.stderr, + ) + self.models[model.semconv_id] = model + except ValidationError as e: + self.errors = True + print("Error parsing {}\n".format(file), file=sys.stderr) + print(e, file=sys.stderr) + + def has_error(self): + return self.errors + + def check_unique_fqns(self): + group_by_fqn: typing.Dict[str, str] = {} + for model in self.models.values(): + for attr in model.attributes: + if not attr.ref: + if group_by_fqn.get(attr.fqn): + self.errors = True + print( + "Attribute {} of Semantic convention '{}' is already defined in {}.".format( + attr.fqn, model.semconv_id, group_by_fqn.get(attr.fqn) + ), + file=sys.stderr, + ) + group_by_fqn[attr.fqn] = model.semconv_id + + def finish(self): + """ Resolves values referenced from other models using `ref` and `extends` attributes AFTER all models were parsed. + Here, sanity checks for `ref/extends` attributes are performed. + """ + semconv: SemanticConvention + # Before resolving attributes, we verify that no duplicate exists. + self.check_unique_fqns() + fixpoint = False + index = 0 + tmp_debug = self.debug + # This is a hot spot for optimizations + while not fixpoint: + fixpoint = True + if index > 0: + self.debug = False + for semconv in self.models.values(): + # Ref first, extends and includes after! + fixpoint_ref = self.resolve_ref(semconv) + fixpoint_inc = self.resolve_include(semconv) + fixpoint = fixpoint and fixpoint_ref and fixpoint_inc + index += 1 + self.debug = tmp_debug + # After we resolve any local dependency, we can resolve parent/child relationship + self._populate_extends() + # From string containing attribute ids to SemanticAttribute objects + self._populate_anyof_attributes() + + def _populate_extends(self): + """ + This internal method goes through every semantic convention to resolve parent/child relationships. + :return: None + """ + unprocessed: typing.Dict[str, SemanticConvention] + unprocessed = self.models.copy() + # Iterate through the list and remove the semantic conventions that have been processed. + while len(unprocessed) > 0: + semconv = next(iter(unprocessed.values())) + self._populate_extends_single(semconv, unprocessed) + + def _populate_extends_single( + self, + semconv: SemanticConvention, + unprocessed: typing.Dict[str, SemanticConvention], + ): + """ + Resolves the parent/child relationship for a single Semantic Convention. If the parent **p** of the input + semantic convention **i** has in turn a parent **pp**, it recursively resolves **pp** before processing **p**. + :param semconv: The semantic convention for which resolve the parent/child relationship. + :param semconvs: The list of remaining semantic conventions to process. + :return: A list of remaining semantic convention to process. + """ + # Resolve parent of current Semantic Convention + if semconv.extends: + extended = self.models.get(semconv.extends) + if extended is None: + raise ValidationError.from_yaml_pos( + semconv._position, + "Semantic Convention {} extends {} but the latter cannot be found!".format( + semconv.semconv_id, semconv.extends + ), + ) + + # Process hierarchy chain + not_yet_processed = extended.extends in unprocessed + if extended.extends and not_yet_processed: + # Recursion on parent if was not already processed + parent_extended = self.models.get(extended.extends) + self._populate_extends_single(parent_extended, unprocessed) + + # inherit prefix and constraints + if not semconv.prefix: + semconv.prefix = extended.prefix + # Constraints + for constraint in extended.constraints: + if constraint not in semconv.constraints and isinstance( + constraint, AnyOf + ): + semconv.constraints += (constraint.inherit_anyof(),) + # Attributes + parent_attributes = {} + for ext_attr in extended.attrs_by_name.values(): + parent_attributes[ext_attr.fqn] = ext_attr.inherit_attribute() + # By induction, parent semconv is already correctly sorted + parent_attributes.update( + SemanticConventionSet._sort_attributes_dict(semconv.attrs_by_name) + ) + semconv.attrs_by_name = parent_attributes + else: # No parent, sort of current attributes + semconv.attrs_by_name = SemanticConventionSet._sort_attributes_dict( + semconv.attrs_by_name + ) + # delete from remaining semantic conventions to process + del unprocessed[semconv.semconv_id] + + @staticmethod + def _sort_attributes_dict( + attributes: typing.Dict[str, SemanticAttribute] + ) -> typing.Dict[str, SemanticAttribute]: + """ + First imported, and then defined attributes. + :param attributes: Dictionary of attributes to sort + :return: A sorted dictionary of attributes + """ + return dict( + sorted(attributes.items(), key=lambda kv: 0 if kv[1].imported else 1) + ) + + def _populate_anyof_attributes(self): + any_of: AnyOf + for semconv in self.models.values(): + for any_of in semconv.constraints: + if isinstance(any_of, AnyOf): + for attr_ids in any_of.choice_list_ids: + constraint_attrs = [] + for attr_id in attr_ids: + ref_attr = self._lookup_attribute(attr_id) + if ref_attr is not None: + constraint_attrs.append(ref_attr) + if constraint_attrs: + any_of.add_attributes(constraint_attrs) + + def resolve_ref(self, semconv: SemanticConvention): + fixpoint_ref = True + attr: SemanticAttribute + for attr in semconv.attributes: + if attr.ref is not None and attr.attr_id is None: + # There are changes + fixpoint_ref = False + ref_attr = self._lookup_attribute(attr.ref) + if not ref_attr: + raise ValidationError.from_yaml_pos( + semconv._position, + "Semantic Convention {} reference `{}` but it cannot be found!".format( + semconv.semconv_id, attr.ref + ), + ) + attr.attr_type = ref_attr.attr_type + if not attr.brief: + attr.brief = ref_attr.brief + if not attr.note: + attr.note = ref_attr.note + if attr.examples is None: + attr.examples = ref_attr.examples + attr.attr_id = attr.ref + return fixpoint_ref + + def resolve_include(self, semconv: SemanticConvention): + fixpoint_inc = True + for constraint in semconv.constraints: + if isinstance(constraint, Include): + include_semconv: SemanticConvention + include_semconv = self.models.get(constraint.semconv_id) + # include required attributes and constraints + if include_semconv is None: + raise ValidationError.from_yaml_pos( + semconv._position, + "Semantic Convention {} includes {} but the latter cannot be found!".format( + semconv.semconv_id, constraint.semconv_id + ), + ) + # We resolve the parent/child relationship of the included semantic convention, if any + self._populate_extends_single( + include_semconv, {include_semconv.semconv_id: include_semconv} + ) + attr: SemanticAttribute + for attr in include_semconv.attributes: + if semconv.contains_attribute(attr): + if self.debug: + print( + "[Includes] {} already contains attribute {}".format( + semconv.semconv_id, attr + ) + ) + continue + # There are changes + fixpoint_inc = False + semconv.attrs_by_name[attr.fqn] = attr.import_attribute() + for inc_constraint in include_semconv.constraints: + if ( + isinstance(inc_constraint, Include) + or inc_constraint in semconv.constraints + ): + # We do not include "include" constraint or the constraint was already imported + continue + # We know the type of the constraint + inc_constraint: AnyOf + # There are changes + fixpoint_inc = False + semconv.constraints += (inc_constraint.inherit_anyof(),) + return fixpoint_inc + + def _lookup_attribute(self, attr_id: str) -> Union[SemanticAttribute, None]: + return next( + ( + attr + for model in self.models.values() + for attr in model.attributes + if attr.fqn == attr_id and attr.ref is None + ), + None, + ) + + def attributes(self): + output = [] + for semconv in self.models.values(): + output.extend(semconv.attributes) + return output diff --git a/semantic-conventions/src/dynatrace/semconv/model/utils.py b/semantic-conventions/src/dynatrace/semconv/model/utils.py new file mode 100644 index 00000000..4b1d22b1 --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/model/utils.py @@ -0,0 +1,50 @@ +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from dynatrace.semconv.model.exceptions import ValidationError + + +ID_RE = re.compile("([a-z](\\.?[a-z0-9_-]+)+)") + + +def validate_id(semconv_id, position): + if not ID_RE.fullmatch(semconv_id): + raise ValidationError.from_yaml_pos( + position, + "Invalid id {}. Semantic Convention ids MUST be {}".format( + semconv_id, ID_RE.pattern + ), + ) + + +def validate_values(yaml, keys, mandatory=()): + """ This method checks only valid keywords and value types are used + """ + unwanted = list(set(yaml) - set(keys)) + if unwanted: + position = yaml.lc.data[unwanted[0]] + msg = "Invalid keys: {}".format(unwanted) + raise ValidationError.from_yaml_pos(position, msg) + if mandatory: + check_no_missing_keys(yaml, mandatory) + + +def check_no_missing_keys(yaml, mandatory): + missing = list(set(mandatory) - set(yaml)) + if missing: + position = yaml.lc.data[list(yaml)[0]] + msg = "Missing keys: {}".format(missing) + raise ValidationError.from_yaml_pos(position, msg) diff --git a/semantic-conventions/src/dynatrace/semconv/templating/__init__.py b/semantic-conventions/src/dynatrace/semconv/templating/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/semantic-conventions/src/dynatrace/semconv/templating/code.py b/semantic-conventions/src/dynatrace/semconv/templating/code.py new file mode 100644 index 00000000..4e8cfd5f --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/templating/code.py @@ -0,0 +1,131 @@ +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import os.path +import re +import typing + +from jinja2 import Environment, FileSystemLoader, select_autoescape + +from dynatrace.semconv.model.semantic_convention import ( + SemanticConventionSet, + SemanticConvention, +) +from dynatrace.semconv.model.utils import ID_RE + + +def to_doc_brief(doc_string: str) -> str: + if doc_string is None: + return "" + doc_string = doc_string.strip() + if doc_string.endswith("."): + return doc_string[:-1] + return doc_string + + +def merge(list: typing.List, elm): + return list.extend(elm) + + +def to_const_name(name: str) -> str: + return name.upper().replace(".", "_").replace("-", "_") + + +def to_camelcase(name: str, first_upper=False) -> str: + first, *rest = name.replace("_", ".").split(".") + if first_upper: + first = first.capitalize() + return first + "".join(word.capitalize() for word in rest) + + +class CodeRenderer: + pattern = "{{{}}}".format(ID_RE.pattern) + matcher = re.compile(pattern) + + parameters: typing.Dict[str, str] + + @staticmethod + def from_commandline_params(parameters: str = ""): + params = {} + if parameters: + for elm in parameters: + pairs = elm.split(",") + for pair in pairs: + (k, v) = pair.split("=") + params.update({k: v}) + return CodeRenderer(params) + + def __init__(self, parameters: typing.Dict[str, str]): + self.parameters = parameters + + def get_data_single_file(self, semconvset, template_path: str) -> dict: + """Returns a dictionary that contains all SemanticConventions to fill the template.""" + data = { + "template": template_path, + "semconvs": semconvset.models, + "attributes": semconvset.attributes(), + } + data.update(self.parameters) + return data + + def get_data_multiple_files(self, semconv, template_path: str) -> dict: + """Returns a dictionary with the data from a single SemanticConvention to fill the template.""" + data = {"template": template_path, "semconv": semconv} + data.update(self.parameters) + return data + + def setup_environment(self, env: Environment): + env.filters["to_doc_brief"] = to_doc_brief + env.filters["to_const_name"] = to_const_name + env.filters["merge"] = merge + env.filters["to_camelcase"] = to_camelcase + + def prefix_output_file( + self, file_name: str, pattern: str, semconv: SemanticConvention + ): + base = os.path.basename(file_name) + dir = os.path.dirname(file_name) + value = getattr(semconv, pattern) + return dir + "/" + to_camelcase(value, True) + base + + def render( + self, + semconvset: SemanticConventionSet, + template_path: str, + output_file, + pattern: str, + ): + file_name = os.path.basename(template_path) + folder = os.path.dirname(template_path) + env = Environment( + loader=FileSystemLoader(searchpath=folder), + autoescape=select_autoescape([""]), + ) + self.setup_environment(env) + if pattern: + semconv: SemanticConvention + for semconv in semconvset.models.values(): + output_name = self.prefix_output_file(output_file, pattern, semconv) + data = self.get_data_multiple_files(semconv, template_path) + template = env.get_template(file_name, data) + template.globals["now"] = datetime.datetime.utcnow() + template.globals["version"] = os.environ.get("ARTIFACT_VERSION", "dev") + template.stream(data).dump(output_name) + else: + data = self.get_data_single_file(semconvset, template_path) + template = env.get_template(file_name, data) + template.globals["now"] = datetime.datetime.utcnow() + template.globals["version"] = os.environ.get("ARTIFACT_VERSION", "dev") + template.stream(data).dump(output_file) diff --git a/semantic-conventions/src/dynatrace/semconv/templating/markdown.py b/semantic-conventions/src/dynatrace/semconv/templating/markdown.py new file mode 100644 index 00000000..b0885751 --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/templating/markdown.py @@ -0,0 +1,404 @@ +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import io +import os +import re +import sys +import typing +from pathlib import PurePath + +from dynatrace.semconv.model.constraints import AnyOf, Include +from dynatrace.semconv.model.semantic_attribute import ( + SemanticAttribute, + EnumAttributeType, + Required, + EnumMember, +) +from dynatrace.semconv.model.semantic_convention import ( + SemanticConventionSet, + SemanticConvention, +) +from dynatrace.semconv.model.utils import ID_RE + + +class RenderContext: + is_full: bool + is_remove_constraint: bool + group_key: str + break_counter: int + enums: list + notes: list + note_index: int + current_md: str + current_semconv: SemanticConvention + + def __init__(self, break_count): + self.is_full = False + self.is_remove_constraint = False + self.group_key = "" + self.break_count = break_count + self.enums = [] + self.notes = [] + self.note_index = 1 + self.current_md = "" + self.current_semconv = None + + def clear_table_generation(self): + self.notes = [] + self.enums = [] + self.note_index = 1 + + def add_note(self, msg: str): + self.note_index += 1 + self.notes.append(msg) + + def add_enum(self, attr: SemanticAttribute): + self.enums.append(attr) + + +class MarkdownRenderer: + p_start = re.compile("") + p_semconv_selector = re.compile( + r"(?P{})(?:\((?P.*)\))?".format(ID_RE.pattern) + ) + p_end = re.compile("") + default_break_conditional_labels = 50 + valid_parameters = ["tag", "full", "remove_constraints"] + + prelude = "\n" + table_headers = "| Attribute | Type | Description | Example | Required |\n|---|---|---|---|---|\n" + + def __init__( + self, + md_folder, + semconvset: SemanticConventionSet, + exclude: list = [], + break_count=default_break_conditional_labels, + check_only=False, + ): + self.render_ctx = RenderContext(break_count) + self.semconvset = semconvset + # We load all markdown files to render + self.file_names = sorted( + set(glob.glob("{}/**/*.md".format(md_folder), recursive=True)) + - set(exclude) + ) + # We build the dict that maps each attribute that has to be rendered to the latest visited file + # that contains it + self.filename_for_attr_fqn = self._create_attribute_location_dict() + self.check_only = check_only + + """ + This method renders attributes as markdown table entry + """ + + def to_markdown_attr( + self, attribute: SemanticAttribute, output: io.StringIO, + ): + name = self.render_attribute_id(attribute.fqn) + attr_type = ( + "enum" + if isinstance(attribute.attr_type, EnumAttributeType) + else attribute.attr_type + ) + description = "" + if attribute.deprecated: + if "deprecated" in attribute.deprecated.lower(): + description = "**{}**
".format(attribute.deprecated) + else: + description = "**Deprecated: {}**
".format(attribute.deprecated) + description += attribute.brief + if attribute.note: + description += " [{}]".format(self.render_ctx.note_index) + self.render_ctx.add_note(attribute.note) + examples = "" + if isinstance(attribute.attr_type, EnumAttributeType): + if attribute.is_local and not attribute.ref: + self.render_ctx.add_enum(attribute) + example_list = attribute.examples if attribute.examples else () + examples = ( + "
".join("`{}`".format(ex) for ex in example_list) + if example_list + else "`{}`".format(attribute.attr_type.members[0].value) + ) + # Add better type info to enum + if attribute.attr_type.custom_values: + attr_type = attribute.attr_type.enum_type + else: + attr_type = "{} enum".format(attribute.attr_type.enum_type) + elif attribute.attr_type: + example_list = attribute.examples if attribute.examples else [] + examples = "
".join("`{}`".format(ex) for ex in example_list) + if attribute.required == Required.ALWAYS: + required = "Yes" + elif attribute.required == Required.CONDITIONAL: + if len(attribute.required_msg) < self.render_ctx.break_count: + required = "Conditional
{}".format(attribute.required_msg) + else: + # We put the condition in the notes after the table + required = "Conditional [{}]".format(self.render_ctx.note_index) + self.render_ctx.add_note(attribute.required_msg) + else: + # check if they are required by some constraint + if ( + not self.render_ctx.is_remove_constraint + and self.render_ctx.current_semconv.has_attribute_constraint(attribute) + ): + required = "See below" + else: + required = "No" + output.write( + "| {} | {} | {} | {} | {} |\n".format( + name, attr_type, description, examples, required + ) + ) + + """ + This method renders anyof constraints into markdown lists + """ + + def to_markdown_anyof(self, anyof: AnyOf, output: io.StringIO): + if anyof.inherited and not self.render_ctx.is_full: + return "" + output.write("\nAt least one of the following is required:\n\n") + for choice in anyof.choice_list_ids: + output.write("* ") + list_of_choice = ", ".join(self.render_attribute_id(c) for c in choice) + output.write(list_of_choice) + output.write("\n") + + def to_markdown_notes(self, output: io.StringIO): + """ Renders notes after a Semantic Convention Table + :return: + """ + counter = 1 + for note in self.render_ctx.notes: + output.write("\n**[{}]:** {}\n".format(counter, note)) + counter += 1 + + def to_markdown_enum(self, output: io.StringIO): + """ Renders enum types after a Semantic Convention Table + :return: + """ + attr: SemanticAttribute + for attr in self.render_ctx.enums: + enum: EnumAttributeType + enum = attr.attr_type + output.write("\n`" + attr.fqn + "` ") + if enum.custom_values: + output.write( + "MUST be one of the following or, if none of the listed values apply, a custom value" + ) + else: + output.write("MUST be one of the following") + output.write(":\n\n") + output.write("| Value | Description |\n|---|---|") + member: EnumMember + counter = 1 + notes = [] + for member in enum.members: + description = member.brief + if member.note: + description += " [{}]".format(counter) + counter += 1 + notes.append(member.note) + output.write("\n| `{}` | {} |".format(member.value, description)) + counter = 1 + if not notes: + output.write("\n") + for note in notes: + output.write("\n\n**[{}]:** {}".format(counter, note)) + counter += 1 + if notes: + output.write("\n") + + """ + Method to render in markdown an attribute id. If the id points to an attribute in another rendered table, a markdown + link is introduced. + """ + + def render_attribute_id(self, attribute_id): + md_file = self.filename_for_attr_fqn.get(attribute_id) + if md_file: + path = PurePath(self.render_ctx.current_md) + if path.as_posix() != PurePath(md_file).as_posix(): + diff = PurePath(os.path.relpath(md_file, start=path.parent)).as_posix() + if diff != ".": + return "[{}]({})".format(attribute_id, diff) + return "`{}`".format(attribute_id) + + """ + Entry method to translate attributes and constraints of a semantic convention into Markdown + """ + + def to_markdown_constraint( + self, obj: typing.Union[AnyOf, Include], output: io.StringIO, + ): + if isinstance(obj, AnyOf): + self.to_markdown_anyof(obj, output) + return + elif isinstance(obj, Include): + return + raise Exception( + "Trying to generate Markdown for a wrong type {}".format(type(obj)) + ) + + def render_md(self): + for md_filename in self.file_names: + with open(md_filename, encoding="utf-8") as md_file: + content = md_file.read() + output = io.StringIO() + self._render_single_file(content, md_filename, output) + if self.check_only: + if content != output.getvalue(): + sys.exit( + "File " + + md_filename + + " contains a table that would be reformatted." + ) + else: + with open(md_filename, "w", encoding="utf-8") as md_file: + md_file.write(output.getvalue()) + if self.check_only: + print("{} files left unchanged.".format(len(self.file_names))) + + """ + This method creates a dictionary that associates each attribute with the latest table in which it is rendered. + This is required by the ref attributes to point to the correct file + """ + + def _create_attribute_location_dict(self): + m = {} + for md in self.file_names: + with open(md, "r", encoding="utf-8") as markdown: + self.current_md = md + content = markdown.read() + for match in self.p_start.finditer(content): + semconv_id, _ = self._parse_semconv_selector(match.group(1).strip()) + semconv = self.semconvset.models.get(semconv_id) + if not semconv: + raise ValueError( + "Semantic Convention ID {} not found".format(semconv_id) + ) + a: SemanticAttribute + valid_attr = ( + a for a in semconv.attributes if a.is_local and not a.ref + ) + for attr in valid_attr: + m[attr.fqn] = md + return m + + def _parse_semconv_selector(self, selector: str): + semconv_id = selector + parameters = {} + m = self.p_semconv_selector.match(selector) + if m: + semconv_id = m.group("semconv_id") + pars = m.group("parameters") + if pars: + for par in pars.split(","): + key_value = par.split("=") + if len(key_value) > 2: + raise ValueError( + "Wrong syntax in " + + m.group(4) + + " in " + + self.render_ctx.current_md + ) + key = key_value[0].strip() + if key not in self.valid_parameters: + raise ValueError( + "Unexpected parameter `" + + key_value[0] + + "` in " + + self.render_ctx.current_md + ) + if key in parameters: + raise ValueError( + "Parameter `" + + key_value[0] + + "` already defined in " + + self.render_ctx.current_md + ) + value = key_value[1] if len(key_value) == 2 else "" + parameters[key] = value + return semconv_id, parameters + + def _render_single_file(self, content: str, md: str, output: io.StringIO): + last_match = 0 + self.render_ctx.current_md = md + # The current implementation swallows nested semconv tags + while True: + match = self.p_start.search(content, last_match) + if not match: + break + semconv_id, parameters = self._parse_semconv_selector( + match.group(1).strip() + ) + semconv = self.semconvset.models.get(semconv_id) + if not semconv: + # We should not fail here since we would detect this earlier + # But better be safe than sorry + raise ValueError( + "Semantic Convention ID {} not found".format(semconv_id) + ) + output.write(content[last_match : match.start(0)]) + self._render_table(semconv, parameters, output) + end_match = self.p_end.search(content, last_match) + if not end_match: + raise ValueError("Missing ending tag") + last_match = end_match.end() + output.write(content[last_match:]) + + def _render_table( + self, semconv: SemanticConvention, parameters, output: io.StringIO + ): + header: str + header = semconv.semconv_id + if parameters: + header += "(" + header += ",".join( + par + "=" + val if val else par for par, val in parameters.items() + ) + header = header + ")" + output.write(MarkdownRenderer.prelude.format(header)) + self.render_ctx.clear_table_generation() + self.render_ctx.current_semconv = semconv + self.render_ctx.is_remove_constraint = "remove_constraints" in parameters + self.render_ctx.group_key = parameters.get("tag") + self.render_ctx.is_full = "full" in parameters + attr_to_print = [] + attr: SemanticAttribute + for attr in sorted( + semconv.attributes, key=lambda a: "" if a.ref is None else a.ref + ): + if self.render_ctx.group_key is not None: + if attr.tag == self.render_ctx.group_key: + attr_to_print.append(attr) + continue + if self.render_ctx.is_full or attr.is_local: + attr_to_print.append(attr) + if attr_to_print: + output.write(MarkdownRenderer.table_headers) + for attr in attr_to_print: + self.to_markdown_attr(attr, output) + self.to_markdown_notes(output) + if not self.render_ctx.is_remove_constraint: + for cnst in semconv.constraints: + self.to_markdown_constraint(cnst, output) + self.to_markdown_enum(output) + + output.write("") diff --git a/semantic-conventions/src/dynatrace/semconv/version.py b/semantic-conventions/src/dynatrace/semconv/version.py new file mode 100644 index 00000000..71a91fef --- /dev/null +++ b/semantic-conventions/src/dynatrace/semconv/version.py @@ -0,0 +1,15 @@ +# Copyright 2020 Dynatrace LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +__version__ = "0.2.0" From d0715d7e8ae1979d18b7f0a5330ace830c3b63f9 Mon Sep 17 00:00:00 2001 From: Giovanni Liva Date: Mon, 10 Aug 2020 08:47:03 +0200 Subject: [PATCH 2/4] Add Github Action --- .github/workflows/protobuf-dockerimage.yml | 4 +-- .github/workflows/semcongen-dockerimage.yml | 33 +++++++++++++++++++++ README.md | 1 + 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/semcongen-dockerimage.yml diff --git a/.github/workflows/protobuf-dockerimage.yml b/.github/workflows/protobuf-dockerimage.yml index 9cef4233..983a1d13 100644 --- a/.github/workflows/protobuf-dockerimage.yml +++ b/.github/workflows/protobuf-dockerimage.yml @@ -1,5 +1,5 @@ name: Protobuf Docker Image -on: +on: push: tags: [ '**' ] branches: [ master ] @@ -9,7 +9,7 @@ on: - .github/workflows/protobuf-dockerimage.yml - protobuf/Dockerfile - protobuf/protoc-wrapper - + jobs: build: runs-on: ubuntu-latest diff --git a/.github/workflows/semcongen-dockerimage.yml b/.github/workflows/semcongen-dockerimage.yml new file mode 100644 index 00000000..1f86e738 --- /dev/null +++ b/.github/workflows/semcongen-dockerimage.yml @@ -0,0 +1,33 @@ +name: Semantic Convention Generator Docker Image +on: + push: + tags: [ '**' ] + branches: [ master ] + pull_request: + branches: [ master ] + paths: + - .github/workflows/semconvgen-dockerimage.yml + - 'semantic-conventions/**' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - name: Build the Docker image + run: docker build semantic-conventions/. -t semconvgen + - name: Push the Docker image + if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/') + run: | + echo "${{ secrets.DOCKER_PASSWORD }}" | docker login -u "${{ secrets.DOCKER_USERNAME }}" --password-stdin + function tag_and_push { + docker tag semconvgen "otel/semconvgen:${1}" && docker push "otel/semconvgen:${1}" + } + if [[ "${GITHUB_REF}" == "refs/heads/master" ]]; then + tag_and_push "latest" + elif [[ "${GITHUB_REF}" =~ refs/tags/v[0-9]+\.[0-9]+\.[0-9]+ ]]; then + TAG="${GITHUB_REF#"refs/tags/v"}" + tag_and_push "${TAG}" + else + tag_and_push "${GITHUB_REF#"refs/tags/"}" + fi \ No newline at end of file diff --git a/README.md b/README.md index 22aff2c7..73826806 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,4 @@ # Build Tools * [Protobuf](./protobuf/README.md) +* [Semantic Convention Generator](./semantic-conventions/README.md) From 690bf1df203c82e5f5828088fffd9f70da807f19 Mon Sep 17 00:00:00 2001 From: Giovanni Liva Date: Tue, 18 Aug 2020 10:34:23 +0200 Subject: [PATCH 3/4] Address feedback --- semantic-conventions/.gitignore | 138 ++++++++++++++++++ semantic-conventions/setup.cfg | 2 +- semantic-conventions/setup.py | 2 +- .../{dynatrace => opentelemetry}/__init__.py | 0 .../semconv/__init__.py | 0 .../semconv/main.py | 23 +-- .../semconv/model/__init__.py | 0 .../semconv/model/constraints.py | 4 +- .../semconv/model/exceptions.py | 2 +- .../semconv/model/semantic_attribute.py | 11 +- .../semconv/model/semantic_convention.py | 12 +- .../semconv/model/utils.py | 4 +- .../semconv/templating/__init__.py | 0 .../semconv/templating/code.py | 32 ++-- .../semconv/templating/markdown.py | 57 +++----- .../semconv/version.py | 2 +- 16 files changed, 211 insertions(+), 78 deletions(-) create mode 100644 semantic-conventions/.gitignore rename semantic-conventions/src/{dynatrace => opentelemetry}/__init__.py (100%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/__init__.py (100%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/main.py (87%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/model/__init__.py (100%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/model/constraints.py (95%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/model/exceptions.py (97%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/model/semantic_attribute.py (98%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/model/semantic_convention.py (97%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/model/utils.py (93%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/templating/__init__.py (100%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/templating/code.py (83%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/templating/markdown.py (92%) rename semantic-conventions/src/{dynatrace => opentelemetry}/semconv/version.py (93%) diff --git a/semantic-conventions/.gitignore b/semantic-conventions/.gitignore new file mode 100644 index 00000000..5391d873 --- /dev/null +++ b/semantic-conventions/.gitignore @@ -0,0 +1,138 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ \ No newline at end of file diff --git a/semantic-conventions/setup.cfg b/semantic-conventions/setup.cfg index c8387d02..969cc85c 100644 --- a/semantic-conventions/setup.cfg +++ b/semantic-conventions/setup.cfg @@ -33,4 +33,4 @@ where = src [options.entry_points] console_scripts = - gen-semconv = dynatrace.semconv.main:main \ No newline at end of file + gen-semconv = opentelemetry.semconv.main:main \ No newline at end of file diff --git a/semantic-conventions/setup.py b/semantic-conventions/setup.py index 4d614926..70a4e6f1 100644 --- a/semantic-conventions/setup.py +++ b/semantic-conventions/setup.py @@ -3,7 +3,7 @@ import setuptools BASE_DIR = os.path.dirname(__file__) -VERSION_FILENAME = os.path.join(BASE_DIR, "src", "dynatrace", "semconv", "version.py") +VERSION_FILENAME = os.path.join(BASE_DIR, "src", "opentelemetry", "semconv", "version.py") PACKAGE_INFO = {} with open(VERSION_FILENAME) as f: exec(f.read(), PACKAGE_INFO) diff --git a/semantic-conventions/src/dynatrace/__init__.py b/semantic-conventions/src/opentelemetry/__init__.py similarity index 100% rename from semantic-conventions/src/dynatrace/__init__.py rename to semantic-conventions/src/opentelemetry/__init__.py diff --git a/semantic-conventions/src/dynatrace/semconv/__init__.py b/semantic-conventions/src/opentelemetry/semconv/__init__.py similarity index 100% rename from semantic-conventions/src/dynatrace/semconv/__init__.py rename to semantic-conventions/src/opentelemetry/semconv/__init__.py diff --git a/semantic-conventions/src/dynatrace/semconv/main.py b/semantic-conventions/src/opentelemetry/semconv/main.py similarity index 87% rename from semantic-conventions/src/dynatrace/semconv/main.py rename to semantic-conventions/src/opentelemetry/semconv/main.py index 2b028627..bf8599b1 100644 --- a/semantic-conventions/src/dynatrace/semconv/main.py +++ b/semantic-conventions/src/opentelemetry/semconv/main.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,18 +17,19 @@ import argparse import glob import sys +from typing import List -from dynatrace.semconv.model.semantic_convention import SemanticConventionSet -from dynatrace.semconv.templating.code import CodeRenderer +from opentelemetry.semconv.model.semantic_convention import SemanticConventionSet +from opentelemetry.semconv.templating.code import CodeRenderer -from dynatrace.semconv.templating.markdown import MarkdownRenderer +from opentelemetry.semconv.templating.markdown import MarkdownRenderer def parse_semconv(args, parser) -> SemanticConventionSet: semconv = SemanticConventionSet(args.debug) find_yaml(args) for file in sorted(args.files): - if not file.endswith(".yaml"): + if not file.endswith(".yaml") and not file.endswith(".yml"): parser.error("{} is not a yaml file.".format(file)) semconv.parse(file) semconv.finish() @@ -37,7 +38,7 @@ def parse_semconv(args, parser) -> SemanticConventionSet: return semconv -def exclude_file_list(folder: str, pattern: str) -> list: +def exclude_file_list(folder: str, pattern: str) -> List[str]: if not pattern: return [] sep = "/" @@ -73,11 +74,11 @@ def find_yaml(args): exclude = set( exclude_file_list(args.yaml_root if args.yaml_root else "", args.exclude) ) - file_names = ( - set(glob.glob("{}/**/*.yaml".format(args.yaml_root), recursive=True)) - - exclude - ) - args.files.extend(sorted(file_names)) + yaml_files = set( + glob.glob("{}/**/*.yaml".format(args.yaml_root), recursive=True) + ).union(set(glob.glob("{}/**/*.yml".format(args.yaml_root), recursive=True))) + file_names = yaml_files - exclude + args.files.extend(file_names) def check_args(arguments, parser): diff --git a/semantic-conventions/src/dynatrace/semconv/model/__init__.py b/semantic-conventions/src/opentelemetry/semconv/model/__init__.py similarity index 100% rename from semantic-conventions/src/dynatrace/semconv/model/__init__.py rename to semantic-conventions/src/opentelemetry/semconv/model/__init__.py diff --git a/semantic-conventions/src/dynatrace/semconv/model/constraints.py b/semantic-conventions/src/opentelemetry/semconv/model/constraints.py similarity index 95% rename from semantic-conventions/src/dynatrace/semconv/model/constraints.py rename to semantic-conventions/src/opentelemetry/semconv/model/constraints.py index 3c7a3455..938bd816 100644 --- a/semantic-conventions/src/dynatrace/semconv/model/constraints.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/constraints.py @@ -1,4 +1,4 @@ -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ from dataclasses import dataclass, field, replace from typing import List, Tuple, Set -from dynatrace.semconv.model.semantic_attribute import SemanticAttribute +from opentelemetry.semconv.model.semantic_attribute import SemanticAttribute # We cannot frozen due to later evaluation of the attributes diff --git a/semantic-conventions/src/dynatrace/semconv/model/exceptions.py b/semantic-conventions/src/opentelemetry/semconv/model/exceptions.py similarity index 97% rename from semantic-conventions/src/dynatrace/semconv/model/exceptions.py rename to semantic-conventions/src/opentelemetry/semconv/model/exceptions.py index 918609ad..f9497023 100644 --- a/semantic-conventions/src/dynatrace/semconv/model/exceptions.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/exceptions.py @@ -1,4 +1,4 @@ -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/semantic-conventions/src/dynatrace/semconv/model/semantic_attribute.py b/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py similarity index 98% rename from semantic-conventions/src/dynatrace/semconv/model/semantic_attribute.py rename to semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py index 34be27c0..8b89b966 100644 --- a/semantic-conventions/src/dynatrace/semconv/model/semantic_attribute.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py @@ -1,4 +1,4 @@ -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ from ruamel.yaml.comments import CommentedMap, CommentedSeq -from dynatrace.semconv.model.exceptions import ValidationError -from dynatrace.semconv.model.utils import ( +from opentelemetry.semconv.model.exceptions import ValidationError +from opentelemetry.semconv.model.utils import ( validate_values, validate_id, check_no_missing_keys, @@ -235,12 +235,9 @@ def equivalent_to(self, other: "SemanticAttribute"): class AttributeType: # https://yaml.org/type/bool.html - bool_type = re.compile( - "y|Y|yes|Yes|YES|n|N|no|No|NO|true|True|TRUE|false|False|FALSE|on|On|ON|off|Off|OFF" - ) - bool_type_true = re.compile("y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON") bool_type_false = re.compile("n|N|no|No|NO|false|False|FALSE|off|Off|OFF") + bool_type = re.compile(bool_type_true.pattern + "|" + bool_type_false.pattern) @staticmethod def get_type(t): diff --git a/semantic-conventions/src/dynatrace/semconv/model/semantic_convention.py b/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py similarity index 97% rename from semantic-conventions/src/dynatrace/semconv/model/semantic_convention.py rename to semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py index d1483545..48c63150 100644 --- a/semantic-conventions/src/dynatrace/semconv/model/semantic_convention.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/semantic_convention.py @@ -1,4 +1,4 @@ -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,10 +21,10 @@ from ruamel.yaml import YAML from ruamel.yaml.comments import CommentedSeq -from dynatrace.semconv.model.constraints import Include, AnyOf -from dynatrace.semconv.model.exceptions import ValidationError -from dynatrace.semconv.model.semantic_attribute import SemanticAttribute, Required -from dynatrace.semconv.model.utils import validate_values, validate_id +from opentelemetry.semconv.model.constraints import Include, AnyOf +from opentelemetry.semconv.model.exceptions import ValidationError +from opentelemetry.semconv.model.semantic_attribute import SemanticAttribute, Required +from opentelemetry.semconv.model.utils import validate_values, validate_id class SpanKind(Enum): @@ -238,7 +238,7 @@ def check_unique_fqns(self): for model in self.models.values(): for attr in model.attributes: if not attr.ref: - if group_by_fqn.get(attr.fqn): + if attr.fqn in group_by_fqn: self.errors = True print( "Attribute {} of Semantic convention '{}' is already defined in {}.".format( diff --git a/semantic-conventions/src/dynatrace/semconv/model/utils.py b/semantic-conventions/src/opentelemetry/semconv/model/utils.py similarity index 93% rename from semantic-conventions/src/dynatrace/semconv/model/utils.py rename to semantic-conventions/src/opentelemetry/semconv/model/utils.py index 4b1d22b1..b0a31781 100644 --- a/semantic-conventions/src/dynatrace/semconv/model/utils.py +++ b/semantic-conventions/src/opentelemetry/semconv/model/utils.py @@ -1,4 +1,4 @@ -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,7 @@ import re -from dynatrace.semconv.model.exceptions import ValidationError +from opentelemetry.semconv.model.exceptions import ValidationError ID_RE = re.compile("([a-z](\\.?[a-z0-9_-]+)+)") diff --git a/semantic-conventions/src/dynatrace/semconv/templating/__init__.py b/semantic-conventions/src/opentelemetry/semconv/templating/__init__.py similarity index 100% rename from semantic-conventions/src/dynatrace/semconv/templating/__init__.py rename to semantic-conventions/src/opentelemetry/semconv/templating/__init__.py diff --git a/semantic-conventions/src/dynatrace/semconv/templating/code.py b/semantic-conventions/src/opentelemetry/semconv/templating/code.py similarity index 83% rename from semantic-conventions/src/dynatrace/semconv/templating/code.py rename to semantic-conventions/src/opentelemetry/semconv/templating/code.py index 4e8cfd5f..aa057759 100644 --- a/semantic-conventions/src/dynatrace/semconv/templating/code.py +++ b/semantic-conventions/src/opentelemetry/semconv/templating/code.py @@ -1,4 +1,4 @@ -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,14 +19,14 @@ from jinja2 import Environment, FileSystemLoader, select_autoescape -from dynatrace.semconv.model.semantic_convention import ( +from opentelemetry.semconv.model.semantic_convention import ( SemanticConventionSet, SemanticConvention, ) -from dynatrace.semconv.model.utils import ID_RE +from opentelemetry.semconv.model.utils import ID_RE -def to_doc_brief(doc_string: str) -> str: +def to_doc_brief(doc_string: typing.Optional[str]) -> str: if doc_string is None: return "" doc_string = doc_string.strip() @@ -57,20 +57,24 @@ class CodeRenderer: parameters: typing.Dict[str, str] @staticmethod - def from_commandline_params(parameters: str = ""): + def from_commandline_params(parameters=None): + if parameters is None: + parameters = [] params = {} if parameters: for elm in parameters: pairs = elm.split(",") for pair in pairs: (k, v) = pair.split("=") - params.update({k: v}) + params[k] = v return CodeRenderer(params) def __init__(self, parameters: typing.Dict[str, str]): self.parameters = parameters - def get_data_single_file(self, semconvset, template_path: str) -> dict: + def get_data_single_file( + self, semconvset: SemanticConventionSet, template_path: str + ) -> dict: """Returns a dictionary that contains all SemanticConventions to fill the template.""" data = { "template": template_path, @@ -80,25 +84,27 @@ def get_data_single_file(self, semconvset, template_path: str) -> dict: data.update(self.parameters) return data - def get_data_multiple_files(self, semconv, template_path: str) -> dict: + def get_data_multiple_files( + self, semconv: SemanticConvention, template_path: str + ) -> dict: """Returns a dictionary with the data from a single SemanticConvention to fill the template.""" data = {"template": template_path, "semconv": semconv} data.update(self.parameters) return data - def setup_environment(self, env: Environment): + @staticmethod + def setup_environment(env: Environment): env.filters["to_doc_brief"] = to_doc_brief env.filters["to_const_name"] = to_const_name env.filters["merge"] = merge env.filters["to_camelcase"] = to_camelcase - def prefix_output_file( - self, file_name: str, pattern: str, semconv: SemanticConvention - ): + @staticmethod + def prefix_output_file(file_name: str, pattern: str, semconv: SemanticConvention): base = os.path.basename(file_name) dir = os.path.dirname(file_name) value = getattr(semconv, pattern) - return dir + "/" + to_camelcase(value, True) + base + return os.path.join(dir, to_camelcase(value, True), base) def render( self, diff --git a/semantic-conventions/src/dynatrace/semconv/templating/markdown.py b/semantic-conventions/src/opentelemetry/semconv/templating/markdown.py similarity index 92% rename from semantic-conventions/src/dynatrace/semconv/templating/markdown.py rename to semantic-conventions/src/opentelemetry/semconv/templating/markdown.py index b0885751..8a2f4da8 100644 --- a/semantic-conventions/src/dynatrace/semconv/templating/markdown.py +++ b/semantic-conventions/src/opentelemetry/semconv/templating/markdown.py @@ -1,4 +1,4 @@ -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,18 +20,18 @@ import typing from pathlib import PurePath -from dynatrace.semconv.model.constraints import AnyOf, Include -from dynatrace.semconv.model.semantic_attribute import ( +from opentelemetry.semconv.model.constraints import AnyOf, Include +from opentelemetry.semconv.model.semantic_attribute import ( SemanticAttribute, EnumAttributeType, Required, EnumMember, ) -from dynatrace.semconv.model.semantic_convention import ( +from opentelemetry.semconv.model.semantic_convention import ( SemanticConventionSet, SemanticConvention, ) -from dynatrace.semconv.model.utils import ID_RE +from opentelemetry.semconv.model.utils import ID_RE class RenderContext: @@ -41,7 +41,6 @@ class RenderContext: break_counter: int enums: list notes: list - note_index: int current_md: str current_semconv: SemanticConvention @@ -52,17 +51,14 @@ def __init__(self, break_count): self.break_count = break_count self.enums = [] self.notes = [] - self.note_index = 1 self.current_md = "" self.current_semconv = None def clear_table_generation(self): self.notes = [] self.enums = [] - self.note_index = 1 def add_note(self, msg: str): - self.note_index += 1 self.notes.append(msg) def add_enum(self, attr: SemanticAttribute): @@ -101,13 +97,12 @@ def __init__( self.filename_for_attr_fqn = self._create_attribute_location_dict() self.check_only = check_only - """ - This method renders attributes as markdown table entry - """ - def to_markdown_attr( self, attribute: SemanticAttribute, output: io.StringIO, ): + """ + This method renders attributes as markdown table entry + """ name = self.render_attribute_id(attribute.fqn) attr_type = ( "enum" @@ -122,8 +117,8 @@ def to_markdown_attr( description = "**Deprecated: {}**
".format(attribute.deprecated) description += attribute.brief if attribute.note: - description += " [{}]".format(self.render_ctx.note_index) self.render_ctx.add_note(attribute.note) + description += " [{}]".format(len(self.render_ctx.notes)) examples = "" if isinstance(attribute.attr_type, EnumAttributeType): if attribute.is_local and not attribute.ref: @@ -149,8 +144,8 @@ def to_markdown_attr( required = "Conditional
{}".format(attribute.required_msg) else: # We put the condition in the notes after the table - required = "Conditional [{}]".format(self.render_ctx.note_index) self.render_ctx.add_note(attribute.required_msg) + required = "Conditional [{}]".format(len(self.render_ctx.notes)) else: # check if they are required by some constraint if ( @@ -166,11 +161,10 @@ def to_markdown_attr( ) ) - """ - This method renders anyof constraints into markdown lists - """ - def to_markdown_anyof(self, anyof: AnyOf, output: io.StringIO): + """ + This method renders anyof constraints into markdown lists + """ if anyof.inherited and not self.render_ctx.is_full: return "" output.write("\nAt least one of the following is required:\n\n") @@ -225,12 +219,11 @@ def to_markdown_enum(self, output: io.StringIO): if notes: output.write("\n") - """ - Method to render in markdown an attribute id. If the id points to an attribute in another rendered table, a markdown - link is introduced. - """ - def render_attribute_id(self, attribute_id): + """ + Method to render in markdown an attribute id. If the id points to an attribute in another rendered table, a + markdown link is introduced. + """ md_file = self.filename_for_attr_fqn.get(attribute_id) if md_file: path = PurePath(self.render_ctx.current_md) @@ -240,13 +233,12 @@ def render_attribute_id(self, attribute_id): return "[{}]({})".format(attribute_id, diff) return "`{}`".format(attribute_id) - """ - Entry method to translate attributes and constraints of a semantic convention into Markdown - """ - def to_markdown_constraint( self, obj: typing.Union[AnyOf, Include], output: io.StringIO, ): + """ + Entry method to translate attributes and constraints of a semantic convention into Markdown + """ if isinstance(obj, AnyOf): self.to_markdown_anyof(obj, output) return @@ -275,12 +267,11 @@ def render_md(self): if self.check_only: print("{} files left unchanged.".format(len(self.file_names))) - """ - This method creates a dictionary that associates each attribute with the latest table in which it is rendered. - This is required by the ref attributes to point to the correct file - """ - def _create_attribute_location_dict(self): + """ + This method creates a dictionary that associates each attribute with the latest table in which it is rendered. + This is required by the ref attributes to point to the correct file + """ m = {} for md in self.file_names: with open(md, "r", encoding="utf-8") as markdown: diff --git a/semantic-conventions/src/dynatrace/semconv/version.py b/semantic-conventions/src/opentelemetry/semconv/version.py similarity index 93% rename from semantic-conventions/src/dynatrace/semconv/version.py rename to semantic-conventions/src/opentelemetry/semconv/version.py index 71a91fef..e26528ae 100644 --- a/semantic-conventions/src/dynatrace/semconv/version.py +++ b/semantic-conventions/src/opentelemetry/semconv/version.py @@ -1,4 +1,4 @@ -# Copyright 2020 Dynatrace LLC +# Copyright The OpenTelemetry Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. From 3b380022cb8449447295df24012cab9cb2ddabae Mon Sep 17 00:00:00 2001 From: Giovanni Liva Date: Tue, 18 Aug 2020 13:15:27 +0200 Subject: [PATCH 4/4] Make output more close to what we currently use --- .../src/opentelemetry/semconv/templating/markdown.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/semantic-conventions/src/opentelemetry/semconv/templating/markdown.py b/semantic-conventions/src/opentelemetry/semconv/templating/markdown.py index 8a2f4da8..8cca971f 100644 --- a/semantic-conventions/src/opentelemetry/semconv/templating/markdown.py +++ b/semantic-conventions/src/opentelemetry/semconv/templating/markdown.py @@ -167,7 +167,8 @@ def to_markdown_anyof(self, anyof: AnyOf, output: io.StringIO): """ if anyof.inherited and not self.render_ctx.is_full: return "" - output.write("\nAt least one of the following is required:\n\n") + output.write("\n**Additional attribute requirements:** At least one of the following sets of attributes is " + "required:\n\n") for choice in anyof.choice_list_ids: output.write("* ") list_of_choice = ", ".join(self.render_attribute_id(c) for c in choice) @@ -230,7 +231,7 @@ def render_attribute_id(self, attribute_id): if path.as_posix() != PurePath(md_file).as_posix(): diff = PurePath(os.path.relpath(md_file, start=path.parent)).as_posix() if diff != ".": - return "[{}]({})".format(attribute_id, diff) + return "[`{}`]({})".format(attribute_id, diff) return "`{}`".format(attribute_id) def to_markdown_constraint(