Run & fix linters flake8, pylint, mypy (#62)

.github/workflows/semconvgen.yml

@@ -12,19 +12,30 @@ on:
 jobs:
   tests:
     runs-on: ubuntu-latest
+    defaults:
+      run: 
+        working-directory: semantic-conventions/
     steps:
     - uses: actions/checkout@v1
     - uses: actions/setup-python@v2
-    - name: install dependencies
+    - name: install dev-dependencies
       run: |
-        pip install -U pip setuptools wheel
-        pip install -r semantic-conventions/dev-requirements.txt
-    - name: Check formatting
-      run: black --check --diff semantic-conventions/
+        pip install -U pip
+        pip install -U setuptools wheel
+        pip install -r dev-requirements.txt
+    - name: Check formatting (black)
+      run: black --check --diff .
+    - name: Fast basic linting (flake8)
+      run: flake8 .
     - name: install
-      run: pip install -U -e ./semantic-conventions/
+      run: pip install -U -e .
     - name: run tests
       run: pytest -v
+    - name: Slow linting (pylint)
+      run: pylint *.py src/
+    - name: Type checking (mypy)
+      run: mypy src/
+
 
   build-and-publish-docker:
     runs-on: ubuntu-latest

semantic-conventions/.flake8

@@ -0,0 +1,15 @@
+# Adapted from https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html#flake8
+[flake8]
+extend-ignore = E203
+max-line-length = 120
+exclude =
+  .bzr
+  .git
+  .hg
+  .svn
+  .tox
+  CVS
+  .venv*/
+  venv*/
+  target
+  __pycache__

semantic-conventions/.pylintrc

@@ -0,0 +1,7 @@
+[MESSAGES CONTROL]
+# TODO: We should actually fix the warnings instead of disabling them
+#   except for the stuff up to fixme, which is fine to ignore / defer to black. 
+disable = bad-whitespace, wrong-import-order, wrong-hanging-indentation, line-too-long, missing-docstring, unused-import, fixme, invalid-name, too-many-instance-attributes, too-many-locals, too-many-statements, too-many-branches, too-many-arguments, too-many-public-methods, too-few-public-methods, protected-access
+
+[FORMAT]
+good-names=e,ex

semantic-conventions/dev-requirements.txt

@@ -1,3 +1,5 @@
-black==21.6b0
-mypy==0.770
-pytest==6.1.1
+black==21.8b0
+mypy==0.910
+pytest==6.2.5
+flake8==3.9.2
+pylint==2.10.2

semantic-conventions/mypy.ini

@@ -0,0 +1,7 @@
+[mypy]
+
+[mypy-jinja2.*]
+ignore_missing_imports = True
+
+[mypy-mistune.*]
+ignore_missing_imports = True

semantic-conventions/setup.py

@@ -8,7 +8,7 @@
 )
 PACKAGE_INFO = {}
 with open(VERSION_FILENAME, encoding="utf-8") as f:
-    exec(f.read(), PACKAGE_INFO)
+    exec(f.read(), PACKAGE_INFO)  # pylint:disable=exec-used
 
 
 VERSION_SUFFIX = os.environ.get("SEMCONGEN_VERSION_SUFFIX")

semantic-conventions/src/opentelemetry/semconv/main.py

@@ -20,12 +20,8 @@
 from typing import List
 
 from opentelemetry.semconv.model.semantic_convention import (
+    CONVENTION_CLS_BY_GROUP_TYPE,
     SemanticConventionSet,
-    SpanSemanticConvention,
-    ResourceSemanticConvention,
-    EventSemanticConvention,
-    MetricSemanticConvention,
-    UnitSemanticConvention,
 )
 from opentelemetry.semconv.templating.code import CodeRenderer
 
@@ -206,13 +202,7 @@ def setup_parser():
     )
     parser.add_argument(
         "--only",
-        choices=[
-            SpanSemanticConvention.GROUP_TYPE_NAME,
-            ResourceSemanticConvention.GROUP_TYPE_NAME,
-            EventSemanticConvention.GROUP_TYPE_NAME,
-            MetricSemanticConvention.GROUP_TYPE_NAME,
-            UnitSemanticConvention.GROUP_TYPE_NAME,
-        ],
+        choices=list(CONVENTION_CLS_BY_GROUP_TYPE.keys()),
         help="Process only semantic conventions of the specified type.",
     )
     parser.add_argument(

semantic-conventions/src/opentelemetry/semconv/model/constraints.py

@@ -12,8 +12,8 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-from dataclasses import dataclass, field, replace
-from typing import List, Tuple, Set
+from dataclasses import dataclass, replace
+from typing import List, Tuple
 
 from opentelemetry.semconv.model.exceptions import ValidationError
 from opentelemetry.semconv.model.semantic_attribute import SemanticAttribute
@@ -39,9 +39,9 @@ class AnyOf:
         _yaml_src_position          Contains the position in the YAML file of the AnyOf attribute
     """
 
-    choice_list_ids: Tuple[Tuple[str, ...]]
+    choice_list_ids: Tuple[Tuple[str, ...], ...]
     inherited: bool = False
-    choice_list_attributes: Tuple[Tuple[SemanticAttribute, ...]] = ()
+    choice_list_attributes: Tuple[Tuple[SemanticAttribute, ...], ...] = ()
     _yaml_src_position: int = 0
 
     def __eq__(self, other):

semantic-conventions/src/opentelemetry/semconv/model/exceptions.py

@@ -28,7 +28,7 @@ def from_yaml_pos(cls, pos, msg):
         return cls(pos[0] + 1, pos[1] + 1, msg)
 
     def __init__(self, line, column, message):
-        super(ValidationError, self).__init__(line, column, message)
+        super().__init__(line, column, message)
         self.message = message
         self.line = line
         self.column = column

semantic-conventions/src/opentelemetry/semconv/model/semantic_attribute.py

@@ -12,12 +12,11 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 
-import numbers
 import re
 from collections.abc import Iterable
 from dataclasses import dataclass, replace
 from enum import Enum
-from typing import List, Union
+from typing import List, Optional, Union, Dict
 
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
@@ -41,20 +40,6 @@ class StabilityLevel(Enum):
     DEPRECATED = 3
 
 
-class HasAttributes:
-    def _set_attributes(self, prefix, stability, node):
-        self.attrs_by_name = SemanticAttribute.parse(
-            prefix, stability, node.get("attributes")
-        )
-
-    @property
-    def attributes(self):
-        if not hasattr(self, "attrs_by_name"):
-            return []
-
-        return list(self.attrs_by_name.values())
-
-
 def unique_attributes(attributes):
     output = []
     for x in attributes:
@@ -97,11 +82,13 @@ def is_enum(self):
         return isinstance(self.attr_type, EnumAttributeType)
 
     @staticmethod
-    def parse(prefix, semconv_stability, yaml_attributes):
+    def parse(
+        prefix, semconv_stability, yaml_attributes
+    ) -> "Dict[str, SemanticAttribute]":
         """This method parses the yaml representation for semantic attributes
         creating the respective SemanticAttribute objects.
         """
-        attributes = {}
+        attributes = {}  # type: Dict[str, SemanticAttribute]
         allowed_keys = (
             "id",
             "type",
@@ -149,6 +136,7 @@ def parse(prefix, semconv_stability, yaml_attributes):
             }
             required_msg = ""
             required_val = attribute.get("required", "")
+            required: Optional[Required]
             if isinstance(required_val, CommentedMap):
                 required = Required.CONDITIONAL
                 required_msg = required_val.get("conditional", None)
@@ -217,7 +205,7 @@ def parse(prefix, semconv_stability, yaml_attributes):
                     "Attribute id "
                     + fqn
                     + " is already present at line "
-                    + str(attributes.get(fqn).position[0] + 1)
+                    + str(attributes[fqn].position[0] + 1)
                 )
                 raise ValidationError.from_yaml_pos(position, msg)
             attributes[fqn] = attr
@@ -397,7 +385,7 @@ def to_bool(key, parent_object):
         if isinstance(yaml_value, str):
             if AttributeType.bool_type_true.fullmatch(yaml_value):
                 return True
-            elif AttributeType.bool_type_false.fullmatch(yaml_value):
+            if AttributeType.bool_type_false.fullmatch(yaml_value):
                 return False
         position = parent_object.lc.data[key]
         msg = "Value '{}' for {} field is not allowed".format(yaml_value, key)
@@ -431,47 +419,46 @@ def parse(attribute_type):
         if isinstance(attribute_type, str):
             if AttributeType.is_simple_type(attribute_type):
                 return attribute_type
-            else:  # Wrong type used - rise the exception and fill the missing data in the parent
+            # Wrong type used - raise the exception and fill the missing data in the parent
+            raise ValidationError(
+                0, 0, "Invalid type: {} is not allowed".format(attribute_type)
+            )
+        allowed_keys = ["allow_custom_values", "members"]
+        mandatory_keys = ["members"]
+        validate_values(attribute_type, allowed_keys, mandatory_keys)
+        custom_values = (
+            bool(attribute_type.get("allow_custom_values"))
+            if "allow_custom_values" in attribute_type
+            else False
+        )
+        members = []
+        if attribute_type["members"] is None or len(attribute_type["members"]) < 1:
+            # Missing members - rise the exception and fill the missing data in the parent
+            raise ValidationError(0, 0, "Enumeration without values!")
+
+        allowed_keys = ["id", "value", "brief", "note"]
+        mandatory_keys = ["id", "value"]
+        for member in attribute_type["members"]:
+            validate_values(member, allowed_keys, mandatory_keys)
+            if not EnumAttributeType.is_valid_enum_value(member["value"]):
                 raise ValidationError(
-                    0, 0, "Invalid type: {} is not allowed".format(attribute_type)
+                    0, 0, "Invalid value used in enum: <{}>".format(member["value"])
                 )
-        else:
-            allowed_keys = ["allow_custom_values", "members"]
-            mandatory_keys = ["members"]
-            validate_values(attribute_type, allowed_keys, mandatory_keys)
-            custom_values = (
-                bool(attribute_type.get("allow_custom_values"))
-                if "allow_custom_values" in attribute_type
-                else False
-            )
-            members = []
-            if attribute_type["members"] is None or len(attribute_type["members"]) < 1:
-                # Missing members - rise the exception and fill the missing data in the parent
-                raise ValidationError(0, 0, "Enumeration without values!")
-
-            allowed_keys = ["id", "value", "brief", "note"]
-            mandatory_keys = ["id", "value"]
-            for member in attribute_type["members"]:
-                validate_values(member, allowed_keys, mandatory_keys)
-                if not EnumAttributeType.is_valid_enum_value(member["value"]):
-                    raise ValidationError(
-                        0, 0, "Invalid value used in enum: <{}>".format(member["value"])
-                    )
-                members.append(
-                    EnumMember(
-                        member_id=member["id"],
-                        value=member["value"],
-                        brief=member.get("brief").strip()
-                        if "brief" in member
-                        else member["id"],
-                        note=member.get("note").strip() if "note" in member else "",
-                    )
+            members.append(
+                EnumMember(
+                    member_id=member["id"],
+                    value=member["value"],
+                    brief=member.get("brief").strip()
+                    if "brief" in member
+                    else member["id"],
+                    note=member.get("note").strip() if "note" in member else "",
                 )
-            enum_type = AttributeType.get_type(members[0].value)
-            for m in members:
-                if enum_type != AttributeType.get_type(m.value):
-                    raise ValidationError(0, 0, "Enumeration type inconsistent!")
-            return EnumAttributeType(custom_values, members, enum_type)
+            )
+        enum_type = AttributeType.get_type(members[0].value)
+        for m in members:
+            if enum_type != AttributeType.get_type(m.value):
+                raise ValidationError(0, 0, "Enumeration type inconsistent!")
+        return EnumAttributeType(custom_values, members, enum_type)
 
 
 @dataclass