Add Configuration Object to Semantic Manifest

.changes/unreleased/Features-20230623-114625.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Add Configuration Object to Semantic Manifest
+time: 2023-06-23T11:46:25.858497-07:00
+custom:
+  Author: plypaul
+  Issue: "94"

dbt_semantic_interfaces/implementations/project_configuration.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+from typing import List, Optional
+
+from importlib_metadata import version
+from pydantic import validator
+from typing_extensions import override
+
+from dbt_semantic_interfaces.implementations.base import (
+    HashableBaseModel,
+    ModelWithMetadataParsing,
+)
+from dbt_semantic_interfaces.implementations.metadata import PydanticMetadata
+from dbt_semantic_interfaces.implementations.semantic_version import (
+    UNKNOWN_VERSION_SENTINEL,
+    PydanticSemanticVersion,
+)
+from dbt_semantic_interfaces.implementations.time_spine_table_configuration import (
+    PydanticTimeSpineTableConfiguration,
+)
+from dbt_semantic_interfaces.protocols import ProtocolHint
+from dbt_semantic_interfaces.protocols.project_configuration import ProjectConfiguration
+
+
+class PydanticProjectConfiguration(HashableBaseModel, ModelWithMetadataParsing, ProtocolHint[ProjectConfiguration]):
+    """Pydantic implementation of ProjectConfiguration."""
+
+    @override
+    def _implements_protocol(self) -> ProjectConfiguration:
+        return self
+
+    time_spine_table_configurations: List[PydanticTimeSpineTableConfiguration]
+    metadata: Optional[PydanticMetadata] = None
+    dsi_package_version: PydanticSemanticVersion = UNKNOWN_VERSION_SENTINEL
+
+    @validator("dsi_package_version", always=True)
+    @classmethod
+    def __create_default_dsi_package_version(cls, value: Optional[PydanticSemanticVersion]) -> PydanticSemanticVersion:
+        """Returns the version of the dbt_semantic_interfaces package that generated this manifest."""
+        if value is not None and value != UNKNOWN_VERSION_SENTINEL:
+            return value
+        return PydanticSemanticVersion.create_from_string(version("dbt_semantic_interfaces"))

dbt_semantic_interfaces/implementations/semantic_manifest.py

@@ -1,11 +1,12 @@
 from typing import List
 
-from importlib_metadata import version
-from pydantic import validator
 from typing_extensions import override
 
 from dbt_semantic_interfaces.implementations.base import HashableBaseModel
 from dbt_semantic_interfaces.implementations.metric import PydanticMetric
+from dbt_semantic_interfaces.implementations.project_configuration import (
+    PydanticProjectConfiguration,
+)
 from dbt_semantic_interfaces.implementations.semantic_model import PydanticSemanticModel
 from dbt_semantic_interfaces.protocols import ProtocolHint, SemanticManifest
 
@@ -14,17 +15,9 @@ class PydanticSemanticManifest(HashableBaseModel, ProtocolHint[SemanticManifest]
     """Model holds all the information the SemanticLayer needs to render a query."""
 
     @override
-    def _implements_protocol(self) -> SemanticManifest:  # noqa: D
+    def _implements_protocol(self) -> SemanticManifest:
         return self
 
     semantic_models: List[PydanticSemanticModel]
     metrics: List[PydanticMetric]
-    interfaces_version: str = ""
-
-    @validator("interfaces_version", always=True)
-    @classmethod
-    def __create_default_interfaces_version(cls, value: str) -> str:  # type: ignore[misc]
-        """Returns the version of the dbt_semantic_interfaces package that generated this manifest."""
-        if value:
-            return value
-        return version("dbt_semantic_interfaces")
+    project_configuration: PydanticProjectConfiguration

dbt_semantic_interfaces/implementations/semantic_version.py

@@ -0,0 +1,49 @@
+from __future__ import annotations
+
+from typing import Optional
+
+from typing_extensions import override
+
+from dbt_semantic_interfaces.implementations.base import (
+    HashableBaseModel,
+    PydanticCustomInputParser,
+    PydanticParseableValueType,
+)
+from dbt_semantic_interfaces.protocols.semantic_version import SemanticVersion
+
+
+class PydanticSemanticVersion(PydanticCustomInputParser, HashableBaseModel):
+    """Pydantic implementation of SemanticVersion."""
+
+    @override
+    def _implements_protocol(self) -> SemanticVersion:
+        return self
+
+    major_version: str
+    minor_version: str
+    patch_version: Optional[str]
+
+    @classmethod
+    @override
+    def _from_yaml_value(cls, input: PydanticParseableValueType) -> PydanticSemanticVersion:
+        if isinstance(input, str):
+            return PydanticSemanticVersion.create_from_string(input)
+        else:
+            raise ValueError(
+                f"{cls.__name__} inputs from YAML files are expected to be of either type string or "
+                f"object (key/value pairs), but got type {type(input)} with value: {input}"
+            )
+
+    @staticmethod
+    def create_from_string(version_str: str) -> PydanticSemanticVersion:  # noqa: D
+        version_str_split = version_str.split(".")
+        if len(version_str_split) < 2:
+            raise ValueError(f"Expected version string to be of the form x.y or x.y.z, but got {version_str}")
+        return PydanticSemanticVersion(
+            major_version=version_str_split[0],
+            minor_version=version_str_split[1],
+            patch_version=".".join(version_str_split[2:]) if len(version_str_split) >= 3 else None,
+        )
+
+
+UNKNOWN_VERSION_SENTINEL = PydanticSemanticVersion(major_version="0", minor_version="0", patch_version="0")

dbt_semantic_interfaces/implementations/time_spine_table_configuration.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing_extensions import override
+
+from dbt_semantic_interfaces.implementations.base import (
+    HashableBaseModel,
+    ModelWithMetadataParsing,
+)
+from dbt_semantic_interfaces.protocols import ProtocolHint
+from dbt_semantic_interfaces.protocols.time_spine_configuration import (
+    TimeSpineTableConfiguration,
+)
+from dbt_semantic_interfaces.type_enums import TimeGranularity
+
+
+class PydanticTimeSpineTableConfiguration(
+    HashableBaseModel, ModelWithMetadataParsing, ProtocolHint[TimeSpineTableConfiguration]
+):
+    """Pydantic implementation of SemanticVersion."""
+
+    @override
+    def _implements_protocol(self) -> TimeSpineTableConfiguration:
+        return self
+
+    location: str
+    column_name: str
+    grain: TimeGranularity

dbt_semantic_interfaces/parsing/dir_to_model.py

@@ -9,13 +9,17 @@
 
 from dbt_semantic_interfaces.errors import ParsingException
 from dbt_semantic_interfaces.implementations.metric import PydanticMetric
+from dbt_semantic_interfaces.implementations.project_configuration import (
+    PydanticProjectConfiguration,
+)
 from dbt_semantic_interfaces.implementations.semantic_manifest import (
     PydanticSemanticManifest,
 )
 from dbt_semantic_interfaces.implementations.semantic_model import PydanticSemanticModel
 from dbt_semantic_interfaces.parsing.objects import Version, YamlConfigFile
 from dbt_semantic_interfaces.parsing.schemas import (
     metric_validator,
+    project_configuration_validator,
     semantic_model_validator,
 )
 from dbt_semantic_interfaces.parsing.yaml_loader import (
@@ -39,7 +43,9 @@
 VERSION_KEY = "mf_config_schema"
 METRIC_TYPE = "metric"
 SEMANTIC_MODEL_TYPE = "semantic_model"
-DOCUMENT_TYPES = [METRIC_TYPE, SEMANTIC_MODEL_TYPE]
+PROJECT_CONFIGURATION_TYPE = "project_configuration"
+
+DOCUMENT_TYPES = [METRIC_TYPE, SEMANTIC_MODEL_TYPE, PROJECT_CONFIGURATION_TYPE]
 
 
 @dataclass(frozen=True)
@@ -58,7 +64,7 @@ class FileParsingResult:
         issues: Issues found when trying to parse the file
     """
 
-    elements: List[Union[PydanticSemanticModel, PydanticMetric]]
+    elements: List[Union[PydanticSemanticModel, PydanticMetric, PydanticProjectConfiguration]]
     issues: List[ValidationIssue]
 
 
@@ -183,6 +189,7 @@ def parse_yaml_files_to_semantic_manifest(
     files: List[YamlConfigFile],
     semantic_model_class: Type[PydanticSemanticModel] = PydanticSemanticModel,
     metric_class: Type[PydanticMetric] = PydanticMetric,
+    project_configuration_class: Type[PydanticProjectConfiguration] = PydanticProjectConfiguration,
 ) -> SemanticManifestBuildResult:
     """Builds SemanticManifest from list of config files (as strings).
 
@@ -193,7 +200,8 @@ def parse_yaml_files_to_semantic_manifest(
     """
     semantic_models = []
     metrics = []
-    valid_object_classes = [semantic_model_class.__name__, metric_class.__name__]
+    project_configurations = []
+    valid_object_classes = [semantic_model_class.__name__, metric_class.__name__, project_configuration_class.__name__]
     issues: List[ValidationIssue] = []
 
     for config_file in files:
@@ -208,6 +216,8 @@ def parse_yaml_files_to_semantic_manifest(
                 semantic_models.append(obj)
             elif isinstance(obj, metric_class):
                 metrics.append(obj)
+            elif isinstance(obj, project_configuration_class):
+                project_configurations.append(obj)
             else:
                 file_issues.append(
                     ValidationError(
@@ -218,10 +228,14 @@ def parse_yaml_files_to_semantic_manifest(
 
         issues += file_issues
 
+    if len(project_configurations) != 1:
+        raise ParsingException(f"Did not find exactly one project configuration. Got: {project_configurations}")
+
     return SemanticManifestBuildResult(
         semantic_manifest=PydanticSemanticManifest(
             semantic_models=semantic_models,
             metrics=metrics,
+            project_configuration=project_configurations[0],
         ),
         issues=SemanticManifestValidationResults.from_issues_sequence(issues),
     )
@@ -231,9 +245,10 @@ def parse_config_yaml(
     config_yaml: YamlConfigFile,
     semantic_model_class: Type[PydanticSemanticModel] = PydanticSemanticModel,
     metric_class: Type[PydanticMetric] = PydanticMetric,
+    project_configuration_class: Type[PydanticProjectConfiguration] = PydanticProjectConfiguration,
 ) -> FileParsingResult:
     """Parses transform config file passed as string - Returns list of model objects."""
-    results: List[Union[PydanticSemanticModel, PydanticMetric]] = []
+    results: List[Union[PydanticSemanticModel, PydanticMetric, PydanticProjectConfiguration]] = []
     ctx: Optional[ParsingContext] = None
     issues: List[ValidationIssue] = []
     try:
@@ -299,6 +314,9 @@ def parse_config_yaml(
                 elif document_type == SEMANTIC_MODEL_TYPE:
                     semantic_model_validator.validate(config_document[document_type])
                     results.append(semantic_model_class.parse_obj(object_cfg))
+                elif document_type == PROJECT_CONFIGURATION_TYPE:
+                    project_configuration_validator.validate(config_document[document_type])
+                    results.append(project_configuration_class.parse_obj(object_cfg))
                 else:
                     issues.append(
                         ValidationError(

dbt_semantic_interfaces/parsing/explicit_schema.py

@@ -9,6 +9,7 @@
     "metric": "metric",
     "semantic_model": "semantic_model",
     "derived_group_by_element_schema": "derived_identifier",
+    "project_configuration": "project_configuration",
 }
 
 BASE_SCHEMA = {

dbt_semantic_interfaces/parsing/schemas.py

@@ -242,6 +242,33 @@
 }
 
 
+time_spine_table_configuration_schema = {
+    "$id": "time_spine_table_configuration",
+    "type": "object",
+    "properties": {
+        "location": {"type": "string"},
+        "column_name": {"type": "string"},
+        "grain": {"enum": time_granularity_values},
+    },
+    "additionalProperties": False,
+    "required": ["location", "column_name", "grain"],
+}
+
+
+project_configuration_schema = {
+    "$id": "project_configuration",
+    "type": "object",
+    "properties": {
+        "time_spine_table_configurations": {
+            "type": "array",
+            "items": {"$ref": "time_spine_table_configuration"},
+        },
+    },
+    "additionalProperties": False,
+    "required": ["time_spine_table_configurations"],
+}
+
+
 semantic_model_schema = {
     "$id": "semantic_model",
     "type": "object",
@@ -284,6 +311,7 @@
     metric_schema["$id"]: metric_schema,
     semantic_model_schema["$id"]: semantic_model_schema,
     derived_group_by_element_schema["$id"]: derived_group_by_element_schema,
+    project_configuration_schema["$id"]: project_configuration_schema,
     # Sub-object schemas
     metric_input_measure_schema["$id"]: metric_input_measure_schema,
     metric_type_params_schema["$id"]: metric_type_params_schema,
@@ -297,10 +325,12 @@
     metric_input_schema["$id"]: metric_input_schema,
     node_relation_schema["$id"]: node_relation_schema,
     semantic_model_defaults_schema["$id"]: semantic_model_defaults_schema,
+    time_spine_table_configuration_schema["$id"]: time_spine_table_configuration_schema,
 }
 
 
 resolver = RefResolver.from_schema(schema=metric_schema, store=schema_store)
 semantic_model_validator = SchemaValidator(semantic_model_schema, resolver=resolver)
 derived_group_by_element_validator = SchemaValidator(derived_group_by_element_schema, resolver=resolver)
 metric_validator = SchemaValidator(metric_schema, resolver=resolver)
+project_configuration_validator = SchemaValidator(project_configuration_schema, resolver=resolver)

dbt_semantic_interfaces/parsing/schemas/default_explicit_schema.json

@@ -404,6 +404,22 @@
             ],
             "type": "object"
         },
+        "project_configuration": {
+            "$id": "project_configuration",
+            "additionalProperties": false,
+            "properties": {
+                "time_spine_table_configurations": {
+                    "items": {
+                        "$ref": "#/definitions/time_spine_table_configuration"
+                    },
+                    "type": "array"
+                }
+            },
+            "required": [
+                "time_spine_table_configurations"
+            ],
+            "type": "object"
+        },
         "semantic_model": {
             "$id": "semantic_model",
             "additionalProperties": false,
@@ -456,6 +472,38 @@
             "required": [],
             "type": "object"
         },
+        "time_spine_table_configuration": {
+            "$id": "time_spine_table_configuration",
+            "additionalProperties": false,
+            "properties": {
+                "column_name": {
+                    "type": "string"
+                },
+                "grain": {
+                    "enum": [
+                        "DAY",
+                        "WEEK",
+                        "MONTH",
+                        "QUARTER",
+                        "YEAR",
+                        "day",
+                        "week",
+                        "month",
+                        "quarter",
+                        "year"
+                    ]
+                },
+                "location": {
+                    "type": "string"
+                }
+            },
+            "required": [
+                "location",
+                "column_name",
+                "grain"
+            ],
+            "type": "object"
+        },
         "validity_params_schema": {
             "$id": "validity_params_schema",
             "additionalProperties": false,
@@ -477,6 +525,9 @@
         "metric": {
             "$ref": "#/definitions/metric"
         },
+        "project_configuration": {
+            "$ref": "#/definitions/project_configuration"
+        },
         "semantic_model": {
             "$ref": "#/definitions/semantic_model"
         }