From c95a3dea432e43f09ab781f47a2d4771161cbf1b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 16 Dec 2021 13:30:52 -0500 Subject: [PATCH] [#4504] Use mashumaro for logging serialization --- CHANGELOG.md | 1 + core/dbt/adapters/cache.py | 5 +- core/dbt/adapters/factory.py | 2 +- core/dbt/context/base.py | 4 +- core/dbt/contracts/graph/parsed.py | 27 ++ core/dbt/events/README.md | 22 +- core/dbt/events/base_types.py | 67 +-- core/dbt/events/functions.py | 19 +- core/dbt/events/serialization.py | 40 ++ core/dbt/events/stubs.py | 72 --- core/dbt/events/test_types.py | 12 +- core/dbt/events/types.py | 756 +++++++++++++---------------- core/dbt/graph/selector.py | 3 +- core/dbt/main.py | 10 +- core/dbt/task/base.py | 6 +- core/dbt/task/freshness.py | 10 +- core/dbt/task/run.py | 14 +- core/dbt/task/runnable.py | 39 +- core/dbt/task/seed.py | 10 +- core/dbt/task/snapshot.py | 4 +- core/dbt/task/test.py | 10 +- core/dbt/utils.py | 33 ++ test/unit/test_events.py | 125 ++--- 23 files changed, 578 insertions(+), 713 deletions(-) create mode 100644 core/dbt/events/serialization.py delete mode 100644 core/dbt/events/stubs.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f3424f8beba..9f5b0feaa4f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Reimplement log message to use adapter name instead of the object method. ([#4501](https://github.com/dbt-labs/dbt-core/pull/4501)) - Issue better error message for incompatible schemas ([#4470](https://github.com/dbt-labs/dbt-core/pull/4442), [#4497](https://github.com/dbt-labs/dbt-core/pull/4497)) - Remove secrets from error related to packages. ([#4507](https://github.com/dbt-labs/dbt-core/pull/4507)) +- Some structured logging serialization cleanup ([#4504](https://github.com/dbt-labs/dbt-core/issues/4504), [#4505](https://github.com/dbt-labs/dbt-core/pull/4505)) ### Docs - Fix missing data on exposures in docs ([#4467](https://github.com/dbt-labs/dbt-core/issues/4467)) diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 6c14c0fdb42..a5112c69ef1 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -291,11 +291,12 @@ def add_link(self, referenced, dependent): :raises InternalError: If either entry does not exist. """ ref_key = _make_key(referenced) + dep_key = _make_key(dependent) if (ref_key.database, ref_key.schema) not in self: # if we have not cached the referenced schema at all, we must be # referring to a table outside our control. There's no need to make # a link - we will never drop the referenced relation during a run. - fire_event(UncachedRelation(dep_key=dependent, ref_key=ref_key)) + fire_event(UncachedRelation(dep_key=dep_key, ref_key=ref_key)) return if ref_key not in self.relations: # Insert a dummy "external" relation. @@ -303,8 +304,6 @@ def add_link(self, referenced, dependent): type=referenced.External ) self.add(referenced) - - dep_key = _make_key(dependent) if dep_key not in self.relations: # Insert a dummy "external" relation. dependent = dependent.replace( diff --git a/core/dbt/adapters/factory.py b/core/dbt/adapters/factory.py index 64021f10536..3c2f07d1db6 100644 --- a/core/dbt/adapters/factory.py +++ b/core/dbt/adapters/factory.py @@ -66,7 +66,7 @@ def load_plugin(self, name: str) -> Type[Credentials]: # if we failed to import the target module in particular, inform # the user about it via a runtime error if exc.name == 'dbt.adapters.' + name: - fire_event(AdapterImportError(exc=exc)) + fire_event(AdapterImportError(exc=str(exc))) raise RuntimeException(f'Could not find adapter type {name}!') # otherwise, the error had to have come from some underlying # library. Log the stack trace. diff --git a/core/dbt/context/base.py b/core/dbt/context/base.py index 0c09786667c..616bd331d4d 100644 --- a/core/dbt/context/base.py +++ b/core/dbt/context/base.py @@ -488,9 +488,9 @@ def log(msg: str, info: bool = False) -> str: {% endmacro %}" """ if info: - fire_event(MacroEventInfo(msg)) + fire_event(MacroEventInfo(msg=msg)) else: - fire_event(MacroEventDebug(msg)) + fire_event(MacroEventDebug(msg=msg)) return '' @contextproperty diff --git a/core/dbt/contracts/graph/parsed.py b/core/dbt/contracts/graph/parsed.py index bcdf4907203..14c966f0fb5 100644 --- a/core/dbt/contracts/graph/parsed.py +++ b/core/dbt/contracts/graph/parsed.py @@ -211,6 +211,20 @@ def write_node(self, target_path: str, subdirectory: str, payload: str): write_file(full_path, payload) return full_path + @property + def node_info(self): + node_info = { + "node_path": self.path, + "node_name": self.name, + "unique_id": self.unique_id, + "resource_type": self.resource_type.value, + "materialized": self.config.get('materialized'), + "node_status": str(self._event_status.get('node_status')), + "node_started_at": self._event_status.get("started_at"), + "node_finished_at": self._event_status.get("finished_at") + } + return node_info + T = TypeVar('T', bound='ParsedNode') @@ -738,6 +752,19 @@ def has_freshness(self): def search_name(self): return f'{self.source_name}.{self.name}' + @property + def node_info(self): + node_info = { + "node_path": self.path, + "node_name": self.name, + "unique_id": self.unique_id, + "resource_type": self.resource_type.value, + "node_status": str(self._event_status.get('node_status')), + "node_started_at": self._event_status.get("started_at"), + "node_finished_at": self._event_status.get("finished_at") + } + return node_info + @dataclass class ParsedExposure(UnparsedBaseNode, HasUniqueID, HasFqn): diff --git a/core/dbt/events/README.md b/core/dbt/events/README.md index 9a9dfe41422..9afe3825022 100644 --- a/core/dbt/events/README.md +++ b/core/dbt/events/README.md @@ -29,28 +29,10 @@ class PartialParsingDeletedExposure(DebugLevel, Cli, File): ## Optional (based on your event) -- Events associated with node status changes must have `report_node_data` passed in and be extended with `NodeInfo` -- define `asdict` if your data is not serializable to json +- Events associated with node status changes must be extended with `NodeInfo` which contains a node_info attribute -Example -``` -@dataclass -class SuperImportantNodeEvent(InfoLevel, File, NodeInfo): - node_name: str - run_result: RunResult - report_node_data: ParsedModelNode # may vary - code: str = "Q036" - - def message(self) -> str: - return f"{self.node_name} had overly verbose result of {run_result}" - - @classmethod - def asdict(cls, data: list) -> dict: - return dict((k, str(v)) for k, v in data) - -``` -All values other than `code` and `report_node_data` will be included in the `data` node of the json log output. +All values other than `code` and `node_info` will be included in the `data` node of the json log output. Once your event has been added, add a dummy call to your new event at the bottom of `types.py` and also add your new Event to the list `sample_values` in `test/unit/test_events.py'. diff --git a/core/dbt/events/base_types.py b/core/dbt/events/base_types.py index 7246740996e..863e0d21b02 100644 --- a/core/dbt/events/base_types.py +++ b/core/dbt/events/base_types.py @@ -1,9 +1,10 @@ -from abc import ABCMeta, abstractmethod, abstractproperty -from dataclasses import dataclass +from abc import ABCMeta +from dataclasses import dataclass, field +from dbt.events.serialization import dbtClassEventMixin from datetime import datetime import os import threading -from typing import Any, Optional +from typing import Any, Optional, Dict # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # These base types define the _required structure_ for the concrete event # @@ -16,19 +17,6 @@ class Cache(): pass -@dataclass -class Node(): - node_path: str - node_name: str - unique_id: str - resource_type: str - materialized: str - node_status: str - node_started_at: datetime - node_finished_at: Optional[datetime] - type: str = 'node_status' - - @dataclass class ShowException(): # N.B.: @@ -42,31 +30,28 @@ def __post_init__(self): # TODO add exhaustiveness checking for subclasses -# can't use ABCs with @dataclass because of https://github.com/python/mypy/issues/5374 # top-level superclass for all events -class Event(metaclass=ABCMeta): +@dataclass +class Event(dbtClassEventMixin, metaclass=ABCMeta): # fields that should be on all events with their default implementations log_version: int = 1 ts: Optional[datetime] = None # use getter for non-optional ts_rfc3339: Optional[str] = None # use getter for non-optional pid: Optional[int] = None # use getter for non-optional - node_info: Optional[Node] # four digit string code that uniquely identifies this type of event # uniqueness and valid characters are enforced by tests - @abstractproperty + @property @staticmethod def code() -> str: raise Exception("code() not implemented for event") # do not define this yourself. inherit it from one of the above level types. - @abstractmethod def level_tag(self) -> str: raise Exception("level_tag not implemented for Event") # Solely the human readable message. Timestamps and formatting will be added by the logger. # Must override yourself - @abstractmethod def message(self) -> str: raise Exception("msg not implemented for Event") @@ -99,21 +84,6 @@ def get_invocation_id(cls) -> str: from dbt.events.functions import get_invocation_id return get_invocation_id() - # default dict factory for all events. can override on concrete classes. - @classmethod - def asdict(cls, data: list) -> dict: - d = dict() - for k, v in data: - # stringify all exceptions - if isinstance(v, Exception) or isinstance(v, BaseException): - d[k] = str(v) - # skip all binary data - elif isinstance(v, bytes): - continue - else: - d[k] = v - return d - # in preparation for #3977 class TestLevel(Event): @@ -141,24 +111,6 @@ def level_tag(self) -> str: return "error" -@dataclass # type: ignore -class NodeInfo(Event, metaclass=ABCMeta): - report_node_data: Any # Union[ParsedModelNode, ...] TODO: resolve circular imports - - def get_node_info(self): - node_info = Node( - node_path=self.report_node_data.path, - node_name=self.report_node_data.name, - unique_id=self.report_node_data.unique_id, - resource_type=self.report_node_data.resource_type.value, - materialized=self.report_node_data.config.get('materialized'), - node_status=str(self.report_node_data._event_status.get('node_status')), - node_started_at=self.report_node_data._event_status.get("started_at"), - node_finished_at=self.report_node_data._event_status.get("finished_at") - ) - return node_info - - # prevents an event from going to the file class NoFile(): pass @@ -167,3 +119,8 @@ class NoFile(): # prevents an event from going to stdout class NoStdOut(): pass + + +@dataclass +class NodeInfo(): + node_info: Dict[str, Any] = field(default_factory=dict) diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index dfacbde8b98..7f52a338061 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -19,7 +19,6 @@ import uuid import threading from typing import Any, Callable, Dict, List, Optional, Union -import dataclasses from collections import deque @@ -138,25 +137,21 @@ def event_to_serializable_dict( ts_fn: Callable[[datetime], str] ) -> Dict[str, Any]: data = dict() - node_info = dict() + node_info: Dict[str, Any] = dict() log_line = dict() try: - log_line = dataclasses.asdict(e, dict_factory=type(e).asdict) - except AttributeError: + log_line = e.to_dict() + except AttributeError as exc: event_type = type(e).__name__ raise Exception( # TODO this may hang async threads - f"type {event_type} is not serializable to json." - f" First make sure that the call sites for {event_type} match the type hints" - f" and if they do, you can override the dataclass method `asdict` in {event_type} in" - " types.py to define your own serialization function to a dictionary of valid json" - " types" + f"type {event_type} is not serializable. {str(exc)}" ) if isinstance(e, NodeInfo): - node_info = dataclasses.asdict(e.get_node_info()) + node_info = e.node_info for field, value in log_line.items(): # type: ignore[attr-defined] - if field not in ["code", "report_node_data"]: + if field not in ["code", "node_info", "log_version", "ts", "ts_rfc3339", "pid"]: data[field] = value event_dict = { @@ -315,7 +310,7 @@ def fire_event(e: Event) -> None: global EVENT_HISTORY if len(EVENT_HISTORY) == (flags.EVENT_BUFFER_SIZE - 1): EVENT_HISTORY.append(e) - fire_event(EventBufferFull()) + EVENT_HISTORY.append(EventBufferFull()) else: EVENT_HISTORY.append(e) diff --git a/core/dbt/events/serialization.py b/core/dbt/events/serialization.py new file mode 100644 index 00000000000..f102e2290b8 --- /dev/null +++ b/core/dbt/events/serialization.py @@ -0,0 +1,40 @@ +from mashumaro import DataClassDictMixin +from mashumaro.config import ( + BaseConfig as MashBaseConfig +) +from mashumaro.types import SerializationStrategy +from datetime import datetime +from dateutil.parser import parse +from typing import cast + + +class ExceptionSerialization(SerializationStrategy): + def serialize(self, value): + out = str(value) + return out + + def deserialize(self, value): + return (Exception(value)) + + +class DateTimeSerialization(SerializationStrategy): + def serialize(self, value): + out = value.isoformat() + # Assume UTC if timezone is missing + if value.tzinfo is None: + out += "Z" + return out + + def deserialize(self, value): + return ( + value if isinstance(value, datetime) else parse(cast(str, value)) + ) + + +class dbtClassEventMixin(DataClassDictMixin): + + class Config(MashBaseConfig): + serialization_strategy = { + Exception: ExceptionSerialization(), + datetime: DateTimeSerialization(), + } diff --git a/core/dbt/events/stubs.py b/core/dbt/events/stubs.py deleted file mode 100644 index 1e34436fb45..00000000000 --- a/core/dbt/events/stubs.py +++ /dev/null @@ -1,72 +0,0 @@ -from typing import ( - Any, - List, - NamedTuple, - Optional, - Dict, -) - -# N.B.: -# These stubs were autogenerated by stubgen and then hacked -# to pieces to ensure we had something other than "Any" types -# where using external classes to instantiate event subclasses -# in events/types.py. -# -# This goes away when we turn mypy on for everything. -# -# Don't trust them too much at all! - - -class _ReferenceKey(NamedTuple): - database: Any - schema: Any - identifier: Any - - -class _CachedRelation: - referenced_by: Any - inner: Any - - -class BaseRelation: - path: Any - type: Optional[Any] - quote_character: str - include_policy: Any - quote_policy: Any - dbt_created: bool - - -class InformationSchema(BaseRelation): - information_schema_view: Optional[str] - - -class CompiledNode(): - compiled_sql: Optional[str] - extra_ctes_injected: bool - extra_ctes: List[Any] - relation_name: Optional[str] - - -class CompiledModelNode(CompiledNode): - resource_type: Any - - -class ParsedModelNode(): - resource_type: Any - - -class ParsedHookNode(): - resource_type: Any - index: Optional[int] - - -class RunResult(): - status: str - timing: List[Any] - thread_id: str - execution_time: float - adapter_response: Dict[str, Any] - message: Optional[str] - failures: Optional[int] - node: Any diff --git a/core/dbt/events/test_types.py b/core/dbt/events/test_types.py index e0563b02165..07cde8684c4 100644 --- a/core/dbt/events/test_types.py +++ b/core/dbt/events/test_types.py @@ -14,7 +14,7 @@ @dataclass class IntegrationTestInfo(InfoLevel, NoFile): - msg: str + msg: str = '' code: str = "T001" def message(self) -> str: @@ -23,7 +23,7 @@ def message(self) -> str: @dataclass class IntegrationTestDebug(DebugLevel, NoFile): - msg: str + msg: str = '' code: str = "T002" def message(self) -> str: @@ -32,7 +32,7 @@ def message(self) -> str: @dataclass class IntegrationTestWarn(WarnLevel, NoFile): - msg: str + msg: str = '' code: str = "T003" def message(self) -> str: @@ -41,7 +41,7 @@ def message(self) -> str: @dataclass class IntegrationTestError(ErrorLevel, NoFile): - msg: str + msg: str = '' code: str = "T004" def message(self) -> str: @@ -50,7 +50,7 @@ def message(self) -> str: @dataclass class IntegrationTestException(ShowException, ErrorLevel, NoFile): - msg: str + msg: str = '' code: str = "T005" def message(self) -> str: @@ -59,7 +59,7 @@ def message(self) -> str: @dataclass class UnitTestInfo(InfoLevel, NoFile): - msg: str + msg: str = '' code: str = "T006" def message(self) -> str: diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index 7844ced6eec..985b84776d0 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -1,13 +1,5 @@ -import argparse -from dataclasses import dataclass -from dbt.adapters.reference_keys import _make_key, _ReferenceKey -from dbt.events.stubs import ( - _CachedRelation, - BaseRelation, - ParsedHookNode, - ParsedModelNode, - RunResult -) +from dataclasses import dataclass, field +from dbt.adapters.reference_keys import _ReferenceKey from dbt import ui from dbt.events.base_types import ( Event, NoFile, DebugLevel, InfoLevel, WarnLevel, ErrorLevel, ShowException, @@ -50,9 +42,9 @@ @dataclass # type: ignore class AdapterEventBase(Event): - name: str - base_msg: str - args: Tuple[Any, ...] + name: str = '' + base_msg: str = '' + args: Tuple[Any, ...] = field(default_factory=tuple) # instead of having this inherit from one of the level classes def level_tag(self) -> str: @@ -109,16 +101,16 @@ def message(self) -> str: @dataclass class MainEncounteredError(ErrorLevel, NoFile): - e: BaseException + e: str = '' code: str = "Z002" def message(self) -> str: - return f"Encountered an error:\n{str(self.e)}" + return f"Encountered an error:\n{self.e}" @dataclass class MainStackTrace(DebugLevel, NoFile): - stack_trace: str + stack_trace: str = '' code: str = "Z003" def message(self) -> str: @@ -127,7 +119,7 @@ def message(self) -> str: @dataclass class MainReportVersion(InfoLevel): - v: str # could be VersionSpecifier instead if we resolved some circular imports + v: str = '' # could be VersionSpecifier instead if we resolved some circular imports code: str = "A001" def message(self): @@ -136,20 +128,16 @@ def message(self): @dataclass class MainReportArgs(DebugLevel): - args: argparse.Namespace + args: Dict[str, str] = field(default_factory=dict) code: str = "A002" def message(self): return f"running dbt with arguments {str(self.args)}" - @classmethod - def asdict(cls, data: list) -> dict: - return dict((k, str(v)) for k, v in data) - @dataclass class MainTrackingUserState(DebugLevel): - user_state: str + user_state: str = '' code: str = "A003" def message(self): @@ -230,7 +218,7 @@ def message(self) -> str: @dataclass class ReportPerformancePath(InfoLevel): - path: str + path: str = '' code: str = "I010" def message(self) -> str: @@ -239,7 +227,7 @@ def message(self) -> str: @dataclass class GitSparseCheckoutSubdirectory(DebugLevel): - subdir: str + subdir: str = '' code: str = "M001" def message(self) -> str: @@ -248,7 +236,7 @@ def message(self) -> str: @dataclass class GitProgressCheckoutRevision(DebugLevel): - revision: str + revision: str = '' code: str = "M002" def message(self) -> str: @@ -257,7 +245,7 @@ def message(self) -> str: @dataclass class GitProgressUpdatingExistingDependency(DebugLevel): - dir: str + dir: str = '' code: str = "M003" def message(self) -> str: @@ -266,7 +254,7 @@ def message(self) -> str: @dataclass class GitProgressPullingNewDependency(DebugLevel): - dir: str + dir: str = '' code: str = "M004" def message(self) -> str: @@ -275,7 +263,7 @@ def message(self) -> str: @dataclass class GitNothingToDo(DebugLevel): - sha: str + sha: str = '' code: str = "M005" def message(self) -> str: @@ -284,8 +272,8 @@ def message(self) -> str: @dataclass class GitProgressUpdatedCheckoutRange(DebugLevel): - start_sha: str - end_sha: str + start_sha: str = '' + end_sha: str = '' code: str = "M006" def message(self) -> str: @@ -294,7 +282,7 @@ def message(self) -> str: @dataclass class GitProgressCheckedOutAt(DebugLevel): - end_sha: str + end_sha: str = '' code: str = "M007" def message(self) -> str: @@ -303,7 +291,7 @@ def message(self) -> str: @dataclass class RegistryProgressMakingGETRequest(DebugLevel): - url: str + url: str = '' code: str = "M008" def message(self) -> str: @@ -312,8 +300,8 @@ def message(self) -> str: @dataclass class RegistryProgressGETResponse(DebugLevel): - url: str - resp_code: int + url: str = '' + resp_code: Optional[int] = None code: str = "M009" def message(self) -> str: @@ -323,7 +311,7 @@ def message(self) -> str: # TODO this was actually `logger.exception(...)` not `logger.error(...)` @dataclass class SystemErrorRetrievingModTime(ErrorLevel): - path: str + path: str = '' code: str = "Z004" def message(self) -> str: @@ -332,9 +320,9 @@ def message(self) -> str: @dataclass class SystemCouldNotWrite(DebugLevel): - path: str - reason: str - exc: Exception + path: str = '' + reason: str = '' + exc: Exception = Exception('') code: str = "Z005" def message(self) -> str: @@ -346,7 +334,7 @@ def message(self) -> str: @dataclass class SystemExecutingCmd(DebugLevel): - cmd: List[str] + cmd: List[str] = field(default_factory=list) code: str = "Z006" def message(self) -> str: @@ -355,7 +343,7 @@ def message(self) -> str: @dataclass class SystemStdOutMsg(DebugLevel): - bmsg: bytes + bmsg: bytes = b"" code: str = "Z007" def message(self) -> str: @@ -364,7 +352,7 @@ def message(self) -> str: @dataclass class SystemStdErrMsg(DebugLevel): - bmsg: bytes + bmsg: bytes = b"" code: str = "Z008" def message(self) -> str: @@ -373,7 +361,7 @@ def message(self) -> str: @dataclass class SystemReportReturnCode(DebugLevel): - returncode: int + returncode: Optional[int] = None code: str = "Z009" def message(self) -> str: @@ -382,22 +370,21 @@ def message(self) -> str: @dataclass class SelectorReportInvalidSelector(InfoLevel): - selector_methods: dict - spec_method: str - raw_spec: str + valid_selectors: str = '' + spec_method: str = '' + raw_spec: str = '' code: str = "M010" def message(self) -> str: - valid_selectors = ", ".join(self.selector_methods) return ( f"The '{self.spec_method}' selector specified in {self.raw_spec} is " - f"invalid. Must be one of [{valid_selectors}]" + f"invalid. Must be one of [{self.valid_selectors}]" ) @dataclass class MacroEventInfo(InfoLevel): - msg: str + msg: str = '' code: str = "M011" def message(self) -> str: @@ -406,7 +393,7 @@ def message(self) -> str: @dataclass class MacroEventDebug(DebugLevel): - msg: str + msg: str = '' code: str = "M012" def message(self) -> str: @@ -415,8 +402,8 @@ def message(self) -> str: @dataclass class NewConnection(DebugLevel): - conn_type: str - conn_name: str + conn_type: str = '' + conn_name: str = '' code: str = "E005" def message(self) -> str: @@ -425,7 +412,7 @@ def message(self) -> str: @dataclass class ConnectionReused(DebugLevel): - conn_name: str + conn_name: str = '' code: str = "E006" def message(self) -> str: @@ -434,7 +421,7 @@ def message(self) -> str: @dataclass class ConnectionLeftOpen(DebugLevel): - conn_name: Optional[str] + conn_name: Optional[str] = None code: str = "E007" def message(self) -> str: @@ -443,7 +430,7 @@ def message(self) -> str: @dataclass class ConnectionClosed(DebugLevel): - conn_name: Optional[str] + conn_name: Optional[str] = None code: str = "E008" def message(self) -> str: @@ -452,7 +439,7 @@ def message(self) -> str: @dataclass class RollbackFailed(ShowException, DebugLevel): - conn_name: Optional[str] + conn_name: Optional[str] = None code: str = "E009" def message(self) -> str: @@ -462,7 +449,7 @@ def message(self) -> str: # TODO: can we combine this with ConnectionClosed? @dataclass class ConnectionClosed2(DebugLevel): - conn_name: Optional[str] + conn_name: Optional[str] = None code: str = "E010" def message(self) -> str: @@ -472,7 +459,7 @@ def message(self) -> str: # TODO: can we combine this with ConnectionLeftOpen? @dataclass class ConnectionLeftOpen2(DebugLevel): - conn_name: Optional[str] + conn_name: Optional[str] = None code: str = "E011" def message(self) -> str: @@ -481,7 +468,7 @@ def message(self) -> str: @dataclass class Rollback(DebugLevel): - conn_name: Optional[str] + conn_name: Optional[str] = None code: str = "E012" def message(self) -> str: @@ -490,9 +477,9 @@ def message(self) -> str: @dataclass class CacheMiss(DebugLevel): - conn_name: str - database: Optional[str] - schema: str + conn_name: str = '' + database: Optional[str] = None + schema: str = '' code: str = "E013" def message(self) -> str: @@ -504,27 +491,19 @@ def message(self) -> str: @dataclass class ListRelations(DebugLevel): - database: Optional[str] - schema: str - relations: List[_ReferenceKey] + database: Optional[str] = None + schema: str = '' + relations: List[_ReferenceKey] = field(default_factory=list) code: str = "E014" def message(self) -> str: return f"with database={self.database}, schema={self.schema}, relations={self.relations}" - @classmethod - def asdict(cls, data: list) -> dict: - d = dict() - for k, v in data: - if type(v) == list: - d[k] = [str(x) for x in v] - return d - @dataclass class ConnectionUsed(DebugLevel): - conn_type: str - conn_name: Optional[str] + conn_type: str = '' + conn_name: Optional[str] = None code: str = "E015" def message(self) -> str: @@ -533,8 +512,8 @@ def message(self) -> str: @dataclass class SQLQuery(DebugLevel): - conn_name: Optional[str] - sql: str + conn_name: Optional[str] = None + sql: str = '' code: str = "E016" def message(self) -> str: @@ -543,8 +522,8 @@ def message(self) -> str: @dataclass class SQLQueryStatus(DebugLevel): - status: str - elapsed: float + status: str = '' + elapsed: Optional[float] = None code: str = "E017" def message(self) -> str: @@ -553,7 +532,7 @@ def message(self) -> str: @dataclass class SQLCommit(DebugLevel): - conn_name: str + conn_name: str = '' code: str = "E018" def message(self) -> str: @@ -562,9 +541,9 @@ def message(self) -> str: @dataclass class ColTypeChange(DebugLevel): - orig_type: str - new_type: str - table: str + orig_type: str = '' + new_type: str = '' + table: str = '' code: str = "E019" def message(self) -> str: @@ -573,7 +552,7 @@ def message(self) -> str: @dataclass class SchemaCreation(DebugLevel): - relation: _ReferenceKey + relation: Optional[_ReferenceKey] = None code: str = "E020" def message(self) -> str: @@ -582,23 +561,19 @@ def message(self) -> str: @dataclass class SchemaDrop(DebugLevel): - relation: _ReferenceKey + relation: Optional[_ReferenceKey] = None code: str = "E021" def message(self) -> str: return f'Dropping schema "{self.relation}".' - @classmethod - def asdict(cls, data: list) -> dict: - return dict((k, str(v)) for k, v in data) - # TODO pretty sure this is only ever called in dead code # see: core/dbt/adapters/cache.py _add_link vs add_link @dataclass class UncachedRelation(DebugLevel, Cache): - dep_key: _ReferenceKey - ref_key: _ReferenceKey + dep_key: Optional[_ReferenceKey] = None + ref_key: Optional[_ReferenceKey] = None code: str = "E022" def message(self) -> str: @@ -611,8 +586,8 @@ def message(self) -> str: @dataclass class AddLink(DebugLevel, Cache): - dep_key: _ReferenceKey - ref_key: _ReferenceKey + dep_key: Optional[_ReferenceKey] = None + ref_key: Optional[_ReferenceKey] = None code: str = "E023" def message(self) -> str: @@ -621,7 +596,7 @@ def message(self) -> str: @dataclass class AddRelation(DebugLevel, Cache): - relation: _ReferenceKey + relation: Optional[_ReferenceKey] = None code: str = "E024" def message(self) -> str: @@ -630,7 +605,7 @@ def message(self) -> str: @dataclass class DropMissingRelation(DebugLevel, Cache): - relation: _ReferenceKey + relation: Optional[_ReferenceKey] = None code: str = "E025" def message(self) -> str: @@ -639,27 +614,17 @@ def message(self) -> str: @dataclass class DropCascade(DebugLevel, Cache): - dropped: _ReferenceKey - consequences: Set[_ReferenceKey] + dropped: Optional[_ReferenceKey] = None + consequences: Set[_ReferenceKey] = field(default_factory=set) code: str = "E026" def message(self) -> str: return f"drop {self.dropped} is cascading to {self.consequences}" - @classmethod - def asdict(cls, data: list) -> dict: - d = dict() - for k, v in data: - if isinstance(v, list): - d[k] = [str(x) for x in v] - else: - d[k] = str(v) # type: ignore - return d - @dataclass class DropRelation(DebugLevel, Cache): - dropped: _ReferenceKey + dropped: Optional[_ReferenceKey] = None code: str = "E027" def message(self) -> str: @@ -668,9 +633,9 @@ def message(self) -> str: @dataclass class UpdateReference(DebugLevel, Cache): - old_key: _ReferenceKey - new_key: _ReferenceKey - cached_key: _ReferenceKey + old_key: Optional[_ReferenceKey] = None + new_key: Optional[_ReferenceKey] = None + cached_key: Optional[_ReferenceKey] = None code: str = "E028" def message(self) -> str: @@ -680,7 +645,7 @@ def message(self) -> str: @dataclass class TemporaryRelation(DebugLevel, Cache): - key: _ReferenceKey + key: Optional[_ReferenceKey] = None code: str = "E029" def message(self) -> str: @@ -689,8 +654,8 @@ def message(self) -> str: @dataclass class RenameSchema(DebugLevel, Cache): - old_key: _ReferenceKey - new_key: _ReferenceKey + old_key: Optional[_ReferenceKey] = None + new_key: Optional[_ReferenceKey] = None code: str = "E030" def message(self) -> str: @@ -700,7 +665,7 @@ def message(self) -> str: @dataclass class DumpBeforeAddGraph(DebugLevel, Cache): # large value. delay not necessary since every debug level message is logged anyway. - dump: Dict[str, List[str]] + dump: Dict[str, List[str]] = field(default_factory=dict) code: str = "E031" def message(self) -> str: @@ -710,7 +675,7 @@ def message(self) -> str: @dataclass class DumpAfterAddGraph(DebugLevel, Cache): # large value. delay not necessary since every debug level message is logged anyway. - dump: Dict[str, List[str]] + dump: Dict[str, List[str]] = field(default_factory=dict) code: str = "E032" def message(self) -> str: @@ -720,7 +685,7 @@ def message(self) -> str: @dataclass class DumpBeforeRenameSchema(DebugLevel, Cache): # large value. delay not necessary since every debug level message is logged anyway. - dump: Dict[str, List[str]] + dump: Dict[str, List[str]] = field(default_factory=dict) code: str = "E033" def message(self) -> str: @@ -730,7 +695,7 @@ def message(self) -> str: @dataclass class DumpAfterRenameSchema(DebugLevel, Cache): # large value. delay not necessary since every debug level message is logged anyway. - dump: Dict[str, List[str]] + dump: Dict[str, List[str]] = field(default_factory=dict) code: str = "E034" def message(self) -> str: @@ -739,16 +704,12 @@ def message(self) -> str: @dataclass class AdapterImportError(InfoLevel): - exc: ModuleNotFoundError + exc: str = '' code: str = "E035" def message(self) -> str: return f"Error importing adapter: {self.exc}" - @classmethod - def asdict(cls, data: list) -> dict: - return dict((k, str(v)) for k, v in data) - @dataclass class PluginLoadError(ShowException, DebugLevel): @@ -760,7 +721,7 @@ def message(self): @dataclass class NewConnectionOpening(DebugLevel): - connection_state: str + connection_state: str = '' code: str = "E037" def message(self) -> str: @@ -777,8 +738,8 @@ def message(self) -> str: @dataclass class MergedFromState(DebugLevel): - nbr_merged: int - sample: List + nbr_merged: Optional[int] = None + sample: List = field(default_factory=list) code: str = "A004" def message(self) -> str: @@ -787,8 +748,8 @@ def message(self) -> str: @dataclass class MissingProfileTarget(InfoLevel): - profile_name: str - target_name: str + profile_name: str = '' + target_name: str = '' code: str = "A005" def message(self) -> str: @@ -797,7 +758,7 @@ def message(self) -> str: @dataclass class ProfileLoadError(ShowException, DebugLevel): - exc: Exception + exc: Exception = Exception('') code: str = "A006" def message(self) -> str: @@ -806,7 +767,7 @@ def message(self) -> str: @dataclass class ProfileNotFound(InfoLevel): - profile_name: Optional[str] + profile_name: Optional[str] = None code: str = "A007" def message(self) -> str: @@ -823,7 +784,7 @@ def message(self) -> str: @dataclass class GenericTestFileParse(DebugLevel): - path: str + path: str = '' code: str = "I011" def message(self) -> str: @@ -832,7 +793,7 @@ def message(self) -> str: @dataclass class MacroFileParse(DebugLevel): - path: str + path: str = '' code: str = "I012" def message(self) -> str: @@ -849,7 +810,7 @@ def message(self) -> str: @dataclass class PartialParsingExceptionFile(DebugLevel): - file: str + file: str = '' code: str = "I014" def message(self) -> str: @@ -858,7 +819,7 @@ def message(self) -> str: @dataclass class PartialParsingFile(DebugLevel): - file_dict: Dict + file_dict: Dict = field(default_factory=dict) code: str = "I015" def message(self) -> str: @@ -867,7 +828,7 @@ def message(self) -> str: @dataclass class PartialParsingException(DebugLevel): - exc_info: Dict + exc_info: Dict = field(default_factory=dict) code: str = "I016" def message(self) -> str: @@ -908,7 +869,7 @@ def message(self) -> str: @dataclass class PartialParsingDeletedMetric(DebugLevel): - id: str + id: str = '' code: str = "I021" def message(self) -> str: @@ -917,7 +878,7 @@ def message(self) -> str: @dataclass class ManifestWrongMetadataVersion(DebugLevel): - version: str + version: str = '' code: str = "I022" def message(self) -> str: @@ -927,8 +888,8 @@ def message(self) -> str: @dataclass class PartialParsingVersionMismatch(InfoLevel): - saved_version: str - current_version: str + saved_version: str = '' + current_version: str = '' code: str = "I023" def message(self) -> str: @@ -980,8 +941,8 @@ def message(self) -> str: @dataclass class ParsedFileLoadFailed(ShowException, DebugLevel): - path: str - exc: Exception + path: str = '' + exc: Exception = Exception('') code: str = "I029" def message(self) -> str: @@ -998,7 +959,7 @@ def message(self) -> str: @dataclass class StaticParserCausedJinjaRendering(DebugLevel): - path: str + path: str = '' code: str = "I031" def message(self) -> str: @@ -1009,7 +970,7 @@ def message(self) -> str: # the `TestLevel` logger once we implement it. Some will probably stay `DebugLevel`. @dataclass class UsingExperimentalParser(DebugLevel): - path: str + path: str = '' code: str = "I032" def message(self) -> str: @@ -1018,7 +979,7 @@ def message(self) -> str: @dataclass class SampleFullJinjaRendering(DebugLevel): - path: str + path: str = '' code: str = "I033" def message(self) -> str: @@ -1027,7 +988,7 @@ def message(self) -> str: @dataclass class StaticParserFallbackJinjaRendering(DebugLevel): - path: str + path: str = '' code: str = "I034" def message(self) -> str: @@ -1036,7 +997,7 @@ def message(self) -> str: @dataclass class StaticParsingMacroOverrideDetected(DebugLevel): - path: str + path: str = '' code: str = "I035" def message(self) -> str: @@ -1045,7 +1006,7 @@ def message(self) -> str: @dataclass class StaticParserSuccess(DebugLevel): - path: str + path: str = '' code: str = "I036" def message(self) -> str: @@ -1054,7 +1015,7 @@ def message(self) -> str: @dataclass class StaticParserFailure(DebugLevel): - path: str + path: str = '' code: str = "I037" def message(self) -> str: @@ -1063,7 +1024,7 @@ def message(self) -> str: @dataclass class ExperimentalParserSuccess(DebugLevel): - path: str + path: str = '' code: str = "I038" def message(self) -> str: @@ -1072,7 +1033,7 @@ def message(self) -> str: @dataclass class ExperimentalParserFailure(DebugLevel): - path: str + path: str = '' code: str = "I039" def message(self) -> str: @@ -1081,9 +1042,9 @@ def message(self) -> str: @dataclass class PartialParsingEnabled(DebugLevel): - deleted: int - added: int - changed: int + deleted: Optional[int] = None + added: Optional[int] = None + changed: Optional[int] = None code: str = "I040" def message(self) -> str: @@ -1095,7 +1056,7 @@ def message(self) -> str: @dataclass class PartialParsingAddedFile(DebugLevel): - file_id: str + file_id: str = '' code: str = "I041" def message(self) -> str: @@ -1104,7 +1065,7 @@ def message(self) -> str: @dataclass class PartialParsingDeletedFile(DebugLevel): - file_id: str + file_id: str = '' code: str = "I042" def message(self) -> str: @@ -1113,7 +1074,7 @@ def message(self) -> str: @dataclass class PartialParsingUpdatedFile(DebugLevel): - file_id: str + file_id: str = '' code: str = "I043" def message(self) -> str: @@ -1122,7 +1083,7 @@ def message(self) -> str: @dataclass class PartialParsingNodeMissingInSourceFile(DebugLevel): - source_file: str + source_file: str = '' code: str = "I044" def message(self) -> str: @@ -1131,7 +1092,7 @@ def message(self) -> str: @dataclass class PartialParsingMissingNodes(DebugLevel): - file_id: str + file_id: str = '' code: str = "I045" def message(self) -> str: @@ -1140,7 +1101,7 @@ def message(self) -> str: @dataclass class PartialParsingChildMapMissingUniqueID(DebugLevel): - unique_id: str + unique_id: str = '' code: str = "I046" def message(self) -> str: @@ -1149,7 +1110,7 @@ def message(self) -> str: @dataclass class PartialParsingUpdateSchemaFile(DebugLevel): - file_id: str + file_id: str = '' code: str = "I047" def message(self) -> str: @@ -1158,7 +1119,7 @@ def message(self) -> str: @dataclass class PartialParsingDeletedSource(DebugLevel): - unique_id: str + unique_id: str = '' code: str = "I048" def message(self) -> str: @@ -1167,7 +1128,7 @@ def message(self) -> str: @dataclass class PartialParsingDeletedExposure(DebugLevel): - unique_id: str + unique_id: str = '' code: str = "I049" def message(self) -> str: @@ -1176,7 +1137,7 @@ def message(self) -> str: @dataclass class InvalidDisabledSourceInTestNode(WarnLevel): - msg: str + msg: str = '' code: str = "I050" def message(self) -> str: @@ -1185,7 +1146,7 @@ def message(self) -> str: @dataclass class InvalidRefInTestNode(WarnLevel): - msg: str + msg: str = '' code: str = "I051" def message(self) -> str: @@ -1194,7 +1155,7 @@ def message(self) -> str: @dataclass class RunningOperationCaughtError(ErrorLevel): - exc: Exception + exc: Exception = Exception('') code: str = "Q001" def message(self) -> str: @@ -1203,7 +1164,7 @@ def message(self) -> str: @dataclass class RunningOperationUncaughtError(ErrorLevel): - exc: Exception + exc: Exception = Exception('') code: str = "FF01" def message(self) -> str: @@ -1220,7 +1181,7 @@ def message(self) -> str: @dataclass class DbtProjectErrorException(ErrorLevel): - exc: Exception + exc: Exception = Exception('') code: str = "A010" def message(self) -> str: @@ -1237,7 +1198,7 @@ def message(self) -> str: @dataclass class DbtProfileErrorException(ErrorLevel): - exc: Exception + exc: Exception = Exception('') code: str = "A012" def message(self) -> str: @@ -1254,7 +1215,7 @@ def message(self) -> str: @dataclass class ListSingleProfile(InfoLevel): - profile: str + profile: str = '' code: str = "A014" def message(self) -> str: @@ -1284,7 +1245,7 @@ def message(self) -> str: @dataclass class CatchableExceptionOnRun(ShowException, DebugLevel): - exc: Exception + exc: Exception = Exception('') code: str = "W002" def message(self) -> str: @@ -1293,8 +1254,8 @@ def message(self) -> str: @dataclass class InternalExceptionOnRun(DebugLevel): - build_path: str - exc: Exception + build_path: str = '' + exc: Exception = Exception('') code: str = "W003" def message(self) -> str: @@ -1323,9 +1284,9 @@ def message(self) -> str: @dataclass class GenericExceptionOnRun(ErrorLevel): - build_path: Optional[str] - unique_id: str - exc: str # TODO: make this the actual exception once we have a better searilization strategy + build_path: Optional[str] = None + unique_id: str = '' + exc: Exception = Exception('') code: str = "W004" def message(self) -> str: @@ -1341,8 +1302,8 @@ def message(self) -> str: @dataclass class NodeConnectionReleaseError(ShowException, DebugLevel): - node_name: str - exc: Exception + node_name: str = '' + exc: Exception = Exception('') code: str = "W005" def message(self) -> str: @@ -1352,7 +1313,7 @@ def message(self) -> str: @dataclass class CheckCleanPath(InfoLevel, NoFile): - path: str + path: str = '' code: str = "Z012" def message(self) -> str: @@ -1361,8 +1322,7 @@ def message(self) -> str: @dataclass class ConfirmCleanPath(InfoLevel, NoFile): - path: str - + path: str = '' code: str = "Z013" def message(self) -> str: @@ -1371,7 +1331,7 @@ def message(self) -> str: @dataclass class ProtectedCleanPath(InfoLevel, NoFile): - path: str + path: str = '' code: str = "Z014" def message(self) -> str: @@ -1388,8 +1348,8 @@ def message(self) -> str: @dataclass class OpenCommand(InfoLevel): - open_cmd: str - profiles_dir: str + open_cmd: str = '' + profiles_dir: str = '' code: str = "Z016" def message(self) -> str: @@ -1414,7 +1374,7 @@ def message(self) -> str: @dataclass class DepsStartPackageInstall(InfoLevel): - package_name: str + package_name: str = '' code: str = "M014" def message(self) -> str: @@ -1423,7 +1383,7 @@ def message(self) -> str: @dataclass class DepsInstallInfo(InfoLevel): - version_name: str + version_name: str = '' code: str = "M015" def message(self) -> str: @@ -1432,7 +1392,7 @@ def message(self) -> str: @dataclass class DepsUpdateAvailable(InfoLevel): - version_latest: str + version_latest: str = '' code: str = "M016" def message(self) -> str: @@ -1449,7 +1409,7 @@ def message(self) -> str: @dataclass class DepsListSubdirectory(InfoLevel): - subdirectory: str + subdirectory: str = '' code: str = "M018" def message(self) -> str: @@ -1458,7 +1418,7 @@ def message(self) -> str: @dataclass class DepsNotifyUpdatesAvailable(InfoLevel): - packages: List[str] + packages: List[str] = field(default_factory=list) code: str = "M019" def message(self) -> str: @@ -1468,7 +1428,7 @@ def message(self) -> str: @dataclass class DatabaseErrorRunning(InfoLevel): - hook_type: str + hook_type: str = '' code: str = "E038" def message(self) -> str: @@ -1485,8 +1445,8 @@ def message(self) -> str: @dataclass class HooksRunning(InfoLevel): - num_hooks: int - hook_type: str + num_hooks: Optional[int] = None + hook_type: str = '' code: str = "E039" def message(self) -> str: @@ -1496,8 +1456,8 @@ def message(self) -> str: @dataclass class HookFinished(InfoLevel): - stat_line: str - execution: str + stat_line: str = '' + execution: str = '' code: str = "E040" def message(self) -> str: @@ -1506,7 +1466,7 @@ def message(self) -> str: @dataclass class WriteCatalogFailure(ErrorLevel): - num_exceptions: int + num_exceptions: Optional[int] = None code: str = "E041" def message(self) -> str: @@ -1516,7 +1476,7 @@ def message(self) -> str: @dataclass class CatalogWritten(InfoLevel): - path: str + path: str = '' code: str = "E042" def message(self) -> str: @@ -1557,8 +1517,8 @@ def message(self) -> str: @dataclass class ServingDocsPort(InfoLevel): - address: str - port: int + address: str = '' + port: Optional[int] = None code: str = "Z018" def message(self) -> str: @@ -1567,7 +1527,7 @@ def message(self) -> str: @dataclass class ServingDocsAccessInfo(InfoLevel): - port: str + port: str = '' code: str = "Z019" def message(self) -> str: @@ -1584,7 +1544,7 @@ def message(self) -> str: @dataclass class SeedHeader(InfoLevel): - header: str + header: str = '' code: str = "Q004" def message(self) -> str: @@ -1592,8 +1552,8 @@ def message(self) -> str: @dataclass -class SeedHeaderSeperator(InfoLevel): - len_header: int +class SeedHeaderSeparator(InfoLevel): + len_header: int = 0 code: str = "Q005" def message(self) -> str: @@ -1602,9 +1562,9 @@ def message(self) -> str: @dataclass class RunResultWarning(WarnLevel): - resource_type: str - node_name: str - path: str + resource_type: str = '' + node_name: str = '' + path: str = '' code: str = "Z021" def message(self) -> str: @@ -1614,9 +1574,9 @@ def message(self) -> str: @dataclass class RunResultFailure(ErrorLevel): - resource_type: str - node_name: str - path: str + resource_type: str = '' + node_name: str = '' + path: str = '' code: str = "Z022" def message(self) -> str: @@ -1626,7 +1586,7 @@ def message(self) -> str: @dataclass class StatsLine(InfoLevel): - stats: Dict + stats: Dict = field(default_factory=dict) code: str = "Z023" def message(self) -> str: @@ -1636,7 +1596,7 @@ def message(self) -> str: @dataclass class RunResultError(ErrorLevel): - msg: str + msg: str = '' code: str = "Z024" def message(self) -> str: @@ -1645,7 +1605,7 @@ def message(self) -> str: @dataclass class RunResultErrorNoMessage(ErrorLevel): - status: str + status: str = '' code: str = "Z025" def message(self) -> str: @@ -1654,7 +1614,7 @@ def message(self) -> str: @dataclass class SQLCompiledPath(InfoLevel): - path: str + path: str = '' code: str = "Z026" def message(self) -> str: @@ -1663,7 +1623,7 @@ def message(self) -> str: @dataclass class SQlRunnerException(ShowException, DebugLevel): - exc: Exception + exc: Exception = Exception('') code: str = "Q006" def message(self) -> str: @@ -1672,7 +1632,7 @@ def message(self) -> str: @dataclass class CheckNodeTestFailure(InfoLevel): - relation_name: str + relation_name: str = '' code: str = "Z027" def message(self) -> str: @@ -1683,7 +1643,7 @@ def message(self) -> str: @dataclass class FirstRunResultError(ErrorLevel): - msg: str + msg: str = '' code: str = "Z028" def message(self) -> str: @@ -1692,7 +1652,7 @@ def message(self) -> str: @dataclass class AfterFirstRunResultError(ErrorLevel): - msg: str + msg: str = '' code: str = "Z029" def message(self) -> str: @@ -1701,8 +1661,8 @@ def message(self) -> str: @dataclass class EndOfRunSummary(InfoLevel): - num_errors: int - num_warnings: int + num_errors: int = 0 + num_warnings: int = 0 keyboard_interrupt: bool = False code: str = "Z030" @@ -1723,10 +1683,9 @@ def message(self) -> str: @dataclass class PrintStartLine(InfoLevel, NodeInfo): - description: str - index: int - total: int - report_node_data: ParsedModelNode + description: str = '' + index: Optional[int] = None + total: Optional[int] = None code: str = "Q033" def message(self) -> str: @@ -1741,11 +1700,9 @@ def message(self) -> str: @dataclass class PrintHookStartLine(InfoLevel, NodeInfo): - statement: str - index: int - total: int - truncate: bool - report_node_data: Any # TODO: resolve ParsedHookNode circular import + statement: str = '' + index: Optional[int] = None + total: Optional[int] = None code: str = "Q032" def message(self) -> str: @@ -1754,18 +1711,16 @@ def message(self) -> str: status='RUN', index=self.index, total=self.total, - truncate=self.truncate) + truncate=True) @dataclass class PrintHookEndLine(InfoLevel, NodeInfo): - statement: str - status: str - index: int - total: int - execution_time: int - truncate: bool - report_node_data: Any # TODO: resolve ParsedHookNode circular import + statement: str = '' + status: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q007" def message(self) -> str: @@ -1775,17 +1730,16 @@ def message(self) -> str: index=self.index, total=self.total, execution_time=self.execution_time, - truncate=self.truncate) + truncate=True) @dataclass class SkippingDetails(InfoLevel, NodeInfo): - resource_type: str - schema: str - node_name: str - index: int - total: int - report_node_data: ParsedModelNode + resource_type: str = '' + schema: str = '' + node_name: str = '' + index: Optional[int] = None + total: Optional[int] = None code: str = "Q034" def message(self) -> str: @@ -1801,11 +1755,10 @@ def message(self) -> str: @dataclass class PrintErrorTestResult(ErrorLevel, NodeInfo): - name: str - index: int - num_models: int - execution_time: int - report_node_data: ParsedModelNode + name: str = '' + index: Optional[int] = None + num_models: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q008" def message(self) -> str: @@ -1820,11 +1773,10 @@ def message(self) -> str: @dataclass class PrintPassTestResult(InfoLevel, NodeInfo): - name: str - index: int - num_models: int - execution_time: int - report_node_data: ParsedModelNode + name: str = '' + index: Optional[int] = None + num_models: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q009" def message(self) -> str: @@ -1839,12 +1791,11 @@ def message(self) -> str: @dataclass class PrintWarnTestResult(WarnLevel, NodeInfo): - name: str - index: int - num_models: int - execution_time: int - failures: List[str] - report_node_data: ParsedModelNode + name: str = '' + index: Optional[int] = None + num_models: Optional[int] = None + execution_time: Optional[int] = None + failures: Optional[int] = None code: str = "Q010" def message(self) -> str: @@ -1859,12 +1810,11 @@ def message(self) -> str: @dataclass class PrintFailureTestResult(ErrorLevel, NodeInfo): - name: str - index: int - num_models: int - execution_time: int - failures: List[str] - report_node_data: ParsedModelNode + name: str = '' + index: Optional[int] = None + num_models: Optional[int] = None + execution_time: Optional[int] = None + failures: Optional[int] = None code: str = "Q011" def message(self) -> str: @@ -1879,10 +1829,10 @@ def message(self) -> str: @dataclass class PrintSkipBecauseError(ErrorLevel): - schema: str - relation: str - index: int - total: int + schema: str = '' + relation: str = '' + index: Optional[int] = None + total: Optional[int] = None code: str = "Z034" def message(self) -> str: @@ -1895,12 +1845,11 @@ def message(self) -> str: @dataclass class PrintModelErrorResultLine(ErrorLevel, NodeInfo): - description: str - status: str - index: int - total: int - execution_time: int - report_node_data: ParsedModelNode + description: str = '' + status: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q035" def message(self) -> str: @@ -1915,12 +1864,11 @@ def message(self) -> str: @dataclass class PrintModelResultLine(InfoLevel, NodeInfo): - description: str - status: str - index: int - total: int - execution_time: int - report_node_data: ParsedModelNode + description: str = '' + status: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q012" def message(self) -> str: @@ -1935,13 +1883,12 @@ def message(self) -> str: @dataclass class PrintSnapshotErrorResultLine(ErrorLevel, NodeInfo): - status: str - description: str - cfg: Dict - index: int - total: int - execution_time: int - report_node_data: ParsedModelNode + status: str = '' + description: str = '' + cfg: Dict[str, Any] = field(default_factory=dict) + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q013" def message(self) -> str: @@ -1956,13 +1903,12 @@ def message(self) -> str: @dataclass class PrintSnapshotResultLine(InfoLevel, NodeInfo): - status: str - description: str - cfg: Dict - index: int - total: int - execution_time: int - report_node_data: ParsedModelNode + status: str = '' + description: str = '' + cfg: Dict[str, Any] = field(default_factory=dict) + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q014" def message(self) -> str: @@ -1977,13 +1923,12 @@ def message(self) -> str: @dataclass class PrintSeedErrorResultLine(ErrorLevel, NodeInfo): - status: str - index: int - total: int - execution_time: int - schema: str - relation: str - report_node_data: ParsedModelNode + status: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None + schema: str = '' + relation: str = '' code: str = "Q015" def message(self) -> str: @@ -1998,13 +1943,12 @@ def message(self) -> str: @dataclass class PrintSeedResultLine(InfoLevel, NodeInfo): - status: str - index: int - total: int - execution_time: int - schema: str - relation: str - report_node_data: ParsedModelNode + status: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None + schema: str = '' + relation: str = '' code: str = "Q016" def message(self) -> str: @@ -2019,12 +1963,11 @@ def message(self) -> str: @dataclass class PrintHookEndErrorLine(ErrorLevel, NodeInfo): - source_name: str - table_name: str - index: int - total: int - execution_time: int - report_node_data: ParsedHookNode + source_name: str = '' + table_name: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q017" def message(self) -> str: @@ -2039,12 +1982,11 @@ def message(self) -> str: @dataclass class PrintHookEndErrorStaleLine(ErrorLevel, NodeInfo): - source_name: str - table_name: str - index: int - total: int - execution_time: int - report_node_data: ParsedHookNode + source_name: str = '' + table_name: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q018" def message(self) -> str: @@ -2059,12 +2001,11 @@ def message(self) -> str: @dataclass class PrintHookEndWarnLine(WarnLevel, NodeInfo): - source_name: str - table_name: str - index: int - total: int - execution_time: int - report_node_data: ParsedHookNode + source_name: str = '' + table_name: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q019" def message(self) -> str: @@ -2079,12 +2020,11 @@ def message(self) -> str: @dataclass class PrintHookEndPassLine(InfoLevel, NodeInfo): - source_name: str - table_name: str - index: int - total: int - execution_time: int - report_node_data: ParsedHookNode + source_name: str = '' + table_name: str = '' + index: Optional[int] = None + total: Optional[int] = None + execution_time: Optional[int] = None code: str = "Q020" def message(self) -> str: @@ -2099,7 +2039,7 @@ def message(self) -> str: @dataclass class PrintCancelLine(ErrorLevel): - conn_name: str + conn_name: str = '' code: str = "Q021" def message(self) -> str: @@ -2112,7 +2052,7 @@ def message(self) -> str: @dataclass class DefaultSelector(InfoLevel): - name: str + name: str = '' code: str = "Q022" def message(self) -> str: @@ -2121,8 +2061,7 @@ def message(self) -> str: @dataclass class NodeStart(DebugLevel, NodeInfo): - unique_id: str - report_node_data: ParsedModelNode + unique_id: str = '' code: str = "Q023" def message(self) -> str: @@ -2131,22 +2070,17 @@ def message(self) -> str: @dataclass class NodeFinished(DebugLevel, NodeInfo): - unique_id: str - report_node_data: ParsedModelNode - run_result: RunResult + unique_id: str = '' + run_result: Dict[str, Any] = field(default_factory=dict) code: str = "Q024" def message(self) -> str: return f"Finished running node {self.unique_id}" - @classmethod - def asdict(cls, data: list) -> dict: - return dict((k, str(v)) for k, v in data) - @dataclass class QueryCancelationUnsupported(InfoLevel): - type: str + type: str = '' code: str = "Q025" def message(self) -> str: @@ -2158,8 +2092,8 @@ def message(self) -> str: @dataclass class ConcurrencyLine(InfoLevel): - num_threads: int - target_name: str + num_threads: Optional[int] = None + target_name: str = '' code: str = "Q026" def message(self) -> str: @@ -2168,8 +2102,7 @@ def message(self) -> str: @dataclass class NodeCompiling(DebugLevel, NodeInfo): - unique_id: str - report_node_data: ParsedModelNode + unique_id: str = '' code: str = "Q030" def message(self) -> str: @@ -2178,8 +2111,7 @@ def message(self) -> str: @dataclass class NodeExecuting(DebugLevel, NodeInfo): - unique_id: str - report_node_data: ParsedModelNode + unique_id: str = '' code: str = "Q031" def message(self) -> str: @@ -2188,7 +2120,7 @@ def message(self) -> str: @dataclass class StarterProjectPath(DebugLevel): - dir: str + dir: str = '' code: str = "A017" def message(self) -> str: @@ -2197,7 +2129,7 @@ def message(self) -> str: @dataclass class ConfigFolderDirectory(InfoLevel): - dir: str + dir: str = '' code: str = "A018" def message(self) -> str: @@ -2206,7 +2138,7 @@ def message(self) -> str: @dataclass class NoSampleProfileFound(InfoLevel): - adapter: str + adapter: str = '' code: str = "A019" def message(self) -> str: @@ -2215,8 +2147,8 @@ def message(self) -> str: @dataclass class ProfileWrittenWithSample(InfoLevel): - name: str - path: str + name: str = '' + path: str = '' code: str = "A020" def message(self) -> str: @@ -2227,8 +2159,8 @@ def message(self) -> str: @dataclass class ProfileWrittenWithTargetTemplateYAML(InfoLevel): - name: str - path: str + name: str = '' + path: str = '' code: str = "A021" def message(self) -> str: @@ -2239,8 +2171,8 @@ def message(self) -> str: @dataclass class ProfileWrittenWithProjectTemplateYAML(InfoLevel): - name: str - path: str + name: str = '' + path: str = '' code: str = "A022" def message(self) -> str: @@ -2267,7 +2199,7 @@ def message(self) -> str: @dataclass class ProjectNameAlreadyExists(InfoLevel): - name: str + name: str = '' code: str = "A025" def message(self) -> str: @@ -2276,7 +2208,7 @@ def message(self) -> str: @dataclass class GetAddendum(InfoLevel): - msg: str + msg: str = '' code: str = "A026" def message(self) -> str: @@ -2285,7 +2217,7 @@ def message(self) -> str: @dataclass class DepsSetDownloadDirectory(DebugLevel): - path: str + path: str = '' code: str = "A027" def message(self) -> str: @@ -2320,7 +2252,7 @@ def message(self) -> str: @dataclass class FoundStats(InfoLevel): - stat_line: str + stat_line: str = '' code: str = "W006" def message(self) -> str: @@ -2330,7 +2262,7 @@ def message(self) -> str: # TODO: should this have NodeInfo on it? @dataclass class CompilingNode(DebugLevel): - unique_id: str + unique_id: str = '' code: str = "Q027" def message(self) -> str: @@ -2339,7 +2271,7 @@ def message(self) -> str: @dataclass class WritingInjectedSQLForNode(DebugLevel): - unique_id: str + unique_id: str = '' code: str = "Q028" def message(self) -> str: @@ -2356,7 +2288,7 @@ def message(self) -> str: @dataclass class SendingEvent(DebugLevel): - kwargs: str + kwargs: str = '' code: str = "Z040" def message(self) -> str: @@ -2397,8 +2329,8 @@ def message(self) -> str: @dataclass class RetryExternalCall(DebugLevel): - attempt: int - max: int + attempt: Optional[int] = None + max: Optional[int] = None code: str = "Z045" def message(self) -> str: @@ -2407,8 +2339,8 @@ def message(self) -> str: @dataclass class GeneralWarningMsg(WarnLevel): - msg: str - log_fmt: str + msg: str = '' + log_fmt: str = '' code: str = "Z046" def message(self) -> str: @@ -2419,8 +2351,8 @@ def message(self) -> str: @dataclass class GeneralWarningException(WarnLevel): - exc: Exception - log_fmt: str + exc: Exception = Exception('') + log_fmt: str = '' code: str = "Z047" def message(self) -> str: @@ -2444,11 +2376,11 @@ def message(self) -> str: # # TODO remove these lines once we run mypy everywhere. if 1 == 0: - MainReportVersion('') + MainReportVersion() MainKeyboardInterrupt() - MainEncounteredError(BaseException('')) - MainStackTrace('') - MainTrackingUserState('') + MainEncounteredError(e=str(BaseException(''))) + MainStackTrace() + MainTrackingUserState() ParsingStart() ParsingCompiling() ParsingWritingManifest() @@ -2472,7 +2404,7 @@ def message(self) -> str: SystemStdOutMsg(bmsg=b"") SystemStdErrMsg(bmsg=b"") SelectorReportInvalidSelector( - selector_methods={"": ""}, spec_method="", raw_spec="" + valid_selectors="", spec_method="", raw_spec="" ) MacroEventInfo(msg="") MacroEventDebug(msg="") @@ -2491,8 +2423,8 @@ def message(self) -> str: SQLQueryStatus(status="", elapsed=0.1) SQLCommit(conn_name="") ColTypeChange(orig_type="", new_type="", table="") - SchemaCreation(relation=_make_key(BaseRelation())) - SchemaDrop(relation=_make_key(BaseRelation())) + SchemaCreation(relation=_ReferenceKey(database="", schema="", identifier="")) + SchemaDrop(relation=_ReferenceKey(database="", schema="", identifier="")) UncachedRelation( dep_key=_ReferenceKey(database="", schema="", identifier=""), ref_key=_ReferenceKey(database="", schema="", identifier=""), @@ -2501,7 +2433,7 @@ def message(self) -> str: dep_key=_ReferenceKey(database="", schema="", identifier=""), ref_key=_ReferenceKey(database="", schema="", identifier=""), ) - AddRelation(relation=_make_key(_CachedRelation())) + AddRelation(relation=_ReferenceKey(database="", schema="", identifier="")) DropMissingRelation(relation=_ReferenceKey(database="", schema="", identifier="")) DropCascade( dropped=_ReferenceKey(database="", schema="", identifier=""), @@ -2517,11 +2449,11 @@ def message(self) -> str: old_key=_ReferenceKey(database="", schema="", identifier=""), new_key=_ReferenceKey(database="", schema="", identifier="") ) - DumpBeforeAddGraph(dict()) - DumpAfterAddGraph(dict()) - DumpBeforeRenameSchema(dict()) - DumpAfterRenameSchema(dict()) - AdapterImportError(ModuleNotFoundError()) + DumpBeforeAddGraph() + DumpAfterAddGraph() + DumpBeforeRenameSchema() + DumpAfterRenameSchema() + AdapterImportError() PluginLoadError() SystemReportReturnCode(returncode=0) NewConnectionOpening(connection_state='') @@ -2545,7 +2477,7 @@ def message(self) -> str: PartialParsingFailedBecauseProfileChange() PartialParsingFailedBecauseNewProjectDependency() PartialParsingFailedBecauseHashChanged() - PartialParsingDeletedMetric('') + PartialParsingDeletedMetric() ParsedFileLoadFailed(path='', exc=Exception('')) PartialParseSaveFileNotFound() StaticParserCausedJinjaRendering(path='') @@ -2581,7 +2513,7 @@ def message(self) -> str: ProfileHelpMessage() CatchableExceptionOnRun(exc=Exception('')) InternalExceptionOnRun(build_path='', exc=Exception('')) - GenericExceptionOnRun(build_path='', unique_id='', exc='') + GenericExceptionOnRun(build_path='', unique_id='', exc=Exception('')) NodeConnectionReleaseError(node_name='', exc=Exception('')) CheckCleanPath(path='') ConfirmCleanPath(path='') @@ -2608,7 +2540,7 @@ def message(self) -> str: ServingDocsAccessInfo(port='') ServingDocsExitInfo() SeedHeader(header='') - SeedHeaderSeperator(len_header=0) + SeedHeaderSeparator(len_header=0) RunResultWarning(resource_type='', node_name='', path='') RunResultFailure(resource_type='', node_name='', path='') StatsLine(stats={}) @@ -2619,13 +2551,12 @@ def message(self) -> str: FirstRunResultError(msg='') AfterFirstRunResultError(msg='') EndOfRunSummary(num_errors=0, num_warnings=0, keyboard_interrupt=False) - PrintStartLine(description='', index=0, total=0, report_node_data=ParsedModelNode()) + PrintStartLine(description='', index=0, total=0, node_info={}) PrintHookStartLine( statement='', index=0, total=0, - truncate=False, - report_node_data=ParsedHookNode() + node_info={}, ) PrintHookEndLine( statement='', @@ -2633,8 +2564,7 @@ def message(self) -> str: index=0, total=0, execution_time=0, - truncate=False, - report_node_data=ParsedHookNode() + node_info={}, ) SkippingDetails( resource_type='', @@ -2642,37 +2572,37 @@ def message(self) -> str: node_name='', index=0, total=0, - report_node_data=ParsedModelNode() + node_info={}, ) PrintErrorTestResult( name='', index=0, num_models=0, execution_time=0, - report_node_data=ParsedModelNode() + node_info={}, ) PrintPassTestResult( name='', index=0, num_models=0, execution_time=0, - report_node_data=ParsedModelNode() + node_info={}, ) PrintWarnTestResult( name='', index=0, num_models=0, execution_time=0, - failures=[], - report_node_data=ParsedModelNode() + failures=0, + node_info={}, ) PrintFailureTestResult( name='', index=0, num_models=0, execution_time=0, - failures=[], - report_node_data=ParsedModelNode() + failures=0, + node_info={}, ) PrintSkipBecauseError(schema='', relation='', index=0, total=0) PrintModelErrorResultLine( @@ -2681,7 +2611,7 @@ def message(self) -> str: index=0, total=0, execution_time=0, - report_node_data=ParsedModelNode() + node_info={}, ) PrintModelResultLine( description='', @@ -2689,7 +2619,7 @@ def message(self) -> str: index=0, total=0, execution_time=0, - report_node_data=ParsedModelNode() + node_info={}, ) PrintSnapshotErrorResultLine( status='', @@ -2698,7 +2628,7 @@ def message(self) -> str: index=0, total=0, execution_time=0, - report_node_data=ParsedModelNode() + node_info={}, ) PrintSnapshotResultLine( status='', @@ -2707,7 +2637,7 @@ def message(self) -> str: index=0, total=0, execution_time=0, - report_node_data=ParsedModelNode() + node_info={}, ) PrintSeedErrorResultLine( status='', @@ -2716,7 +2646,7 @@ def message(self) -> str: execution_time=0, schema='', relation='', - report_node_data=ParsedModelNode() + node_info={}, ) PrintSeedResultLine( status='', @@ -2725,7 +2655,7 @@ def message(self) -> str: execution_time=0, schema='', relation='', - report_node_data=ParsedModelNode() + node_info={}, ) PrintHookEndErrorLine( source_name='', @@ -2733,7 +2663,7 @@ def message(self) -> str: index=0, total=0, execution_time=0, - report_node_data=ParsedHookNode() + node_info={}, ) PrintHookEndErrorStaleLine( source_name='', @@ -2741,7 +2671,7 @@ def message(self) -> str: index=0, total=0, execution_time=0, - report_node_data=ParsedHookNode() + node_info={}, ) PrintHookEndWarnLine( source_name='', @@ -2749,7 +2679,7 @@ def message(self) -> str: index=0, total=0, execution_time=0, - report_node_data=ParsedHookNode() + node_info={}, ) PrintHookEndPassLine( source_name='', @@ -2757,16 +2687,16 @@ def message(self) -> str: index=0, total=0, execution_time=0, - report_node_data=ParsedHookNode() + node_info={}, ) PrintCancelLine(conn_name='') DefaultSelector(name='') - NodeStart(report_node_data=ParsedModelNode(), unique_id='') - NodeFinished(report_node_data=ParsedModelNode(), unique_id='', run_result=RunResult()) + NodeStart(node_info={}, unique_id='') + NodeFinished(node_info={}, unique_id='', run_result={}) QueryCancelationUnsupported(type='') ConcurrencyLine(num_threads=0, target_name='') - NodeCompiling(report_node_data=ParsedModelNode(), unique_id='') - NodeExecuting(report_node_data=ParsedModelNode(), unique_id='') + NodeCompiling(node_info={}, unique_id='') + NodeExecuting(node_info={}, unique_id='') StarterProjectPath(dir='') ConfigFolderDirectory(dir='') NoSampleProfileFound(adapter='') diff --git a/core/dbt/graph/selector.py b/core/dbt/graph/selector.py index 1bcd7e5b1d7..642bb5d2a36 100644 --- a/core/dbt/graph/selector.py +++ b/core/dbt/graph/selector.py @@ -86,8 +86,9 @@ def get_nodes_from_criteria( try: collected = self.select_included(nodes, spec) except InvalidSelectorException: + valid_selectors = ", ".join(self.SELECTOR_METHODS) fire_event(SelectorReportInvalidSelector( - selector_methods=self.SELECTOR_METHODS, + valid_selectors=valid_selectors, spec_method=spec.method, raw_spec=spec.raw )) diff --git a/core/dbt/main.py b/core/dbt/main.py index 19f7c8d0e28..92d2e5a7e9e 100644 --- a/core/dbt/main.py +++ b/core/dbt/main.py @@ -36,7 +36,7 @@ import dbt.tracking -from dbt.utils import ExitCodes +from dbt.utils import ExitCodes, args_to_dict from dbt.config.profile import DEFAULT_PROFILES_DIR, read_user_config from dbt.exceptions import ( InternalException, @@ -140,7 +140,7 @@ def main(args=None): exit_code = e.code except BaseException as e: - fire_event(MainEncounteredError(e=e)) + fire_event(MainEncounteredError(e=str(e))) fire_event(MainStackTrace(stack_trace=traceback.format_exc())) exit_code = ExitCodes.UnhandledError.value @@ -205,7 +205,7 @@ def track_run(task): ) except (NotImplementedException, FailedToConnectException) as e: - fire_event(MainEncounteredError(e=e)) + fire_event(MainEncounteredError(e=str(e))) dbt.tracking.track_invocation_end( config=task.config, args=task.args, result_type="error" ) @@ -235,10 +235,10 @@ def run_from_args(parsed): setup_event_logger(log_path or 'logs', level_override) fire_event(MainReportVersion(v=str(dbt.version.installed))) - fire_event(MainReportArgs(args=parsed)) + fire_event(MainReportArgs(args=args_to_dict(parsed))) if dbt.tracking.active_user is not None: # mypy appeasement, always true - fire_event(MainTrackingUserState(dbt.tracking.active_user.state())) + fire_event(MainTrackingUserState(user_state=dbt.tracking.active_user.state())) results = None diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index 347b2846360..7c986936dcc 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -290,7 +290,7 @@ def compile_and_execute(self, manifest, ctx): ctx.node._event_status['node_status'] = RunningStatus.Compiling fire_event( NodeCompiling( - report_node_data=ctx.node, + node_info=ctx.node.node_info, unique_id=ctx.node.unique_id, ) ) @@ -306,7 +306,7 @@ def compile_and_execute(self, manifest, ctx): ctx.node._event_status['node_status'] = RunningStatus.Executing fire_event( NodeExecuting( - report_node_data=ctx.node, + node_info=ctx.node.node_info, unique_id=ctx.node.unique_id, ) ) @@ -448,7 +448,7 @@ def on_skip(self): node_name=node_name, index=self.node_index, total=self.num_nodes, - report_node_data=self.node + node_info=self.node.node_info ) ) diff --git a/core/dbt/task/freshness.py b/core/dbt/task/freshness.py index 5edb7b476ca..8bae7f85d1b 100644 --- a/core/dbt/task/freshness.py +++ b/core/dbt/task/freshness.py @@ -41,7 +41,7 @@ def before_execute(self): description=description, index=self.node_index, total=self.num_nodes, - report_node_data=self.node + node_info=self.node.node_info ) ) @@ -60,7 +60,7 @@ def after_execute(self, result): index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, - report_node_data=self.node + node_info=self.node.node_info ) ) elif result.status == FreshnessStatus.Error: @@ -71,7 +71,7 @@ def after_execute(self, result): index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, - report_node_data=self.node + node_info=self.node.node_info ) ) elif result.status == FreshnessStatus.Warn: @@ -82,7 +82,7 @@ def after_execute(self, result): index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, - report_node_data=self.node + node_info=self.node.node_info ) ) else: @@ -93,7 +93,7 @@ def after_execute(self, result): index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, - report_node_data=self.node + node_info=self.node.node_info ) ) diff --git a/core/dbt/task/run.py b/core/dbt/task/run.py index e9db0345c4c..1c50e320b55 100644 --- a/core/dbt/task/run.py +++ b/core/dbt/task/run.py @@ -178,7 +178,7 @@ def print_start_line(self): description=self.describe_node(), index=self.node_index, total=self.num_nodes, - report_node_data=self.node + node_info=self.node.node_info ) ) @@ -192,7 +192,7 @@ def print_result_line(self, result): index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, - report_node_data=self.node + node_info=self.node.node_info ) ) else: @@ -203,7 +203,7 @@ def print_result_line(self, result): index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, - report_node_data=self.node + node_info=self.node.node_info ) ) @@ -357,8 +357,7 @@ def run_hooks(self, adapter, hook_type: RunHookType, extra_context): statement=hook_text, index=idx, total=num_hooks, - truncate=True, - report_node_data=hook + node_info=hook.node_info ) ) @@ -380,8 +379,7 @@ def run_hooks(self, adapter, hook_type: RunHookType, extra_context): index=idx, total=num_hooks, execution_time=timer.elapsed, - truncate=True, - report_node_data=hook + node_info=hook.node_info ) ) # `_event_status` dict is only used for logging. Make sure @@ -401,7 +399,7 @@ def safe_run_hooks( try: self.run_hooks(adapter, hook_type, extra_context) except RuntimeException: - fire_event(DatabaseErrorRunning(hook_type)) + fire_event(DatabaseErrorRunning(hook_type=hook_type.value)) raise def print_results_line(self, results, execution_time): diff --git a/core/dbt/task/runnable.py b/core/dbt/task/runnable.py index 8e3a0945f3b..95e831de479 100644 --- a/core/dbt/task/runnable.py +++ b/core/dbt/task/runnable.py @@ -6,7 +6,6 @@ from datetime import datetime from multiprocessing.dummy import Pool as ThreadPool from typing import Optional, Dict, List, Set, Tuple, Iterable, AbstractSet -from pathlib import PosixPath, WindowsPath from .printer import ( print_run_result_error, @@ -216,7 +215,7 @@ def call_runner(self, runner): with startctx, extended_metadata: fire_event( NodeStart( - report_node_data=runner.node, + node_info=runner.node.node_info, unique_id=runner.node.unique_id, ) ) @@ -231,9 +230,9 @@ def call_runner(self, runner): with finishctx, DbtModelState(status): fire_event( NodeFinished( - report_node_data=runner.node, + node_info=runner.node.node_info, unique_id=runner.node.unique_id, - run_result=result + run_result=result.to_dict(), ) ) # `_event_status` dict is only used for logging. Make sure @@ -591,38 +590,8 @@ def get_result(self, results, elapsed_time, generated_at): results=results, elapsed_time=elapsed_time, generated_at=generated_at, - args=self.args_to_dict(), + args=dbt.utils.args_to_dict(self.args), ) - def args_to_dict(self): - var_args = vars(self.args).copy() - # update the args with the flags, which could also come from environment - # variables or user_config - flag_dict = flags.get_flag_dict() - var_args.update(flag_dict) - dict_args = {} - # remove args keys that clutter up the dictionary - for key in var_args: - if key == 'cls': - continue - if var_args[key] is None: - continue - # TODO: add more default_false_keys - default_false_keys = ( - 'debug', 'full_refresh', 'fail_fast', 'warn_error', - 'single_threaded', 'log_cache_events', - 'use_experimental_parser', - ) - if key in default_false_keys and var_args[key] is False: - continue - if key == 'vars' and var_args[key] == '{}': - continue - # this was required for a test case - if (isinstance(var_args[key], PosixPath) or - isinstance(var_args[key], WindowsPath)): - var_args[key] = str(var_args[key]) - dict_args[key] = var_args[key] - return dict_args - def task_end_messages(self, results): print_run_end_messages(results) diff --git a/core/dbt/task/seed.py b/core/dbt/task/seed.py index 4a222a50705..7bfd91ecd83 100644 --- a/core/dbt/task/seed.py +++ b/core/dbt/task/seed.py @@ -11,7 +11,7 @@ from dbt.logger import TextOnly from dbt.events.functions import fire_event from dbt.events.types import ( - SeedHeader, SeedHeaderSeperator, EmptyLine, PrintSeedErrorResultLine, + SeedHeader, SeedHeaderSeparator, EmptyLine, PrintSeedErrorResultLine, PrintSeedResultLine, PrintStartLine ) from dbt.node_types import NodeType @@ -28,7 +28,7 @@ def before_execute(self): description=self.describe_node(), index=self.node_index, total=self.num_nodes, - report_node_data=self.node + node_info=self.node.node_info ) ) @@ -52,7 +52,7 @@ def print_result_line(self, result): execution_time=result.execution_time, schema=self.node.schema, relation=model.alias, - report_node_data=model + node_info=model.node_info ) ) else: @@ -64,7 +64,7 @@ def print_result_line(self, result): execution_time=result.execution_time, schema=self.node.schema, relation=model.alias, - report_node_data=model + node_info=model.node_info ) ) @@ -109,7 +109,7 @@ def show_table(self, result): with TextOnly(): fire_event(EmptyLine()) fire_event(SeedHeader(header=header)) - fire_event(SeedHeaderSeperator(len_header=len(header))) + fire_event(SeedHeaderSeparator(len_header=len(header))) rand_table.print_table(max_rows=10, max_columns=None) with TextOnly(): diff --git a/core/dbt/task/snapshot.py b/core/dbt/task/snapshot.py index 7e36c4882dc..11b37b0bf47 100644 --- a/core/dbt/task/snapshot.py +++ b/core/dbt/task/snapshot.py @@ -24,7 +24,7 @@ def print_result_line(self, result): index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, - report_node_data=model + node_info=model.node_info ) ) else: @@ -36,7 +36,7 @@ def print_result_line(self, result): index=self.node_index, total=self.num_nodes, execution_time=result.execution_time, - report_node_data=model + node_info=model.node_info ) ) diff --git a/core/dbt/task/test.py b/core/dbt/task/test.py index 5680926f929..4267ed629d9 100644 --- a/core/dbt/task/test.py +++ b/core/dbt/task/test.py @@ -75,7 +75,7 @@ def print_result_line(self, result): index=self.node_index, num_models=self.num_nodes, execution_time=result.execution_time, - report_node_data=model + node_info=model.node_info ) ) elif result.status == TestStatus.Pass: @@ -85,7 +85,7 @@ def print_result_line(self, result): index=self.node_index, num_models=self.num_nodes, execution_time=result.execution_time, - report_node_data=model + node_info=model.node_info ) ) elif result.status == TestStatus.Warn: @@ -96,7 +96,7 @@ def print_result_line(self, result): num_models=self.num_nodes, execution_time=result.execution_time, failures=result.failures, - report_node_data=model + node_info=model.node_info ) ) elif result.status == TestStatus.Fail: @@ -107,7 +107,7 @@ def print_result_line(self, result): num_models=self.num_nodes, execution_time=result.execution_time, failures=result.failures, - report_node_data=model + node_info=model.node_info ) ) else: @@ -119,7 +119,7 @@ def print_start_line(self): description=self.describe_node(), index=self.node_index, total=self.num_nodes, - report_node_data=self.node + node_info=self.node.node_info ) ) diff --git a/core/dbt/utils.py b/core/dbt/utils.py index 658ce227e83..e6e6d0180d7 100644 --- a/core/dbt/utils.py +++ b/core/dbt/utils.py @@ -11,11 +11,13 @@ import os import requests import time +from pathlib import PosixPath, WindowsPath from contextlib import contextmanager from dbt.exceptions import ConnectionException from dbt.events.functions import fire_event from dbt.events.types import RetryExternalCall +from dbt import flags from enum import Enum from typing_extensions import Protocol from typing import ( @@ -613,3 +615,34 @@ def _connection_exception_retry(fn, max_attempts: int, attempt: int = 0): _connection_exception_retry(fn, max_attempts, attempt + 1) else: raise ConnectionException('External connection exception occurred: ' + str(exc)) + + +def args_to_dict(args): + var_args = vars(args).copy() + # update the args with the flags, which could also come from environment + # variables or user_config + flag_dict = flags.get_flag_dict() + var_args.update(flag_dict) + dict_args = {} + # remove args keys that clutter up the dictionary + for key in var_args: + if key == 'cls': + continue + if var_args[key] is None: + continue + # TODO: add more default_false_keys + default_false_keys = ( + 'debug', 'full_refresh', 'fail_fast', 'warn_error', + 'single_threaded', 'log_cache_events', + 'use_experimental_parser', + ) + if key in default_false_keys and var_args[key] is False: + continue + if key == 'vars' and var_args[key] == '{}': + continue + # this was required for a test case + if (isinstance(var_args[key], PosixPath) or + isinstance(var_args[key], WindowsPath)): + var_args[key] = str(var_args[key]) + dict_args[key] = var_args[key] + return dict_args diff --git a/test/unit/test_events.py b/test/unit/test_events.py index 5c5ff574d9d..9185cba3b5e 100644 --- a/test/unit/test_events.py +++ b/test/unit/test_events.py @@ -1,12 +1,11 @@ - +from dbt.adapters.reference_keys import _ReferenceKey from dbt.events.test_types import UnitTestInfo -from argparse import Namespace from dbt.events import AdapterLogger from dbt.events.functions import event_to_serializable_dict +from dbt.events.base_types import NodeInfo from dbt.events.types import * from dbt.events.test_types import * -from dbt.events.base_types import Event -from dbt.events.stubs import _CachedRelation, BaseRelation, _ReferenceKey, ParsedModelNode +from dbt.events.base_types import Event, TestLevel, DebugLevel, WarnLevel, InfoLevel, ErrorLevel from importlib import reload import dbt.events.functions as event_funcs import dbt.flags as flags @@ -22,6 +21,8 @@ def get_all_subclasses(cls): all_subclasses = [] for subclass in cls.__subclasses__(): + if subclass in [NodeInfo, AdapterEventBase, TestLevel, DebugLevel, WarnLevel, InfoLevel, ErrorLevel]: + continue all_subclasses.append(subclass) all_subclasses.extend(get_all_subclasses(subclass)) return set(all_subclasses) @@ -97,17 +98,21 @@ def setUp(self) -> None: def test_buffer_populates(self): event_funcs.fire_event(UnitTestInfo(msg="Test Event 1")) event_funcs.fire_event(UnitTestInfo(msg="Test Event 2")) + event1 = event_funcs.EVENT_HISTORY[-2] self.assertTrue( - event_funcs.EVENT_HISTORY.count(UnitTestInfo(msg='Test Event 1', code='T006')) == 1 + event_funcs.EVENT_HISTORY.count(event1) == 1 ) # ensure events drop from the front of the buffer when buffer maxsize is reached def test_buffer_FIFOs(self): - for n in range(0,(flags.EVENT_BUFFER_SIZE + 1)): + event_funcs.EVENT_HISTORY.clear() + for n in range(1,(flags.EVENT_BUFFER_SIZE + 1)): event_funcs.fire_event(UnitTestInfo(msg=f"Test Event {n}")) + event_full = event_funcs.EVENT_HISTORY[-1] + self.assertEqual(event_full.code, 'Z048') self.assertTrue( - event_funcs.EVENT_HISTORY.count(EventBufferFull(code='Z048')) == 1 + event_funcs.EVENT_HISTORY.count(event_full) == 1 ) self.assertTrue( event_funcs.EVENT_HISTORY.count(UnitTestInfo(msg='Test Event 1', code='T006')) == 0 @@ -149,11 +154,11 @@ def MockNode(): sample_values = [ - MainReportVersion(''), + MainReportVersion(), MainKeyboardInterrupt(), - MainEncounteredError(BaseException('')), - MainStackTrace(''), - MainTrackingUserState(''), + MainEncounteredError(e=str(BaseException(''))), + MainStackTrace(), + MainTrackingUserState(), ParsingStart(), ParsingCompiling(), ParsingWritingManifest(), @@ -172,12 +177,12 @@ def MockNode(): GitProgressUpdatedCheckoutRange(start_sha="", end_sha=""), GitProgressCheckedOutAt(end_sha=""), SystemErrorRetrievingModTime(path=""), - SystemCouldNotWrite(path="", reason="", exc=Exception("")), + SystemCouldNotWrite(path="", reason="", exc=""), SystemExecutingCmd(cmd=[""]), SystemStdOutMsg(bmsg=b""), SystemStdErrMsg(bmsg=b""), SelectorReportInvalidSelector( - selector_methods={"": ""}, spec_method="", raw_spec="" + valid_selectors="", spec_method="", raw_spec="" ), MacroEventInfo(msg=""), MacroEventDebug(msg=""), @@ -222,18 +227,18 @@ def MockNode(): old_key=_ReferenceKey(database="", schema="", identifier=""), new_key=_ReferenceKey(database="", schema="", identifier="") ), - DumpBeforeAddGraph(dict()), - DumpAfterAddGraph(dict()), - DumpBeforeRenameSchema(dict()), - DumpAfterRenameSchema(dict()), - AdapterImportError(ModuleNotFoundError()), + DumpBeforeAddGraph(dump=dict()), + DumpAfterAddGraph(dump=dict()), + DumpBeforeRenameSchema(dump=dict()), + DumpAfterRenameSchema(dump=dict()), + AdapterImportError(exc=str(ModuleNotFoundError())), PluginLoadError(), SystemReportReturnCode(returncode=0), NewConnectionOpening(connection_state=''), TimingInfoCollected(), MergedFromState(nbr_merged=0, sample=[]), MissingProfileTarget(profile_name='', target_name=''), - ProfileLoadError(exc=Exception('')), + ProfileLoadError(), ProfileNotFound(profile_name=''), InvalidVarsYAML(), GenericTestFileParse(path=''), @@ -250,8 +255,8 @@ def MockNode(): PartialParsingFailedBecauseProfileChange(), PartialParsingFailedBecauseNewProjectDependency(), PartialParsingFailedBecauseHashChanged(), - PartialParsingDeletedMetric(''), - ParsedFileLoadFailed(path='', exc=Exception('')), + PartialParsingDeletedMetric(), + ParsedFileLoadFailed(path='', exc=''), PartialParseSaveFileNotFound(), StaticParserCausedJinjaRendering(path=''), UsingExperimentalParser(path=''), @@ -274,7 +279,7 @@ def MockNode(): PartialParsingDeletedExposure(unique_id=''), InvalidDisabledSourceInTestNode(msg=''), InvalidRefInTestNode(msg=''), - RunningOperationCaughtError(exc=Exception('')), + RunningOperationCaughtError(exc=''), RunningOperationUncaughtError(exc=Exception('')), DbtProjectError(), DbtProjectErrorException(exc=Exception('')), @@ -313,7 +318,7 @@ def MockNode(): ServingDocsAccessInfo(port=''), ServingDocsExitInfo(), SeedHeader(header=''), - SeedHeaderSeperator(len_header=0), + SeedHeaderSeparator(len_header=0), RunResultWarning(resource_type='', node_name='', path=''), RunResultFailure(resource_type='', node_name='', path=''), StatsLine(stats={'pass':0, 'warn':0, 'error':0, 'skip':0, 'total':0}), @@ -324,37 +329,37 @@ def MockNode(): FirstRunResultError(msg=''), AfterFirstRunResultError(msg=''), EndOfRunSummary(num_errors=0, num_warnings=0, keyboard_interrupt=False), - PrintStartLine(description='', index=0, total=0, report_node_data=MockNode()), - PrintHookStartLine(statement='', index=0, total=0, truncate=False, report_node_data=MockNode()), - PrintHookEndLine(statement='', status='', index=0, total=0, execution_time=0, truncate=False, report_node_data=MockNode()), - SkippingDetails(resource_type='', schema='', node_name='', index=0, total=0, report_node_data=MockNode()), - PrintErrorTestResult(name='', index=0, num_models=0, execution_time=0, report_node_data=MockNode()), - PrintPassTestResult(name='', index=0, num_models=0, execution_time=0, report_node_data=MockNode()), - PrintWarnTestResult(name='', index=0, num_models=0, execution_time=0, failures=[], report_node_data=MockNode()), - PrintFailureTestResult(name='', index=0, num_models=0, execution_time=0, failures=[], report_node_data=MockNode()), + PrintStartLine(description='', index=0, total=0, node_info={}), + PrintHookStartLine(statement='', index=0, total=0, node_info={}), + PrintHookEndLine(statement='', status='', index=0, total=0, execution_time=0, node_info={}), + SkippingDetails(resource_type='', schema='', node_name='', index=0, total=0, node_info={}), + PrintErrorTestResult(name='', index=0, num_models=0, execution_time=0, node_info={}), + PrintPassTestResult(name='', index=0, num_models=0, execution_time=0, node_info={}), + PrintWarnTestResult(name='', index=0, num_models=0, execution_time=0, failures=0, node_info={}), + PrintFailureTestResult(name='', index=0, num_models=0, execution_time=0, failures=0, node_info={}), PrintSkipBecauseError(schema='', relation='', index=0, total=0), - PrintModelErrorResultLine(description='', status='', index=0, total=0, execution_time=0, report_node_data=MockNode()), - PrintModelResultLine(description='', status='', index=0, total=0, execution_time=0, report_node_data=MockNode()), + PrintModelErrorResultLine(description='', status='', index=0, total=0, execution_time=0, node_info={}), + PrintModelResultLine(description='', status='', index=0, total=0, execution_time=0, node_info={}), PrintSnapshotErrorResultLine(status='', description='', cfg={}, index=0, total=0, execution_time=0, - report_node_data=MockNode()), - PrintSnapshotResultLine(status='', description='', cfg={}, index=0, total=0, execution_time=0, report_node_data=MockNode()), - PrintSeedErrorResultLine(status='', index=0, total=0, execution_time=0, schema='', relation='', report_node_data=MockNode()), - PrintSeedResultLine(status='', index=0, total=0, execution_time=0, schema='', relation='', report_node_data=MockNode()), - PrintHookEndErrorLine(source_name='', table_name='', index=0, total=0, execution_time=0, report_node_data=MockNode()), - PrintHookEndErrorStaleLine(source_name='', table_name='', index=0, total=0, execution_time=0, report_node_data=MockNode()), - PrintHookEndWarnLine(source_name='', table_name='', index=0, total=0, execution_time=0, report_node_data=MockNode()), - PrintHookEndPassLine(source_name='', table_name='', index=0, total=0, execution_time=0, report_node_data=MockNode()), + node_info={}), + PrintSnapshotResultLine(status='', description='', cfg={}, index=0, total=0, execution_time=0, node_info={}), + PrintSeedErrorResultLine(status='', index=0, total=0, execution_time=0, schema='', relation='', node_info={}), + PrintSeedResultLine(status='', index=0, total=0, execution_time=0, schema='', relation='', node_info={}), + PrintHookEndErrorLine(source_name='', table_name='', index=0, total=0, execution_time=0, node_info={}), + PrintHookEndErrorStaleLine(source_name='', table_name='', index=0, total=0, execution_time=0, node_info={}), + PrintHookEndWarnLine(source_name='', table_name='', index=0, total=0, execution_time=0, node_info={}), + PrintHookEndPassLine(source_name='', table_name='', index=0, total=0, execution_time=0, node_info={}), PrintCancelLine(conn_name=''), DefaultSelector(name=''), - NodeStart(unique_id='', report_node_data=MockNode()), - NodeCompiling(unique_id='', report_node_data=MockNode()), - NodeExecuting(unique_id='', report_node_data=MockNode()), - NodeFinished(unique_id='', report_node_data=MockNode(), run_result=''), + NodeStart(unique_id='', node_info={}), + NodeCompiling(unique_id='', node_info={}), + NodeExecuting(unique_id='', node_info={}), + NodeFinished(unique_id='', node_info={}, run_result={}), QueryCancelationUnsupported(type=''), ConcurrencyLine(num_threads=0, target_name=''), StarterProjectPath(dir=''), @@ -384,26 +389,26 @@ def MockNode(): GeneralWarningMsg(msg='', log_fmt=''), GeneralWarningException(exc=Exception(''), log_fmt=''), PartialParsingProfileEnvVarsChanged(), - AdapterEventDebug('', '', ()), - AdapterEventInfo('', '', ()), - AdapterEventWarning('', '', ()), - AdapterEventError('', '', ()), + AdapterEventDebug(), + AdapterEventInfo(), + AdapterEventWarning(), + AdapterEventError(), PrintDebugStackTrace(), - MainReportArgs(Namespace()), - RegistryProgressMakingGETRequest(''), + MainReportArgs(args={}), + RegistryProgressMakingGETRequest(), DepsUTD(), PartialParsingNotEnabled(), - SQlRunnerException(Exception('')), - DropRelation(''), + SQlRunnerException(exc=Exception('')), + DropRelation(), PartialParsingProjectEnvVarsChanged(), - RegistryProgressGETResponse('', ''), - IntegrationTestDebug(''), - IntegrationTestInfo(''), - IntegrationTestWarn(''), - IntegrationTestError(''), - IntegrationTestException(''), + RegistryProgressGETResponse(), + IntegrationTestDebug(), + IntegrationTestInfo(), + IntegrationTestWarn(), + IntegrationTestError(), + IntegrationTestException(), EventBufferFull(), - UnitTestInfo('') + UnitTestInfo(), ]