From 0dbdecef10bb85ba951e58e6f372686bd6bc0449 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Tue, 10 Jan 2023 15:56:52 -0500 Subject: [PATCH] Ct 1716 cleanup logging events (#6561) * Combine DbtProfileError log events * Combine DbtProjectErrorException with DbtProjectError * Combine cache logging events * Changie * fix ticket number * Ooops. Add another file. * fix serialization of profile names --- .../Under the Hood-20230110-114233.yaml | 6 + core/dbt/adapters/cache.py | 77 ++--- core/dbt/events/proto_types.py | 272 ++---------------- core/dbt/events/types.proto | 211 ++------------ core/dbt/events/types.py | 213 ++++---------- core/dbt/task/base.py | 29 +- tests/unit/test_events.py | 41 +-- 7 files changed, 158 insertions(+), 691 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20230110-114233.yaml diff --git a/.changes/unreleased/Under the Hood-20230110-114233.yaml b/.changes/unreleased/Under the Hood-20230110-114233.yaml new file mode 100644 index 00000000000..c18a26d4a03 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20230110-114233.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Combine some logging events +time: 2023-01-10T11:42:33.580756-05:00 +custom: + Author: gshank + Issue: 1716 1717 1719 diff --git a/core/dbt/adapters/cache.py b/core/dbt/adapters/cache.py index 90c4cab27fb..430c79d3b3a 100644 --- a/core/dbt/adapters/cache.py +++ b/core/dbt/adapters/cache.py @@ -16,21 +16,7 @@ TruncatedModelNameCausedCollision, ) from dbt.events.functions import fire_event, fire_event_if -from dbt.events.types import ( - AddLink, - AddRelation, - DropCascade, - DropMissingRelation, - DropRelation, - DumpAfterAddGraph, - DumpAfterRenameSchema, - DumpBeforeAddGraph, - DumpBeforeRenameSchema, - RenameSchema, - TemporaryRelation, - UncachedRelation, - UpdateReference, -) +from dbt.events.types import CacheAction, CacheDumpGraph import dbt.flags as flags from dbt.utils import lowercase @@ -281,7 +267,7 @@ def _add_link(self, referenced_key, dependent_key): referenced.add_reference(dependent) - # TODO: Is this dead code? I can't seem to find it grepping the codebase. + # This is called in plugins/postgres/dbt/adapters/postgres/impl.py def add_link(self, referenced, dependent): """Add a link between two relations to the database. If either relation does not exist, it will be added as an "external" relation. @@ -303,9 +289,9 @@ def add_link(self, referenced, dependent): # referring to a table outside our control. There's no need to make # a link - we will never drop the referenced relation during a run. fire_event( - UncachedRelation( - dep_key=_make_msg_from_ref_key(dep_key), + CacheAction( ref_key=_make_msg_from_ref_key(ref_key), + ref_key_2=_make_msg_from_ref_key(dep_key), ) ) return @@ -318,8 +304,10 @@ def add_link(self, referenced, dependent): dependent = dependent.replace(type=referenced.External) self.add(dependent) fire_event( - AddLink( - dep_key=_make_msg_from_ref_key(dep_key), ref_key=_make_msg_from_ref_key(ref_key) + CacheAction( + action="add_link", + ref_key=_make_msg_from_ref_key(dep_key), + ref_key_2=_make_msg_from_ref_key(ref_key), ) ) with self.lock: @@ -332,12 +320,18 @@ def add(self, relation): :param BaseRelation relation: The underlying relation. """ cached = _CachedRelation(relation) - fire_event(AddRelation(relation=_make_ref_key_msg(cached))) - fire_event_if(flags.LOG_CACHE_EVENTS, lambda: DumpBeforeAddGraph(dump=self.dump_graph())) + fire_event_if( + flags.LOG_CACHE_EVENTS, + lambda: CacheDumpGraph(before_after="before", action="adding", dump=self.dump_graph()), + ) + fire_event(CacheAction(action="add_relation", ref_key=_make_ref_key_msg(cached))) with self.lock: self._setdefault(cached) - fire_event_if(flags.LOG_CACHE_EVENTS, lambda: DumpAfterAddGraph(dump=self.dump_graph())) + fire_event_if( + flags.LOG_CACHE_EVENTS, + lambda: CacheDumpGraph(before_after="after", action="adding", dump=self.dump_graph()), + ) def _remove_refs(self, keys): """Removes all references to all entries in keys. This does not @@ -365,16 +359,19 @@ def drop(self, relation): """ dropped_key = _make_ref_key(relation) dropped_key_msg = _make_ref_key_msg(relation) - fire_event(DropRelation(dropped=dropped_key_msg)) + fire_event(CacheAction(action="drop_relation", ref_key=dropped_key_msg)) with self.lock: if dropped_key not in self.relations: - fire_event(DropMissingRelation(relation=dropped_key_msg)) + fire_event(CacheAction(action="drop_missing_relation", ref_key=dropped_key_msg)) return consequences = self.relations[dropped_key].collect_consequences() # convert from a list of _ReferenceKeys to a list of ReferenceKeyMsgs consequence_msgs = [_make_msg_from_ref_key(key) for key in consequences] - - fire_event(DropCascade(dropped=dropped_key_msg, consequences=consequence_msgs)) + fire_event( + CacheAction( + action="drop_cascade", ref_key=dropped_key_msg, ref_list=consequence_msgs + ) + ) self._remove_refs(consequences) def _rename_relation(self, old_key, new_relation): @@ -397,12 +394,14 @@ def _rename_relation(self, old_key, new_relation): for cached in self.relations.values(): if cached.is_referenced_by(old_key): fire_event( - UpdateReference( - old_key=_make_ref_key_msg(old_key), - new_key=_make_ref_key_msg(new_key), - cached_key=_make_ref_key_msg(cached.key()), + CacheAction( + action="update_reference", + ref_key=_make_ref_key_msg(old_key), + ref_key_2=_make_ref_key_msg(new_key), + ref_key_3=_make_ref_key_msg(cached.key()), ) ) + cached.rename_key(old_key, new_key) self.relations[new_key] = relation @@ -430,7 +429,9 @@ def _check_rename_constraints(self, old_key, new_key): raise TruncatedModelNameCausedCollision(new_key, self.relations) if old_key not in self.relations: - fire_event(TemporaryRelation(key=_make_msg_from_ref_key(old_key))) + fire_event( + CacheAction(action="temporary_relation", ref_key=_make_msg_from_ref_key(old_key)) + ) return False return True @@ -449,13 +450,16 @@ def rename(self, old, new): old_key = _make_ref_key(old) new_key = _make_ref_key(new) fire_event( - RenameSchema( - old_key=_make_msg_from_ref_key(old_key), new_key=_make_msg_from_ref_key(new) + CacheAction( + action="rename_relation", + ref_key=_make_msg_from_ref_key(old_key), + ref_key_2=_make_msg_from_ref_key(new), ) ) fire_event_if( - flags.LOG_CACHE_EVENTS, lambda: DumpBeforeRenameSchema(dump=self.dump_graph()) + flags.LOG_CACHE_EVENTS, + lambda: CacheDumpGraph(before_after="before", action="rename", dump=self.dump_graph()), ) with self.lock: @@ -465,7 +469,8 @@ def rename(self, old, new): self._setdefault(_CachedRelation(new)) fire_event_if( - flags.LOG_CACHE_EVENTS, lambda: DumpAfterRenameSchema(dump=self.dump_graph()) + flags.LOG_CACHE_EVENTS, + lambda: CacheDumpGraph(before_after="after", action="rename", dump=self.dump_graph()), ) def get_relations(self, database: Optional[str], schema: Optional[str]) -> List[Any]: diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index db61362f428..746ce294067 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -172,107 +172,30 @@ class InvalidVarsYAMLMsg(betterproto.Message): @dataclass -class DbtProjectError(betterproto.Message): +class LogDbtProjectError(betterproto.Message): """A009""" - pass - - -@dataclass -class DbtProjectErrorMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DbtProjectError" = betterproto.message_field(2) - - -@dataclass -class DbtProjectErrorException(betterproto.Message): - """A010""" - exc: str = betterproto.string_field(1) @dataclass -class DbtProjectErrorExceptionMsg(betterproto.Message): +class LogDbtProjectErrorMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DbtProjectErrorException" = betterproto.message_field(2) + data: "LogDbtProjectError" = betterproto.message_field(2) @dataclass -class DbtProfileError(betterproto.Message): +class LogDbtProfileError(betterproto.Message): """A011""" - pass - - -@dataclass -class DbtProfileErrorMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DbtProfileError" = betterproto.message_field(2) - - -@dataclass -class DbtProfileErrorException(betterproto.Message): - """A012""" - exc: str = betterproto.string_field(1) + profiles: List[str] = betterproto.string_field(2) @dataclass -class DbtProfileErrorExceptionMsg(betterproto.Message): +class LogDbtProfileErrorMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DbtProfileErrorException" = betterproto.message_field(2) - - -@dataclass -class ProfileListTitle(betterproto.Message): - """A013""" - - pass - - -@dataclass -class ProfileListTitleMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "ProfileListTitle" = betterproto.message_field(2) - - -@dataclass -class ListSingleProfile(betterproto.Message): - """A014""" - - profile: str = betterproto.string_field(1) - - -@dataclass -class ListSingleProfileMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "ListSingleProfile" = betterproto.message_field(2) - - -@dataclass -class NoDefinedProfiles(betterproto.Message): - """A015""" - - pass - - -@dataclass -class NoDefinedProfilesMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "NoDefinedProfiles" = betterproto.message_field(2) - - -@dataclass -class ProfileHelpMessage(betterproto.Message): - """A016""" - - pass - - -@dataclass -class ProfileHelpMessageMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "ProfileHelpMessage" = betterproto.message_field(2) + data: "LogDbtProfileError" = betterproto.message_field(2) @dataclass @@ -509,10 +432,10 @@ class ExposureNameDeprecationMsg(betterproto.Message): class FunctionDeprecated(betterproto.Message): """D008""" - function_name: str = betterproto.string_field(2) - reason: str = betterproto.string_field(3) - suggested_action: str = betterproto.string_field(4) - version: str = betterproto.string_field(5) + function_name: str = betterproto.string_field(1) + reason: str = betterproto.string_field(2) + suggested_action: str = betterproto.string_field(3) + version: str = betterproto.string_field(4) @dataclass @@ -828,186 +751,37 @@ class SchemaDropMsg(betterproto.Message): @dataclass -class UncachedRelation(betterproto.Message): +class CacheAction(betterproto.Message): """E022""" - dep_key: "ReferenceKeyMsg" = betterproto.message_field(1) - ref_key: "ReferenceKeyMsg" = betterproto.message_field(2) - - -@dataclass -class UncachedRelationMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "UncachedRelation" = betterproto.message_field(2) - - -@dataclass -class AddLink(betterproto.Message): - """E023""" - - dep_key: "ReferenceKeyMsg" = betterproto.message_field(1) + action: str = betterproto.string_field(1) ref_key: "ReferenceKeyMsg" = betterproto.message_field(2) + ref_key_2: "ReferenceKeyMsg" = betterproto.message_field(3) + ref_key_3: "ReferenceKeyMsg" = betterproto.message_field(4) + ref_list: List["ReferenceKeyMsg"] = betterproto.message_field(5) @dataclass -class AddLinkMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "AddLink" = betterproto.message_field(2) - - -@dataclass -class AddRelation(betterproto.Message): - """E024""" - - relation: "ReferenceKeyMsg" = betterproto.message_field(1) - - -@dataclass -class AddRelationMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "AddRelation" = betterproto.message_field(2) - - -@dataclass -class DropMissingRelation(betterproto.Message): - """E025""" - - relation: "ReferenceKeyMsg" = betterproto.message_field(1) - - -@dataclass -class DropMissingRelationMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DropMissingRelation" = betterproto.message_field(2) - - -@dataclass -class DropCascade(betterproto.Message): - """E026""" - - dropped: "ReferenceKeyMsg" = betterproto.message_field(1) - consequences: List["ReferenceKeyMsg"] = betterproto.message_field(2) - - -@dataclass -class DropCascadeMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DropCascade" = betterproto.message_field(2) - - -@dataclass -class DropRelation(betterproto.Message): - """E027""" - - dropped: "ReferenceKeyMsg" = betterproto.message_field(1) - - -@dataclass -class DropRelationMsg(betterproto.Message): +class CacheActionMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DropRelation" = betterproto.message_field(2) + data: "CacheAction" = betterproto.message_field(2) @dataclass -class UpdateReference(betterproto.Message): - """E028""" - - old_key: "ReferenceKeyMsg" = betterproto.message_field(1) - new_key: "ReferenceKeyMsg" = betterproto.message_field(2) - cached_key: "ReferenceKeyMsg" = betterproto.message_field(3) - - -@dataclass -class UpdateReferenceMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "UpdateReference" = betterproto.message_field(2) - - -@dataclass -class TemporaryRelation(betterproto.Message): - """E029""" - - key: "ReferenceKeyMsg" = betterproto.message_field(1) - - -@dataclass -class TemporaryRelationMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "TemporaryRelation" = betterproto.message_field(2) - - -@dataclass -class RenameSchema(betterproto.Message): - """E030""" - - old_key: "ReferenceKeyMsg" = betterproto.message_field(1) - new_key: "ReferenceKeyMsg" = betterproto.message_field(2) - - -@dataclass -class RenameSchemaMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "RenameSchema" = betterproto.message_field(2) - - -@dataclass -class DumpBeforeAddGraph(betterproto.Message): +class CacheDumpGraph(betterproto.Message): """E031""" dump: Dict[str, "ListOfStrings"] = betterproto.map_field( 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE ) + before_after: str = betterproto.string_field(2) + action: str = betterproto.string_field(3) @dataclass -class DumpBeforeAddGraphMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DumpBeforeAddGraph" = betterproto.message_field(2) - - -@dataclass -class DumpAfterAddGraph(betterproto.Message): - """E032""" - - dump: Dict[str, "ListOfStrings"] = betterproto.map_field( - 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE - ) - - -@dataclass -class DumpAfterAddGraphMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DumpAfterAddGraph" = betterproto.message_field(2) - - -@dataclass -class DumpBeforeRenameSchema(betterproto.Message): - """E033""" - - dump: Dict[str, "ListOfStrings"] = betterproto.map_field( - 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE - ) - - -@dataclass -class DumpBeforeRenameSchemaMsg(betterproto.Message): - info: "EventInfo" = betterproto.message_field(1) - data: "DumpBeforeRenameSchema" = betterproto.message_field(2) - - -@dataclass -class DumpAfterRenameSchema(betterproto.Message): - """E034""" - - dump: Dict[str, "ListOfStrings"] = betterproto.map_field( - 1, betterproto.TYPE_STRING, betterproto.TYPE_MESSAGE - ) - - -@dataclass -class DumpAfterRenameSchemaMsg(betterproto.Message): +class CacheDumpGraphMsg(betterproto.Message): info: "EventInfo" = betterproto.message_field(1) - data: "DumpAfterRenameSchema" = betterproto.message_field(2) + data: "CacheDumpGraph" = betterproto.message_field(2) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index a03e70fd364..85d46692089 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -133,79 +133,29 @@ message InvalidVarsYAMLMsg { } // A009 -message DbtProjectError { -} - -message DbtProjectErrorMsg { - EventInfo info = 1; - DbtProjectError data = 2; -} - -// A010 -message DbtProjectErrorException { +message LogDbtProjectError { string exc = 1; } -message DbtProjectErrorExceptionMsg { +message LogDbtProjectErrorMsg { EventInfo info = 1; - DbtProjectErrorException data = 2; -} - -// A011 -message DbtProfileError { + LogDbtProjectError data = 2; } -message DbtProfileErrorMsg { - EventInfo info = 1; - DbtProfileError data = 2; -} +// Skipped A010 -// A012 -message DbtProfileErrorException { +// A011 +message LogDbtProfileError { string exc = 1; + repeated string profiles = 2; } -message DbtProfileErrorExceptionMsg { +message LogDbtProfileErrorMsg { EventInfo info = 1; - DbtProfileErrorException data = 2; + LogDbtProfileError data = 2; } -// A013 -message ProfileListTitle { -} - -message ProfileListTitleMsg { - EventInfo info = 1; - ProfileListTitle data = 2; -} - -// A014 -message ListSingleProfile { - string profile = 1; -} - -message ListSingleProfileMsg { - EventInfo info = 1; - ListSingleProfile data = 2; -} - -// A015 -message NoDefinedProfiles { -} - -message NoDefinedProfilesMsg { - EventInfo info = 1; - NoDefinedProfiles data = 2; -} - -// A016 -message ProfileHelpMessage { -} - -message ProfileHelpMessageMsg { - EventInfo info = 1; - ProfileHelpMessage data = 2; -} +// Skipped A012, A013, A014, A015, A016 // A017 message StarterProjectPath { @@ -387,10 +337,10 @@ message ExposureNameDeprecationMsg { //D008 message FunctionDeprecated { - string function_name = 2; - string reason = 3; - string suggested_action = 4; - string version = 5; + string function_name = 1; + string reason = 2; + string suggested_action = 3; + string version = 4; } message FunctionDeprecatedMsg { @@ -644,140 +594,35 @@ message SchemaDropMsg { } // E022 -message UncachedRelation { - ReferenceKeyMsg dep_key = 1; - ReferenceKeyMsg ref_key = 2; -} - -message UncachedRelationMsg { - EventInfo info = 1; - UncachedRelation data = 2; -} - -// E023 -message AddLink { - ReferenceKeyMsg dep_key = 1; +message CacheAction { + string action = 1; ReferenceKeyMsg ref_key = 2; + ReferenceKeyMsg ref_key_2 = 3; + ReferenceKeyMsg ref_key_3 = 4; + repeated ReferenceKeyMsg ref_list = 5; } -message AddLinkMsg { +message CacheActionMsg { EventInfo info = 1; - AddLink data = 2; -} - -// E024 -message AddRelation { - ReferenceKeyMsg relation = 1; + CacheAction data = 2; } -message AddRelationMsg { - EventInfo info = 1; - AddRelation data = 2; -} - -// E025 -message DropMissingRelation { - ReferenceKeyMsg relation = 1; -} - -message DropMissingRelationMsg { - EventInfo info = 1; - DropMissingRelation data = 2; -} - -// E026 -message DropCascade { - ReferenceKeyMsg dropped = 1; - repeated ReferenceKeyMsg consequences = 2; -} - -message DropCascadeMsg { - EventInfo info = 1; - DropCascade data = 2; -} - -// E027 -message DropRelation { - ReferenceKeyMsg dropped = 1; -} - -message DropRelationMsg { - EventInfo info = 1; - DropRelation data = 2; -} - -// E028 -message UpdateReference { - ReferenceKeyMsg old_key = 1; - ReferenceKeyMsg new_key = 2; - ReferenceKeyMsg cached_key = 3; -} - -message UpdateReferenceMsg { - EventInfo info = 1; - UpdateReference data = 2; -} - -// E029 -message TemporaryRelation { - ReferenceKeyMsg key = 1; -} - -message TemporaryRelationMsg { - EventInfo info = 1; - TemporaryRelation data = 2; -} - -// E030 -message RenameSchema { - ReferenceKeyMsg old_key = 1; - ReferenceKeyMsg new_key = 2; -} - -message RenameSchemaMsg { - EventInfo info = 1; - RenameSchema data = 2; -} +// Skipping E023, E024, E025, E026, E027, E028, E029, E0230 // E031 -message DumpBeforeAddGraph { - map dump = 1; -} - -message DumpBeforeAddGraphMsg { - EventInfo info = 1; - DumpBeforeAddGraph data = 2; -} - -// E032 -message DumpAfterAddGraph { +message CacheDumpGraph { map dump = 1; + string before_after = 2; + string action = 3; } -message DumpAfterAddGraphMsg { +message CacheDumpGraphMsg { EventInfo info = 1; - DumpAfterAddGraph data = 2; -} - -// E033 -message DumpBeforeRenameSchema { - map dump = 1; + CacheDumpGraph data = 2; } -message DumpBeforeRenameSchemaMsg { - EventInfo info = 1; - DumpBeforeRenameSchema data = 2; -} -// E034 -message DumpAfterRenameSchema { - map dump = 1; -} - -message DumpAfterRenameSchemaMsg { - EventInfo info = 1; - DumpAfterRenameSchema data = 2; -} +// Skipping E032, E033, E034 // E035 message AdapterImportError { diff --git a/core/dbt/events/types.py b/core/dbt/events/types.py index e2d108419ca..b76188a8c97 100644 --- a/core/dbt/events/types.py +++ b/core/dbt/events/types.py @@ -117,79 +117,40 @@ def message(self) -> str: @dataclass -class DbtProjectError(ErrorLevel, pt.DbtProjectError): +class LogDbtProjectError(ErrorLevel, pt.LogDbtProjectError): def code(self): return "A009" def message(self) -> str: - return "Encountered an error while reading the project:" + msg = "Encountered an error while reading the project:" + if self.exc: + msg += f" ERROR: {str(self.exc)}" + return msg -@dataclass -class DbtProjectErrorException(ErrorLevel, pt.DbtProjectErrorException): - def code(self): - return "A010" - - def message(self) -> str: - return f" ERROR: {str(self.exc)}" +# Skipped A010 @dataclass -class DbtProfileError(ErrorLevel, pt.DbtProfileError): +class LogDbtProfileError(ErrorLevel, pt.LogDbtProfileError): def code(self): return "A011" def message(self) -> str: - return "Encountered an error while reading profiles:" - - -@dataclass -class DbtProfileErrorException(ErrorLevel, pt.DbtProfileErrorException): - def code(self): - return "A012" - - def message(self) -> str: - return f" ERROR: {str(self.exc)}" - - -@dataclass -class ProfileListTitle(InfoLevel, pt.ProfileListTitle): - def code(self): - return "A013" - - def message(self) -> str: - return "Defined profiles:" - - -@dataclass -class ListSingleProfile(InfoLevel, pt.ListSingleProfile): - def code(self): - return "A014" - - def message(self) -> str: - return f" - {self.profile}" - - -@dataclass -class NoDefinedProfiles(InfoLevel, pt.NoDefinedProfiles): - def code(self): - return "A015" - - def message(self) -> str: - return "There are no profiles defined in your profiles.yml file" - - -@dataclass -class ProfileHelpMessage(InfoLevel, pt.ProfileHelpMessage): - def code(self): - return "A016" + msg = "Encountered an error while reading profiles:\n" f" ERROR: {str(self.exc)}" + if self.profiles: + msg += "Defined profiles:\n" + for profile in self.profiles: + msg += f" - {profile}" + else: + msg += "There are no profiles defined in your profiles.yml file" - def message(self) -> str: - return """ + msg += """ For more information on configuring profiles, please consult the dbt docs: https://docs.getdbt.com/docs/configure-your-profile """ + return msg @dataclass @@ -631,130 +592,54 @@ def message(self) -> str: return f'Dropping schema "{self.relation}".' -# TODO pretty sure this is only ever called in dead code -# see: core/dbt/adapters/cache.py _add_link vs add_link @dataclass -class UncachedRelation(DebugLevel, Cache, pt.UncachedRelation): +class CacheAction(DebugLevel, Cache, pt.CacheAction): def code(self): return "E022" - def message(self) -> str: - return ( - f"{self.dep_key} references {str(self.ref_key)} " - f"but {self.ref_key.database}.{self.ref_key.schema}" - "is not in the cache, skipping assumed external relation" - ) - - -@dataclass -class AddLink(DebugLevel, Cache, pt.AddLink): - def code(self): - return "E023" - - def message(self) -> str: - return f"adding link, {self.dep_key} references {self.ref_key}" - - -@dataclass -class AddRelation(DebugLevel, Cache, pt.AddRelation): - def code(self): - return "E024" - - def message(self) -> str: - return f"Adding relation: {str(self.relation)}" - - -@dataclass -class DropMissingRelation(DebugLevel, Cache, pt.DropMissingRelation): - def code(self): - return "E025" - - def message(self) -> str: - return f"dropped a nonexistent relationship: {str(self.relation)}" - - -@dataclass -class DropCascade(DebugLevel, Cache, pt.DropCascade): - def code(self): - return "E026" - - def message(self) -> str: - return f"drop {self.dropped} is cascading to {self.consequences}" - - -@dataclass -class DropRelation(DebugLevel, Cache, pt.DropRelation): - def code(self): - return "E027" - - def message(self) -> str: - return f"Dropping relation: {self.dropped}" - - -@dataclass -class UpdateReference(DebugLevel, Cache, pt.UpdateReference): - def code(self): - return "E028" - - def message(self) -> str: - return ( - f"updated reference from {self.old_key} -> {self.cached_key} to " - f"{self.new_key} -> {self.cached_key}" - ) - - -@dataclass -class TemporaryRelation(DebugLevel, Cache, pt.TemporaryRelation): - def code(self): - return "E029" + def message(self): + if self.action == "add_link": + return f"adding link, {self.ref_key} references {self.ref_key_2}" + elif self.action == "add_relation": + return f"adding relation: {str(self.ref_key)}" + elif self.action == "drop_missing_relation": + return f"dropped a nonexistent relationship: {str(self.ref_key)}" + elif self.action == "drop_cascade": + return f"drop {self.ref_key} is cascading to {self.ref_list}" + elif self.action == "drop_relation": + return f"Dropping relation: {self.ref_key}" + elif self.action == "update_reference": + return ( + f"updated reference from {self.ref_key} -> {self.ref_key_3} to " + f"{self.ref_key_2} -> {self.ref_key_3}" + ) + elif self.action == "temporary_relation": + return f"old key {self.ref_key} not found in self.relations, assuming temporary" + elif self.action == "rename_relation": + return f"Renaming relation {self.ref_key} to {self.ref_key_2}" + elif self.action == "uncached_relation": + return ( + f"{self.ref_key_2} references {str(self.ref_key)} " + f"but {self.ref_key.database}.{self.ref_key.schema}" + "is not in the cache, skipping assumed external relation" + ) + else: + return f"{self.ref_key}" - def message(self) -> str: - return f"old key {self.key} not found in self.relations, assuming temporary" - -@dataclass -class RenameSchema(DebugLevel, Cache, pt.RenameSchema): - def code(self): - return "E030" - - def message(self) -> str: - return f"Renaming relation {self.old_key} to {self.new_key}" +# Skipping E023, E024, E025, E026, E027, E028, E029, E030 @dataclass -class DumpBeforeAddGraph(DebugLevel, Cache, pt.DumpBeforeAddGraph): +class CacheDumpGraph(DebugLevel, Cache, pt.CacheDumpGraph): def code(self): return "E031" def message(self) -> str: - return f"before adding : {self.dump}" - - -@dataclass -class DumpAfterAddGraph(DebugLevel, Cache, pt.DumpAfterAddGraph): - def code(self): - return "E032" - - def message(self) -> str: - return f"after adding: {self.dump}" + return f"{self.before_after} {self.action} : {self.dump}" -@dataclass -class DumpBeforeRenameSchema(DebugLevel, Cache, pt.DumpBeforeRenameSchema): - def code(self): - return "E033" - - def message(self) -> str: - return f"before rename: {self.dump}" - - -@dataclass -class DumpAfterRenameSchema(DebugLevel, Cache, pt.DumpAfterRenameSchema): - def code(self): - return "E034" - - def message(self) -> str: - return f"after rename: {self.dump}" +# Skipping E032, E033, E034 @dataclass diff --git a/core/dbt/task/base.py b/core/dbt/task/base.py index e448a15c1d2..b7ababdd067 100644 --- a/core/dbt/task/base.py +++ b/core/dbt/task/base.py @@ -24,14 +24,8 @@ from dbt.logger import log_manager from dbt.events.functions import fire_event from dbt.events.types import ( - DbtProjectError, - DbtProjectErrorException, - DbtProfileError, - DbtProfileErrorException, - ProfileListTitle, - ListSingleProfile, - NoDefinedProfiles, - ProfileHelpMessage, + LogDbtProjectError, + LogDbtProfileError, CatchableExceptionOnRun, InternalExceptionOnRun, GenericExceptionOnRun, @@ -102,26 +96,13 @@ def from_args(cls, args): # for the clean or deps tasks config = cls.ConfigType.from_args(args) except dbt.exceptions.DbtProjectError as exc: - fire_event(DbtProjectError()) - fire_event(DbtProjectErrorException(exc=str(exc))) + fire_event(LogDbtProjectError(exc=str(exc))) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException("Could not run dbt") from exc except dbt.exceptions.DbtProfileError as exc: - fire_event(DbtProfileError()) - fire_event(DbtProfileErrorException(exc=str(exc))) - - all_profiles = read_profiles(flags.PROFILES_DIR).keys() - - if len(all_profiles) > 0: - fire_event(ProfileListTitle()) - for profile in all_profiles: - fire_event(ListSingleProfile(profile=profile)) - else: - fire_event(NoDefinedProfiles()) - - fire_event(ProfileHelpMessage()) - + all_profile_names = list(read_profiles(flags.PROFILES_DIR).keys()) + fire_event(LogDbtProfileError(exc=str(exc), profiles=all_profile_names)) tracking.track_invalid_invocation(args=args, result_type=exc.result_type) raise dbt.exceptions.RuntimeException("Could not run dbt") from exc return cls(args, config) diff --git a/tests/unit/test_events.py b/tests/unit/test_events.py index aa1579da9d1..a7056a729b2 100644 --- a/tests/unit/test_events.py +++ b/tests/unit/test_events.py @@ -112,14 +112,8 @@ def test_event_codes(self): MergedFromState(num_merged=0, sample=[]), MissingProfileTarget(profile_name="", target_name=""), InvalidVarsYAML(), - DbtProjectError(), - DbtProjectErrorException(exc=""), - DbtProfileError(), - DbtProfileErrorException(exc=""), - ProfileListTitle(), - ListSingleProfile(profile=""), - NoDefinedProfiles(), - ProfileHelpMessage(), + LogDbtProjectError(), + LogDbtProfileError(), StarterProjectPath(dir=""), ConfigFolderDirectory(dir=""), NoSampleProfileFound(adapter=""), @@ -165,35 +159,12 @@ def test_event_codes(self): ), SchemaCreation(relation=ReferenceKeyMsg(database="", schema="", identifier="")), SchemaDrop(relation=ReferenceKeyMsg(database="", schema="", identifier="")), - UncachedRelation( - dep_key=ReferenceKeyMsg(database="", schema="", identifier=""), + CacheAction( + action="adding_relation", ref_key=ReferenceKeyMsg(database="", schema="", identifier=""), + ref_key_2=ReferenceKeyMsg(database="", schema="", identifier=""), ), - AddLink( - dep_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ref_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ), - AddRelation(relation=ReferenceKeyMsg(database="", schema="", identifier="")), - DropMissingRelation(relation=ReferenceKeyMsg(database="", schema="", identifier="")), - DropCascade( - dropped=ReferenceKeyMsg(database="", schema="", identifier=""), - consequences=[ReferenceKeyMsg(database="", schema="", identifier="")], - ), - DropRelation(dropped=ReferenceKeyMsg()), - UpdateReference( - old_key=ReferenceKeyMsg(database="", schema="", identifier=""), - new_key=ReferenceKeyMsg(database="", schema="", identifier=""), - cached_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ), - TemporaryRelation(key=ReferenceKeyMsg(database="", schema="", identifier="")), - RenameSchema( - old_key=ReferenceKeyMsg(database="", schema="", identifier=""), - new_key=ReferenceKeyMsg(database="", schema="", identifier=""), - ), - DumpBeforeAddGraph(dump=dict()), - DumpAfterAddGraph(dump=dict()), - DumpBeforeRenameSchema(dump=dict()), - DumpAfterRenameSchema(dump=dict()), + CacheDumpGraph(before_after="before", action="rename", dump=dict()), AdapterImportError(exc=""), PluginLoadError(exc_info=""), NewConnectionOpening(connection_state=""),