From 24d51457ed43c7e11513dc5a8ad0856a956b933e Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 3 Oct 2022 15:44:00 -0500 Subject: [PATCH 01/13] WIP --- core/dbt/constants.py | 1 + core/dbt/contracts/util.py | 7 +------ core/dbt/events/base_types.py | 5 +++++ core/dbt/events/functions.py | 16 +++++++++++++++- core/dbt/events/proto_types.py | 5 ++++- core/dbt/events/types.proto | 3 ++- tests/unit/test_proto_events.py | 2 +- 7 files changed, 29 insertions(+), 10 deletions(-) diff --git a/core/dbt/constants.py b/core/dbt/constants.py index c5b949f83aa..1599df3e335 100644 --- a/core/dbt/constants.py +++ b/core/dbt/constants.py @@ -1,2 +1,3 @@ SECRET_ENV_PREFIX = "DBT_ENV_SECRET_" DEFAULT_ENV_PLACEHOLDER = "DBT_DEFAULT_PLACEHOLDER" +METADATA_ENV_PREFIX = "DBT_ENV_CUSTOM_ENV_" diff --git a/core/dbt/contracts/util.py b/core/dbt/contracts/util.py index bc6c32ab237..25893a10ff3 100644 --- a/core/dbt/contracts/util.py +++ b/core/dbt/contracts/util.py @@ -4,6 +4,7 @@ from typing import List, Tuple, ClassVar, Type, TypeVar, Dict, Any, Optional from dbt.clients.system import write_json, read_json +from dbt.constants import METADATA_ENV_PREFIX from dbt import deprecations from dbt.exceptions import ( InternalException, @@ -148,12 +149,6 @@ def __str__(self) -> str: return BASE_SCHEMAS_URL + self.path -SCHEMA_VERSION_KEY = "dbt_schema_version" - - -METADATA_ENV_PREFIX = "DBT_ENV_CUSTOM_ENV_" - - def get_metadata_env() -> Dict[str, str]: return { k[len(METADATA_ENV_PREFIX) :]: v diff --git a/core/dbt/events/base_types.py b/core/dbt/events/base_types.py index 5ed447a48cc..74bd887fdc6 100644 --- a/core/dbt/events/base_types.py +++ b/core/dbt/events/base_types.py @@ -14,6 +14,10 @@ class Cache: # Events with this class will only be logged when the `--log-cache-events` flag is passed pass +def get_metadata_vars() -> dict: + from dbt.events.functions import get_metadata_vars + return get_metadata_vars() + def get_invocation_id() -> str: from dbt.events.functions import get_invocation_id @@ -48,6 +52,7 @@ def __post_init__(self): if not hasattr(self.info, "msg") or not self.info.msg: self.info.msg = self.message() self.info.invocation_id = get_invocation_id() + self.info.metadata = get_metadata_vars() self.info.ts = datetime.utcnow() self.info.pid = get_pid() self.info.thread = get_thread_name() diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 8ebb6b648c6..44b49b02cfc 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -1,5 +1,7 @@ import betterproto from colorama import Style +# from dbt.contracts.util import get_metadata_env +from dbt.constants import METADATA_ENV_PREFIX import dbt.events.functions as this # don't worry I hate it too. from dbt.events.base_types import NoStdOut, BaseEvent, NoFile, Cache from dbt.events.types import EventBufferFull, MainReportVersion, EmptyLine @@ -20,7 +22,7 @@ import os import uuid import threading -from typing import List, Optional, Union, Callable +from typing import List, Optional, Union, Callable, Dict from collections import deque LOG_VERSION = 3 @@ -42,6 +44,7 @@ format_color = True format_json = False invocation_id: Optional[str] = None +metadata_vars: Optional[Dict[str, str]] = None def setup_event_logger(log_path, level_override=None): @@ -263,6 +266,17 @@ def fire_event(e: BaseEvent) -> None: if log_line: send_to_logger(STDOUT_LOG, level_tag=e.level_tag(), log_line=log_line) +def get_metadata_vars() -> str: + global metadata_vars + if metadata_vars is None: + # todo: centralize this with contracts.utils logic - caused circular import + metadata_vars = { + k[len(METADATA_ENV_PREFIX) :]: v + for k, v in os.environ.items() + if k.startswith(METADATA_ENV_PREFIX) + } + return metadata_vars + def get_invocation_id() -> str: global invocation_id diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index f529784743a..4e382dfca1b 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -20,9 +20,12 @@ class EventInfo(betterproto.Message): pid: int = betterproto.int32_field(6) thread: str = betterproto.string_field(7) ts: datetime = betterproto.message_field(8) - extra: Dict[str, str] = betterproto.map_field( + metadata: Dict[str, str] = betterproto.map_field( 9, betterproto.TYPE_STRING, betterproto.TYPE_STRING ) + extra: Dict[str, str] = betterproto.map_field( + 10, betterproto.TYPE_STRING, betterproto.TYPE_STRING + ) @dataclass diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index 0d952cce521..ba9c20ce26e 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -14,7 +14,8 @@ message EventInfo { int32 pid = 6; string thread = 7; google.protobuf.Timestamp ts = 8; - map extra = 9; + map metatdata_vars = 9; + map extra = 10; } // TimingInfo diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py index 34f6ab5d019..6a6a1dcaee6 100644 --- a/tests/unit/test_proto_events.py +++ b/tests/unit/test_proto_events.py @@ -12,7 +12,7 @@ from dbt.version import installed -info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra"} +info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "metadata", "extra"} def test_events(): From 86b5371385040adf1064d9043cb334ee7d09308a Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 3 Oct 2022 16:03:51 -0500 Subject: [PATCH 02/13] use extra dict --- core/dbt/context/providers.py | 4 ++-- core/dbt/contracts/util.py | 14 ++------------ core/dbt/events/base_types.py | 4 +++- core/dbt/events/functions.py | 7 ++++--- core/dbt/events/proto_types.py | 5 +---- core/dbt/events/types.proto | 3 +-- tests/unit/test_proto_events.py | 2 +- 7 files changed, 14 insertions(+), 25 deletions(-) diff --git a/core/dbt/context/providers.py b/core/dbt/context/providers.py index fde8554c7d9..ee3700bd44d 100644 --- a/core/dbt/context/providers.py +++ b/core/dbt/context/providers.py @@ -41,7 +41,7 @@ ParsedSourceDefinition, ) from dbt.contracts.graph.metrics import MetricReference, ResolvedMetricReference -from dbt.contracts.util import get_metadata_env +from dbt.events.functions import get_metadata_vars from dbt.exceptions import ( CompilationException, ParsingException, @@ -713,7 +713,7 @@ def _get_namespace_builder(self): @contextproperty def dbt_metadata_envs(self) -> Dict[str, str]: - return get_metadata_env() + return get_metadata_vars() @contextproperty def invocation_args_dict(self): diff --git a/core/dbt/contracts/util.py b/core/dbt/contracts/util.py index 25893a10ff3..f0975fda10b 100644 --- a/core/dbt/contracts/util.py +++ b/core/dbt/contracts/util.py @@ -1,10 +1,8 @@ import dataclasses -import os from datetime import datetime from typing import List, Tuple, ClassVar, Type, TypeVar, Dict, Any, Optional from dbt.clients.system import write_json, read_json -from dbt.constants import METADATA_ENV_PREFIX from dbt import deprecations from dbt.exceptions import ( InternalException, @@ -12,7 +10,7 @@ IncompatibleSchemaException, ) from dbt.version import __version__ -from dbt.events.functions import get_invocation_id +from dbt.events.functions import get_invocation_id, get_metadata_vars from dbt.dataclass_schema import dbtClassMixin from dbt.dataclass_schema import ( @@ -149,14 +147,6 @@ def __str__(self) -> str: return BASE_SCHEMAS_URL + self.path -def get_metadata_env() -> Dict[str, str]: - return { - k[len(METADATA_ENV_PREFIX) :]: v - for k, v in os.environ.items() - if k.startswith(METADATA_ENV_PREFIX) - } - - # This is used in the ManifestMetadata, RunResultsMetadata, RunOperationResultMetadata, # FreshnessMetadata, and CatalogMetadata classes @dataclasses.dataclass @@ -165,7 +155,7 @@ class BaseArtifactMetadata(dbtClassMixin): dbt_version: str = __version__ generated_at: datetime = dataclasses.field(default_factory=datetime.utcnow) invocation_id: Optional[str] = dataclasses.field(default_factory=get_invocation_id) - env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_env) + env: Dict[str, str] = dataclasses.field(default_factory=get_metadata_vars) def __post_serialize__(self, dct): dct = super().__post_serialize__(dct) diff --git a/core/dbt/events/base_types.py b/core/dbt/events/base_types.py index 74bd887fdc6..b6f128dfc97 100644 --- a/core/dbt/events/base_types.py +++ b/core/dbt/events/base_types.py @@ -14,8 +14,10 @@ class Cache: # Events with this class will only be logged when the `--log-cache-events` flag is passed pass + def get_metadata_vars() -> dict: from dbt.events.functions import get_metadata_vars + return get_metadata_vars() @@ -52,7 +54,7 @@ def __post_init__(self): if not hasattr(self.info, "msg") or not self.info.msg: self.info.msg = self.message() self.info.invocation_id = get_invocation_id() - self.info.metadata = get_metadata_vars() + self.info.extra = get_metadata_vars() self.info.ts = datetime.utcnow() self.info.pid = get_pid() self.info.thread = get_thread_name() diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 44b49b02cfc..35e73461e3a 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -1,5 +1,6 @@ import betterproto from colorama import Style + # from dbt.contracts.util import get_metadata_env from dbt.constants import METADATA_ENV_PREFIX import dbt.events.functions as this # don't worry I hate it too. @@ -266,11 +267,11 @@ def fire_event(e: BaseEvent) -> None: if log_line: send_to_logger(STDOUT_LOG, level_tag=e.level_tag(), log_line=log_line) -def get_metadata_vars() -> str: + +def get_metadata_vars() -> Dict[str, str]: global metadata_vars if metadata_vars is None: - # todo: centralize this with contracts.utils logic - caused circular import - metadata_vars = { + metadata_vars = { k[len(METADATA_ENV_PREFIX) :]: v for k, v in os.environ.items() if k.startswith(METADATA_ENV_PREFIX) diff --git a/core/dbt/events/proto_types.py b/core/dbt/events/proto_types.py index 4e382dfca1b..f529784743a 100644 --- a/core/dbt/events/proto_types.py +++ b/core/dbt/events/proto_types.py @@ -20,11 +20,8 @@ class EventInfo(betterproto.Message): pid: int = betterproto.int32_field(6) thread: str = betterproto.string_field(7) ts: datetime = betterproto.message_field(8) - metadata: Dict[str, str] = betterproto.map_field( - 9, betterproto.TYPE_STRING, betterproto.TYPE_STRING - ) extra: Dict[str, str] = betterproto.map_field( - 10, betterproto.TYPE_STRING, betterproto.TYPE_STRING + 9, betterproto.TYPE_STRING, betterproto.TYPE_STRING ) diff --git a/core/dbt/events/types.proto b/core/dbt/events/types.proto index ba9c20ce26e..0d952cce521 100644 --- a/core/dbt/events/types.proto +++ b/core/dbt/events/types.proto @@ -14,8 +14,7 @@ message EventInfo { int32 pid = 6; string thread = 7; google.protobuf.Timestamp ts = 8; - map metatdata_vars = 9; - map extra = 10; + map extra = 9; } // TimingInfo diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py index 6a6a1dcaee6..34f6ab5d019 100644 --- a/tests/unit/test_proto_events.py +++ b/tests/unit/test_proto_events.py @@ -12,7 +12,7 @@ from dbt.version import installed -info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "metadata", "extra"} +info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra"} def test_events(): From fe032a1bdd4bb971469b13ee16042c2bff8b1ae2 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 3 Oct 2022 16:19:54 -0500 Subject: [PATCH 03/13] remove commented out code --- core/dbt/events/functions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 35e73461e3a..c985f8dec4a 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -1,7 +1,6 @@ import betterproto from colorama import Style -# from dbt.contracts.util import get_metadata_env from dbt.constants import METADATA_ENV_PREFIX import dbt.events.functions as this # don't worry I hate it too. from dbt.events.base_types import NoStdOut, BaseEvent, NoFile, Cache From 4f4dc1f8fcfd6b06fa3655c1778ec4a62c368e85 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 4 Oct 2022 07:54:46 -0500 Subject: [PATCH 04/13] fix duplicates --- core/dbt/events/base_types.py | 4 ++-- core/dbt/events/functions.py | 16 ++++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/core/dbt/events/base_types.py b/core/dbt/events/base_types.py index b6f128dfc97..1f785446c33 100644 --- a/core/dbt/events/base_types.py +++ b/core/dbt/events/base_types.py @@ -16,9 +16,9 @@ class Cache: def get_metadata_vars() -> dict: - from dbt.events.functions import get_metadata_vars + from dbt.events.functions import global_metadata_vars - return get_metadata_vars() + return global_metadata_vars() def get_invocation_id() -> str: diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index c985f8dec4a..4e3c06751e3 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -267,17 +267,21 @@ def fire_event(e: BaseEvent) -> None: send_to_logger(STDOUT_LOG, level_tag=e.level_tag(), log_line=log_line) -def get_metadata_vars() -> Dict[str, str]: +def global_metadata_vars() -> Dict[str, str]: global metadata_vars if metadata_vars is None: - metadata_vars = { - k[len(METADATA_ENV_PREFIX) :]: v - for k, v in os.environ.items() - if k.startswith(METADATA_ENV_PREFIX) - } + metadata_vars = get_metadata_vars() return metadata_vars +def get_metadata_vars() -> Dict[str, str]: + return { + k[len(METADATA_ENV_PREFIX) :]: v + for k, v in os.environ.items() + if k.startswith(METADATA_ENV_PREFIX) + } + + def get_invocation_id() -> str: global invocation_id if invocation_id is None: From 39b39452402e854baaf052171651bf3139de894e Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 4 Oct 2022 13:59:24 -0500 Subject: [PATCH 05/13] add test --- tests/unit/test_proto_events.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py index 34f6ab5d019..c1b9a3e057b 100644 --- a/tests/unit/test_proto_events.py +++ b/tests/unit/test_proto_events.py @@ -1,3 +1,4 @@ +import os import sys from dbt.events.types import ( MainReportVersion, @@ -15,6 +16,27 @@ info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra"} +def test_extra_dict_on_event(): + + os.environ["DBT_ENV_CUSTOM_ENV_env_key"] = "env_value" + + event = MainReportVersion(version=str(installed), log_version=LOG_VERSION) + event_dict = event_to_dict(event) + assert set(event_dict["info"].keys()) == info_keys + assert event.info.extra == {"env_key": "env_value"} + serialized = bytes(event) + + # Extract EventInfo from serialized message + generic_event = pl.GenericMessage().parse(serialized) + assert generic_event.info.code == "A001" + # get the message class for the real message from the generic message + message_class = getattr(sys.modules["dbt.events.proto_types"], generic_event.info.name) + new_event = message_class().parse(serialized) + assert new_event.info.extra == event.info.extra + + del os.environ["DBT_ENV_CUSTOM_ENV_env_key"] + + def test_events(): # A001 event From c89ab433efd9d6b655e19ffbad26ffd474b84acf Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 5 Oct 2022 08:44:10 -0500 Subject: [PATCH 06/13] tests wip --- .../context_methods/test_custom_env_vars.py | 33 +++++++++++++++++++ tests/unit/test_event_extra_dict.py | 28 ++++++++++++++++ tests/unit/test_proto_events.py | 22 +------------ 3 files changed, 62 insertions(+), 21 deletions(-) create mode 100644 tests/functional/context_methods/test_custom_env_vars.py create mode 100644 tests/unit/test_event_extra_dict.py diff --git a/tests/functional/context_methods/test_custom_env_vars.py b/tests/functional/context_methods/test_custom_env_vars.py new file mode 100644 index 00000000000..5236268e807 --- /dev/null +++ b/tests/functional/context_methods/test_custom_env_vars.py @@ -0,0 +1,33 @@ +import pytest +import json +import os + +from dbt.constants import METADATA_ENV_PREFIX +from dbt.tests.util import run_dbt_and_capture + + +def parse_json_logs(json_log_output): + parsed_logs = [] + for line in json_log_output.split("\n"): + try: + log = json.loads(line) + except ValueError: + continue + + parsed_logs.append(log) + + return parsed_logs + + +class TestCustomVarInLogs: + @pytest.fixture(scope="class", autouse=True) + def setup(self): + os.environ[METADATA_ENV_PREFIX + "some_var"] = "value" + yield + del os.environ[METADATA_ENV_PREFIX + "some_var"] + + def test_extra_filled(self, project): + _, log_output = run_dbt_and_capture(['--log-format=json', 'deps'],) + logs = parse_json_logs(log_output) + for log in logs: + assert log['info'].get('extra') == {"some_var": "value"} diff --git a/tests/unit/test_event_extra_dict.py b/tests/unit/test_event_extra_dict.py new file mode 100644 index 00000000000..dc479c67d35 --- /dev/null +++ b/tests/unit/test_event_extra_dict.py @@ -0,0 +1,28 @@ +import os +import sys +from dbt.events.types import BuildingCatalog +from dbt.events import proto_types as pl +from dbt.events.functions import event_to_dict + + + +class TestExtraEventDict: + def test_extra_dict_on_event(self): + os.environ["DBT_ENV_CUSTOM_ENV_env_key"] = "env_value" + + event = BuildingCatalog() + event_dict = event_to_dict(event) + serialized = bytes(event) + assert "extra" in event_dict["info"].keys() + assert event.info.extra == {"env_key": "env_value"} + + # Extract EventInfo from serialized message + generic_event = pl.GenericMessage().parse(serialized) + assert generic_event.info.code == "E044" + # get the message class for the real message from the generic message + message_class = getattr(sys.modules["dbt.events.proto_types"], generic_event.info.name) + new_event = message_class().parse(serialized) + assert new_event.info.extra == event.info.extra + + #cleanup + del os.environ["DBT_ENV_CUSTOM_ENV_env_key"] diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py index c1b9a3e057b..6e66315945c 100644 --- a/tests/unit/test_proto_events.py +++ b/tests/unit/test_proto_events.py @@ -7,6 +7,7 @@ MainEncounteredError, PluginLoadError, PrintStartLine, + BuildingCatalog ) from dbt.events.functions import event_to_dict, LOG_VERSION from dbt.events import proto_types as pl @@ -16,27 +17,6 @@ info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra"} -def test_extra_dict_on_event(): - - os.environ["DBT_ENV_CUSTOM_ENV_env_key"] = "env_value" - - event = MainReportVersion(version=str(installed), log_version=LOG_VERSION) - event_dict = event_to_dict(event) - assert set(event_dict["info"].keys()) == info_keys - assert event.info.extra == {"env_key": "env_value"} - serialized = bytes(event) - - # Extract EventInfo from serialized message - generic_event = pl.GenericMessage().parse(serialized) - assert generic_event.info.code == "A001" - # get the message class for the real message from the generic message - message_class = getattr(sys.modules["dbt.events.proto_types"], generic_event.info.name) - new_event = message_class().parse(serialized) - assert new_event.info.extra == event.info.extra - - del os.environ["DBT_ENV_CUSTOM_ENV_env_key"] - - def test_events(): # A001 event From 204872c0d75ac0c0ae07d4687da5d14206e305f1 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Mon, 10 Oct 2022 15:14:47 -0500 Subject: [PATCH 07/13] consolidate --- tests/unit/test_event_extra_dict.py | 28 ---------------------------- tests/unit/test_proto_events.py | 21 +++++++++++++++++++-- 2 files changed, 19 insertions(+), 30 deletions(-) delete mode 100644 tests/unit/test_event_extra_dict.py diff --git a/tests/unit/test_event_extra_dict.py b/tests/unit/test_event_extra_dict.py deleted file mode 100644 index dc479c67d35..00000000000 --- a/tests/unit/test_event_extra_dict.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import sys -from dbt.events.types import BuildingCatalog -from dbt.events import proto_types as pl -from dbt.events.functions import event_to_dict - - - -class TestExtraEventDict: - def test_extra_dict_on_event(self): - os.environ["DBT_ENV_CUSTOM_ENV_env_key"] = "env_value" - - event = BuildingCatalog() - event_dict = event_to_dict(event) - serialized = bytes(event) - assert "extra" in event_dict["info"].keys() - assert event.info.extra == {"env_key": "env_value"} - - # Extract EventInfo from serialized message - generic_event = pl.GenericMessage().parse(serialized) - assert generic_event.info.code == "E044" - # get the message class for the real message from the generic message - message_class = getattr(sys.modules["dbt.events.proto_types"], generic_event.info.name) - new_event = message_class().parse(serialized) - assert new_event.info.extra == event.info.extra - - #cleanup - del os.environ["DBT_ENV_CUSTOM_ENV_env_key"] diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py index 6e66315945c..fba7ad8b8cf 100644 --- a/tests/unit/test_proto_events.py +++ b/tests/unit/test_proto_events.py @@ -1,4 +1,3 @@ -import os import sys from dbt.events.types import ( MainReportVersion, @@ -7,7 +6,6 @@ MainEncounteredError, PluginLoadError, PrintStartLine, - BuildingCatalog ) from dbt.events.functions import event_to_dict, LOG_VERSION from dbt.events import proto_types as pl @@ -17,6 +15,25 @@ info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra"} +def test_extra_dict_on_event(monkeypatch): + + monkeypatch.setenv("DBT_ENV_CUSTOM_ENV_env_key", "env_value") + + event = MainReportVersion(version=str(installed), log_version=LOG_VERSION) + event_dict = event_to_dict(event) + assert set(event_dict["info"].keys()) == info_keys + assert event.info.extra == {"env_key": "env_value"} + serialized = bytes(event) + + # Extract EventInfo from serialized message + generic_event = pl.GenericMessage().parse(serialized) + assert generic_event.info.code == "A001" + # get the message class for the real message from the generic message + message_class = getattr(sys.modules["dbt.events.proto_types"], generic_event.info.name) + new_event = message_class().parse(serialized) + assert new_event.info.extra == event.info.extra + + def test_events(): # A001 event From 768825b0982d242b4efeec5a1d30775f69b56982 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Tue, 11 Oct 2022 16:30:31 -0500 Subject: [PATCH 08/13] working, but needs some cleanup --- core/dbt/events/base_types.py | 46 ++++++++++++++++++++++++++++++--- core/dbt/events/functions.py | 7 ----- tests/unit/test_proto_events.py | 45 ++++++++++++++++++-------------- 3 files changed, 68 insertions(+), 30 deletions(-) diff --git a/core/dbt/events/base_types.py b/core/dbt/events/base_types.py index 1f785446c33..434ca8fef4b 100644 --- a/core/dbt/events/base_types.py +++ b/core/dbt/events/base_types.py @@ -3,6 +3,8 @@ import threading from datetime import datetime +import functools +import collections # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # These base types define the _required structure_ for the concrete event # @@ -10,15 +12,51 @@ # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# TODO: this should not live here - duplicated and modified from utils.py because of circular imports for now +class memoized: + """Decorator. Caches a function's return value each time it is called. If + called later with the same arguments, the cached value is returned (not + reevaluated). + + Taken from https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize""" + + def __init__(self, func): + self.func = func + self.cache = {} + + def __call__(self, *args): + if not isinstance(args, collections.abc.Hashable): + # uncacheable. a list, for instance. + # better to not cache than blow up. + return self.func(*args) + if args in self.cache: + return self.cache[args] + value = self.func(*args) + self.cache[args] = value + return value + + def __repr__(self): + """Return the function's docstring.""" + return self.func.__doc__ + + def __get__(self, obj, objtype): + """Support instance methods.""" + return functools.partial(self.__call__, obj) + + def reset(self): + self.cache = {} + + class Cache: # Events with this class will only be logged when the `--log-cache-events` flag is passed pass -def get_metadata_vars() -> dict: - from dbt.events.functions import global_metadata_vars +@memoized +def get_global_metadata_vars() -> dict: + from dbt.events.functions import get_metadata_vars - return global_metadata_vars() + return get_metadata_vars() def get_invocation_id() -> str: @@ -54,7 +92,7 @@ def __post_init__(self): if not hasattr(self.info, "msg") or not self.info.msg: self.info.msg = self.message() self.info.invocation_id = get_invocation_id() - self.info.extra = get_metadata_vars() + self.info.extra = get_global_metadata_vars() self.info.ts = datetime.utcnow() self.info.pid = get_pid() self.info.thread = get_thread_name() diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 4e3c06751e3..5af3b5a2f4c 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -267,13 +267,6 @@ def fire_event(e: BaseEvent) -> None: send_to_logger(STDOUT_LOG, level_tag=e.level_tag(), log_line=log_line) -def global_metadata_vars() -> Dict[str, str]: - global metadata_vars - if metadata_vars is None: - metadata_vars = get_metadata_vars() - return metadata_vars - - def get_metadata_vars() -> Dict[str, str]: return { k[len(METADATA_ENV_PREFIX) :]: v diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py index fba7ad8b8cf..bcd412296a8 100644 --- a/tests/unit/test_proto_events.py +++ b/tests/unit/test_proto_events.py @@ -8,6 +8,7 @@ PrintStartLine, ) from dbt.events.functions import event_to_dict, LOG_VERSION +from dbt.events.base_types import get_global_metadata_vars from dbt.events import proto_types as pl from dbt.version import installed @@ -15,25 +16,6 @@ info_keys = {"name", "code", "msg", "level", "invocation_id", "pid", "thread", "ts", "extra"} -def test_extra_dict_on_event(monkeypatch): - - monkeypatch.setenv("DBT_ENV_CUSTOM_ENV_env_key", "env_value") - - event = MainReportVersion(version=str(installed), log_version=LOG_VERSION) - event_dict = event_to_dict(event) - assert set(event_dict["info"].keys()) == info_keys - assert event.info.extra == {"env_key": "env_value"} - serialized = bytes(event) - - # Extract EventInfo from serialized message - generic_event = pl.GenericMessage().parse(serialized) - assert generic_event.info.code == "A001" - # get the message class for the real message from the generic message - message_class = getattr(sys.modules["dbt.events.proto_types"], generic_event.info.name) - new_event = message_class().parse(serialized) - assert new_event.info.extra == event.info.extra - - def test_events(): # A001 event @@ -116,3 +98,28 @@ def test_node_info_events(): ) assert event assert event.node_info.node_path == "some_path" + + +def test_extra_dict_on_event(monkeypatch): + + monkeypatch.setenv("DBT_ENV_CUSTOM_ENV_env_key", "env_value") + + # need to reset the memoized values so they get filled appropriately with the expected env var + get_global_metadata_vars.reset() + + event = MainReportVersion(version=str(installed), log_version=LOG_VERSION) + event_dict = event_to_dict(event) + assert set(event_dict["info"].keys()) == info_keys + assert event.info.extra == {"env_key": "env_value"} + serialized = bytes(event) + + # Extract EventInfo from serialized message + generic_event = pl.GenericMessage().parse(serialized) + assert generic_event.info.code == "A001" + # get the message class for the real message from the generic message + message_class = getattr(sys.modules["dbt.events.proto_types"], generic_event.info.name) + new_event = message_class().parse(serialized) + assert new_event.info.extra == event.info.extra + + # clean up + get_global_metadata_vars.reset() From c5ed871a40e57cf4e49f61bc99f1a116715588ed Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 12 Oct 2022 11:47:35 -0500 Subject: [PATCH 09/13] convert back to globals --- core/dbt/events/base_types.py | 39 --------------------------------- core/dbt/events/functions.py | 18 ++++++++++----- core/dbt/lib.py | 1 + tests/unit/test_proto_events.py | 8 +++---- 4 files changed, 17 insertions(+), 49 deletions(-) diff --git a/core/dbt/events/base_types.py b/core/dbt/events/base_types.py index 434ca8fef4b..489b70cb1ad 100644 --- a/core/dbt/events/base_types.py +++ b/core/dbt/events/base_types.py @@ -3,56 +3,17 @@ import threading from datetime import datetime -import functools -import collections - # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # These base types define the _required structure_ for the concrete event # # types defined in types.py # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # -# TODO: this should not live here - duplicated and modified from utils.py because of circular imports for now -class memoized: - """Decorator. Caches a function's return value each time it is called. If - called later with the same arguments, the cached value is returned (not - reevaluated). - - Taken from https://wiki.python.org/moin/PythonDecoratorLibrary#Memoize""" - - def __init__(self, func): - self.func = func - self.cache = {} - - def __call__(self, *args): - if not isinstance(args, collections.abc.Hashable): - # uncacheable. a list, for instance. - # better to not cache than blow up. - return self.func(*args) - if args in self.cache: - return self.cache[args] - value = self.func(*args) - self.cache[args] = value - return value - - def __repr__(self): - """Return the function's docstring.""" - return self.func.__doc__ - - def __get__(self, obj, objtype): - """Support instance methods.""" - return functools.partial(self.__call__, obj) - - def reset(self): - self.cache = {} - - class Cache: # Events with this class will only be logged when the `--log-cache-events` flag is passed pass -@memoized def get_global_metadata_vars() -> dict: from dbt.events.functions import get_metadata_vars diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 5af3b5a2f4c..0784b868e2b 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -268,11 +268,19 @@ def fire_event(e: BaseEvent) -> None: def get_metadata_vars() -> Dict[str, str]: - return { - k[len(METADATA_ENV_PREFIX) :]: v - for k, v in os.environ.items() - if k.startswith(METADATA_ENV_PREFIX) - } + global metadata_vars + if metadata_vars is None: + metadata_vars = { + k[len(METADATA_ENV_PREFIX) :]: v + for k, v in os.environ.items() + if k.startswith(METADATA_ENV_PREFIX) + } + return metadata_vars + + +def reset_metadata_vars() -> None: + global metadata_vars + metadata_vars = None def get_invocation_id() -> str: diff --git a/core/dbt/lib.py b/core/dbt/lib.py index 11d4e07e524..ff8f06c88a8 100644 --- a/core/dbt/lib.py +++ b/core/dbt/lib.py @@ -90,6 +90,7 @@ def get_dbt_config(project_dir, args=None, single_threaded=False): # Make sure we have a valid invocation_id dbt.events.functions.set_invocation_id() + dbt.events.functions.reset_metadata_vars() return config diff --git a/tests/unit/test_proto_events.py b/tests/unit/test_proto_events.py index bcd412296a8..46e9479ef39 100644 --- a/tests/unit/test_proto_events.py +++ b/tests/unit/test_proto_events.py @@ -7,8 +7,7 @@ PluginLoadError, PrintStartLine, ) -from dbt.events.functions import event_to_dict, LOG_VERSION -from dbt.events.base_types import get_global_metadata_vars +from dbt.events.functions import event_to_dict, LOG_VERSION, reset_metadata_vars from dbt.events import proto_types as pl from dbt.version import installed @@ -104,8 +103,7 @@ def test_extra_dict_on_event(monkeypatch): monkeypatch.setenv("DBT_ENV_CUSTOM_ENV_env_key", "env_value") - # need to reset the memoized values so they get filled appropriately with the expected env var - get_global_metadata_vars.reset() + reset_metadata_vars() event = MainReportVersion(version=str(installed), log_version=LOG_VERSION) event_dict = event_to_dict(event) @@ -122,4 +120,4 @@ def test_extra_dict_on_event(monkeypatch): assert new_event.info.extra == event.info.extra # clean up - get_global_metadata_vars.reset() + reset_metadata_vars() From fa072705960e565c417975d3eb5f7ee7726aab10 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 12 Oct 2022 14:43:07 -0500 Subject: [PATCH 10/13] reset globals in tests --- core/dbt/tests/util.py | 5 ++++- test/unit/test_context.py | 6 ++++++ test/unit/test_manifest.py | 6 +++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/core/dbt/tests/util.py b/core/dbt/tests/util.py index b4fb24fdd50..af837c18b17 100644 --- a/core/dbt/tests/util.py +++ b/core/dbt/tests/util.py @@ -11,7 +11,7 @@ from dbt.main import handle_and_check from dbt.logger import log_manager from dbt.contracts.graph.manifest import Manifest -from dbt.events.functions import fire_event, capture_stdout_logs, stop_capture_stdout_logs +from dbt.events.functions import fire_event, capture_stdout_logs, stop_capture_stdout_logs, reset_metadata_vars from dbt.events.test_types import IntegrationTestDebug # ============================================================================= @@ -63,6 +63,9 @@ def run_dbt(args: List[str] = None, expect_pass=True): # Ignore logbook warnings warnings.filterwarnings("ignore", category=DeprecationWarning, module="logbook") + # reset global vars + reset_metadata_vars() + # The logger will complain about already being initialized if # we don't do this. log_manager.reset_handlers() diff --git a/test/unit/test_context.py b/test/unit/test_context.py index e1737103e1f..668d76cc525 100644 --- a/test/unit/test_context.py +++ b/test/unit/test_context.py @@ -19,6 +19,7 @@ from dbt.config.project import VarProvider from dbt.context import base, target, configured, providers, docs, manifest, macros from dbt.contracts.files import FileHash +from dbt.events.functions import reset_metadata_vars from dbt.node_types import NodeType import dbt.exceptions from .utils import ( @@ -503,6 +504,8 @@ def test_macro_namespace(config_postgres, manifest_fx): assert result["some_macro"].macro is package_macro def test_dbt_metadata_envs(monkeypatch, config_postgres, manifest_fx, get_adapter, get_include_paths): + reset_metadata_vars() + envs = { "DBT_ENV_CUSTOM_ENV_RUN_ID": 1234, "DBT_ENV_CUSTOM_ENV_JOB_ID": 5678, @@ -519,3 +522,6 @@ def test_dbt_metadata_envs(monkeypatch, config_postgres, manifest_fx, get_adapte ) assert ctx["dbt_metadata_envs"] == {'JOB_ID': 5678, 'RUN_ID': 1234} + + # cleanup + reset_metadata_vars() diff --git a/test/unit/test_manifest.py b/test/unit/test_manifest.py index e6a83ac0759..cbce93fc052 100644 --- a/test/unit/test_manifest.py +++ b/test/unit/test_manifest.py @@ -34,7 +34,8 @@ ) from dbt.contracts.graph.compiled import CompiledModelNode -from dbt.events.functions import get_invocation_id +from dbt.events.functions import reset_metadata_vars + from dbt.node_types import NodeType import freezegun @@ -60,6 +61,8 @@ class ManifestTest(unittest.TestCase): def setUp(self): + reset_metadata_vars() + # TODO: why is this needed for tests in this module to pass? tracking.active_user = None @@ -304,6 +307,7 @@ def setUp(self): def tearDown(self): del os.environ['DBT_ENV_CUSTOM_ENV_key'] + reset_metadata_vars() @freezegun.freeze_time('2018-02-14T09:15:13Z') def test__no_nodes(self): From 9160b51dbfaef1135a5ba378cc8cad01835edb37 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 12 Oct 2022 14:54:40 -0500 Subject: [PATCH 11/13] fix up some imports --- core/dbt/events/functions.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/dbt/events/functions.py b/core/dbt/events/functions.py index 834069d37a4..3276ec467ed 100644 --- a/core/dbt/events/functions.py +++ b/core/dbt/events/functions.py @@ -1,13 +1,9 @@ import betterproto from colorama import Style - -from dbt.constants import METADATA_ENV_PREFIX -import dbt.events.functions as this # don't worry I hate it too. - from dbt.events.base_types import NoStdOut, BaseEvent, NoFile, Cache from dbt.events.types import EventBufferFull, MainReportVersion, EmptyLine import dbt.flags as flags -from dbt.constants import SECRET_ENV_PREFIX +from dbt.constants import SECRET_ENV_PREFIX, METADATA_ENV_PREFIX from dbt.logger import make_log_dir_if_missing, GLOBAL_LOGGER from datetime import datetime From eb3770b00e2e1f08c7905cdce4ec88ea382c46c6 Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Wed, 12 Oct 2022 15:31:56 -0500 Subject: [PATCH 12/13] fix text for windows --- tests/functional/context_methods/test_custom_env_vars.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/functional/context_methods/test_custom_env_vars.py b/tests/functional/context_methods/test_custom_env_vars.py index 5236268e807..95f406b02e9 100644 --- a/tests/functional/context_methods/test_custom_env_vars.py +++ b/tests/functional/context_methods/test_custom_env_vars.py @@ -2,7 +2,6 @@ import json import os -from dbt.constants import METADATA_ENV_PREFIX from dbt.tests.util import run_dbt_and_capture @@ -22,9 +21,9 @@ def parse_json_logs(json_log_output): class TestCustomVarInLogs: @pytest.fixture(scope="class", autouse=True) def setup(self): - os.environ[METADATA_ENV_PREFIX + "some_var"] = "value" + os.environ["DBT_ENV_CUSTOM_ENV_some_var"] = "value" yield - del os.environ[METADATA_ENV_PREFIX + "some_var"] + del os.environ["DBT_ENV_CUSTOM_ENV_some_var"] def test_extra_filled(self, project): _, log_output = run_dbt_and_capture(['--log-format=json', 'deps'],) From d6196722286210dac275bcac84129be15960cd3d Mon Sep 17 00:00:00 2001 From: Emily Rockman Date: Thu, 13 Oct 2022 07:11:18 -0500 Subject: [PATCH 13/13] modify test for windows --- tests/functional/context_methods/test_custom_env_vars.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/functional/context_methods/test_custom_env_vars.py b/tests/functional/context_methods/test_custom_env_vars.py index 95f406b02e9..413789c7676 100644 --- a/tests/functional/context_methods/test_custom_env_vars.py +++ b/tests/functional/context_methods/test_custom_env_vars.py @@ -21,12 +21,13 @@ def parse_json_logs(json_log_output): class TestCustomVarInLogs: @pytest.fixture(scope="class", autouse=True) def setup(self): - os.environ["DBT_ENV_CUSTOM_ENV_some_var"] = "value" + # on windows, python uppercases env var names because windows is case insensitive + os.environ["DBT_ENV_CUSTOM_ENV_SOME_VAR"] = "value" yield - del os.environ["DBT_ENV_CUSTOM_ENV_some_var"] + del os.environ["DBT_ENV_CUSTOM_ENV_SOME_VAR"] def test_extra_filled(self, project): _, log_output = run_dbt_and_capture(['--log-format=json', 'deps'],) logs = parse_json_logs(log_output) for log in logs: - assert log['info'].get('extra') == {"some_var": "value"} + assert log['info'].get('extra') == {"SOME_VAR": "value"}