From 77e37b0b1ca3fa81bfe4fdbe48e8b11a300e8c76 Mon Sep 17 00:00:00 2001 From: Paolo Quadri Date: Fri, 24 Nov 2023 01:00:57 +0100 Subject: [PATCH] feat: configurable metabase api timeotus (#187) Co-authored-by: Paolo Quadri --- dbtmetabase/__init__.py | 16 ++++++++++++++++ dbtmetabase/metabase.py | 7 ++++--- dbtmetabase/models/interface.py | 3 +++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/dbtmetabase/__init__.py b/dbtmetabase/__init__.py index c9fc959e..da2f2da0 100644 --- a/dbtmetabase/__init__.py +++ b/dbtmetabase/__init__.py @@ -24,6 +24,7 @@ "MB_HOST", "MB_DATABASE", "MB_SESSION_TOKEN", + "MB_HTTP_TIMEOUT", ] @@ -289,6 +290,15 @@ def shared_opts(func: Callable) -> Callable: multiple=True, help="Additional HTTP request header to be sent to Metabase.", ) + @click.option( + "--metabase_http_timeout", + cls=OptionAcceptableFromConfig, + type=int, + default=15, + envvar="MB_HTTP_TIMEOUT", + show_envvar=True, + help="Set the value for single requests timeout", + ) @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) @@ -559,6 +569,7 @@ def models( metabase_sync: bool = True, metabase_sync_timeout: Optional[int] = None, metabase_exclude_sources: bool = False, + metabase_http_timeout: int = 15, dbt_include_tags: bool = True, dbt_docs_url: Optional[str] = None, verbose: bool = False, @@ -585,6 +596,7 @@ def models( metabase_sync (bool, optional): Attempt to synchronize Metabase schema with local models. Defaults to True. metabase_sync_timeout (Optional[int], optional): Synchronization timeout (in secs). If set, we will fail hard on synchronization failure; if not set, we will proceed after attempting sync regardless of success. Only valid if sync is enabled. Defaults to None. metabase_exclude_sources (bool, optional): Flag to skip exporting sources to Metabase. Defaults to False. + metabase_http_timeout (int, optional): Set the timeout for the single Metabase requests. Defaults to 15. dbt_include_tags (bool, optional): Flag to append tags to table descriptions in Metabase. Defaults to True. dbt_docs_url (Optional[str], optional): Pass in URL to dbt docs site. Appends dbt docs URL for each model to Metabase table description. Defaults to None. http_extra_headers (Optional[str], optional): Additional HTTP request headers to be sent to Metabase. Defaults to None. @@ -625,6 +637,7 @@ def models( sync_timeout=metabase_sync_timeout, exclude_sources=metabase_exclude_sources, http_extra_headers=http_extra_headers, + http_timeout=metabase_http_timeout, ) # Load client @@ -685,6 +698,7 @@ def exposures( metabase_verify: Optional[str] = None, metabase_sync: bool = True, metabase_sync_timeout: Optional[int] = None, + metabase_http_timeout: int = 15, output_path: str = ".", output_name: str = "metabase_exposures.yml", include_personal_collections: bool = False, @@ -711,6 +725,7 @@ def exposures( metabase_verify (Optional[str], optional): Path to custom certificate bundle to be used by Metabase client. Defaults to None. metabase_sync (bool, optional): Attempt to synchronize Metabase schema with local models. Defaults to True. metabase_sync_timeout (Optional[int], optional): Synchronization timeout (in secs). If set, we will fail hard on synchronization failure; if not set, we will proceed after attempting sync regardless of success. Only valid if sync is enabled. Defaults to None. + metabase_http_timeout (int, optional): Set the timeout for the single Metabase requests. Default 15 output_path (str): Output path for generated exposure yaml. Defaults to "." local dir. output_name (str): Output name for generated exposure yaml. Defaults to metabase_exposures.yml. include_personal_collections (bool, optional): Flag to include Personal Collections during exposure parsing. Defaults to False. @@ -749,6 +764,7 @@ def exposures( sync=metabase_sync, sync_timeout=metabase_sync_timeout, http_extra_headers=http_extra_headers, + http_timeout=metabase_http_timeout, ) # Load client diff --git a/dbtmetabase/metabase.py b/dbtmetabase/metabase.py index e4100755..f880d79b 100644 --- a/dbtmetabase/metabase.py +++ b/dbtmetabase/metabase.py @@ -128,6 +128,7 @@ def __init__( sync_timeout: Optional[int] = None, exclude_sources: bool = False, http_extra_headers: Optional[dict] = None, + http_timeout: int = 15, ): """Constructor. @@ -165,7 +166,7 @@ def __init__( self.table_map: MutableMapping = {} self.models_exposed: List = [] self.native_query: str = "" - + self.http_timeout = http_timeout # This regex is looking for from and join clauses, and extracting the table part. # It won't recognize some valid sql table references, such as `from "table with spaces"`. self.exposure_parser = re.compile(r"[FfJj][RrOo][OoIi][MmNn]\s+([\w.\"]+)") @@ -723,7 +724,7 @@ def increase_indent(self, flow=False, indentless=False): creator = self.api("get", f"/api/user/{exposure['creator_id']}") except requests.exceptions.HTTPError as error: creator = {} - if error.response.status_code != 404: + if error.response is None or error.response.status_code != 404: raise creator_email = creator.get("email") @@ -982,7 +983,7 @@ def api( response = self.session.request( method, f"{self.base_url}{path}", - timeout=15, + timeout=self.http_timeout, **kwargs, ) diff --git a/dbtmetabase/models/interface.py b/dbtmetabase/models/interface.py index 436ac73b..cdeb37e2 100644 --- a/dbtmetabase/models/interface.py +++ b/dbtmetabase/models/interface.py @@ -33,6 +33,7 @@ def __init__( sync_timeout: Optional[int] = None, exclude_sources: bool = False, http_extra_headers: Optional[dict] = None, + http_timeout: int = 15, ): """Constructor. @@ -62,6 +63,7 @@ def __init__( self.verify = verify self.cert = cert self.http_extra_headers = dict(http_extra_headers) if http_extra_headers else {} + self.http_timeout = http_timeout # Metabase Sync self.sync = sync self.sync_timeout = sync_timeout @@ -109,6 +111,7 @@ def prepare_metabase_client(self, dbt_models: Optional[List[MetabaseModel]] = No sync=self.sync, sync_timeout=self.sync_timeout, exclude_sources=self.exclude_sources, + http_timeout=self.http_timeout, ) # Sync and attempt schema alignment prior to execution; if timeout is not explicitly set, proceed regardless of success