From 27932139fb7f6effeacbc949d682a1fc6cff0a18 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Fri, 6 May 2022 19:59:37 -0600 Subject: [PATCH 1/2] Check exceptions in force_flush too Fixes #2665 --- .../sdk/_metrics/_internal/__init__.py | 37 +++++++++++++++++-- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/_internal/__init__.py index eebcb0b13cf..b3e81075903 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/_internal/__init__.py @@ -373,12 +373,41 @@ def __init__( def force_flush(self, timeout_millis: float = 10_000) -> bool: deadline_ns = _time_ns() + timeout_millis * 10**6 + metric_reader_error = {} + for metric_reader in self._sdk_config.metric_readers: current_ts = _time_ns() - if current_ts >= deadline_ns: - raise Exception("Timed out while flushing metric readers") - metric_reader.collect( - timeout_millis=(deadline_ns - current_ts) / 10**6 + try: + if current_ts >= deadline_ns: + raise Exception("Timed out while flushing metric readers") + metric_reader.collect( + timeout_millis=(deadline_ns - current_ts) / 10**6 + ) + + # pylint: disable=broad-except + except Exception as error: + + metric_reader_error[metric_reader] = error + + if self._atexit_handler is not None: + unregister(self._atexit_handler) + self._atexit_handler = None + + if metric_reader_error: + + metric_reader_error_string = "\n".join( + [ + f"{metric_reader.__class__.__name__}: {repr(error)}" + for metric_reader, error in metric_reader_error.items() + ] + ) + + raise Exception( + ( + "MeterProvider.force_flush failed because the following " + "metric readers failed during collect:\n" + f"{metric_reader_error_string}" + ) ) return True From e338c7f9cdb14e4568967ba24b87ac0469f15e67 Mon Sep 17 00:00:00 2001 From: Diego Hurtado Date: Sat, 7 May 2022 17:36:51 -0600 Subject: [PATCH 2/2] Address feedback --- .../opentelemetry/sdk/_metrics/_internal/__init__.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/_internal/__init__.py b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/_internal/__init__.py index b3e81075903..733f84e6c20 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/_internal/__init__.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/_metrics/_internal/__init__.py @@ -389,10 +389,6 @@ def force_flush(self, timeout_millis: float = 10_000) -> bool: metric_reader_error[metric_reader] = error - if self._atexit_handler is not None: - unregister(self._atexit_handler) - self._atexit_handler = None - if metric_reader_error: metric_reader_error_string = "\n".join( @@ -403,11 +399,9 @@ def force_flush(self, timeout_millis: float = 10_000) -> bool: ) raise Exception( - ( - "MeterProvider.force_flush failed because the following " - "metric readers failed during collect:\n" - f"{metric_reader_error_string}" - ) + "MeterProvider.force_flush failed because the following " + "metric readers failed during collect:\n" + f"{metric_reader_error_string}" ) return True