From 7f894998885a2be0e984d70689bca138c719f893 Mon Sep 17 00:00:00 2001 From: Ildus Date: Tue, 25 Jan 2022 15:18:37 +0300 Subject: [PATCH 01/10] logging fixes for log storing update --- core/logging/logger_utils.py | 1 + core/model/base_user.py | 4 ++++ scenarios/user/user_model.py | 4 ++-- smart_kit/handlers/handle_respond.py | 2 +- smart_kit/management/app_manager.py | 1 + smart_kit/start_points/base_main_loop.py | 2 +- smart_kit/start_points/main_loop_async_http.py | 2 +- smart_kit/start_points/main_loop_kafka.py | 2 +- smart_kit/template/static/configs/logging_config.yml | 3 ++- 9 files changed, 14 insertions(+), 7 deletions(-) diff --git a/core/logging/logger_utils.py b/core/logging/logger_utils.py index 4687ee0b..f5359aaf 100644 --- a/core/logging/logger_utils.py +++ b/core/logging/logger_utils.py @@ -80,6 +80,7 @@ def log(message, user=None, params=None, level="INFO", exc_info=None, log_store_ except AttributeError: message_maker = LoggerMessageCreator + log_store_for = logging.log_store_for_map.get(params.get(log_const.KEY_NAME), log_store_for) if instance is not None: params = message_maker.make_message(user, params, instance.__class__.__name__, log_store_for) else: diff --git a/core/model/base_user.py b/core/model/base_user.py index 176abae1..7dfffe8b 100644 --- a/core/model/base_user.py +++ b/core/model/base_user.py @@ -62,6 +62,10 @@ def raw(self): "uid": str(self.id)}) return super(BaseUser, self).raw + @property + def raw_without_log(self): + return json.dumps(super(BaseUser, self).raw) + @property def raw_str(self): return json.dumps(self.raw) diff --git a/scenarios/user/user_model.py b/scenarios/user/user_model.py index 9cdd9939..75af4818 100644 --- a/scenarios/user/user_model.py +++ b/scenarios/user/user_model.py @@ -48,8 +48,8 @@ def _initialize(self): db_version = self.variables.get(self.USER_DB_VERSION, default=0) self.variables.set(self.USER_DB_VERSION, db_version + 1) log("%(class_name)s.__init__ USER %(uid)s LOAD db_version = %(db_version)s.", self, - {"db_version": str(db_version), - "uid": str(self.id)}) + params={"db_version": str(db_version), + "uid": str(self.id), log_const.KEY_NAME:"user_load"}) self.behaviors.initialize() @property diff --git a/smart_kit/handlers/handle_respond.py b/smart_kit/handlers/handle_respond.py index 35bb6988..d4d2cf9c 100644 --- a/smart_kit/handlers/handle_respond.py +++ b/smart_kit/handlers/handle_respond.py @@ -29,7 +29,7 @@ def run(self, payload, user): action_params = self.get_action_params(payload, user) action_name = self.get_action_name(payload, user) params = { - log_const.KEY_NAME: "handling_respond", + log_const.KEY_NAME: "process_time", "callback_id": str(callback_id), "process_time": self.get_processing_time(user), "action_name": action_name diff --git a/smart_kit/management/app_manager.py b/smart_kit/management/app_manager.py index c0af60e3..eaab9f26 100644 --- a/smart_kit/management/app_manager.py +++ b/smart_kit/management/app_manager.py @@ -15,6 +15,7 @@ def init_logger(app_config): logger_config = LoggerConfig(app_config.CONFIGS_PATH) logger_config.init() config = logger_config[LOGGING_CONFIG] + logging.log_store_for_map = config.pop('log_store_for_map') logging.config.dictConfig(config) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index e997a7a3..f7ed60e5 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -120,7 +120,7 @@ def save_user(self, db_uid, user, message): no_collisions = True try: - str_data = user.raw_str + str_data = user.raw_without_log log("Saving User %(uid)s. Serialized utf8 json length is %(user_length)s symbols.", user=user, params={"uid": user.id, log_const.KEY_NAME: "user_save", diff --git a/smart_kit/start_points/main_loop_async_http.py b/smart_kit/start_points/main_loop_async_http.py index e893b9ea..ad908208 100644 --- a/smart_kit/start_points/main_loop_async_http.py +++ b/smart_kit/start_points/main_loop_async_http.py @@ -69,7 +69,7 @@ async def save_user(self, db_uid, user, message): no_collisions = True try: - str_data = user.raw_str + str_data = user.raw_without_log log("Saving User %(uid)s. Serialized utf8 json length is %(user_length)s symbols.", user=user, params={"uid": user.id, log_const.KEY_NAME: "user_save", diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index 1b1e68a5..abab1525 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -411,7 +411,7 @@ def _send_request(self, user, answer, mq_message): def _log_request(self, user, request, answer, original_mq_message): log("OUTGOING TO TOPIC_KEY: %(topic_key)s DATA: %(data)s", - params={log_const.KEY_NAME: "outgoing_message", + params={log_const.KEY_NAME: "outgoing_message" + request.topic_key, "topic_key": request.topic_key, "headers": str(request._get_new_headers(original_mq_message)), "data": answer.masked_value}, user=user) diff --git a/smart_kit/template/static/configs/logging_config.yml b/smart_kit/template/static/configs/logging_config.yml index 58eba28d..9005efb6 100644 --- a/smart_kit/template/static/configs/logging_config.yml +++ b/smart_kit/template/static/configs/logging_config.yml @@ -46,4 +46,5 @@ formatters: format: "[{levelname}] {name} {asctime} {module} {args[uid]} {args[message_id]} {args[logging_uuid]} {args[class_name]}: {message}" console_simple_formatter: - format: "[%(levelname)s] %(name)s %(asctime)s %(module)s: %(message)s" \ No newline at end of file + format: "[%(levelname)s] %(name)s %(asctime)s %(module)s: %(message)s" +log_store_for_map: \ No newline at end of file From d8af45a795de1bc6ecc01ed5d1093e84e7480bec Mon Sep 17 00:00:00 2001 From: Ildus Date: Wed, 26 Jan 2022 17:23:43 +0300 Subject: [PATCH 02/10] logging fixes for log storing update --- core/logging/logger_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/logging/logger_utils.py b/core/logging/logger_utils.py index f5359aaf..ac5a9ec0 100644 --- a/core/logging/logger_utils.py +++ b/core/logging/logger_utils.py @@ -80,7 +80,7 @@ def log(message, user=None, params=None, level="INFO", exc_info=None, log_store_ except AttributeError: message_maker = LoggerMessageCreator - log_store_for = logging.log_store_for_map.get(params.get(log_const.KEY_NAME), log_store_for) + log_store_for = logging.log_store_for_map.get(params.get(log_const.KEY_NAME, {}), log_store_for) if instance is not None: params = message_maker.make_message(user, params, instance.__class__.__name__, log_store_for) else: From 89a5a52ec6b81236ca54f4a8658e5c280f162158 Mon Sep 17 00:00:00 2001 From: Ildus Date: Fri, 4 Feb 2022 17:01:54 +0300 Subject: [PATCH 03/10] DPNLPF-1389: add log store map checking --- core/logging/logger_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/logging/logger_utils.py b/core/logging/logger_utils.py index ac5a9ec0..ae8aea3c 100644 --- a/core/logging/logger_utils.py +++ b/core/logging/logger_utils.py @@ -80,7 +80,10 @@ def log(message, user=None, params=None, level="INFO", exc_info=None, log_store_ except AttributeError: message_maker = LoggerMessageCreator - log_store_for = logging.log_store_for_map.get(params.get(log_const.KEY_NAME, {}), log_store_for) + log_store_for_map = getattr(logging,"log_store_for_map") + if log_store_for_map is not None and params is not None: + log_store_for = log_store_for_map.get(params.get(log_const.KEY_NAME), log_store_for) + if instance is not None: params = message_maker.make_message(user, params, instance.__class__.__name__, log_store_for) else: From afaa4698735d222d7f3c4ff6b210ea3404892e61 Mon Sep 17 00:00:00 2001 From: Ildus Date: Mon, 7 Feb 2022 11:53:08 +0300 Subject: [PATCH 04/10] DPNLPF-1389: config get store map add --- smart_kit/management/app_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/smart_kit/management/app_manager.py b/smart_kit/management/app_manager.py index eaab9f26..36ec01c4 100644 --- a/smart_kit/management/app_manager.py +++ b/smart_kit/management/app_manager.py @@ -15,7 +15,7 @@ def init_logger(app_config): logger_config = LoggerConfig(app_config.CONFIGS_PATH) logger_config.init() config = logger_config[LOGGING_CONFIG] - logging.log_store_for_map = config.pop('log_store_for_map') + logging.log_store_for_map = config.get('log_store_for_map', None) logging.config.dictConfig(config) From 41770c265e7336436f2ee17567c51d070e3001a7 Mon Sep 17 00:00:00 2001 From: Ildus Date: Tue, 8 Feb 2022 06:39:34 +0300 Subject: [PATCH 05/10] DPNLPF-1389: add none for gettatr --- core/logging/logger_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/logging/logger_utils.py b/core/logging/logger_utils.py index a67c1d40..1b0fb7e4 100644 --- a/core/logging/logger_utils.py +++ b/core/logging/logger_utils.py @@ -83,7 +83,7 @@ def log(message, user=None, params=None, level="INFO", exc_info=None, log_store_ except AttributeError: message_maker = LoggerMessageCreator - log_store_for_map = getattr(logging,"log_store_for_map") + log_store_for_map = getattr(logging,"log_store_for_map", None) if log_store_for_map is not None and params is not None: log_store_for = log_store_for_map.get(params.get(log_const.KEY_NAME), log_store_for) From 9ac837ee3462fdbf1ed0769ca09e884fca5df35c Mon Sep 17 00:00:00 2001 From: Ildus Date: Thu, 10 Feb 2022 08:58:04 +0300 Subject: [PATCH 06/10] DPNLPF-1389: raw log fix, topic name in log fix --- core/model/base_user.py | 4 ---- smart_kit/start_points/base_main_loop.py | 6 +----- smart_kit/start_points/main_loop_async_http.py | 6 +----- smart_kit/start_points/main_loop_kafka.py | 6 +++++- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/core/model/base_user.py b/core/model/base_user.py index 7dfffe8b..176abae1 100644 --- a/core/model/base_user.py +++ b/core/model/base_user.py @@ -62,10 +62,6 @@ def raw(self): "uid": str(self.id)}) return super(BaseUser, self).raw - @property - def raw_without_log(self): - return json.dumps(super(BaseUser, self).raw) - @property def raw_str(self): return json.dumps(self.raw) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 2449bfce..7275f31a 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -123,11 +123,7 @@ def save_user(self, db_uid, user, message): no_collisions = True try: - str_data = user.raw_without_log - log("Saving User %(uid)s. Serialized utf8 json length is %(user_length)s symbols.", user=user, - params={"uid": user.id, - log_const.KEY_NAME: "user_save", - "user_length": len(str_data)}) + str_data = user.raw if user.initial_db_data and self.user_save_check_for_collisions: no_collisions = self.db_adapter.replace_if_equals(db_uid, sample=user.initial_db_data, diff --git a/smart_kit/start_points/main_loop_async_http.py b/smart_kit/start_points/main_loop_async_http.py index ad908208..77c40f11 100644 --- a/smart_kit/start_points/main_loop_async_http.py +++ b/smart_kit/start_points/main_loop_async_http.py @@ -69,11 +69,7 @@ async def save_user(self, db_uid, user, message): no_collisions = True try: - str_data = user.raw_without_log - log("Saving User %(uid)s. Serialized utf8 json length is %(user_length)s symbols.", user=user, - params={"uid": user.id, - log_const.KEY_NAME: "user_save", - "user_length": len(str_data)}) + str_data = user.raw if self.db_adapter.IS_ASYNC: if user.initial_db_data and self.user_save_check_for_collisions: diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index c706a321..f9b32e8e 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -405,8 +405,12 @@ def _send_request(self, user, answer, mq_message): self._log_request(user, request, answer, mq_message) def _log_request(self, user, request, answer, original_mq_message): + if request.topic_key is not None: + topic = request.topic_key + else: + topic = request.topic log("OUTGOING TO TOPIC_KEY: %(topic_key)s DATA: %(data)s", - params={log_const.KEY_NAME: "outgoing_message" + request.topic_key, + params={log_const.KEY_NAME: f"outgoing_message_{topic}", "topic_key": request.topic_key, "headers": str(request._get_new_headers(original_mq_message)), "data": answer.masked_value, From fe285ff0ff269487dfaec77ac8bd2fa514b1fe7e Mon Sep 17 00:00:00 2001 From: Ildus Date: Thu, 10 Feb 2022 09:11:59 +0300 Subject: [PATCH 07/10] DPNLPF-1389: raw log fix, topic name in log fix --- smart_kit/start_points/base_main_loop.py | 2 +- smart_kit/start_points/main_loop_async_http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/smart_kit/start_points/base_main_loop.py b/smart_kit/start_points/base_main_loop.py index 7275f31a..1be19481 100644 --- a/smart_kit/start_points/base_main_loop.py +++ b/smart_kit/start_points/base_main_loop.py @@ -123,7 +123,7 @@ def save_user(self, db_uid, user, message): no_collisions = True try: - str_data = user.raw + str_data = user.raw_str if user.initial_db_data and self.user_save_check_for_collisions: no_collisions = self.db_adapter.replace_if_equals(db_uid, sample=user.initial_db_data, diff --git a/smart_kit/start_points/main_loop_async_http.py b/smart_kit/start_points/main_loop_async_http.py index 77c40f11..b4bbd5ce 100644 --- a/smart_kit/start_points/main_loop_async_http.py +++ b/smart_kit/start_points/main_loop_async_http.py @@ -69,7 +69,7 @@ async def save_user(self, db_uid, user, message): no_collisions = True try: - str_data = user.raw + str_data = user.raw_str if self.db_adapter.IS_ASYNC: if user.initial_db_data and self.user_save_check_for_collisions: From 594d3ceb40313c31193d3348ca4ece92a0e04fe3 Mon Sep 17 00:00:00 2001 From: Ildus Date: Thu, 10 Feb 2022 16:40:03 +0300 Subject: [PATCH 08/10] DPNLPF-1389: topic key remove from key name --- smart_kit/start_points/main_loop_kafka.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/smart_kit/start_points/main_loop_kafka.py b/smart_kit/start_points/main_loop_kafka.py index f9b32e8e..a956241e 100644 --- a/smart_kit/start_points/main_loop_kafka.py +++ b/smart_kit/start_points/main_loop_kafka.py @@ -405,12 +405,8 @@ def _send_request(self, user, answer, mq_message): self._log_request(user, request, answer, mq_message) def _log_request(self, user, request, answer, original_mq_message): - if request.topic_key is not None: - topic = request.topic_key - else: - topic = request.topic log("OUTGOING TO TOPIC_KEY: %(topic_key)s DATA: %(data)s", - params={log_const.KEY_NAME: f"outgoing_message_{topic}", + params={log_const.KEY_NAME: "outgoing_message", "topic_key": request.topic_key, "headers": str(request._get_new_headers(original_mq_message)), "data": answer.masked_value, From eac84409a9aa32d80f3f8c81a3cf7d5c85466062 Mon Sep 17 00:00:00 2001 From: Ildus Date: Tue, 15 Feb 2022 13:00:57 +0300 Subject: [PATCH 09/10] DPNLPF-1389: move raw to add user length to log --- core/model/base_user.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/core/model/base_user.py b/core/model/base_user.py index 176abae1..9b68ba5f 100644 --- a/core/model/base_user.py +++ b/core/model/base_user.py @@ -57,10 +57,11 @@ def fields(self): @property def raw(self): + raw = super(BaseUser, self).raw log("%(class_name)s.raw USER %(uid)s SAVE db_version = %(db_version)s.", self, {"db_version": str(self.variables.get(self.USER_DB_VERSION)), - "uid": str(self.id)}) - return super(BaseUser, self).raw + "uid": str(self.id), "user_length":len(raw)}) + return raw @property def raw_str(self): From a4a8539e76ac17eef0ae6131f3e6096815c011da Mon Sep 17 00:00:00 2001 From: Ildus Date: Wed, 16 Feb 2022 10:52:04 +0300 Subject: [PATCH 10/10] DPNLPF-1389: move raw log to add user length to log --- core/model/base_user.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/model/base_user.py b/core/model/base_user.py index 9b68ba5f..838a5cff 100644 --- a/core/model/base_user.py +++ b/core/model/base_user.py @@ -57,16 +57,16 @@ def fields(self): @property def raw(self): - raw = super(BaseUser, self).raw + return super(BaseUser, self).raw + + @property + def raw_str(self): + raw = json.dumps(self.raw) log("%(class_name)s.raw USER %(uid)s SAVE db_version = %(db_version)s.", self, {"db_version": str(self.variables.get(self.USER_DB_VERSION)), "uid": str(self.id), "user_length":len(raw)}) return raw - @property - def raw_str(self): - return json.dumps(self.raw) - def expire(self): self.counters.expire() self.variables.expire()