From dc412f2f7251a47e1146ef6422a7b24d0b7ab8e9 Mon Sep 17 00:00:00 2001 From: knguyen Date: Thu, 28 Oct 2021 14:11:00 +0700 Subject: [PATCH 1/3] refs #57802: Add function to check log update delay exceeded --- server/app/routes.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/server/app/routes.py b/server/app/routes.py index 92b2c530..0840a1f7 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -9,6 +9,7 @@ import traceback from collections import Counter from copy import deepcopy +from datetime import datetime from functools import wraps import flask @@ -2345,6 +2346,16 @@ def post_log(task_id): return flask.jsonify(200) +def check_exceed_log_update_delay(log_file): + with open(log_file, 'r') as f: + last_log_time_str = f.readlines()[-1].split('UTC')[0].strip() + last_log_time = datetime.strptime(last_log_time_str, '%Y-%b-%d %H:%M:%S.%f') + time_delta = (datetime.utcnow() - last_log_time).total_seconds() + if time_delta > app.get_other_config(['default', 'log_update_delay'], fallback=3600): + return True + return False + + @app.route("/task/stat/", methods=["POST"]) @filter_request("POST/task/stat") @task_request From 88230720514417a31385fede3eae11cf5f670819 Mon Sep 17 00:00:00 2001 From: knguyen Date: Thu, 28 Oct 2021 17:48:28 +0700 Subject: [PATCH 2/3] refs #57802: Send warning email if log update delay is exceeded --- server/app/routes.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/server/app/routes.py b/server/app/routes.py index 0840a1f7..0e855fa6 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -11,6 +11,7 @@ from copy import deepcopy from datetime import datetime from functools import wraps +from threading import Thread import flask import semver @@ -30,6 +31,7 @@ TaskScoring, TASK_RELEASE_TYPE from utils.common_utils import is_resource_train_restricted, check_permission_access_train_restricted from utils.storage_utils import StorageUtils +from utils.email_utils import send_task_status_notification_email GLOBAL_POOL_NAME = "global_pool" SYSTRAN_BASE_STORAGE = "shared_testdata" @@ -2264,6 +2266,12 @@ def terminate_internal(task_id): @filter_request("PUT/task/beat") @task_write_control def task_beat(task_id): + log_file = os.path.join(taskfile_dir, task_id, 'log') + if os.path.isfile(log_file) and check_exceed_log_update_delay(log_file) and app.get_other_config( + ['email', 'allow_send_mail'], fallback=True): + infos = task.info(redis_db, taskfile_dir, task_id, ["type", "content", "eval_model", "model"]) + Thread(target=send_task_status_notification_email, + args=({**infos, **{"id": task_id}}, 'running', g.user.receive_mail)).start() duration = flask.request.args.get('duration') try: if duration is not None: From d1898b76cfac178b4294fede224641c10d462652 Mon Sep 17 00:00:00 2001 From: knguyen Date: Fri, 29 Oct 2021 09:12:48 +0700 Subject: [PATCH 3/3] refs #57802: Update api task/beat --- server/app/routes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/server/app/routes.py b/server/app/routes.py index 0e855fa6..ff45254e 100644 --- a/server/app/routes.py +++ b/server/app/routes.py @@ -2266,12 +2266,6 @@ def terminate_internal(task_id): @filter_request("PUT/task/beat") @task_write_control def task_beat(task_id): - log_file = os.path.join(taskfile_dir, task_id, 'log') - if os.path.isfile(log_file) and check_exceed_log_update_delay(log_file) and app.get_other_config( - ['email', 'allow_send_mail'], fallback=True): - infos = task.info(redis_db, taskfile_dir, task_id, ["type", "content", "eval_model", "model"]) - Thread(target=send_task_status_notification_email, - args=({**infos, **{"id": task_id}}, 'running', g.user.receive_mail)).start() duration = flask.request.args.get('duration') try: if duration is not None: @@ -2283,6 +2277,12 @@ def task_beat(task_id): task.beat(redis_db, task_id, duration, container_id) except Exception as e: abort(flask.make_response(flask.jsonify(message=str(e)), 400)) + log_file = os.path.join(taskfile_dir, task_id, 'log') + if os.path.isfile(log_file) and check_exceed_log_update_delay(log_file) and app.get_other_config( + ['email', 'allow_send_mail'], fallback=True): + infos = task.info(redis_db, taskfile_dir, task_id, ["type", "content", "eval_model", "model"]) + Thread(target=send_task_status_notification_email, + args=({**infos, **{"id": task_id}}, 'running', g.user.receive_mail)).start() return flask.jsonify(200)