From f97c35106f1472a5246e732bd8e779cf157975c1 Mon Sep 17 00:00:00 2001 From: Tristan Gueguen Date: Mon, 25 Mar 2024 16:06:43 +0100 Subject: [PATCH 1/3] helper(scalingo-logs): print the N longest recent queries --- app/commands.py | 6 ++++ app/helpers/scalingo_logs.py | 58 ++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 app/helpers/scalingo_logs.py diff --git a/app/commands.py b/app/commands.py index c29571d0..9b7e6afa 100644 --- a/app/commands.py +++ b/app/commands.py @@ -15,6 +15,7 @@ from app.domain.regulations import compute_regulation_for_user from app.domain.vehicle import find_vehicle from app.helpers.oauth.models import ThirdPartyApiKey +from app.helpers.scalingo_logs import get_long_requests from app.helpers.xml.greco import temp_write_greco_xml from app.models.controller_control import ControllerControl from app.models.user import User @@ -303,3 +304,8 @@ def load_company_stats(): def temp_command_generate_xm_control(id): control = ControllerControl.query.get(id) temp_write_greco_xml(control) + + +@app.cli.command("log_long_requests", with_appcontext=False) +def log_long_requests(): + get_long_requests(nb_results=20, nb_lines=100000) diff --git a/app/helpers/scalingo_logs.py b/app/helpers/scalingo_logs.py new file mode 100644 index 00000000..b7c5bfef --- /dev/null +++ b/app/helpers/scalingo_logs.py @@ -0,0 +1,58 @@ +import re +import subprocess + + +def extract_timestamp_from_log(log): + return log[:19] + + +def extract_endpoint_from_log(log): + endpoint_pattern = r'/graphql "(\w+)"' + endpoint_match = re.search(endpoint_pattern, log) + return endpoint_match.group(1) if endpoint_match else "" + + +def extract_time_from_log(log): + time_pattern = r"time=(\d+)ms" + time_match = re.search(time_pattern, log) + return int(time_match.group(1)) if time_match else None + + +def extract_user_from_log(log): + username_pattern = r"user=([\w\s]+) user_id" + userid_pattern = r"user_id=([\d]+)" + + username_match = re.search(username_pattern, log) + userid_match = re.search(userid_pattern, log) + + username = username_match.group(1) if username_match else "" + userid = userid_match.group(1) if userid_match else "" + + return (username, userid) + + +def get_long_requests(nb_results=20, nb_lines=10000): + cmd = f"scalingo --region osc-fr1 --app mobilic-api logs --lines {nb_lines} | grep '\[web-' | awk 'NF'" + + lines = [ + l.decode("utf-8") + for l in subprocess.check_output(cmd, shell=True).splitlines() + ] + + data = [] + for line in lines: + time = extract_time_from_log(line) + if time is None: + continue + data.append((time, line)) + data.sort(key=lambda l: l[0], reverse=True) + for d in data[:nb_results]: + (ms, text) = d + (username, userid) = extract_user_from_log(text) + ts = extract_timestamp_from_log(text) + endpoint = extract_endpoint_from_log(text) + print( + " - ".join( + [f"{ms} ms", ts, endpoint, f"id={userid}", f"name={username}"] + ) + ) From 30b1166f09b0f937ccaf9b483a07d47b0aa5dd2a Mon Sep 17 00:00:00 2001 From: Tristan Gueguen Date: Mon, 25 Mar 2024 16:26:27 +0100 Subject: [PATCH 2/3] helper(scalingo-logs): print last requests from user --- app/commands.py | 8 +++++++- app/helpers/scalingo_logs.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/commands.py b/app/commands.py index 9b7e6afa..b516848f 100644 --- a/app/commands.py +++ b/app/commands.py @@ -15,7 +15,7 @@ from app.domain.regulations import compute_regulation_for_user from app.domain.vehicle import find_vehicle from app.helpers.oauth.models import ThirdPartyApiKey -from app.helpers.scalingo_logs import get_long_requests +from app.helpers.scalingo_logs import get_long_requests, get_user_requests from app.helpers.xml.greco import temp_write_greco_xml from app.models.controller_control import ControllerControl from app.models.user import User @@ -309,3 +309,9 @@ def temp_command_generate_xm_control(id): @app.cli.command("log_long_requests", with_appcontext=False) def log_long_requests(): get_long_requests(nb_results=20, nb_lines=100000) + + +@app.cli.command("log_user_requests", with_appcontext=False) +@click.argument("user_id", required=True) +def log_user_requests(user_id): + get_user_requests(user_id=user_id, nb_results=20, nb_lines=1000000) diff --git a/app/helpers/scalingo_logs.py b/app/helpers/scalingo_logs.py index b7c5bfef..ac0bb447 100644 --- a/app/helpers/scalingo_logs.py +++ b/app/helpers/scalingo_logs.py @@ -56,3 +56,15 @@ def get_long_requests(nb_results=20, nb_lines=10000): [f"{ms} ms", ts, endpoint, f"id={userid}", f"name={username}"] ) ) + + +def get_user_requests(user_id, nb_results=20, nb_lines=10000): + cmd = f"scalingo --region osc-fr1 --app mobilic-api logs --lines {nb_lines} | grep 'user_id={user_id}'" + + lines = [ + l.decode("utf-8") + for l in subprocess.check_output(cmd, shell=True).splitlines() + ] + + for l in lines[:nb_results]: + print(l) From 05bf815a94a625ba5c1fba4bbe4e7cceb9dd80c2 Mon Sep 17 00:00:00 2001 From: Gaspard Lonchampt Date: Wed, 3 Apr 2024 18:40:47 +0200 Subject: [PATCH 3/3] helper(scalingo-log): add status code 5xx to search --- app/commands.py | 21 ++++++++++- app/helpers/scalingo_logs.py | 70 ++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 1 deletion(-) diff --git a/app/commands.py b/app/commands.py index b516848f..3904592c 100644 --- a/app/commands.py +++ b/app/commands.py @@ -15,7 +15,11 @@ from app.domain.regulations import compute_regulation_for_user from app.domain.vehicle import find_vehicle from app.helpers.oauth.models import ThirdPartyApiKey -from app.helpers.scalingo_logs import get_long_requests, get_user_requests +from app.helpers.scalingo_logs import ( + get_long_requests, + get_user_requests, + get_status_5xx_requests_all, +) from app.helpers.xml.greco import temp_write_greco_xml from app.models.controller_control import ControllerControl from app.models.user import User @@ -315,3 +319,18 @@ def log_long_requests(): @click.argument("user_id", required=True) def log_user_requests(user_id): get_user_requests(user_id=user_id, nb_results=20, nb_lines=1000000) + + +@app.cli.command("log_status_5xx_requests_all", with_appcontext=False) +@click.argument("nb_lines", required=True, type=int) +@click.option( + "--source", + required=True, + type=click.Choice(["scalingo", "file"]), + help="'scalingo' or 'file'.", +) +@click.option("--file_path", required=False, type=str, help="File path.") +def log_status_5xx_requests_all(nb_lines, source, file_path): + get_status_5xx_requests_all( + nb_lines=nb_lines, source_type=source, file_path=file_path + ) diff --git a/app/helpers/scalingo_logs.py b/app/helpers/scalingo_logs.py index ac0bb447..460515f2 100644 --- a/app/helpers/scalingo_logs.py +++ b/app/helpers/scalingo_logs.py @@ -12,12 +12,30 @@ def extract_endpoint_from_log(log): return endpoint_match.group(1) if endpoint_match else "" +def extract_path_from_log(log): + path_pattern = r'path="([^"]+)"' + path_match = re.search(path_pattern, log) + return path_match.group(1) if path_match else "" + + +def extract_referer_from_log(log): + referer_pattern = r'referer="([^"]+)"' + referer_match = re.search(referer_pattern, log) + return referer_match.group(1) if referer_match else "" + + def extract_time_from_log(log): time_pattern = r"time=(\d+)ms" time_match = re.search(time_pattern, log) return int(time_match.group(1)) if time_match else None +def extract_duration_from_log(log): + time_pattern = r"duration=([\d.]+)s" + time_match = re.search(time_pattern, log) + return float(time_match.group(1)) if time_match else None + + def extract_user_from_log(log): username_pattern = r"user=([\w\s]+) user_id" userid_pattern = r"user_id=([\d]+)" @@ -31,6 +49,18 @@ def extract_user_from_log(log): return (username, userid) +def extract_status_code_from_log(log): + status_code_pattern = r"status=(5\d{2})" + status_code_match = re.search(status_code_pattern, log) + return status_code_match.group(1) if status_code_match else "" + + +def extract_request_id_from_log(log): + request_id_pattern = r"request_id=([^ ]+)" + request_id_match = re.search(request_id_pattern, log) + return request_id_match.group(1) if request_id_match else "" + + def get_long_requests(nb_results=20, nb_lines=10000): cmd = f"scalingo --region osc-fr1 --app mobilic-api logs --lines {nb_lines} | grep '\[web-' | awk 'NF'" @@ -68,3 +98,43 @@ def get_user_requests(user_id, nb_results=20, nb_lines=10000): for l in lines[:nb_results]: print(l) + + +def get_status_5xx_requests_all(nb_lines, source_type, file_path=None): + if source_type == "scalingo": + cmd = f"scalingo --region osc-fr1 --app mobilic-api logs --lines {nb_lines} | grep 'status=5' || true" + elif source_type == "file" and file_path: + cmd = f"cat {file_path} | grep 'status=5'" + else: + raise ValueError( + "source_type must be 'scalingo' or 'file' with a valid file_path." + ) + + lines = [ + l.decode("utf-8") + for l in subprocess.check_output(cmd, shell=True).splitlines() + ] + + lines_sorted = sorted(lines, key=lambda x: x[:19]) + + for l in lines_sorted: + ts = extract_timestamp_from_log(l) + status_code = extract_status_code_from_log(l) + duration = extract_duration_from_log(l) + path = extract_path_from_log(l) + referer = extract_referer_from_log(l) + request_id = extract_request_id_from_log(l) + + duration_str = f"{duration}s" if duration is not None else "N/A" + print( + " - ".join( + [ + ts, + f"status={status_code}", + f"duration={duration_str}", + f"path={path}", + f"referer={referer}", + f"id={request_id}", + ] + ) + )