From e4b902262218dd0de043f92c791eb0dfac3ee9e9 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Wed, 22 May 2024 14:10:35 +0200 Subject: [PATCH 1/2] fix(prepro): Log instead of print, reduce sleep time to 2s from 10 --- preprocessing/dummy/main.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/preprocessing/dummy/main.py b/preprocessing/dummy/main.py index b15a4ed72..171cb59aa 100644 --- a/preprocessing/dummy/main.py +++ b/preprocessing/dummy/main.py @@ -1,6 +1,7 @@ import argparse import dataclasses import json +import logging import random import time from dataclasses import dataclass, field @@ -8,6 +9,8 @@ import requests +logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser() parser.add_argument("--backend-host", type=str, default="http://127.0.0.1:8079", help="Host address of the loculus backend") @@ -68,7 +71,7 @@ def fetch_unprocessed_sequences(n: int) -> List[Sequence]: response = requests.post(url, data=params, headers=headers) if not response.ok: if response.status_code == 422: - print("{}. Sleeping for a while.".format(response.text)) + logging.debug("{}. Sleeping for a while.".format(response.text)) time.sleep(60 * 10) return [] raise Exception("Fetching unprocessed data failed. Status code: {}".format(response.status_code), response.text) @@ -171,7 +174,7 @@ def main(): locally_processed = 0 if watch_mode: - print("Started in watch mode - waiting 10 seconds before fetching data.") + logging.debug("Started in watch mode - waiting 10 seconds before fetching data.") time.sleep(10) if args.maxSequences and args.maxSequences < 100: @@ -183,8 +186,8 @@ def main(): unprocessed = fetch_unprocessed_sequences(sequences_to_fetch) if len(unprocessed) == 0: if watch_mode: - print("Processed {} sequences. Sleeping for 10 seconds.".format(locally_processed)) - time.sleep(10) + logging.debug("Processed {} sequences. Sleeping for 10 seconds.".format(locally_processed)) + time.sleep(2) locally_processed = 0 continue else: @@ -196,7 +199,7 @@ def main(): if args.maxSequences and total_processed >= args.maxSequences: break - print("Total processed sequences: {}".format(total_processed)) + logging.debug("Total processed sequences: {}".format(total_processed)) if __name__ == "__main__": From 6cb23e52df1780bf72ba4655d9c16c4a10682c24 Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Wed, 22 May 2024 14:12:55 +0200 Subject: [PATCH 2/2] fix: log at debug --- preprocessing/dummy/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/preprocessing/dummy/main.py b/preprocessing/dummy/main.py index 171cb59aa..7129cb8b9 100644 --- a/preprocessing/dummy/main.py +++ b/preprocessing/dummy/main.py @@ -9,7 +9,7 @@ import requests -logging.basicConfig(level=logging.INFO) +logging.basicConfig(level=logging.DEBUG) parser = argparse.ArgumentParser() parser.add_argument("--backend-host", type=str, default="http://127.0.0.1:8079",