Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(prepro/dummy): Log instead of print, reduce sleep time to 2s from 10 #2006

Merged
merged 2 commits into from
May 22, 2024
Merged

fix(prepro/dummy): Log instead of print, reduce sleep time to 2s from 10 #2006

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e4b9022
fix(prepro): Log instead of print, reduce sleep time to 2s from 10
corneliusroemer May 22, 2024
6cb23e5
fix: log at debug
corneliusroemer May 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions preprocessing/dummy/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
import argparse
import dataclasses
import json
import logging
import random
import time
from dataclasses import dataclass, field
from typing import List, Optional

import requests

logging.basicConfig(level=logging.DEBUG)

parser = argparse.ArgumentParser()
parser.add_argument("--backend-host", type=str, default="http://127.0.0.1:8079",
help="Host address of the loculus backend")
Expand Down Expand Up @@ -68,7 +71,7 @@ def fetch_unprocessed_sequences(n: int) -> List[Sequence]:
response = requests.post(url, data=params, headers=headers)
if not response.ok:
if response.status_code == 422:
print("{}. Sleeping for a while.".format(response.text))
logging.debug("{}. Sleeping for a while.".format(response.text))
time.sleep(60 * 10)
return []
raise Exception("Fetching unprocessed data failed. Status code: {}".format(response.status_code), response.text)
Expand Down Expand Up @@ -171,7 +174,7 @@ def main():
locally_processed = 0

if watch_mode:
print("Started in watch mode - waiting 10 seconds before fetching data.")
logging.debug("Started in watch mode - waiting 10 seconds before fetching data.")
time.sleep(10)

if args.maxSequences and args.maxSequences < 100:
Expand All @@ -183,8 +186,8 @@ def main():
unprocessed = fetch_unprocessed_sequences(sequences_to_fetch)
if len(unprocessed) == 0:
if watch_mode:
print("Processed {} sequences. Sleeping for 10 seconds.".format(locally_processed))
time.sleep(10)
logging.debug("Processed {} sequences. Sleeping for 10 seconds.".format(locally_processed))
time.sleep(2)
locally_processed = 0
continue
else:
Expand All @@ -196,7 +199,7 @@ def main():

if args.maxSequences and total_processed >= args.maxSequences:
break
print("Total processed sequences: {}".format(total_processed))
logging.debug("Total processed sequences: {}".format(total_processed))


if __name__ == "__main__":
Expand Down