Add logs.

biothings · Oct 16, 2024 · b523127 · b523127
1 parent d609bbd
commit b523127
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/src/hub/dataload/sources/civic/civic_dump.py b/src/hub/dataload/sources/civic/civic_dump.py
@@ -15,8 +15,8 @@ class CivicDumper(HTTPDumper):
     API_URL = "https://civicdb.org/api/graphql"
     SRC_ROOT_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, SRC_NAME)
     SCHEDULE = "0 22 1 * *"
-    MAX_PARALLEL_DUMP = 2
-    SLEEP_BETWEEN_DOWNLOAD = 1.0
+    MAX_PARALLEL_DUMP = 1
+    SLEEP_BETWEEN_DOWNLOAD = 3.0
 
     def set_release(self):
         self.release = datetime.date.today().strftime("%Y-%m-%d")
@@ -52,6 +52,7 @@ def download(self, remoteurl, localfile, headers={}):
         self.prepare_local_folders(localfile)
         variant_id = remoteurl
 
+        self.logger.info(f"Downloading data for variant id: {variant_id}")
         variant_data = GraphqlDump().dump_variant(variant_id=variant_id, api_url=self.API_URL)
 
         with open(localfile, "w") as f:

diff --git a/src/hub/dataload/sources/civic/graphql_dump.py b/src/hub/dataload/sources/civic/graphql_dump.py
@@ -20,6 +20,7 @@ def get_variants_list(self, api_url: str):
                 hasNextPage = response_data["data"]["browseVariants"]["pageInfo"][
                     "hasNextPage"
                 ]
+        print(f"INFO:dump_civic:Count variant IDs = {len(ids)}")
         return ids
 
     def dump_variant(self, api_url: str, variant_id: int):