Merge pull request #488 from axif0/403forbidden

retry mechanism for total and fix overwrite

src/scribe_data/cli/total.py

@@ -20,6 +20,9 @@
     -->
 """
 
+from http.client import IncompleteRead
+from urllib.error import HTTPError
+
 from SPARQLWrapper import JSON
 
 from scribe_data.utils import (
@@ -244,7 +247,28 @@ def get_total_lexemes(language, data_type, doPrint=True):
 
     sparql.setQuery(query)
     sparql.setReturnFormat(JSON)
-    results = sparql.query().convert()
+    try_count = 0
+    max_retries = 2
+    results = None
+
+    while try_count <= max_retries and results is None:
+        try:
+            results = sparql.query().convert()
+
+        except HTTPError as http_err:
+            print(f"HTTPError occurred: {http_err}")
+
+        except IncompleteRead as read_err:
+            print(f"Incomplete read error occurred: {read_err}")
+
+        try_count += 1
+
+        if results is None:
+            if try_count <= max_retries:
+                print("The query will be retried ...")
+
+            else:
+                print("Query failed after retries.")
 
     # Check if the query returned any results.
     if (

src/scribe_data/wikidata/query_data.py

@@ -181,13 +181,13 @@ def query_data(
                         for file in existing_files:
                             file.unlink()
 
-                # elif choice in ["k", "K"]:
-                #     timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
-                #     file_name = f"{target_type}_{timestamp}.json"
+                    # elif choice in ["k", "K"]:
+                    #     timestamp = datetime.now().strftime("%Y_%m_%d_%H_%M_%S")
+                    #     file_name = f"{target_type}_{timestamp}.json"
 
-                else:
-                    print(f"Skipping update for {lang} {target_type}.")
-                    continue
+                    else:
+                        print(f"Skipping update for {lang} {target_type}.")
+                        break
 
         print(f"Querying and formatting {lang} {target_type}")