retry testing and logging

E3SM-Project · Sep 23, 2022 · 50a1ce5 · 50a1ce5
1 parent b87006e
commit 50a1ce5
Show file tree

Hide file tree

Showing 2 changed files with 16 additions and 3 deletions.
diff --git a/tests/test_check.py b/tests/test_check.py
@@ -189,7 +189,7 @@ def testCheckKeepTars(self):
         )
         self.assertEqualOrStop(
             output + err,
-            'INFO: Opening tar archive {}/000000.tar\nINFO: Checking file1.txt\nINFO: Checking file2.txt\nINFO: No failures detected when checking the files. If you have a log file, run "grep -i Exception <log-file>" to double check.\n'.format(
+            'INFO: zstash/000000.tar exists. Checking expected size matches actual size.\nINFO: Opening tar archive {}/000000.tar\nINFO: Checking file1.txt\nINFO: Checking file2.txt\nINFO: No failures detected when checking the files. If you have a log file, run "grep -i Exception <log-file>" to double check.\n'.format(
                 self.cache
             ),
         )

diff --git a/zstash/extract.py b/zstash/extract.py
@@ -446,25 +446,38 @@ def extractFiles(  # noqa: C901
             else:
                 raise TypeError("Invalid config.hpss={}".format(config.hpss))
             tries = args.retries + 1
+            # Set to True to test the `--retries` option with a forced failure.
+            # Then run `python -m unittest tests.test_extract.TestExtract.testExtractRetries`
+            test_retry = False
             while tries > 0:
                 tries -= 1
                 try:
+                    if test_retry:
+                        test_retry = False
+                        raise RuntimeError
                     if not os.path.exists(tfname):
                         # Will need to retrieve from HPSS
                         hpss_get(hpss, tfname, cache)
                     elif cur and tars_table_exists(cur):
+                        logger.info(
+                            f"{tfname} exists. Checking expected size matches actual size."
+                        )
                         actual_size = os.path.getsize(tfname)
                         name_only = os.path.split(tfname)[1]
                         cur.execute(
                             f"select size from tars where name is '{name_only}';"
                         )
                         expected_size: int = cur.fetchall()[0][0]
                         if expected_size > actual_size:
+                            logger.info(
+                                f"{name_only}: expected size={expected_size} > {actual_size}=actual_size"
+                            )
                             hpss_get(hpss, tfname, cache)
-                    else:
-                        break
+                    # `get` successful or not needed: no more tries needed
+                    break
                 except RuntimeError as e:
                     if tries > 0:
+                        logger.info(f"Retrying HPSS get: {tries} tries remaining.")
                         # Run the try-except block again
                         continue
                     else: