fix(client): make sure evaluation log flushed to storage

star-whale · Oct 19, 2023 · 2bf6686 · 2bf6686
1 parent c4eb7df
commit 2bf6686
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 5 deletions.
diff --git a/client/starwhale/api/_impl/data_store.py b/client/starwhale/api/_impl/data_store.py
@@ -1880,7 +1880,7 @@ def _dump_mem_table(self, root_path: Path, mem_table: MemoryTable) -> None:
             manifest.key_column_type = key_type_str
 
         data_blocks = self._load_data_blocks()
-        console.trace(f"load {len(data_blocks)} data blocks")
+        console.trace(f"load {len(data_blocks)} data blocks in {self.table_name}")
 
         key_column = self.key_column
         if key_column is None:
@@ -1914,7 +1914,9 @@ def _dump_mem_table(self, root_path: Path, mem_table: MemoryTable) -> None:
                 last_range_broken = False
 
         for start, end in key_range_not_found:
-            console.trace(f"create new block for key range {start} - {end}")
+            console.trace(
+                f"create new block for key range {start} - {end} in table {self.table_name}"
+            )
             block_id = manifest.next_block_id
             file = self._get_block_file_name(
                 root_path, manifest.block_config.block_name_prefix, block_id
@@ -1965,7 +1967,7 @@ def _dump_mem_table(self, root_path: Path, mem_table: MemoryTable) -> None:
                         manifest.next_block_id,
                     ),
                 )
-                console.trace(f"dump block {block.file}")
+                console.trace(f"dump block {block.file} in table {self.table_name}")
                 block.dump(items, self.compressor, str(root_path))
                 min_rev, max_rev = revision_range_of_items(items)
                 manifest.blocks.append(
@@ -1996,10 +1998,13 @@ def _dump_mem_table(self, root_path: Path, mem_table: MemoryTable) -> None:
         mem_table.records.clear()
 
     def dump(self) -> None:
+        console.trace(f"full dump triggered of table {self.table_name}")
         if self._immutable_memory_table is not None:
+            console.trace(f"dump immutable memory table of {self.table_name}")
             self._dump(self.root_path, self._immutable_memory_table)
             self._immutable_memory_table = None
         if self.memory_table is not None:
+            console.trace(f"dump memory table of {self.table_name}")
             self._dump(self.root_path, self.memory_table)
             self.memory_table = None
 

diff --git a/client/starwhale/base/artifact/base.py b/client/starwhale/base/artifact/base.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import queue
+import atexit
 import typing as t
 import threading
 from abc import ABC, abstractmethod
@@ -51,6 +52,7 @@ def __init__(
             alignment_bytes_size=self._blob_alignment_bytes_size,
             volume_bytes_size=self._blob_volume_bytes_size,
         )
+        atexit.register(self.close)
 
     def __enter__(self) -> AsyncArtifactWriterBase:
         return self

diff --git a/scripts/client_test/cmds/artifacts_cmd.py b/scripts/client_test/cmds/artifacts_cmd.py
@@ -72,7 +72,7 @@ def run_in_host(
         version = gen_uniq_version()
         cmd = [
             CLI,
-            "-vvv",
+            "-vvvv",
             "model",
             "run",
             "--uri",

diff --git a/scripts/client_test/cmds/job_cmd.py b/scripts/client_test/cmds/job_cmd.py
@@ -9,7 +9,9 @@ class Job:
     _cmd = "job"
 
     def info(self, version: str) -> t.Any:
-        _ret_code, _res = invoke_output([CLI, "-o", "json", self._cmd, "info", version])
+        _ret_code, _res = invoke_output(
+            [CLI, "-o", "json", self._cmd, "info", version], log=True
+        )
         try:
             return json.loads(_res) if _ret_code == 0 else {}
         except Exception as e: