chore(client): prefer using builtin list when building dataset (#2913)

star-whale · Oct 27, 2023 · 872e4b1 · 872e4b1
1 parent 5a4073b
commit 872e4b1
Show file tree

Hide file tree

Showing 3 changed files with 13 additions and 38 deletions.
diff --git a/client/starwhale/api/_impl/data_store.py b/client/starwhale/api/_impl/data_store.py
@@ -571,6 +571,9 @@ def __eq__(self, other: Any) -> bool:
             )
         return False
 
+    def __hash__(self) -> int:
+        return hash(str(self))
+
 
 class SwTupleType(SwListType):
     def __init__(
@@ -580,6 +583,9 @@ def __init__(
     ) -> None:
         super().__init__(element_types, sparse_types, True)
 
+    def __hash__(self) -> int:
+        return hash(str(self))
+
 
 class SwMapType(SwCompositeType):
     def __init__(self, key_type: SwType, value_type: SwType) -> None:

diff --git a/client/starwhale/core/dataset/tabular.py b/client/starwhale/core/dataset/tabular.py
@@ -13,12 +13,7 @@
 
 from typing_extensions import Protocol
 
-from starwhale.utils import (
-    console,
-    gen_uniq_version,
-    validate_obj_name,
-    check_python_scalar_type,
-)
+from starwhale.utils import console, gen_uniq_version, validate_obj_name
 from starwhale.consts import ENV_POD_NAME
 from starwhale.base.mixin import ASDictMixin, _do_asdict_convert
 from starwhale.utils.error import (
@@ -216,17 +211,7 @@ def _transform(data: t.Any) -> t.Any:
             elif isinstance(data, dict):
                 return {k: _transform(v) for k, v in data.items()}
             elif isinstance(data, (list, tuple)):
-                data = type(data)([_transform(v) for v in data])
-                types = set(type(i) for i in data)
-                if (
-                    not types
-                    or len(types) == 1
-                    and check_python_scalar_type(types.pop())
-                ):
-                    # Only keep the simple format(the scalar type in list/tuple) the original type for dataset viewer, such as: [1,2,3]
-                    return data
-                else:
-                    return Sequence(data=data, auto_convert=True)
+                return type(data)([_transform(v) for v in data])
             else:
                 return data
 

diff --git a/client/tests/sdk/test_dataset.py b/client/tests/sdk/test_dataset.py
@@ -232,29 +232,13 @@ def test_upload(self, rm: Mocker) -> None:
 
         assert {
             "attributes": [
-                {"name": "_type", "type": "STRING"},
-                {"name": "sequence_type", "type": "STRING"},
-                {"name": "_cnt", "type": "INT64"},
-                {
-                    "attributes": [
-                        {"name": "i0", "type": "INT64"},
-                        {"name": "i1", "type": "STRING"},
-                        {"name": "i2", "type": "FLOAT64"},
-                        {
-                            "elementType": {"type": "STRING"},
-                            "name": "i3",
-                            "type": "TUPLE",
-                        },
-                    ],
-                    "name": "data",
-                    "pythonType": "starwhale.base.data_type.JsonDict",
-                    "type": "OBJECT",
-                },
-                {"name": "auto_convert", "type": "BOOL"},
+                {"index": 0, "type": "INT64"},
+                {"index": 1, "type": "STRING"},
+                {"index": 2, "type": "FLOAT64"},
             ],
+            "elementType": {"elementType": {"type": "STRING"}, "type": "TUPLE"},
             "name": "features/mixed_types_tuple",
-            "pythonType": "starwhale.base.data_type.Sequence",
-            "type": "OBJECT",
+            "type": "TUPLE",
         } in content["tableSchemaDesc"]["columnSchemaList"]
 
         assert {