Skip to content

Commit

Permalink
enhance(client): refactor model/dataset/runtime list for server and c…
Browse files Browse the repository at this point in the history 
…loud instance (#2625)
  • Loading branch information
@tianweidut
tianweidut authored Aug 16, 2023
1 parent 9a18a95 commit 8c61f28
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 145 deletions.
70 changes: 49 additions & 21 deletions client/starwhale/base/cloud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@


class CloudRequestMixed:
def fmt_timestamp(self, ts: t.Union[float, str]) -> str:
@classmethod
def fmt_timestamp(cls, ts: t.Union[float, str]) -> str:
return datetime.fromtimestamp(float(ts) / 1000.0).strftime(FMT_DATETIME).strip()

def fmt_duration(self, ts: t.Union[float, str]) -> str:
Expand Down Expand Up @@ -191,6 +192,15 @@ def parse_pager(self, resp: t.Dict[str, t.Any]) -> t.Dict[str, t.Any]:
total=_d["total"],
current=_d["size"],
remain=_d["total"] - _d["size"],
page={
"page_num": _d["pageNum"],
"page_size": _d["pageSize"],
"pages": _d["pages"],
"pre_page": _d["prePage"],
"next_page": _d["nextPage"],
"has_pre_page": _d["hasPreviousPage"],
"has_next_page": _d["hasNextPage"],
},
)

def _fetch_bundle_info(
Expand Down Expand Up @@ -269,30 +279,48 @@ def _fetch_bundle_all_list(
filter_dict = filter_dict or {}
_params = {"pageNum": page, "pageSize": size}
_params.update(filter_dict)
r = self.do_http_request(
resp = self.do_http_request(
f"/project/{project_uri.name}/{uri_typ.value}",
params=_params,
instance=project_uri.instance,
).json()
objects = {}

_page = page
_size = size
if filter_dict.get("latest") is not None:
_page = 1
_size = 1

for o in r["data"]["list"]:
_name = f"[{o['id']}] {o['name']}"
objects[_name] = self._fetch_bundle_history(
name=o["id"],
project_uri=project_uri,
typ=uri_typ,
page=_page,
size=_size,
)[0]
)
resp.raise_for_status()
r = resp.json()

bundles_map = dict()
for d in r["data"]["list"]:
_v = d["version"]
tags = [_v["alias"]]
if _v["latest"]:
tags.append("latest")
if _v["tags"]:
tags.extend(_v["tags"])

bundles_map[d["name"]] = {
"id": d["id"],
"name": d["name"],
"version": _v["name"],
CREATED_AT_KEY: self.fmt_timestamp(_v["createdTime"]),
"tags": tags,
"is_removed": False,
}
# TODO: make a better meta ingest
if "meta" in _v:
_meta = yaml.safe_load(_v["meta"])
# for dataset meta
if "dataset_summary" in _meta:
bundles_map[d["name"]]["rows"] = _meta["dataset_summary"].get(
"rows"
)
# for runtime meta
if "environment" in _meta:
bundles_map[d["name"]]["mode"] = _meta["environment"]["mode"]
bundles_map[d["name"]]["python"] = _meta["environment"]["python"]

if "size" in _v and _v["size"] is not None:
bundles_map[d["name"]]["size"] = _v["size"]

return objects, self.parse_pager(r)
return bundles_map, self.parse_pager(r)

def get_bundle_size_from_resp(self, typ: ResourceType, item: t.Dict) -> int:
default_size = 0
Expand Down
53 changes: 29 additions & 24 deletions client/starwhale/base/view.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,14 +41,15 @@ def _pager(func: t.Callable) -> t.Callable:
@wraps(func)
def _wrapper(*args: t.Any, **kwargs: t.Any) -> None:
def _print(_r: t.Dict[str, t.Any]) -> None:
p = Panel(
(
f"Counts: [green] {_r['current']}/{_r['total']} [/] :sheep: ,"
f"[red] {_r['remain']} [/] items does not show."
),
title="Count Details",
title_align="left",
content = (
f":sheep: Counts: [green] {_r['current']}/{_r['total']}[/], "
f"[red]{_r['remain']}[/] items does not show."
)
if "page" in _r:
content += "\n:butterfly: Pages: " + ", ".join(
[f"{k}[{v}]" for k, v in _r["page"].items()]
)
p = Panel(content, title="Count Details", title_align="left")
console.print(p)

rt = func(*args, **kwargs) # type: ignore
Expand Down Expand Up @@ -221,27 +222,31 @@ def list_data(
) -> t.List[t.Dict[str, t.Any]]:
result = []
for _name, _versions in _bundles.items():
# compatible with standalone and cloud
# TODO: use a better way to handle this
if not isinstance(_versions, (list, tuple)):
_versions = [_versions]

for _v in _versions:
if show_removed ^ _v["is_removed"]:
if show_removed ^ _v.get("is_removed", False):
continue

_version = (
_v["version"]
if fullname or show_removed
else _v["version"][:SHORT_VERSION_CNT]
)
if _v.get("id"):
_version = f"[{_v['id']:2}] {_version}"
_info = {
"version": (
_v["version"]
if fullname or show_removed
else _v["version"][:SHORT_VERSION_CNT]
),
"name": _name,
CREATED_AT_KEY: _v[CREATED_AT_KEY],
"tags": _v.get("tags"),
}

result.append(
{
"name": _name,
"version": _version,
"tags": _v.get("tags", []),
"size": _v.get("size", 0),
CREATED_AT_KEY: _v.get(CREATED_AT_KEY),
}
)
for k in ("rows", "mode", "python", "size"):
if k in _v:
_info[k] = _v[k]

result.append(_info)

order_keys = [Order("name"), Order(CREATED_AT_KEY, True)]
return sort_obj_list(result, order_keys)
Expand Down
1 change: 1 addition & 0 deletions client/starwhale/core/dataset/view.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,6 +317,7 @@ def list(
"tags": lambda x: ",".join(x),
"size": lambda x: pretty_bytes(x),
"runtime": cls.place_holder_for_empty(),
"rows": lambda x: str(x),
}

cls.print_header()
Expand Down
165 changes: 65 additions & 100 deletions client/tests/base/test_cloud.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,14 @@


class TestCloudRequestMixed(TestCase):
def setUp(self) -> None:
super().setUp()

sw = SWCliConfigMixed()
sw.update_instance(
uri="http://1.1.1.1", user_name="test", sw_token="123", alias="test"
)

def test_get_bundle_size_from_resp(self):
ins = CloudRequestMixed()

Expand All @@ -30,120 +38,77 @@ def test_get_bundle_size_from_resp(self):

@Mocker()
def test_bundle_list(self, rm: Mocker) -> None:
sw = SWCliConfigMixed()
sw.update_instance(
uri="http://1.1.1.1", user_name="test", sw_token="123", alias="test"
)

rm.request(
HTTPMethod.GET,
"http://1.1.1.1/api/v1/project/sw/model?pageNum=1&pageSize=20",
json={
"data": {
"list": [
{"id": 2, "name": "mnist"},
{"id": 1, "name": "text_cls"},
],
"total": 2,
"size": 2,
}
},
status_code=HTTPStatus.OK,
)

rm.request(
req = rm.request(
HTTPMethod.GET,
"http://1.1.1.1/api/v1/project/sw/model?name=mnist",
"http://1.1.1.1/api/v1/project/sw/model",
json={
"data": {
"list": [
{"id": 2, "name": "mnist"},
],
"total": 1,
"size": 1,
}
},
status_code=HTTPStatus.OK,
)

rm.request(
HTTPMethod.GET,
"http://1.1.1.1/api/v1/project/sw/model/1/version",
json={
"data": {
"list": [
{"id": 2, "name": "tc_v2", "createdTime": 3000},
{"id": 1, "name": "tc_v1", "createdTime": 1000},
{
"id": 2,
"name": "mnist",
"version": {
"alias": "v1",
"latest": True,
"tags": ["t1", "t2"],
"name": "version1",
"createdTime": 0,
},
},
{
"id": 1,
"name": "text_cls",
"version": {
"alias": "v2",
"latest": False,
"tags": None,
"name": "version2",
"createdTime": 0,
"meta": yaml.safe_dump(
{
"dataset_summary": {"rows": 10},
"environment": {
"mode": "venv",
"python": "3.7",
},
}
),
},
},
],
"total": 2,
"size": 2,
}
},
status_code=HTTPStatus.OK,
)

rm.request(
HTTPMethod.GET,
"http://1.1.1.1/api/v1/project/sw/model/2/version",
json={
"data": {
"list": [
{"id": 5, "name": "mnist_v3", "createdTime": 2000},
{"id": 4, "name": "mnist_v2", "createdTime": 2000},
{"id": 3, "name": "mnist_v1", "createdTime": 2000},
],
"total": 3,
"size": 3,
}
},
status_code=HTTPStatus.OK,
)

rm.request(
HTTPMethod.GET,
"http://1.1.1.1/api/v1/project/sw/model/2/version?pageNum=1&pageSize=1",
json={
"data": {
"list": [
{"id": 5, "name": "mnist_v3", "createdTime": 2000},
],
"total": 1,
"size": 1,
"pageNum": 1,
"pageSize": 10,
"pages": 1,
"prePage": 0,
"nextPage": 0,
"hasNextPage": False,
"hasPreviousPage": False,
}
},
status_code=HTTPStatus.OK,
)

cbm = CloudBundleModelMixin()
_uri = Project("http://1.1.1.1/project/sw")
_models, _pager = cbm._fetch_bundle_all_list(_uri, uri_typ=ResourceType.model)

assert len(_models.items()) == 2
assert len(_models["[2] mnist"]) == 3
assert len(_models["[1] text_cls"]) == 2
assert _pager["current"] == 2
assert _pager["remain"] == 0

cbm = CloudBundleModelMixin()
_uri = Project("http://1.1.1.1/project/sw")
_models, _pager = cbm._fetch_bundle_all_list(
_uri, uri_typ=ResourceType.model, filter_dict={"name": "mnist"}
_uri, uri_typ=ResourceType.model, page=10, size=1
)

assert len(_models.items()) == 1
assert len(_models["[2] mnist"]) == 3
assert _pager["current"] == 1
assert _pager["remain"] == 0

cbm = CloudBundleModelMixin()
_uri = Project("http://1.1.1.1/project/sw")
_models, _pager = cbm._fetch_bundle_all_list(
_uri,
uri_typ=ResourceType.model,
filter_dict={"name": "mnist", "latest": True},
)

assert len(_models.items()) == 1
assert len(_models["[2] mnist"]) == 1
assert _pager["current"] == 1
assert len(_models) == 2
assert _models["mnist"]["id"] == 2
assert _models["mnist"]["name"] == "mnist"
assert set(_models["mnist"]["tags"]) == set(["t1", "t2", "v1", "latest"])
assert _models["text_cls"]["id"] == 1
assert _models["text_cls"]["tags"] == ["v2"]
assert _pager["current"] == 2
assert _pager["remain"] == 0
assert _pager["page"]["page_num"] == 1
assert _models["text_cls"]["mode"] == "venv"
assert "mode" not in _models["mnist"]
assert _models["text_cls"]["python"] == "3.7"
assert _models["text_cls"]["rows"] == 10

assert req.call_count == 1
assert req.last_request.qs["pagenum"] == ["10"]
assert req.last_request.qs["pagesize"] == ["1"]

0 comments on commit 8c61f28

Please sign in to comment.