Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Made LS Cient requests tenacious #579

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open

Made LS Cient requests tenacious #579

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
cfc3790
made client requests tenacious, fixed incorrect function call
jinensetpal Jan 8, 2025
86c1480
linter fixes
jinensetpal Jan 8, 2025
4c83253
reworked wrapper to work with streamed requests
jinensetpal Jan 8, 2025
26d0faa
removed needless import
jinensetpal Jan 8, 2025
5a1e5af
added ls sdk to dev reqs for unit tests
jinensetpal Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dagshub/common/api/repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def __init__(self, repo: str, host: Optional[str] = None, auth: Optional[Any] =

@retry(retry=retry_if_exception_type(LSInitializingError), wait=wait_fixed(3), stop=stop_after_attempt(5))
def _tenacious_ls_request(self, *args, **kwargs):
res = self.http_request(*args, **kwargs)
res = http_request(*args, **kwargs)
if res.text.startswith("<!DOCTYPE html>"):
raise LSInitializingError()
elif res.status_code // 100 != 2:
Expand Down
50 changes: 49 additions & 1 deletion dagshub/ls_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,51 @@
from tenacity import retry, wait_fixed, stop_after_attempt, retry_if_exception_type
from dagshub.data_engine.model.errors import LSInitializingError
from contextlib import _GeneratorContextManager
from dagshub.common.util import lazy_load
from json import JSONDecodeError
from typing import Optional
from itertools import tee
import importlib.util
import semver
import types

from dagshub.common.api.repo import RepoAPI
from dagshub.auth import get_token
from dagshub.common import config


ls_sdk = lazy_load("label_studio_sdk")


class _TenaciousLSCLientWrapper:
def __init__(self, func):
self.func = func

@retry(
retry=retry_if_exception_type((LSInitializingError, JSONDecodeError, ls_sdk.core.ApiError)),
wait=wait_fixed(3),
stop=stop_after_attempt(5),
)
def wrapped_func(self, *args, **kwargs):
res = self.func(*args, **kwargs)

if isinstance(res, types.GeneratorType):
proxy, res = tee(res)
if next(proxy).startswith(b"<!DOCTYPE html>"):
raise LSInitializingError()
elif isinstance(res, _GeneratorContextManager):
return res
elif isinstance(res, bytes):
if res.startswith("<!DOCTYPE html>"):
raise LSInitializingError()
else:
if res.text.startswith("<!DOCTYPE html>"):
raise LSInitializingError()
elif res.status_code // 100 != 2:
raise RuntimeError(f"Process failed! Server Response: {res.text}")
return res


def _use_legacy_client():
"""
https://github.com/HumanSignal/label-studio/releases/tag/1.13.0, \
Expand Down Expand Up @@ -61,4 +98,15 @@ def get_label_studio_client(
"api_key": token if token is not None else get_token(host=host),
}

return LabelStudio(**kwargs)
ls_client = LabelStudio(**kwargs)
if legacy_client:
ls_client.make_request = _TenaciousLSCLientWrapper(ls_client.make_request).wrapped_func
else:
ls_client._client_wrapper.httpx_client.request = _TenaciousLSCLientWrapper(
ls_client._client_wrapper.httpx_client.request
).wrapped_func
ls_client.projects.exports.create_export = _TenaciousLSCLientWrapper(
ls_client.projects.exports.create_export
).wrapped_func

return ls_client
1 change: 1 addition & 0 deletions requirements-dev.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ pytest-mock==3.14.0
fiftyone==0.23.8
datasets==2.19.1
ultralytics==8.3.47
label-studio-sdk==1.0.8
Loading