From 07813b1fda942adaee49359c99f1f2838f54a4c1 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 1 Nov 2024 15:30:57 +0200 Subject: [PATCH 1/7] tmp --- cvat-core/src/lambda-manager.ts | 32 ++++++++++++------- cvat-core/src/server-proxy.ts | 8 ++--- cvat-ui/src/actions/models-actions.ts | 2 +- cvat-ui/src/reducers/notifications-reducer.ts | 2 +- 4 files changed, 26 insertions(+), 18 deletions(-) diff --git a/cvat-core/src/lambda-manager.ts b/cvat-core/src/lambda-manager.ts index 66733d7ed23..bb564202d8f 100644 --- a/cvat-core/src/lambda-manager.ts +++ b/cvat-core/src/lambda-manager.ts @@ -141,6 +141,7 @@ class LambdaManager { // already listening, avoid sending extra requests return; } + const timeoutCallback = (): void => { serverProxy.lambda.status(requestID).then((response) => { const { status } = response; @@ -148,30 +149,37 @@ class LambdaManager { // check it was not cancelled const { onUpdate } = this.listening[requestID]; if ([RQStatus.QUEUED, RQStatus.STARTED].includes(status)) { - onUpdate.forEach((update) => update(status, response.progress || 0)); + onUpdate.forEach((update) => update(status, response.progress ?? 0)); this.listening[requestID].timeout = window .setTimeout(timeoutCallback, status === RQStatus.QUEUED ? 30000 : 10000); } else { delete this.listening[requestID]; if (status === RQStatus.FINISHED) { - onUpdate - .forEach((update) => update(status, response.progress || 100)); + onUpdate.forEach((update) => update(status, response.progress ?? 100)); + } else if (status === RQStatus.FAILED) { + onUpdate.forEach((update) => update( + status, + response.progress ?? 0, + `The process has ${RQStatus.FAILED}. ${response.exc_info}`, + )); } else { - onUpdate - .forEach((update) => update(status, response.progress || 0, response.exc_info || '')); + onUpdate.forEach((update) => update( + status, + response.progress ?? 0, + `Unexpected status received: ${status}.`, + )); } } } - }).catch((error) => { + }).catch((error: unknown) => { if (requestID in this.listening) { // check it was not cancelled const { onUpdate } = this.listening[requestID]; - onUpdate - .forEach((update) => update( - RQStatus.UNKNOWN, - 0, - `Could not get a status of the request ${requestID}. ${error.toString()}`, - )); + onUpdate.forEach((update) => update( + RQStatus.UNKNOWN, + 0, + `Could not get a status of the request ${requestID}. ${error instanceof Error ? error.message : ''}`, + )); } }).finally(() => { if (requestID in this.listening) { diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 37f2337c0e5..6cc3847c43d 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -175,10 +175,7 @@ function filterPythonTraceback(data: string): string { if (typeof data === 'string' && data.trim().startsWith('Traceback')) { const lastRow = data.split('\n').findLastIndex((str) => str.trim().length); let errorText = `${data.split('\n').slice(lastRow, lastRow + 1)[0]}`; - if (errorText.includes('CvatDatasetNotFoundError')) { - errorText = errorText.replace(/.*CvatDatasetNotFoundError: /, ''); - } - return errorText; + return errorText.split(':').slice(1).join(':').trim(); } return data; @@ -1695,6 +1692,9 @@ async function getLambdaRequestStatus(requestID) { try { const response = await Axios.get(`${backendAPI}/lambda/requests/${requestID}`); + if (response.data.status === RQStatus.FAILED) { + response.data.exc_info = filterPythonTraceback(response.data.exc_info ?? ''); + } return response.data; } catch (errorData) { throw generateError(errorData); diff --git a/cvat-ui/src/actions/models-actions.ts b/cvat-ui/src/actions/models-actions.ts index 38a99181623..98cd80355cf 100644 --- a/cvat-ui/src/actions/models-actions.ts +++ b/cvat-ui/src/actions/models-actions.ts @@ -138,7 +138,7 @@ function listen(inferenceMeta: InferenceMeta, dispatch: (action: ModelsActions) error: message as string, id: requestID, }, - new Error(`Inference status for the task ${taskID} is ${status}. ${message}`), + new Error(message as string), ), ); diff --git a/cvat-ui/src/reducers/notifications-reducer.ts b/cvat-ui/src/reducers/notifications-reducer.ts index 5bc4becb223..13d167d72b2 100644 --- a/cvat-ui/src/reducers/notifications-reducer.ts +++ b/cvat-ui/src/reducers/notifications-reducer.ts @@ -885,7 +885,7 @@ export default function (state = defaultState, action: AnyAction): Notifications models: { ...state.errors.models, inferenceStatusFetching: { - message: `Fetching inference status for the [task ${taskID}](/tasks/${taskID})`, + message: `Automatic annotation for the [task ${taskID}](/tasks/${taskID})`, reason: action.payload.error, shouldLog: shouldLog(action.payload.error), }, From 2f5a34c431caf43fae170011ab167a5211213dd3 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 1 Nov 2024 19:09:35 +0200 Subject: [PATCH 2/7] Do not pass traceback from the server --- cvat-core/src/server-proxy.ts | 18 +----------------- cvat/apps/engine/views.py | 3 ++- cvat/apps/lambda_manager/views.py | 8 ++++++-- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/cvat-core/src/server-proxy.ts b/cvat-core/src/server-proxy.ts index 6cc3847c43d..e3c1c0f01ae 100644 --- a/cvat-core/src/server-proxy.ts +++ b/cvat-core/src/server-proxy.ts @@ -171,23 +171,10 @@ async function chunkUpload(file: File, uploadConfig): Promise<{ uploadSentSize: }); } -function filterPythonTraceback(data: string): string { - if (typeof data === 'string' && data.trim().startsWith('Traceback')) { - const lastRow = data.split('\n').findLastIndex((str) => str.trim().length); - let errorText = `${data.split('\n').slice(lastRow, lastRow + 1)[0]}`; - return errorText.split(':').slice(1).join(':').trim(); - } - - return data; -} - function generateError(errorData: AxiosError): ServerError { if (errorData.response) { if (errorData.response.status >= 500 && typeof errorData.response.data === 'string') { - return new ServerError( - filterPythonTraceback(errorData.response.data), - errorData.response.status, - ); + return new ServerError(errorData.response.data, errorData.response.status); } if (errorData.response.status >= 400 && errorData.response.data) { @@ -1692,9 +1679,6 @@ async function getLambdaRequestStatus(requestID) { try { const response = await Axios.get(`${backendAPI}/lambda/requests/${requestID}`); - if (response.data.status === RQStatus.FAILED) { - response.data.exc_info = filterPythonTraceback(response.data.exc_info ?? ''); - } return response.data; } catch (errorData) { throw generateError(errorData); diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index ac046b1d0b2..2493a3e4cde 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -3252,8 +3252,9 @@ def perform_destroy(self, instance): target.touch() def rq_exception_handler(rq_job, exc_type, exc_value, tb): - rq_job.meta[RQJobMetaField.FORMATTED_EXCEPTION] = "".join( + formatted_exception = "".join( traceback.format_exception_only(exc_type, exc_value)) + rq_job.meta[RQJobMetaField.FORMATTED_EXCEPTION] = parse_exception_message(formatted_exception) rq_job.save_meta() return True diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index 143537985fd..4f0b8083aa0 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -38,6 +38,7 @@ ) from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField from cvat.apps.engine.serializers import LabeledDataSerializer +from cvat.apps.engine.utils import parse_exception_message from cvat.apps.lambda_manager.models import FunctionKind from cvat.apps.lambda_manager.permissions import LambdaPermission from cvat.apps.lambda_manager.serializers import ( @@ -621,11 +622,15 @@ def to_dict(self): "enqueued": self.job.enqueued_at, "started": self.job.started_at, "ended": self.job.ended_at, - "exc_info": self.job.exc_info + "exc_info": self.job.exc_info, } + if dict_['status'] == rq.job.JobStatus.DEFERRED: dict_['status'] = rq.job.JobStatus.QUEUED.value + if dict_['status'] == rq.job.JobStatus.FAILED: + dict_['exc_info'] = self.job.meta.get(RQJobMetaField.FORMATTED_EXCEPTION, dict_['exc_info']) + return dict_ def get_task(self): @@ -1203,7 +1208,6 @@ def retrieve(self, request, pk): self.check_object_permissions(request, pk) queue = LambdaQueue() rq_job = queue.fetch_job(pk) - response_serializer = FunctionCallSerializer(rq_job.to_dict()) return response_serializer.data From e83cb9a5249fd3f7fc6dd2a3febd8cf240a3e210 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 1 Nov 2024 19:15:36 +0200 Subject: [PATCH 3/7] minor updates --- cvat-core/src/lambda-manager.ts | 2 +- cvat/apps/lambda_manager/views.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cvat-core/src/lambda-manager.ts b/cvat-core/src/lambda-manager.ts index bb564202d8f..a81f93f7d21 100644 --- a/cvat-core/src/lambda-manager.ts +++ b/cvat-core/src/lambda-manager.ts @@ -160,7 +160,7 @@ class LambdaManager { onUpdate.forEach((update) => update( status, response.progress ?? 0, - `The process has ${RQStatus.FAILED}. ${response.exc_info}`, + `The process has failed. ${response.exc_info}`, )); } else { onUpdate.forEach((update) => update( diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index 4f0b8083aa0..6cb39cfdfef 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -38,7 +38,6 @@ ) from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField from cvat.apps.engine.serializers import LabeledDataSerializer -from cvat.apps.engine.utils import parse_exception_message from cvat.apps.lambda_manager.models import FunctionKind from cvat.apps.lambda_manager.permissions import LambdaPermission from cvat.apps.lambda_manager.serializers import ( @@ -622,7 +621,7 @@ def to_dict(self): "enqueued": self.job.enqueued_at, "started": self.job.started_at, "ended": self.job.ended_at, - "exc_info": self.job.exc_info, + "exc_info": self.job.exc_info } if dict_['status'] == rq.job.JobStatus.DEFERRED: From b6164a017be19dd40f3e50075226601d1417fece Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 1 Nov 2024 19:43:00 +0200 Subject: [PATCH 4/7] Show error --- cvat-core/src/lambda-manager.ts | 75 ++++++++++--------- .../automatic-annotation-progress.tsx | 11 ++- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/cvat-core/src/lambda-manager.ts b/cvat-core/src/lambda-manager.ts index a81f93f7d21..283a89d34ce 100644 --- a/cvat-core/src/lambda-manager.ts +++ b/cvat-core/src/lambda-manager.ts @@ -137,43 +137,18 @@ class LambdaManager { } if (requestID in this.listening) { - this.listening[requestID].onUpdate.push(callback); // already listening, avoid sending extra requests + this.listening[requestID].onUpdate.push(callback); return; } - const timeoutCallback = (): void => { - serverProxy.lambda.status(requestID).then((response) => { - const { status } = response; - if (requestID in this.listening) { - // check it was not cancelled - const { onUpdate } = this.listening[requestID]; - if ([RQStatus.QUEUED, RQStatus.STARTED].includes(status)) { - onUpdate.forEach((update) => update(status, response.progress ?? 0)); - this.listening[requestID].timeout = window - .setTimeout(timeoutCallback, status === RQStatus.QUEUED ? 30000 : 10000); - } else { - delete this.listening[requestID]; - if (status === RQStatus.FINISHED) { - onUpdate.forEach((update) => update(status, response.progress ?? 100)); - } else if (status === RQStatus.FAILED) { - onUpdate.forEach((update) => update( - status, - response.progress ?? 0, - `The process has failed. ${response.exc_info}`, - )); - } else { - onUpdate.forEach((update) => update( - status, - response.progress ?? 0, - `Unexpected status received: ${status}.`, - )); - } - } - } - }).catch((error: unknown) => { + const timeoutCallback = async (): Promise => { + let response = null; + try { + response = await serverProxy.lambda.status(requestID); + } catch (error: unknown) { if (requestID in this.listening) { - // check it was not cancelled + // check the request was not cancelled const { onUpdate } = this.listening[requestID]; onUpdate.forEach((update) => update( RQStatus.UNKNOWN, @@ -181,11 +156,39 @@ class LambdaManager { `Could not get a status of the request ${requestID}. ${error instanceof Error ? error.message : ''}`, )); } - }).finally(() => { - if (requestID in this.listening) { - this.listening[requestID].timeout = null; + } + + if (!(requestID in this.listening)) { + // request was already canceled + return; + } + + const { status } = response; + const { onUpdate } = this.listening[requestID]; + if ([RQStatus.QUEUED, RQStatus.STARTED].includes(status)) { + onUpdate.forEach((update) => update(status, response.progress ?? 0)); + this.listening[requestID].timeout = window + .setTimeout(timeoutCallback, status === RQStatus.QUEUED ? 30000 : 10000); + } else { + delete this.listening[requestID]; + if (status === RQStatus.FINISHED) { + onUpdate.forEach((update) => update(status, response.progress ?? 100)); + } else if (status === RQStatus.FAILED) { + onUpdate.forEach((update) => update( + status, + response.progress ?? 0, + `The process has failed. ${response.exc_info}`, + )); + } else { + onUpdate.forEach((update) => update( + status, + response.progress ?? 0, + `Unexpected status received: ${status}.`, + )); } - }); + } + + this.listening[requestID].timeout = null; }; this.listening[requestID] = { diff --git a/cvat-ui/src/components/tasks-page/automatic-annotation-progress.tsx b/cvat-ui/src/components/tasks-page/automatic-annotation-progress.tsx index 8b2ecd50768..d0c6c248869 100644 --- a/cvat-ui/src/components/tasks-page/automatic-annotation-progress.tsx +++ b/cvat-ui/src/components/tasks-page/automatic-annotation-progress.tsx @@ -5,7 +5,7 @@ import React from 'react'; import { Row, Col } from 'antd/lib/grid'; -import { CloseOutlined, LoadingOutlined } from '@ant-design/icons'; +import { CloseOutlined, LoadingOutlined, QuestionCircleOutlined } from '@ant-design/icons'; import Text from 'antd/lib/typography/Text'; import Progress from 'antd/lib/progress'; import Modal from 'antd/lib/modal'; @@ -56,7 +56,14 @@ export default function AutomaticAnnotationProgress(props: Props): JSX.Element | } if (activeInference.status === RQStatus.FAILED) { - return (<>Automatic annotation failed); + return ( + <> + Automatic annotation failed + + + + + ); } if (activeInference.status === RQStatus.UNKNOWN) { From 7da4a589b7c64403dae8a265d98049a7d26ad3b1 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 1 Nov 2024 20:38:14 +0200 Subject: [PATCH 5/7] Avoid passing traceback in corner cases --- cvat/apps/engine/serializers.py | 7 ++----- cvat/apps/lambda_manager/views.py | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 5b3845f8260..22cdc04fac2 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -3230,12 +3230,9 @@ def get_message(self, rq_job: RQJob) -> str: message = '' if RQJobStatus.STARTED == rq_job_status: - message = rq_job.meta.get(RQJobMetaField.STATUS, '') + message = rq_job.meta.get(RQJobMetaField.STATUS, "") elif RQJobStatus.FAILED == rq_job_status: - message = rq_job.meta.get( - RQJobMetaField.FORMATTED_EXCEPTION, - parse_exception_message(str(rq_job.exc_info or "Unknown error")), - ) + message = rq_job.meta.get(RQJobMetaField.FORMATTED_EXCEPTION, "Unknown error") return message diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index 6cb39cfdfef..2082a787b4c 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -621,14 +621,14 @@ def to_dict(self): "enqueued": self.job.enqueued_at, "started": self.job.started_at, "ended": self.job.ended_at, - "exc_info": self.job.exc_info + "exc_info": None } if dict_['status'] == rq.job.JobStatus.DEFERRED: dict_['status'] = rq.job.JobStatus.QUEUED.value if dict_['status'] == rq.job.JobStatus.FAILED: - dict_['exc_info'] = self.job.meta.get(RQJobMetaField.FORMATTED_EXCEPTION, dict_['exc_info']) + dict_['exc_info'] = self.job.meta.get(RQJobMetaField.FORMATTED_EXCEPTION, 'Unknown error') return dict_ From d6517799e5562f089b8f48e84d10854e19b80597 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Fri, 1 Nov 2024 20:51:34 +0200 Subject: [PATCH 6/7] Removed unused import --- cvat/apps/engine/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 22cdc04fac2..f1c0b426c35 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -29,7 +29,7 @@ from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine.frame_provider import TaskFrameProvider -from cvat.apps.engine.utils import format_list, parse_exception_message +from cvat.apps.engine.utils import format_list from cvat.apps.engine import field_validation, models from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status from cvat.apps.engine.log import ServerLogManager From 43fabfc474976d140aa00774d3ac113b91bddbe2 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Tue, 5 Nov 2024 10:26:35 +0200 Subject: [PATCH 7/7] Run CI