Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hide traceback when automatic annotation has failed #8636

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 45 additions & 34 deletions cvat-core/src/lambda-manager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -137,47 +137,58 @@ class LambdaManager {
}

if (requestID in this.listening) {
this.listening[requestID].onUpdate.push(callback);
// already listening, avoid sending extra requests
this.listening[requestID].onUpdate.push(callback);
return;
}
const timeoutCallback = (): void => {
serverProxy.lambda.status(requestID).then((response) => {
const { status } = response;
if (requestID in this.listening) {
// check it was not cancelled
const { onUpdate } = this.listening[requestID];
if ([RQStatus.QUEUED, RQStatus.STARTED].includes(status)) {
onUpdate.forEach((update) => update(status, response.progress || 0));
this.listening[requestID].timeout = window
.setTimeout(timeoutCallback, status === RQStatus.QUEUED ? 30000 : 10000);
} else {
delete this.listening[requestID];
if (status === RQStatus.FINISHED) {
onUpdate
.forEach((update) => update(status, response.progress || 100));
} else {
onUpdate
.forEach((update) => update(status, response.progress || 0, response.exc_info || ''));
}
}
}
}).catch((error) => {

const timeoutCallback = async (): Promise<void> => {
let response = null;
try {
response = await serverProxy.lambda.status(requestID);
} catch (error: unknown) {
if (requestID in this.listening) {
// check it was not cancelled
// check the request was not cancelled
const { onUpdate } = this.listening[requestID];
onUpdate
.forEach((update) => update(
RQStatus.UNKNOWN,
0,
`Could not get a status of the request ${requestID}. ${error.toString()}`,
));
onUpdate.forEach((update) => update(
RQStatus.UNKNOWN,
0,
`Could not get a status of the request ${requestID}. ${error instanceof Error ? error.message : ''}`,
));
}
}).finally(() => {
if (requestID in this.listening) {
this.listening[requestID].timeout = null;
}

if (!(requestID in this.listening)) {
// request was already canceled
return;
}

const { status } = response;
const { onUpdate } = this.listening[requestID];
if ([RQStatus.QUEUED, RQStatus.STARTED].includes(status)) {
onUpdate.forEach((update) => update(status, response.progress ?? 0));
this.listening[requestID].timeout = window
.setTimeout(timeoutCallback, status === RQStatus.QUEUED ? 30000 : 10000);
} else {
delete this.listening[requestID];
if (status === RQStatus.FINISHED) {
onUpdate.forEach((update) => update(status, response.progress ?? 100));
} else if (status === RQStatus.FAILED) {
onUpdate.forEach((update) => update(
status,
response.progress ?? 0,
`The process has failed. ${response.exc_info}`,
));
} else {
onUpdate.forEach((update) => update(
status,
response.progress ?? 0,
`Unexpected status received: ${status}.`,
));
}
});
}

this.listening[requestID].timeout = null;
};

this.listening[requestID] = {
Expand Down
18 changes: 1 addition & 17 deletions cvat-core/src/server-proxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -171,26 +171,10 @@
});
}

function filterPythonTraceback(data: string): string {
if (typeof data === 'string' && data.trim().startsWith('Traceback')) {
const lastRow = data.split('\n').findLastIndex((str) => str.trim().length);
let errorText = `${data.split('\n').slice(lastRow, lastRow + 1)[0]}`;
if (errorText.includes('CvatDatasetNotFoundError')) {
errorText = errorText.replace(/.*CvatDatasetNotFoundError: /, '');
}
return errorText;
}

return data;
}

function generateError(errorData: AxiosError): ServerError {
if (errorData.response) {
if (errorData.response.status >= 500 && typeof errorData.response.data === 'string') {
return new ServerError(
filterPythonTraceback(errorData.response.data),
errorData.response.status,
);
return new ServerError(errorData.response.data, errorData.response.status);
}

if (errorData.response.status >= 400 && errorData.response.data) {
Expand Down Expand Up @@ -244,7 +228,7 @@
return new ServerError(message, 0);
}

function prepareData(details) {

Check warning on line 231 in cvat-core/src/server-proxy.ts

View workflow job for this annotation

GitHub Actions / Linter

Missing return type on function
const data = new FormData();
for (const [key, value] of Object.entries(details)) {
if (Array.isArray(value)) {
Expand Down Expand Up @@ -286,7 +270,7 @@
return requestId++;
}

async function get(url: string, requestConfig) {

Check warning on line 273 in cvat-core/src/server-proxy.ts

View workflow job for this annotation

GitHub Actions / Linter

Missing return type on function
return new Promise((resolve, reject) => {
const newRequestId = getRequestId();
requests[newRequestId] = { resolve, reject };
Expand Down Expand Up @@ -819,7 +803,7 @@
save_images: saveImages,
};
return new Promise<string | void>((resolve, reject) => {
async function request() {

Check warning on line 806 in cvat-core/src/server-proxy.ts

View workflow job for this annotation

GitHub Actions / Linter

Missing return type on function
Axios.post(baseURL, {}, {
params,
})
Expand Down Expand Up @@ -914,7 +898,7 @@
const url = `${backendAPI}/tasks/${id}/backup/export`;

return new Promise<string | void>((resolve, reject) => {
async function request() {

Check warning on line 901 in cvat-core/src/server-proxy.ts

View workflow job for this annotation

GitHub Actions / Linter

Missing return type on function
try {
const response = await Axios.post(url, {}, {
params,
Expand Down
2 changes: 1 addition & 1 deletion cvat-ui/src/actions/models-actions.ts
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ function listen(inferenceMeta: InferenceMeta, dispatch: (action: ModelsActions)
error: message as string,
id: requestID,
},
new Error(`Inference status for the task ${taskID} is ${status}. ${message}`),
new Error(message as string),
),
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import React from 'react';
import { Row, Col } from 'antd/lib/grid';
import { CloseOutlined, LoadingOutlined } from '@ant-design/icons';
import { CloseOutlined, LoadingOutlined, QuestionCircleOutlined } from '@ant-design/icons';
import Text from 'antd/lib/typography/Text';
import Progress from 'antd/lib/progress';
import Modal from 'antd/lib/modal';
Expand Down Expand Up @@ -56,7 +56,14 @@ export default function AutomaticAnnotationProgress(props: Props): JSX.Element |
}

if (activeInference.status === RQStatus.FAILED) {
return (<>Automatic annotation failed</>);
return (
<>
Automatic annotation failed
<CVATTooltip title={activeInference.error}>
<QuestionCircleOutlined />
</CVATTooltip>
</>
);
}

if (activeInference.status === RQStatus.UNKNOWN) {
Expand Down
2 changes: 1 addition & 1 deletion cvat-ui/src/reducers/notifications-reducer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -885,7 +885,7 @@ export default function (state = defaultState, action: AnyAction): Notifications
models: {
...state.errors.models,
inferenceStatusFetching: {
message: `Fetching inference status for the [task ${taskID}](/tasks/${taskID})`,
message: `Automatic annotation for the [task ${taskID}](/tasks/${taskID})`,
reason: action.payload.error,
shouldLog: shouldLog(action.payload.error),
},
Expand Down
9 changes: 3 additions & 6 deletions cvat/apps/engine/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

from cvat.apps.dataset_manager.formats.utils import get_label_color
from cvat.apps.engine.frame_provider import TaskFrameProvider
from cvat.apps.engine.utils import format_list, parse_exception_message
from cvat.apps.engine.utils import format_list
from cvat.apps.engine import field_validation, models
from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status
from cvat.apps.engine.log import ServerLogManager
Expand Down Expand Up @@ -3230,12 +3230,9 @@ def get_message(self, rq_job: RQJob) -> str:
message = ''

if RQJobStatus.STARTED == rq_job_status:
message = rq_job.meta.get(RQJobMetaField.STATUS, '')
message = rq_job.meta.get(RQJobMetaField.STATUS, "")
elif RQJobStatus.FAILED == rq_job_status:
message = rq_job.meta.get(
RQJobMetaField.FORMATTED_EXCEPTION,
parse_exception_message(str(rq_job.exc_info or "Unknown error")),
)
message = rq_job.meta.get(RQJobMetaField.FORMATTED_EXCEPTION, "Unknown error")

return message

Expand Down
3 changes: 2 additions & 1 deletion cvat/apps/engine/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -3252,8 +3252,9 @@ def perform_destroy(self, instance):
target.touch()

def rq_exception_handler(rq_job, exc_type, exc_value, tb):
rq_job.meta[RQJobMetaField.FORMATTED_EXCEPTION] = "".join(
formatted_exception = "".join(
traceback.format_exception_only(exc_type, exc_value))
rq_job.meta[RQJobMetaField.FORMATTED_EXCEPTION] = parse_exception_message(formatted_exception)
rq_job.save_meta()

return True
Expand Down
7 changes: 5 additions & 2 deletions cvat/apps/lambda_manager/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -621,11 +621,15 @@ def to_dict(self):
"enqueued": self.job.enqueued_at,
"started": self.job.started_at,
"ended": self.job.ended_at,
"exc_info": self.job.exc_info
"exc_info": None
}

if dict_['status'] == rq.job.JobStatus.DEFERRED:
dict_['status'] = rq.job.JobStatus.QUEUED.value

if dict_['status'] == rq.job.JobStatus.FAILED:
dict_['exc_info'] = self.job.meta.get(RQJobMetaField.FORMATTED_EXCEPTION, 'Unknown error')

return dict_

def get_task(self):
Expand Down Expand Up @@ -1203,7 +1207,6 @@ def retrieve(self, request, pk):
self.check_object_permissions(request, pk)
queue = LambdaQueue()
rq_job = queue.fetch_job(pk)

response_serializer = FunctionCallSerializer(rq_job.to_dict())
return response_serializer.data

Expand Down
Loading