From 03eeae928bff8e478626a557daf827403fb27e46 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Thu, 3 Sep 2020 16:34:59 +0300 Subject: [PATCH 1/7] tmp --- .../src/typescript/interactionHandler.ts | 4 - cvat-ui/src/actions/annotation-actions.ts | 21 +- .../controls-side-bar/controls-side-bar.tsx | 4 +- .../controls-side-bar/tools-control.tsx | 318 ++++++++++++++++-- .../standard-workspace/styles.scss | 3 +- cvat-ui/src/reducers/annotation-reducer.ts | 2 +- cvat-ui/src/reducers/interfaces.ts | 2 +- 7 files changed, 317 insertions(+), 37 deletions(-) diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index 76237cf06ed4..7ede21e3265b 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -155,10 +155,6 @@ export class InteractionHandlerImpl implements InteractionHandler { this.shapesWereUpdated = true; this.canvas.off('mousedown.interaction', eventListener); - if (this.shouldRaiseEvent(false)) { - this.onInteraction(this.prepareResult(), true, false); - } - this.interact({ enabled: false }); }).addClass('cvat_canvas_shape_drawing').attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index efeaa81e8ff0..a862a0757265 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -1424,12 +1424,21 @@ export function repeatDrawShapeAsync(): ThunkAction { let activeControl = ActiveControl.CURSOR; if (activeInteractor) { - canvasInstance.interact({ - enabled: true, - shapeType: 'points', - minPosVertices: 4, // TODO: Add parameter to interactor - }); - dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + if (activeInteractor.type === 'tracker') { + canvasInstance.interact({ + enabled: true, + shapeType: 'rectangle', + }); + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + } else { + canvasInstance.interact({ + enabled: true, + shapeType: 'points', + minPosVertices: 4, // TODO: Add parameter to interactor + }); + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + } + return; } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx index a202d5c99ac4..ef4b457e11d5 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx @@ -85,7 +85,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { preventDefault(event); const drawing = [ActiveControl.DRAW_POINTS, ActiveControl.DRAW_POLYGON, ActiveControl.DRAW_POLYLINE, ActiveControl.DRAW_RECTANGLE, - ActiveControl.DRAW_CUBOID, ActiveControl.INTERACTION].includes(activeControl); + ActiveControl.DRAW_CUBOID, ActiveControl.AI_TOOLS].includes(activeControl); if (!drawing) { canvasInstance.cancel(); @@ -98,7 +98,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { repeatDrawShape(); } } else { - if (activeControl === ActiveControl.INTERACTION) { + if (activeControl === ActiveControl.AI_TOOLS) { // separated API method canvasInstance.interact({ enabled: false }); return; diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 195466b26b64..99bc0fb8ef8a 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -32,6 +32,7 @@ import { } from 'actions/annotation-actions'; import { InteractionResult } from 'cvat-canvas/src/typescript/canvas'; import DetectorRunner from 'components/model-runner-modal/detector-runner'; +import InputNumber from 'antd/lib/input-number'; interface StateToProps { canvasInstance: Canvas; @@ -39,10 +40,12 @@ interface StateToProps { states: any[]; activeLabelID: number; jobInstance: any; - isInteraction: boolean; + isActivated: boolean; frame: number; interactors: Model[]; detectors: Model[]; + trackers: Model[]; + curZOrder: number; } interface DispatchToProps { @@ -60,18 +63,20 @@ function mapStateToProps(state: CombinedState): StateToProps { const { instance: jobInstance } = annotation.job; const { instance: canvasInstance, activeControl } = annotation.canvas; const { models } = state; - const { interactors, detectors } = models; + const { interactors, detectors, trackers } = models; return { interactors, detectors, - isInteraction: activeControl === ActiveControl.INTERACTION, + trackers, + isActivated: activeControl === ActiveControl.AI_TOOLS, activeLabelID: annotation.drawing.activeLabelID, labels: annotation.job.labels, states: annotation.annotations.states, canvasInstance, jobInstance, frame, + curZOrder: annotation.annotations.zLayer.cur, }; } @@ -103,7 +108,10 @@ interface State { activeInteractor: Model | null; activeLabelID: number; interactiveStateID: number | null; + activeTracker: Model | null; + trackingFrames: number; fetching: boolean; + mode: 'detection' | 'interaction' | 'tracking'; } class ToolsControlComponent extends React.PureComponent { @@ -114,9 +122,12 @@ class ToolsControlComponent extends React.PureComponent { super(props); this.state = { activeInteractor: props.interactors.length ? props.interactors[0] : null, + activeTracker: props.trackers.length ? props.trackers[0] : null, activeLabelID: props.labels[0].id, interactiveStateID: null, + trackingFrames: 10, fetching: false, + mode: 'interaction', }; this.interactionIsAborted = false; @@ -130,10 +141,11 @@ class ToolsControlComponent extends React.PureComponent { } public componentDidUpdate(prevProps: Props): void { - const { isInteraction } = this.props; - if (prevProps.isInteraction && !isInteraction) { + const { isActivated } = this.props; + if (prevProps.isActivated && !isActivated) { window.removeEventListener('contextmenu', this.contextmenuDisabler); - } else if (!prevProps.isInteraction && isInteraction) { + } else if (!prevProps.isActivated && isActivated) { + // reset flags when start interaction/tracking this.interactionIsDone = false; this.interactionIsAborted = false; window.addEventListener('contextmenu', this.contextmenuDisabler); @@ -162,14 +174,14 @@ class ToolsControlComponent extends React.PureComponent { private cancelListener = async (): Promise => { const { - isInteraction, + isActivated, jobInstance, frame, fetchAnnotations, } = this.props; const { interactiveStateID, fetching } = this.state; - if (isInteraction) { + if (isActivated) { if (fetching && !this.interactionIsDone) { // user pressed ESC this.setState({ fetching: false }); @@ -187,12 +199,13 @@ class ToolsControlComponent extends React.PureComponent { } }; - private interactionListener = async (e: Event): Promise => { + private onInteraction = async (e: Event): Promise => { const { frame, labels, + curZOrder, jobInstance, - isInteraction, + isActivated, activeLabelID, fetchAnnotations, updateAnnotations, @@ -200,8 +213,8 @@ class ToolsControlComponent extends React.PureComponent { const { activeInteractor, interactiveStateID, fetching } = this.state; try { - if (!isInteraction) { - throw Error('Canvas raises event "canvas.interacted" when interaction is off'); + if (!isActivated) { + throw Error('Canvas raises event "canvas.interacted" when interaction with it is off'); } if (fetching) { @@ -216,7 +229,6 @@ class ToolsControlComponent extends React.PureComponent { this.setState({ fetching: true }); try { result = await core.lambda.call(jobInstance.task, interactor, { - task: jobInstance.task, frame, points: convertShapesForInteractor((e as CustomEvent).detail.shapes), }); @@ -241,7 +253,7 @@ class ToolsControlComponent extends React.PureComponent { shapeType: ShapeType.POLYGON, points: result.flat(), occluded: false, - zOrder: (e as CustomEvent).detail.zOrder, + zOrder: curZOrder, }); await jobInstance.annotations.put([object]); @@ -260,7 +272,7 @@ class ToolsControlComponent extends React.PureComponent { shapeType: ShapeType.POLYGON, points: result.flat(), occluded: false, - zOrder: (e as CustomEvent).detail.zOrder, + zOrder: curZOrder, }); // need a clientID of a created object to interact with it further // so, we do not use createAnnotationAction @@ -302,6 +314,121 @@ class ToolsControlComponent extends React.PureComponent { } }; + private onTracking = async (e: Event): Promise => { + const { + isActivated, + jobInstance, + frame, + curZOrder, + fetchAnnotations, + } = this.props; + const { activeTracker, activeLabelID, trackingFrames } = this.state; + + if (!(e as CustomEvent).detail.isDone) { + return; + } + + this.interactionIsDone = true; + try { + if (!isActivated) { + throw Error('Canvas raises event "canvas.interacted" when interaction with it is off'); + } + + this.setState({ + fetching: true, + }); + + const tracker = activeTracker as Model; + const { points } = (e as CustomEvent).detail.shapes[0]; + + const state = new core.classes.ObjectState({ + shapeType: ShapeType.POLYGON, + objectType: ObjectType.TRACK, + zOrder: curZOrder, + label: jobInstance.task.labels + .filter( + (label: any): boolean => label.id === activeLabelID, + )[0], + points: [points[0], points[1], points[2], points[1], points[2], points[3], points[0], points[3]], + frame, + occluded: false, + source: 'auto', + attributes: {}, + }); + + const [clientID] = await jobInstance.annotations.put([state]); + + // update annotations on a canvas + fetchAnnotations(); + + let response = await core.lambda.call(jobInstance.task, tracker, { + task: jobInstance.task, + frame, + shape: (e as CustomEvent).detail.shapes[0].points, + }); + + for (let i = 1; i <= trackingFrames; i++) { + /* eslint-disable no-await-in-loop */ + const [objectState] = await jobInstance.annotations.get(frame + i, true, [`clientID == ${clientID}`]); + + response = await core.lambda.call(jobInstance.task, tracker, { + task: jobInstance.task, + frame: frame + i, + shape: response.points, + state: response.state, + }); + + // let xtl = Number.MAX_SAFE_INTEGER; + // let ytl = Number.MAX_SAFE_INTEGER; + // let xbr = Number.MIN_SAFE_INTEGER; + // let ybr = Number.MIN_SAFE_INTEGER; + + // for (let j = 0; j < response.shape.length; j++) { + // if (j % 2) { // y + // ytl = Math.min(ytl, response.shape[j]); + // ybr = Math.max(ybr, response.shape[j]); + // } else { // x + // xtl = Math.min(xtl, response.shape[j]); + // xbr = Math.max(xbr, response.shape[j]); + // } + // } + + objectState.points = response.shape; + await objectState.save(); + } + + this.setState({ + fetching: false, + }); + + // add a track, get it back + // get it from the next frame + // + + + // start server request + // get result + // push track to collection + } catch (err) { + notification.error({ + description: err.toString(), + message: 'Tracking error occured', + }); + } + }; + + private interactionListener = async (e: Event): Promise => { + const { mode } = this.state; + + if (mode === 'interaction') { + await this.onInteraction(e); + } + + if (mode === 'tracking') { + await this.onTracking(e); + } + }; + private setActiveInteractor = (key: string): void => { const { interactors } = this.props; this.setState({ @@ -311,6 +438,15 @@ class ToolsControlComponent extends React.PureComponent { }); }; + private setActiveTracker = (key: string): void => { + const { trackers } = this.props; + this.setState({ + activeTracker: trackers.filter( + (tracker: Model) => tracker.id === key, + )[0], + }); + }; + private renderLabelBlock(): JSX.Element { const { labels } = this.props; const { activeLabelID } = this.state; @@ -355,10 +491,119 @@ class ToolsControlComponent extends React.PureComponent { ); } + private renderTrackerBlock(): JSX.Element { + const { + trackers, + canvasInstance, + jobInstance, + frame, + onInteractionStart, + } = this.props; + const { + activeTracker, + activeLabelID, + fetching, + trackingFrames, + } = this.state; + + if (!trackers.length) { + return ( + + + No available trackers found + + + ); + } + + return ( + <> + + + Tracker + + + + + + + + + + Tracking frames + + + { + if (typeof (value) !== 'undefined') { + this.setState({ + trackingFrames: value, + }); + } + }} + /> + + + + + + + + + ); + } + private renderInteractorBlock(): JSX.Element { const { interactors, canvasInstance, onInteractionStart } = this.props; const { activeInteractor, activeLabelID, fetching } = this.state; + if (!interactors.length) { + return ( + + + No available interactors found + + + ); + } + return ( <> @@ -389,6 +634,10 @@ class ToolsControlComponent extends React.PureComponent { className='cvat-tools-interact-button' disabled={!activeInteractor || fetching} onClick={() => { + this.setState({ + mode: 'interaction', + }); + if (activeInteractor) { canvasInstance.cancel(); canvasInstance.interact({ @@ -413,10 +662,21 @@ class ToolsControlComponent extends React.PureComponent { const { jobInstance, detectors, + curZOrder, frame, fetchAnnotations, } = this.props; + if (!detectors.length) { + return ( + + + No available interactors found + + + ); + } + return ( { task={jobInstance.task} runInference={async (task: any, model: Model, body: object) => { try { + this.setState({ + mode: 'detection', + }); + this.setState({ fetching: true }); const result = await core.lambda.call(task, model, { ...body, @@ -444,7 +708,7 @@ class ToolsControlComponent extends React.PureComponent { occluded: false, source: 'auto', attributes: {}, - zOrder: 0, // TODO: get current z order + zOrder: curZOrder, }) )); @@ -471,7 +735,7 @@ class ToolsControlComponent extends React.PureComponent { AI Tools - + { this.renderLabelBlock() } { this.renderInteractorBlock() } @@ -479,24 +743,34 @@ class ToolsControlComponent extends React.PureComponent { { this.renderDetectorBlock() } + + { this.renderLabelBlock() } + { this.renderTrackerBlock() } + ); } public render(): JSX.Element | null { - const { interactors, isInteraction, canvasInstance } = this.props; + const { + interactors, + detectors, + trackers, + isActivated, + canvasInstance, + } = this.props; const { fetching } = this.state; - if (!interactors.length) return null; + if (![...interactors, ...detectors, ...trackers].length) return null; - const dynamcPopoverPros = isInteraction ? { + const dynamcPopoverPros = isActivated ? { overlayStyle: { display: 'none', }, } : {}; - const dynamicIconProps = isInteraction ? { + const dynamicIconProps = isActivated ? { className: 'cvat-active-canvas-control cvat-tools-control', onClick: (): void => { canvasInstance.interact({ enabled: false }); @@ -522,7 +796,7 @@ class ToolsControlComponent extends React.PureComponent { {...dynamcPopoverPros} placement='right' overlayClassName='cvat-tools-control-popover' - content={interactors.length && this.renderPopoverContent()} + content={this.renderPopoverContent()} > diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss index ee5dba37d4e2..9dcfdd2b28e2 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss +++ b/cvat-ui/src/components/annotation-page/standard-workspace/styles.scss @@ -92,6 +92,7 @@ } } +.cvat-tools-track-button, .cvat-tools-interact-button { width: 100%; margin-top: 10px; @@ -102,7 +103,7 @@ } .cvat-tools-control-popover-content { - width: 350px; + width: fit-content; padding: 10px; border-radius: 5px; background: $background-color-2; diff --git a/cvat-ui/src/reducers/annotation-reducer.ts b/cvat-ui/src/reducers/annotation-reducer.ts index ca9567e1221d..3227834ad791 100644 --- a/cvat-ui/src/reducers/annotation-reducer.ts +++ b/cvat-ui/src/reducers/annotation-reducer.ts @@ -1058,7 +1058,7 @@ export default (state = defaultState, action: AnyAction): AnnotationState => { }, canvas: { ...state.canvas, - activeControl: ActiveControl.INTERACTION, + activeControl: ActiveControl.AI_TOOLS, }, }; } diff --git a/cvat-ui/src/reducers/interfaces.ts b/cvat-ui/src/reducers/interfaces.ts index 8f9e7af4fff9..327504a71133 100644 --- a/cvat-ui/src/reducers/interfaces.ts +++ b/cvat-ui/src/reducers/interfaces.ts @@ -275,7 +275,7 @@ export enum ActiveControl { GROUP = 'group', SPLIT = 'split', EDIT = 'edit', - INTERACTION = 'interaction', + AI_TOOLS = 'ai_tools', } export enum ShapeType { From 25a0f5122207b8e2178bd8bc051784888cfc5e21 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 7 Sep 2020 10:09:24 +0300 Subject: [PATCH 2/7] Refactored --- .../controls-side-bar/tools-control.tsx | 84 ++++++++++--------- cvat-ui/src/utils/range.ts | 27 ++++++ 2 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 cvat-ui/src/utils/range.ts diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 99bc0fb8ef8a..9bcddf30e2a2 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -13,9 +13,11 @@ import Text from 'antd/lib/typography/Text'; import Tabs from 'antd/lib/tabs'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; +import Progress from 'antd/lib/progress'; import { AIToolsIcon } from 'icons'; import { Canvas } from 'cvat-canvas-wrapper'; +import range from 'utils/range'; import getCore from 'cvat-core-wrapper'; import { CombinedState, @@ -109,6 +111,7 @@ interface State { activeLabelID: number; interactiveStateID: number | null; activeTracker: Model | null; + trackingProgress: number | null; trackingFrames: number; fetching: boolean; mode: 'detection' | 'interaction' | 'tracking'; @@ -125,6 +128,7 @@ class ToolsControlComponent extends React.PureComponent { activeTracker: props.trackers.length ? props.trackers[0] : null, activeLabelID: props.labels[0].id, interactiveStateID: null, + trackingProgress: null, trackingFrames: 10, fetching: false, mode: 'interaction', @@ -323,6 +327,9 @@ class ToolsControlComponent extends React.PureComponent { fetchAnnotations, } = this.props; const { activeTracker, activeLabelID, trackingFrames } = this.state; + const [label] = jobInstance.task.labels.filter( + (_label: any): boolean => _label.id === activeLabelID, + ); if (!(e as CustomEvent).detail.isDone) { return; @@ -336,20 +343,18 @@ class ToolsControlComponent extends React.PureComponent { this.setState({ fetching: true, + trackingProgress: 0, }); const tracker = activeTracker as Model; const { points } = (e as CustomEvent).detail.shapes[0]; const state = new core.classes.ObjectState({ - shapeType: ShapeType.POLYGON, + shapeType: ShapeType.RECTANGLE, objectType: ObjectType.TRACK, zOrder: curZOrder, - label: jobInstance.task.labels - .filter( - (label: any): boolean => label.id === activeLabelID, - )[0], - points: [points[0], points[1], points[2], points[1], points[2], points[3], points[0], points[3]], + label, + points, frame, occluded: false, source: 'auto', @@ -367,53 +372,49 @@ class ToolsControlComponent extends React.PureComponent { shape: (e as CustomEvent).detail.shapes[0].points, }); - for (let i = 1; i <= trackingFrames; i++) { + for (const offset of range(1, trackingFrames + 1)) { /* eslint-disable no-await-in-loop */ - const [objectState] = await jobInstance.annotations.get(frame + i, true, [`clientID == ${clientID}`]); - + const states = await jobInstance.annotations.get(frame + offset); + const [objectState] = states + .filter((_state: any): boolean => _state.clientID === clientID); response = await core.lambda.call(jobInstance.task, tracker, { task: jobInstance.task, - frame: frame + i, + frame: frame + offset, shape: response.points, state: response.state, }); - // let xtl = Number.MAX_SAFE_INTEGER; - // let ytl = Number.MAX_SAFE_INTEGER; - // let xbr = Number.MIN_SAFE_INTEGER; - // let ybr = Number.MIN_SAFE_INTEGER; - - // for (let j = 0; j < response.shape.length; j++) { - // if (j % 2) { // y - // ytl = Math.min(ytl, response.shape[j]); - // ybr = Math.max(ybr, response.shape[j]); - // } else { // x - // xtl = Math.min(xtl, response.shape[j]); - // xbr = Math.max(xbr, response.shape[j]); - // } - // } - - objectState.points = response.shape; + const reduced = response.shape + .reduce((acc: number[], value: number, index: number): number[] => { + if (index % 2) { // y + acc[1] = Math.min(acc[1], value); + acc[3] = Math.max(acc[3], value); + } else { // x + acc[0] = Math.min(acc[0], value); + acc[2] = Math.max(acc[2], value); + } + return acc; + }, [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, + Number.MIN_SAFE_INTEGER, Number.MIN_SAFE_INTEGER, + ]); + + objectState.points = reduced; await objectState.save(); - } - - this.setState({ - fetching: false, - }); - // add a track, get it back - // get it from the next frame - // - - - // start server request - // get result - // push track to collection + this.setState({ + trackingProgress: offset / trackingFrames, + }); + } } catch (err) { notification.error({ description: err.toString(), message: 'Tracking error occured', }); + } finally { + this.setState({ + fetching: false, + trackingProgress: null, + }); } }; @@ -760,7 +761,7 @@ class ToolsControlComponent extends React.PureComponent { isActivated, canvasInstance, } = this.props; - const { fetching } = this.state; + const { fetching, trackingProgress } = this.state; if (![...interactors, ...detectors, ...trackers].length) return null; @@ -791,6 +792,9 @@ class ToolsControlComponent extends React.PureComponent { > Waiting for a server response.. + { trackingProgress !== null && ( + + )} = y) { + throw new Error(`Range() expects the first argument less or equal than the second. Got ${x}, ${y}`); + } + + return Array.from(Array(y - x), (_: number, i: number) => i + x); + } + + if (typeof (x) !== 'undefined') { + if (typeof (x) !== 'number') { + throw new Error(`Range() expects number arguments. Got ${typeof (x)}`); + } + + return [...Array(x).keys()]; + } + + return []; +} From 0e42edab158451645c4f3a5a1cd39488cce622b4 Mon Sep 17 00:00:00 2001 From: Boris Sekachev Date: Mon, 7 Sep 2020 11:26:22 +0300 Subject: [PATCH 3/7] Refactoring & added button to context menu --- cvat-ui/src/actions/annotation-actions.ts | 12 ++ .../controls-side-bar/tools-control.tsx | 122 ++++++++++-------- .../objects-side-bar/object-item-basics.tsx | 3 + .../objects-side-bar/object-item-menu.tsx | 32 +++-- .../objects-side-bar/object-item.tsx | 3 + .../standard-workspace/standard-workspace.tsx | 1 + .../objects-side-bar/object-item.tsx | 14 +- cvat-ui/src/reducers/annotation-reducer.ts | 2 + cvat-ui/src/reducers/interfaces.ts | 2 + 9 files changed, 126 insertions(+), 65 deletions(-) diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index a862a0757265..e6b3ef2f6bcf 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -27,6 +27,7 @@ import getCore from 'cvat-core-wrapper'; import logger, { LogType } from 'cvat-logger'; import { RectDrawingMethod } from 'cvat-canvas-wrapper'; import { getCVATStore } from 'cvat-store'; +import { MutableRefObject } from 'react'; interface AnnotationsParameters { filters: string[]; @@ -189,6 +190,7 @@ export enum AnnotationActionTypes { SAVE_LOGS_SUCCESS = 'SAVE_LOGS_SUCCESS', SAVE_LOGS_FAILED = 'SAVE_LOGS_FAILED', INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS', + SET_AI_TOOLS_REF = 'SET_AI_TOOLS_REF', } export function saveLogsAsync(): ThunkAction { @@ -1397,6 +1399,16 @@ export function interactWithCanvas(activeInteractor: Model, activeLabelID: numbe }; } +export function setAIToolsRef(ref: MutableRefObject): AnyAction { + return { + type: AnnotationActionTypes.SET_AI_TOOLS_REF, + payload: { + aiToolsRef: ref, + }, + }; +} + + export function repeatDrawShapeAsync(): ThunkAction { return async (dispatch: ActionCreator): Promise => { const { diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 9bcddf30e2a2..1a9215d0a456 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -import React from 'react'; +import React, { MutableRefObject } from 'react'; import { connect } from 'react-redux'; import Icon from 'antd/lib/icon'; import Popover from 'antd/lib/popover'; @@ -48,6 +48,7 @@ interface StateToProps { detectors: Model[]; trackers: Model[]; curZOrder: number; + aiToolsRef: MutableRefObject; } interface DispatchToProps { @@ -79,6 +80,7 @@ function mapStateToProps(state: CombinedState): StateToProps { jobInstance, frame, curZOrder: annotation.annotations.zLayer.cur, + aiToolsRef: annotation.aiToolsRef, }; } @@ -117,7 +119,7 @@ interface State { mode: 'detection' | 'interaction' | 'tracking'; } -class ToolsControlComponent extends React.PureComponent { +export class ToolsControlComponent extends React.PureComponent { private interactionIsAborted: boolean; private interactionIsDone: boolean; @@ -139,7 +141,8 @@ class ToolsControlComponent extends React.PureComponent { } public componentDidMount(): void { - const { canvasInstance } = this.props; + const { canvasInstance, aiToolsRef } = this.props; + aiToolsRef.current = this; canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener); canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener); } @@ -157,7 +160,8 @@ class ToolsControlComponent extends React.PureComponent { } public componentWillUnmount(): void { - const { canvasInstance } = this.props; + const { canvasInstance, aiToolsRef } = this.props; + aiToolsRef.current = undefined; canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener); canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener); } @@ -326,7 +330,7 @@ class ToolsControlComponent extends React.PureComponent { curZOrder, fetchAnnotations, } = this.props; - const { activeTracker, activeLabelID, trackingFrames } = this.state; + const { activeLabelID } = this.state; const [label] = jobInstance.task.labels.filter( (_label: any): boolean => _label.id === activeLabelID, ); @@ -341,14 +345,7 @@ class ToolsControlComponent extends React.PureComponent { throw Error('Canvas raises event "canvas.interacted" when interaction with it is off'); } - this.setState({ - fetching: true, - trackingProgress: 0, - }); - - const tracker = activeTracker as Model; const { points } = (e as CustomEvent).detail.shapes[0]; - const state = new core.classes.ObjectState({ shapeType: ShapeType.RECTANGLE, objectType: ObjectType.TRACK, @@ -366,10 +363,60 @@ class ToolsControlComponent extends React.PureComponent { // update annotations on a canvas fetchAnnotations(); + const states = await jobInstance.annotations.get(frame); + const [objectState] = states + .filter((_state: any): boolean => _state.clientID === clientID); + await this.trackState(objectState); + } catch (err) { + notification.error({ + description: err.toString(), + message: 'Tracking error occured', + }); + } + }; + + private interactionListener = async (e: Event): Promise => { + const { mode } = this.state; + + if (mode === 'interaction') { + await this.onInteraction(e); + } + + if (mode === 'tracking') { + await this.onTracking(e); + } + }; + + private setActiveInteractor = (key: string): void => { + const { interactors } = this.props; + this.setState({ + activeInteractor: interactors.filter( + (interactor: Model) => interactor.id === key, + )[0], + }); + }; + + private setActiveTracker = (key: string): void => { + const { trackers } = this.props; + this.setState({ + activeTracker: trackers.filter( + (tracker: Model) => tracker.id === key, + )[0], + }); + }; + + public async trackState(state: any): Promise { + const { jobInstance, frame } = this.props; + const { activeTracker, trackingFrames } = this.state; + const { clientID, points } = state; + + const tracker = activeTracker as Model; + try { + this.setState({ trackingProgress: 0, fetching: true }); let response = await core.lambda.call(jobInstance.task, tracker, { task: jobInstance.task, frame, - shape: (e as CustomEvent).detail.shapes[0].points, + shape: points, }); for (const offset of range(1, trackingFrames + 1)) { @@ -401,52 +448,19 @@ class ToolsControlComponent extends React.PureComponent { objectState.points = reduced; await objectState.save(); - this.setState({ - trackingProgress: offset / trackingFrames, - }); + this.setState({ trackingProgress: offset / trackingFrames }); } - } catch (err) { - notification.error({ - description: err.toString(), - message: 'Tracking error occured', - }); } finally { - this.setState({ - fetching: false, - trackingProgress: null, - }); - } - }; - - private interactionListener = async (e: Event): Promise => { - const { mode } = this.state; - - if (mode === 'interaction') { - await this.onInteraction(e); - } - - if (mode === 'tracking') { - await this.onTracking(e); + this.setState({ trackingProgress: null, fetching: false }); } - }; - - private setActiveInteractor = (key: string): void => { - const { interactors } = this.props; - this.setState({ - activeInteractor: interactors.filter( - (interactor: Model) => interactor.id === key, - )[0], - }); - }; + } - private setActiveTracker = (key: string): void => { + public trackingAvailable(): boolean { + const { activeTracker, trackingFrames } = this.state; const { trackers } = this.props; - this.setState({ - activeTracker: trackers.filter( - (tracker: Model) => tracker.id === key, - )[0], - }); - }; + + return !!trackingFrames && !!trackers.length && activeTracker !== null; + } private renderLabelBlock(): JSX.Element { const { labels } = this.props; diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx index bbfc0e034ff9..9976518754e9 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx @@ -41,6 +41,7 @@ interface Props { toBackground(): void; toForeground(): void; resetCuboidPerspective(): void; + activateTracking(): void; } function ItemTopComponent(props: Props): JSX.Element { @@ -72,6 +73,7 @@ function ItemTopComponent(props: Props): JSX.Element { toBackground, toForeground, resetCuboidPerspective, + activateTracking, } = props; const [menuVisible, setMenuVisible] = useState(false); @@ -150,6 +152,7 @@ function ItemTopComponent(props: Props): JSX.Element { toForeground, resetCuboidPerspective, changeColorPickerVisible, + activateTracking, })} > diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx index 99db3b848917..c4283e3e178b 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx @@ -33,16 +33,17 @@ interface Props { toBackgroundShortcut: string; toForegroundShortcut: string; removeShortcut: string; - changeColor: (value: string) => void; - copy: (() => void); - remove: (() => void); - propagate: (() => void); - createURL: (() => void); - switchOrientation: (() => void); - toBackground: (() => void); - toForeground: (() => void); - resetCuboidPerspective: (() => void); - changeColorPickerVisible: (visible: boolean) => void; + changeColor(value: string): void; + copy(): void; + remove(): void; + propagate(): void; + createURL(): void; + switchOrientation(): void; + toBackground(): void; + toForeground(): void; + resetCuboidPerspective(): void; + changeColorPickerVisible(visible: boolean): void; + activateTracking(): void; } export default function ItemMenu(props: Props): JSX.Element { @@ -71,6 +72,7 @@ export default function ItemMenu(props: Props): JSX.Element { toForeground, resetCuboidPerspective, changeColorPickerVisible, + activateTracking, } = props; return ( @@ -94,6 +96,16 @@ export default function ItemMenu(props: Props): JSX.Element { + {objectType === ObjectType.TRACK && shapeType === ShapeType.RECTANGLE && ( + + + + + + )} { [ShapeType.POLYGON, ShapeType.POLYLINE, ShapeType.CUBOID].includes(shapeType) && (