diff --git a/CHANGELOG.md b/CHANGELOG.md index 844cc02fe6f5..6015d0ea2090 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Ability to work with data on the fly (https://github.com/opencv/cvat/pull/2007) - Annotation in process outline color wheel () - On the fly annotation using DL detectors () +- Automatic tracking of bounding boxes using serverless functions () - [Datumaro] CLI command for dataset equality comparison () - [Datumaro] Merging of datasets with different labels () diff --git a/cvat-canvas/src/typescript/interactionHandler.ts b/cvat-canvas/src/typescript/interactionHandler.ts index 76237cf06ed4..7ede21e3265b 100644 --- a/cvat-canvas/src/typescript/interactionHandler.ts +++ b/cvat-canvas/src/typescript/interactionHandler.ts @@ -155,10 +155,6 @@ export class InteractionHandlerImpl implements InteractionHandler { this.shapesWereUpdated = true; this.canvas.off('mousedown.interaction', eventListener); - if (this.shouldRaiseEvent(false)) { - this.onInteraction(this.prepareResult(), true, false); - } - this.interact({ enabled: false }); }).addClass('cvat_canvas_shape_drawing').attr({ 'stroke-width': consts.BASE_STROKE_WIDTH / this.geometry.scale, diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 91ac8a0f8221..55038b63beae 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.6.0", + "version": "3.6.1", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index 4418f1574495..869464ac5c19 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.6.0", + "version": "3.6.1", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-ui/package-lock.json b/cvat-ui/package-lock.json index 5f7a0b86a676..5c8e3541d842 100644 --- a/cvat-ui/package-lock.json +++ b/cvat-ui/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.9.2", + "version": "1.9.3", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-ui/package.json b/cvat-ui/package.json index f91eabd335a4..f504a65bbd7f 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.9.2", + "version": "1.9.3", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/actions/annotation-actions.ts b/cvat-ui/src/actions/annotation-actions.ts index efeaa81e8ff0..e6b3ef2f6bcf 100644 --- a/cvat-ui/src/actions/annotation-actions.ts +++ b/cvat-ui/src/actions/annotation-actions.ts @@ -27,6 +27,7 @@ import getCore from 'cvat-core-wrapper'; import logger, { LogType } from 'cvat-logger'; import { RectDrawingMethod } from 'cvat-canvas-wrapper'; import { getCVATStore } from 'cvat-store'; +import { MutableRefObject } from 'react'; interface AnnotationsParameters { filters: string[]; @@ -189,6 +190,7 @@ export enum AnnotationActionTypes { SAVE_LOGS_SUCCESS = 'SAVE_LOGS_SUCCESS', SAVE_LOGS_FAILED = 'SAVE_LOGS_FAILED', INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS', + SET_AI_TOOLS_REF = 'SET_AI_TOOLS_REF', } export function saveLogsAsync(): ThunkAction { @@ -1397,6 +1399,16 @@ export function interactWithCanvas(activeInteractor: Model, activeLabelID: numbe }; } +export function setAIToolsRef(ref: MutableRefObject): AnyAction { + return { + type: AnnotationActionTypes.SET_AI_TOOLS_REF, + payload: { + aiToolsRef: ref, + }, + }; +} + + export function repeatDrawShapeAsync(): ThunkAction { return async (dispatch: ActionCreator): Promise => { const { @@ -1424,12 +1436,21 @@ export function repeatDrawShapeAsync(): ThunkAction { let activeControl = ActiveControl.CURSOR; if (activeInteractor) { - canvasInstance.interact({ - enabled: true, - shapeType: 'points', - minPosVertices: 4, // TODO: Add parameter to interactor - }); - dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + if (activeInteractor.type === 'tracker') { + canvasInstance.interact({ + enabled: true, + shapeType: 'rectangle', + }); + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + } else { + canvasInstance.interact({ + enabled: true, + shapeType: 'points', + minPosVertices: 4, // TODO: Add parameter to interactor + }); + dispatch(interactWithCanvas(activeInteractor, activeLabelID)); + } + return; } diff --git a/cvat-ui/src/actions/plugins-actions.ts b/cvat-ui/src/actions/plugins-actions.ts index 6b5ca2c93add..c1c739f6aa36 100644 --- a/cvat-ui/src/actions/plugins-actions.ts +++ b/cvat-ui/src/actions/plugins-actions.ts @@ -30,19 +30,16 @@ export function checkPluginsAsync(): ThunkAction { const plugins: PluginObjects = { ANALYTICS: false, GIT_INTEGRATION: false, - DEXTR_SEGMENTATION: false, }; const promises: Promise[] = [ // check must return true/false with no exceptions PluginChecker.check(SupportedPlugins.ANALYTICS), PluginChecker.check(SupportedPlugins.GIT_INTEGRATION), - PluginChecker.check(SupportedPlugins.DEXTR_SEGMENTATION), ]; const values = await Promise.all(promises); - [plugins.ANALYTICS, plugins.GIT_INTEGRATION, - plugins.DEXTR_SEGMENTATION] = values; + [plugins.ANALYTICS, plugins.GIT_INTEGRATION] = values; dispatch(pluginActions.checkedAllPlugins(plugins)); }; } diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx index a202d5c99ac4..ef4b457e11d5 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/controls-side-bar.tsx @@ -85,7 +85,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { preventDefault(event); const drawing = [ActiveControl.DRAW_POINTS, ActiveControl.DRAW_POLYGON, ActiveControl.DRAW_POLYLINE, ActiveControl.DRAW_RECTANGLE, - ActiveControl.DRAW_CUBOID, ActiveControl.INTERACTION].includes(activeControl); + ActiveControl.DRAW_CUBOID, ActiveControl.AI_TOOLS].includes(activeControl); if (!drawing) { canvasInstance.cancel(); @@ -98,7 +98,7 @@ export default function ControlsSideBarComponent(props: Props): JSX.Element { repeatDrawShape(); } } else { - if (activeControl === ActiveControl.INTERACTION) { + if (activeControl === ActiveControl.AI_TOOLS) { // separated API method canvasInstance.interact({ enabled: false }); return; diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx index 195466b26b64..83db38e21c04 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/controls-side-bar/tools-control.tsx @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -import React from 'react'; +import React, { MutableRefObject } from 'react'; import { connect } from 'react-redux'; import Icon from 'antd/lib/icon'; import Popover from 'antd/lib/popover'; @@ -13,9 +13,11 @@ import Text from 'antd/lib/typography/Text'; import Tabs from 'antd/lib/tabs'; import { Row, Col } from 'antd/lib/grid'; import notification from 'antd/lib/notification'; +import Progress from 'antd/lib/progress'; import { AIToolsIcon } from 'icons'; import { Canvas } from 'cvat-canvas-wrapper'; +import range from 'utils/range'; import getCore from 'cvat-core-wrapper'; import { CombinedState, @@ -32,6 +34,7 @@ import { } from 'actions/annotation-actions'; import { InteractionResult } from 'cvat-canvas/src/typescript/canvas'; import DetectorRunner from 'components/model-runner-modal/detector-runner'; +import InputNumber from 'antd/lib/input-number'; interface StateToProps { canvasInstance: Canvas; @@ -39,10 +42,13 @@ interface StateToProps { states: any[]; activeLabelID: number; jobInstance: any; - isInteraction: boolean; + isActivated: boolean; frame: number; interactors: Model[]; detectors: Model[]; + trackers: Model[]; + curZOrder: number; + aiToolsRef: MutableRefObject; } interface DispatchToProps { @@ -60,18 +66,21 @@ function mapStateToProps(state: CombinedState): StateToProps { const { instance: jobInstance } = annotation.job; const { instance: canvasInstance, activeControl } = annotation.canvas; const { models } = state; - const { interactors, detectors } = models; + const { interactors, detectors, trackers } = models; return { interactors, detectors, - isInteraction: activeControl === ActiveControl.INTERACTION, + trackers, + isActivated: activeControl === ActiveControl.AI_TOOLS, activeLabelID: annotation.drawing.activeLabelID, labels: annotation.job.labels, states: annotation.annotations.states, canvasInstance, jobInstance, frame, + curZOrder: annotation.annotations.zLayer.cur, + aiToolsRef: annotation.aiToolsRef, }; } @@ -103,10 +112,14 @@ interface State { activeInteractor: Model | null; activeLabelID: number; interactiveStateID: number | null; + activeTracker: Model | null; + trackingProgress: number | null; + trackingFrames: number; fetching: boolean; + mode: 'detection' | 'interaction' | 'tracking'; } -class ToolsControlComponent extends React.PureComponent { +export class ToolsControlComponent extends React.PureComponent { private interactionIsAborted: boolean; private interactionIsDone: boolean; @@ -114,9 +127,13 @@ class ToolsControlComponent extends React.PureComponent { super(props); this.state = { activeInteractor: props.interactors.length ? props.interactors[0] : null, + activeTracker: props.trackers.length ? props.trackers[0] : null, activeLabelID: props.labels[0].id, interactiveStateID: null, + trackingProgress: null, + trackingFrames: 10, fetching: false, + mode: 'interaction', }; this.interactionIsAborted = false; @@ -124,16 +141,18 @@ class ToolsControlComponent extends React.PureComponent { } public componentDidMount(): void { - const { canvasInstance } = this.props; + const { canvasInstance, aiToolsRef } = this.props; + aiToolsRef.current = this; canvasInstance.html().addEventListener('canvas.interacted', this.interactionListener); canvasInstance.html().addEventListener('canvas.canceled', this.cancelListener); } public componentDidUpdate(prevProps: Props): void { - const { isInteraction } = this.props; - if (prevProps.isInteraction && !isInteraction) { + const { isActivated } = this.props; + if (prevProps.isActivated && !isActivated) { window.removeEventListener('contextmenu', this.contextmenuDisabler); - } else if (!prevProps.isInteraction && isInteraction) { + } else if (!prevProps.isActivated && isActivated) { + // reset flags when start interaction/tracking this.interactionIsDone = false; this.interactionIsAborted = false; window.addEventListener('contextmenu', this.contextmenuDisabler); @@ -141,7 +160,8 @@ class ToolsControlComponent extends React.PureComponent { } public componentWillUnmount(): void { - const { canvasInstance } = this.props; + const { canvasInstance, aiToolsRef } = this.props; + aiToolsRef.current = undefined; canvasInstance.html().removeEventListener('canvas.interacted', this.interactionListener); canvasInstance.html().removeEventListener('canvas.canceled', this.cancelListener); } @@ -162,14 +182,14 @@ class ToolsControlComponent extends React.PureComponent { private cancelListener = async (): Promise => { const { - isInteraction, + isActivated, jobInstance, frame, fetchAnnotations, } = this.props; const { interactiveStateID, fetching } = this.state; - if (isInteraction) { + if (isActivated) { if (fetching && !this.interactionIsDone) { // user pressed ESC this.setState({ fetching: false }); @@ -187,12 +207,13 @@ class ToolsControlComponent extends React.PureComponent { } }; - private interactionListener = async (e: Event): Promise => { + private onInteraction = async (e: Event): Promise => { const { frame, labels, + curZOrder, jobInstance, - isInteraction, + isActivated, activeLabelID, fetchAnnotations, updateAnnotations, @@ -200,8 +221,8 @@ class ToolsControlComponent extends React.PureComponent { const { activeInteractor, interactiveStateID, fetching } = this.state; try { - if (!isInteraction) { - throw Error('Canvas raises event "canvas.interacted" when interaction is off'); + if (!isActivated) { + throw Error('Canvas raises event "canvas.interacted" when interaction with it is off'); } if (fetching) { @@ -216,7 +237,6 @@ class ToolsControlComponent extends React.PureComponent { this.setState({ fetching: true }); try { result = await core.lambda.call(jobInstance.task, interactor, { - task: jobInstance.task, frame, points: convertShapesForInteractor((e as CustomEvent).detail.shapes), }); @@ -241,7 +261,7 @@ class ToolsControlComponent extends React.PureComponent { shapeType: ShapeType.POLYGON, points: result.flat(), occluded: false, - zOrder: (e as CustomEvent).detail.zOrder, + zOrder: curZOrder, }); await jobInstance.annotations.put([object]); @@ -260,7 +280,7 @@ class ToolsControlComponent extends React.PureComponent { shapeType: ShapeType.POLYGON, points: result.flat(), occluded: false, - zOrder: (e as CustomEvent).detail.zOrder, + zOrder: curZOrder, }); // need a clientID of a created object to interact with it further // so, we do not use createAnnotationAction @@ -302,6 +322,71 @@ class ToolsControlComponent extends React.PureComponent { } }; + private onTracking = async (e: Event): Promise => { + const { + isActivated, + jobInstance, + frame, + curZOrder, + fetchAnnotations, + } = this.props; + const { activeLabelID } = this.state; + const [label] = jobInstance.task.labels.filter( + (_label: any): boolean => _label.id === activeLabelID, + ); + + if (!(e as CustomEvent).detail.isDone) { + return; + } + + this.interactionIsDone = true; + try { + if (!isActivated) { + throw Error('Canvas raises event "canvas.interacted" when interaction with it is off'); + } + + const { points } = (e as CustomEvent).detail.shapes[0]; + const state = new core.classes.ObjectState({ + shapeType: ShapeType.RECTANGLE, + objectType: ObjectType.TRACK, + zOrder: curZOrder, + label, + points, + frame, + occluded: false, + source: 'auto', + attributes: {}, + }); + + const [clientID] = await jobInstance.annotations.put([state]); + + // update annotations on a canvas + fetchAnnotations(); + + const states = await jobInstance.annotations.get(frame); + const [objectState] = states + .filter((_state: any): boolean => _state.clientID === clientID); + await this.trackState(objectState); + } catch (err) { + notification.error({ + description: err.toString(), + message: 'Tracking error occured', + }); + } + }; + + private interactionListener = async (e: Event): Promise => { + const { mode } = this.state; + + if (mode === 'interaction') { + await this.onInteraction(e); + } + + if (mode === 'tracking') { + await this.onTracking(e); + } + }; + private setActiveInteractor = (key: string): void => { const { interactors } = this.props; this.setState({ @@ -311,6 +396,72 @@ class ToolsControlComponent extends React.PureComponent { }); }; + private setActiveTracker = (key: string): void => { + const { trackers } = this.props; + this.setState({ + activeTracker: trackers.filter( + (tracker: Model) => tracker.id === key, + )[0], + }); + }; + + public async trackState(state: any): Promise { + const { jobInstance, frame } = this.props; + const { activeTracker, trackingFrames } = this.state; + const { clientID, points } = state; + + const tracker = activeTracker as Model; + try { + this.setState({ trackingProgress: 0, fetching: true }); + let response = await core.lambda.call(jobInstance.task, tracker, { + task: jobInstance.task, + frame, + shape: points, + }); + + for (const offset of range(1, trackingFrames + 1)) { + /* eslint-disable no-await-in-loop */ + const states = await jobInstance.annotations.get(frame + offset); + const [objectState] = states + .filter((_state: any): boolean => _state.clientID === clientID); + response = await core.lambda.call(jobInstance.task, tracker, { + task: jobInstance.task, + frame: frame + offset, + shape: response.points, + state: response.state, + }); + + const reduced = response.shape + .reduce((acc: number[], value: number, index: number): number[] => { + if (index % 2) { // y + acc[1] = Math.min(acc[1], value); + acc[3] = Math.max(acc[3], value); + } else { // x + acc[0] = Math.min(acc[0], value); + acc[2] = Math.max(acc[2], value); + } + return acc; + }, [Number.MAX_SAFE_INTEGER, Number.MAX_SAFE_INTEGER, + Number.MIN_SAFE_INTEGER, Number.MIN_SAFE_INTEGER, + ]); + + objectState.points = reduced; + await objectState.save(); + + this.setState({ trackingProgress: offset / trackingFrames }); + } + } finally { + this.setState({ trackingProgress: null, fetching: false }); + } + } + + public trackingAvailable(): boolean { + const { activeTracker, trackingFrames } = this.state; + const { trackers } = this.props; + + return !!trackingFrames && !!trackers.length && activeTracker !== null; + } + private renderLabelBlock(): JSX.Element { const { labels } = this.props; const { activeLabelID } = this.state; @@ -355,10 +506,119 @@ class ToolsControlComponent extends React.PureComponent { ); } + private renderTrackerBlock(): JSX.Element { + const { + trackers, + canvasInstance, + jobInstance, + frame, + onInteractionStart, + } = this.props; + const { + activeTracker, + activeLabelID, + fetching, + trackingFrames, + } = this.state; + + if (!trackers.length) { + return ( + + + No available trackers found + + + ); + } + + return ( + <> + + + Tracker + + + + + + + + + + Tracking frames + + + { + if (typeof (value) !== 'undefined') { + this.setState({ + trackingFrames: value, + }); + } + }} + /> + + + + + + + + + ); + } + private renderInteractorBlock(): JSX.Element { const { interactors, canvasInstance, onInteractionStart } = this.props; const { activeInteractor, activeLabelID, fetching } = this.state; + if (!interactors.length) { + return ( + + + No available interactors found + + + ); + } + return ( <> @@ -389,6 +649,10 @@ class ToolsControlComponent extends React.PureComponent { className='cvat-tools-interact-button' disabled={!activeInteractor || fetching} onClick={() => { + this.setState({ + mode: 'interaction', + }); + if (activeInteractor) { canvasInstance.cancel(); canvasInstance.interact({ @@ -413,10 +677,21 @@ class ToolsControlComponent extends React.PureComponent { const { jobInstance, detectors, + curZOrder, frame, fetchAnnotations, } = this.props; + if (!detectors.length) { + return ( + + + No available detectors found + + + ); + } + return ( { task={jobInstance.task} runInference={async (task: any, model: Model, body: object) => { try { + this.setState({ + mode: 'detection', + }); + this.setState({ fetching: true }); const result = await core.lambda.call(task, model, { ...body, @@ -444,7 +723,7 @@ class ToolsControlComponent extends React.PureComponent { occluded: false, source: 'auto', attributes: {}, - zOrder: 0, // TODO: get current z order + zOrder: curZOrder, }) )); @@ -471,7 +750,7 @@ class ToolsControlComponent extends React.PureComponent { AI Tools - + { this.renderLabelBlock() } { this.renderInteractorBlock() } @@ -479,24 +758,34 @@ class ToolsControlComponent extends React.PureComponent { { this.renderDetectorBlock() } + + { this.renderLabelBlock() } + { this.renderTrackerBlock() } + ); } public render(): JSX.Element | null { - const { interactors, isInteraction, canvasInstance } = this.props; - const { fetching } = this.state; + const { + interactors, + detectors, + trackers, + isActivated, + canvasInstance, + } = this.props; + const { fetching, trackingProgress } = this.state; - if (!interactors.length) return null; + if (![...interactors, ...detectors, ...trackers].length) return null; - const dynamcPopoverPros = isInteraction ? { + const dynamcPopoverPros = isActivated ? { overlayStyle: { display: 'none', }, } : {}; - const dynamicIconProps = isInteraction ? { + const dynamicIconProps = isActivated ? { className: 'cvat-active-canvas-control cvat-tools-control', onClick: (): void => { canvasInstance.interact({ enabled: false }); @@ -517,12 +806,15 @@ class ToolsControlComponent extends React.PureComponent { > Waiting for a server response.. + { trackingProgress !== null && ( + + )} diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx index bbfc0e034ff9..9976518754e9 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-basics.tsx @@ -41,6 +41,7 @@ interface Props { toBackground(): void; toForeground(): void; resetCuboidPerspective(): void; + activateTracking(): void; } function ItemTopComponent(props: Props): JSX.Element { @@ -72,6 +73,7 @@ function ItemTopComponent(props: Props): JSX.Element { toBackground, toForeground, resetCuboidPerspective, + activateTracking, } = props; const [menuVisible, setMenuVisible] = useState(false); @@ -150,6 +152,7 @@ function ItemTopComponent(props: Props): JSX.Element { toForeground, resetCuboidPerspective, changeColorPickerVisible, + activateTracking, })} > diff --git a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx index 99db3b848917..5fe8af312559 100644 --- a/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx +++ b/cvat-ui/src/components/annotation-page/standard-workspace/objects-side-bar/object-item-menu.tsx @@ -33,16 +33,17 @@ interface Props { toBackgroundShortcut: string; toForegroundShortcut: string; removeShortcut: string; - changeColor: (value: string) => void; - copy: (() => void); - remove: (() => void); - propagate: (() => void); - createURL: (() => void); - switchOrientation: (() => void); - toBackground: (() => void); - toForeground: (() => void); - resetCuboidPerspective: (() => void); - changeColorPickerVisible: (visible: boolean) => void; + changeColor(value: string): void; + copy(): void; + remove(): void; + propagate(): void; + createURL(): void; + switchOrientation(): void; + toBackground(): void; + toForeground(): void; + resetCuboidPerspective(): void; + changeColorPickerVisible(visible: boolean): void; + activateTracking(): void; } export default function ItemMenu(props: Props): JSX.Element { @@ -71,6 +72,7 @@ export default function ItemMenu(props: Props): JSX.Element { toForeground, resetCuboidPerspective, changeColorPickerVisible, + activateTracking, } = props; return ( @@ -94,6 +96,16 @@ export default function ItemMenu(props: Props): JSX.Element { + {objectType === ObjectType.TRACK && shapeType === ShapeType.RECTANGLE && ( + + + + + + )} { [ShapeType.POLYGON, ShapeType.POLYLINE, ShapeType.CUBOID].includes(shapeType) && (