diff --git a/CHANGELOG.md b/CHANGELOG.md index 5872e8e68ac8..86f28f3fe29b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 + +## \[2.11.2\] - 2024-03-11 + +### Changed + +- Sped up resource updates when there are no matching webhooks + () + +### Fixed + +- Job and task `updated_date` are no longer bumped twice when updating + annotations + () + +- Sending `PATCH /jobs/{id}/data/meta` on each job save even if nothing changed in meta data + () +- Sending `GET /jobs/{id}/data/meta` twice on each job load + () + +- Made analytics report update job scheduling more efficient + () + +- Fixed being unable to connect to in-mem Redis + when the password includes URL-unsafe characters + () + +- Segment anything decoder is loaded anytime when CVAT is opened, but might be not required + () + ## \[2.11.1\] - 2024-03-05 diff --git a/cvat-cli/requirements/base.txt b/cvat-cli/requirements/base.txt index 84eed943a020..a96ad22040e4 100644 --- a/cvat-cli/requirements/base.txt +++ b/cvat-cli/requirements/base.txt @@ -1,3 +1,3 @@ -cvat-sdk~=2.11.1 +cvat-sdk~=2.11.2 Pillow>=10.1.0 setuptools>=65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability diff --git a/cvat-cli/src/cvat_cli/version.py b/cvat-cli/src/cvat_cli/version.py index fd36d7a74976..7ccf0045facb 100644 --- a/cvat-cli/src/cvat_cli/version.py +++ b/cvat-cli/src/cvat_cli/version.py @@ -1 +1 @@ -VERSION = "2.11.1" +VERSION = "2.11.2" diff --git a/cvat-core/package.json b/cvat-core/package.json index 81a1937817f8..5e57ad15cc68 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "15.0.0", + "version": "15.0.1", "type": "module", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "src/api.ts", diff --git a/cvat-core/src/common.ts b/cvat-core/src/common.ts index b0e5f78ee4ce..2792661f3222 100644 --- a/cvat-core/src/common.ts +++ b/cvat-core/src/common.ts @@ -95,6 +95,14 @@ export function checkObjectType(name, value, type, instance?): boolean { export class FieldUpdateTrigger { #updatedFlags: Record = {}; + get(key: string): boolean { + return this.#updatedFlags[key] || false; + } + + resetField(key: string): void { + delete this.#updatedFlags[key]; + } + reset(): void { this.#updatedFlags = {}; } diff --git a/cvat-core/src/frames.ts b/cvat-core/src/frames.ts index a6aa0a668238..b45883fce03e 100644 --- a/cvat-core/src/frames.ts +++ b/cvat-core/src/frames.ts @@ -11,10 +11,11 @@ import PluginRegistry from './plugins'; import serverProxy from './server-proxy'; import { SerializedFramesMetaData } from './server-response-types'; import { Exception, ArgumentError, DataError } from './exceptions'; +import { FieldUpdateTrigger } from './common'; // frame storage by job id const frameDataCache: Record & { deleted_frames: Record }; + meta: FramesMetaData; chunkSize: number; mode: 'annotation' | 'interpolation'; startFrame: number; @@ -36,9 +37,12 @@ const frameDataCache: Record Promise; }> = {}; +// frame meta data storage by job id +const frameMetaCache: Record> = {}; + export class FramesMetaData { public chunkSize: number; - public deletedFrames: number[]; + public deletedFrames: Record; public includedFrames: number[]; public frameFilter: string; public frames: { @@ -52,10 +56,12 @@ export class FramesMetaData { public startFrame: number; public stopFrame: number; - constructor(initialData: SerializedFramesMetaData) { - const data: SerializedFramesMetaData = { + #updateTrigger: FieldUpdateTrigger; + + constructor(initialData: Omit & { deleted_frames: Record }) { + const data: typeof initialData = { chunk_size: undefined, - deleted_frames: [], + deleted_frames: {}, included_frames: [], frame_filter: undefined, frames: [], @@ -65,9 +71,35 @@ export class FramesMetaData { stop_frame: undefined, }; + this.#updateTrigger = new FieldUpdateTrigger(); + for (const property in data) { if (Object.prototype.hasOwnProperty.call(data, property) && property in initialData) { - data[property] = initialData[property]; + if (property === 'deleted_frames') { + const update = (frame: string, remove: boolean): void => { + if (this.#updateTrigger.get(`deletedFrames:${frame}:${!remove}`)) { + this.#updateTrigger.resetField(`deletedFrames:${frame}:${!remove}`); + } else { + this.#updateTrigger.update(`deletedFrames:${frame}:${remove}`); + } + }; + + const handler = { + set: (target, prop, value) => { + update(prop, value); + return Reflect.set(target, prop, value); + }, + deleteProperty: (target, prop) => { + if (prop in target) { + update(prop, false); + } + return Reflect.deleteProperty(target, prop); + }, + }; + data[property] = new Proxy(initialData[property], handler); + } else { + data[property] = initialData[property]; + } } } @@ -104,6 +136,14 @@ export class FramesMetaData { }), ); } + + getUpdated(): Record { + return this.#updateTrigger.getUpdated(this); + } + + resetUpdated(): void { + this.#updateTrigger.reset(); + } } export class FrameData { @@ -379,6 +419,36 @@ Object.defineProperty(FrameData.prototype.data, 'implementation', { writable: false, }); +async function getJobMeta(jobID: number): Promise { + if (!frameMetaCache[jobID]) { + frameMetaCache[jobID] = serverProxy.frames.getMeta('job', jobID) + .then((serverMeta) => new FramesMetaData({ + ...serverMeta, + deleted_frames: Object.fromEntries(serverMeta.deleted_frames.map((_frame) => [_frame, true])), + })) + .catch((error) => { + delete frameMetaCache[jobID]; + throw error; + }); + } + return frameMetaCache[jobID]; +} + +async function saveJobMeta(meta: FramesMetaData, jobID: number): Promise { + frameMetaCache[jobID] = serverProxy.frames.saveMeta('job', jobID, { + deleted_frames: Object.keys(meta.deletedFrames).map((frame) => +frame), + }) + .then((serverMeta) => new FramesMetaData({ + ...serverMeta, + deleted_frames: Object.fromEntries(serverMeta.deleted_frames.map((_frame) => [_frame, true])), + })) + .catch((error) => { + delete frameMetaCache[jobID]; + throw error; + }); + return frameMetaCache[jobID]; +} + function getFrameMeta(jobID, frame): SerializedFramesMetaData['frames'][0] { const { meta, mode, startFrame } = frameDataCache[jobID]; let frameMeta = null; @@ -386,7 +456,7 @@ function getFrameMeta(jobID, frame): SerializedFramesMetaData['frames'][0] { // video tasks have 1 frame info, but image tasks will have many infos [frameMeta] = meta.frames; } else if (mode === 'annotation' || (mode === 'interpolation' && meta.frames.length > 1)) { - if (frame > meta.stop_frame) { + if (frame > meta.stopFrame) { throw new ArgumentError(`Meta information about frame ${frame} can't be received from the server`); } frameMeta = meta.frames[frame - startFrame]; @@ -502,15 +572,12 @@ export async function getFrame( ): Promise { if (!(jobID in frameDataCache)) { const blockType = chunkType === 'video' ? BlockType.MP4VIDEO : BlockType.ARCHIVE; - const meta = await serverProxy.frames.getMeta('job', jobID); - const updatedMeta = { - ...meta, - deleted_frames: Object.fromEntries(meta.deleted_frames.map((_frame) => [_frame, true])), - }; - const mean = updatedMeta.frames.reduce((a, b) => a + b.width * b.height, 0) / updatedMeta.frames.length; + const meta = await getJobMeta(jobID); + + const mean = meta.frames.reduce((a, b) => a + b.width * b.height, 0) / meta.frames.length; const stdDev = Math.sqrt( - updatedMeta.frames.map((x) => (x.width * x.height - mean) ** 2).reduce((a, b) => a + b) / - updatedMeta.frames.length, + meta.frames.map((x) => (x.width * x.height - mean) ** 2).reduce((a, b) => a + b) / + meta.frames.length, ); // limit of decoded frames cache by 2GB @@ -518,7 +585,7 @@ export async function getFrame( Math.floor((2048 * 1024 * 1024) / ((mean + stdDev) * 4 * chunkSize)) || 1, 10, ); frameDataCache[jobID] = { - meta: updatedMeta, + meta, chunkSize, mode, startFrame, @@ -553,7 +620,7 @@ export async function getFrame( name: frameMeta.name, related_files: frameMeta.related_files, frameNumber: frame, - deleted: frame in frameDataCache[jobID].meta.deleted_frames, + deleted: frame in frameDataCache[jobID].meta.deletedFrames, jobID, }); } @@ -561,7 +628,7 @@ export async function getFrame( export async function getDeletedFrames(instanceType: 'job' | 'task', id): Promise> { if (instanceType === 'job') { const { meta } = frameDataCache[id]; - return meta.deleted_frames; + return meta.deletedFrames; } if (instanceType === 'task') { @@ -574,52 +641,29 @@ export async function getDeletedFrames(instanceType: 'job' | 'task', id): Promis export function deleteFrame(jobID: number, frame: number): void { const { meta } = frameDataCache[jobID]; - meta.deleted_frames[frame] = true; + meta.deletedFrames[frame] = true; } export function restoreFrame(jobID: number, frame: number): void { const { meta } = frameDataCache[jobID]; - if (frame in meta.deleted_frames) { - delete meta.deleted_frames[frame]; - } + delete meta.deletedFrames[frame]; } export async function patchMeta(jobID: number): Promise { const { meta } = frameDataCache[jobID]; - const newMeta = await serverProxy.frames.saveMeta('job', jobID, { - deleted_frames: Object.keys(meta.deleted_frames).map((frame) => +frame), - }); - const prevDeletedFrames = meta.deleted_frames; + const updatedFields = meta.getUpdated(); - // it is important do not overwrite the object, it is why we working on keys in two loops below - for (const frame of Object.keys(prevDeletedFrames)) { - delete prevDeletedFrames[frame]; + if (Object.keys(updatedFields).length) { + const newMeta = await saveJobMeta(meta, jobID); + frameDataCache[jobID].meta = newMeta; } - for (const frame of newMeta.deleted_frames) { - prevDeletedFrames[frame] = true; - } - - frameDataCache[jobID].meta = { - ...newMeta, - deleted_frames: prevDeletedFrames, - }; - frameDataCache[jobID].meta.deleted_frames = prevDeletedFrames; } export async function findFrame( jobID: number, frameFrom: number, frameTo: number, filters: { offset?: number, notDeleted: boolean }, ): Promise { const offset = filters.offset || 1; - let meta; - if (!frameDataCache[jobID]) { - const serverMeta = await serverProxy.frames.getMeta('job', jobID); - meta = { - ...serverMeta, - deleted_frames: Object.fromEntries(serverMeta.deleted_frames.map((_frame) => [_frame, true])), - }; - } else { - meta = frameDataCache[jobID].meta; - } + const meta = await getJobMeta(jobID); const sign = Math.sign(frameTo - frameFrom); const predicate = sign > 0 ? (frame) => frame <= frameTo : (frame) => frame >= frameTo; @@ -627,12 +671,12 @@ export async function findFrame( let framesCounter = 0; let lastUndeletedFrame = null; const check = (frame): boolean => { - if (meta.included_frames) { - return (meta.included_frames.includes(frame)) && - (!filters.notDeleted || !(frame in meta.deleted_frames)); + if (meta.includedFrames) { + return (meta.includedFrames.includes(frame)) && + (!filters.notDeleted || !(frame in meta.deletedFrames)); } if (filters.notDeleted) { - return !(frame in meta.deleted_frames); + return !(frame in meta.deletedFrames); } return true; }; @@ -660,5 +704,6 @@ export function getCachedChunks(jobID): number[] { export function clear(jobID: number): void { if (jobID in frameDataCache) { delete frameDataCache[jobID]; + delete frameMetaCache[jobID]; } } diff --git a/cvat-sdk/gen/generate.sh b/cvat-sdk/gen/generate.sh index 0c0e04c70fe4..ccce877b2f4d 100755 --- a/cvat-sdk/gen/generate.sh +++ b/cvat-sdk/gen/generate.sh @@ -8,7 +8,7 @@ set -e GENERATOR_VERSION="v6.0.1" -VERSION="2.11.1" +VERSION="2.11.2" LIB_NAME="cvat_sdk" LAYER1_LIB_NAME="${LIB_NAME}/api_client" DST_DIR="$(cd "$(dirname -- "$0")/.." && pwd)" diff --git a/cvat-ui/plugins/sam/src/ts/index.tsx b/cvat-ui/plugins/sam/src/ts/index.tsx index 461a093bf66d..0473ee35dbec 100644 --- a/cvat-ui/plugins/sam/src/ts/index.tsx +++ b/cvat-ui/plugins/sam/src/ts/index.tsx @@ -1,11 +1,12 @@ -// Copyright (C) 2023 CVAT.ai Corporation +// Copyright (C) 2023-2024 CVAT.ai Corporation // // SPDX-License-Identifier: MIT -import { InferenceSession, Tensor } from 'onnxruntime-web'; +import { Tensor } from 'onnxruntime-web'; import { LRUCache } from 'lru-cache'; -import { Job } from 'cvat-core-wrapper'; +import { CVATCore, MLModel, Job } from 'cvat-core-wrapper'; import { PluginEntryPoint, APIWrapperEnterOptions, ComponentBuilder } from 'components/plugins-entrypoint'; +import { InitBody, DecodeBody, WorkerAction } from './inference.worker'; interface SAMPlugin { name: string; @@ -16,14 +17,14 @@ interface SAMPlugin { enter: ( plugin: SAMPlugin, taskID: number, - model: any, + model: MLModel, args: any, ) => Promise; leave: ( plugin: SAMPlugin, - result: any, + result: object, taskID: number, - model: any, + model: MLModel, args: any, ) => Promise; }; @@ -39,42 +40,32 @@ interface SAMPlugin { }; }; data: { - core: any; - jobs: Record; + initialized: boolean; + worker: Worker; + core: CVATCore | null; + jobs: Record; modelID: string; modelURL: string; embeddings: LRUCache; lowResMasks: LRUCache; - session: InferenceSession | null; + lastClicks: ClickType[]; }; callbacks: { onStatusChange: ((status: string) => void) | null; }; } -interface ONNXInput { - image_embeddings: Tensor; - point_coords: Tensor; - point_labels: Tensor; - orig_im_size: Tensor; - mask_input: Tensor; - has_mask_input: Tensor; - readonly [name: string]: Tensor; -} - interface ClickType { - clickType: -1 | 0 | 1, - height: number | null, - width: number | null, - x: number, - y: number, + clickType: 0 | 1; + x: number; + y: number; } function getModelScale(w: number, h: number): number { // Input images to SAM must be resized so the longest side is 1024 const LONG_SIDE_LENGTH = 1024; - const samScale = LONG_SIDE_LENGTH / Math.max(h, w); - return samScale; + const scale = LONG_SIDE_LENGTH / Math.max(h, w); + return scale; } function modelData( @@ -83,39 +74,27 @@ function modelData( }: { clicks: ClickType[]; tensor: Tensor; - modelScale: { height: number; width: number; samScale: number }; + modelScale: { height: number; width: number; scale: number }; maskInput: Tensor | null; }, -): ONNXInput { +): DecodeBody { const imageEmbedding = tensor; const n = clicks.length; - // If there is no box input, a single padding point with - // label -1 and coordinates (0.0, 0.0) should be concatenated - // so initialize the array to support (n + 1) points. - const pointCoords = new Float32Array(2 * (n + 1)); - const pointLabels = new Float32Array(n + 1); + const pointCoords = new Float32Array(2 * n); + const pointLabels = new Float32Array(n); - // Add clicks and scale to what SAM expects + // Scale and add clicks for (let i = 0; i < n; i++) { - pointCoords[2 * i] = clicks[i].x * modelScale.samScale; - pointCoords[2 * i + 1] = clicks[i].y * modelScale.samScale; + pointCoords[2 * i] = clicks[i].x * modelScale.scale; + pointCoords[2 * i + 1] = clicks[i].y * modelScale.scale; pointLabels[i] = clicks[i].clickType; } - // Add in the extra point/label when only clicks and no box - // The extra point is at (0, 0) with label -1 - pointCoords[2 * n] = 0.0; - pointCoords[2 * n + 1] = 0.0; - pointLabels[n] = -1.0; - // Create the tensor - const pointCoordsTensor = new Tensor('float32', pointCoords, [1, n + 1, 2]); - const pointLabelsTensor = new Tensor('float32', pointLabels, [1, n + 1]); - const imageSizeTensor = new Tensor('float32', [ - modelScale.height, - modelScale.width, - ]); + const pointCoordsTensor = new Tensor('float32', pointCoords, [1, n, 2]); + const pointLabelsTensor = new Tensor('float32', pointLabels, [1, n]); + const imageSizeTensor = new Tensor('float32', [modelScale.height, modelScale.width]); const prevMask = maskInput || new Tensor('float32', new Float32Array(256 * 256), [1, 1, 256, 256]); @@ -154,27 +133,55 @@ const samPlugin: SAMPlugin = { async enter( plugin: SAMPlugin, taskID: number, - model: any, { frame }: { frame: number }, + model: MLModel, { frame }: { frame: number }, ): Promise { - if (model.id === plugin.data.modelID) { - if (!plugin.data.session) { - throw new Error('SAM plugin is not ready, session was not initialized'); + return new Promise((resolve, reject) => { + function resolvePromise(): void { + const key = `${taskID}_${frame}`; + if (plugin.data.embeddings.has(key)) { + resolve({ preventMethodCall: true }); + } else { + resolve(null); + } } - const key = `${taskID}_${frame}`; - if (plugin.data.embeddings.has(key)) { - return { preventMethodCall: true }; + if (model.id === plugin.data.modelID) { + if (!plugin.data.initialized) { + samPlugin.data.worker.postMessage({ + action: WorkerAction.INIT, + payload: { + decoderURL: samPlugin.data.modelURL, + } as InitBody, + }); + + samPlugin.data.worker.onmessage = (e: MessageEvent) => { + if (e.data.action !== WorkerAction.INIT) { + reject(new Error( + `Caught unexpected action response from worker: ${e.data.action}`, + )); + } + + if (!e.data.error) { + samPlugin.data.initialized = true; + resolvePromise(); + } else { + reject(new Error(`SAM worker was not initialized. ${e.data.error}`)); + } + }; + } else { + resolvePromise(); + } + } else { + resolve(null); } - } - - return null; + }); }, async leave( plugin: SAMPlugin, result: any, taskID: number, - model: any, + model: MLModel, { frame, pos_points, neg_points }: { frame: number, pos_points: number[][], neg_points: number[][], }, @@ -183,106 +190,136 @@ const samPlugin: SAMPlugin = { mask: number[][]; bounds: [number, number, number, number]; }> { - if (model.id !== plugin.data.modelID) { - return result; - } - - const job = Object.values(plugin.data.jobs).find((_job) => ( - _job.taskId === taskID && frame >= _job.startFrame && frame <= _job.stopFrame - )) as Job; - if (!job) { - throw new Error('Could not find a job corresponding to the request'); - } - - const { height: imHeight, width: imWidth } = await job.frames.get(frame); - const key = `${taskID}_${frame}`; - - if (result) { - const bin = window.atob(result.blob); - const uint8Array = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) { - uint8Array[i] = bin.charCodeAt(i); + return new Promise((resolve, reject) => { + if (model.id !== plugin.data.modelID) { + resolve(result); } - const float32Arr = new Float32Array(uint8Array.buffer); - plugin.data.embeddings.set(key, new Tensor('float32', float32Arr, [1, 256, 64, 64])); - } - - const modelScale = { - width: imWidth, - height: imHeight, - samScale: getModelScale(imWidth, imHeight), - }; - - const composedClicks = [...pos_points, ...neg_points].map(([x, y], index) => ({ - clickType: index < pos_points.length ? 1 : 0 as 0 | 1 | -1, - height: null, - width: null, - x, - y, - })); - - const feeds = modelData({ - clicks: composedClicks, - tensor: plugin.data.embeddings.get(key) as Tensor, - modelScale, - maskInput: plugin.data.lowResMasks.has(key) ? plugin.data.lowResMasks.get(key) as Tensor : null, - }); - function toMatImage(input: number[], width: number, height: number): number[][] { - const image = Array(height).fill(0); - for (let i = 0; i < image.length; i++) { - image[i] = Array(width).fill(0); - } + const job = Object.values(plugin.data.jobs).find((_job) => ( + _job.taskId === taskID && frame >= _job.startFrame && frame <= _job.stopFrame + )) as Job; - for (let i = 0; i < input.length; i++) { - const row = Math.floor(i / width); - const col = i % width; - image[row][col] = input[i] * 255; + if (!job) { + throw new Error('Could not find a job corresponding to the request'); } - return image; - } - - function onnxToImage(input: any, width: number, height: number): number[][] { - return toMatImage(input, width, height); - } - - const data = await (plugin.data.session as InferenceSession).run(feeds); - const { masks, low_res_masks: lowResMasks } = data; - const imageData = onnxToImage(masks.data, masks.dims[3], masks.dims[2]); - plugin.data.lowResMasks.set(key, lowResMasks); - - const xtl = Number(data.xtl.data[0]); - const xbr = Number(data.xbr.data[0]); - const ytl = Number(data.ytl.data[0]); - const ybr = Number(data.ybr.data[0]); - - return { - mask: imageData, - bounds: [xtl, ytl, xbr, ybr], - }; + plugin.data.jobs = { + // we do not need to store old job instances + [job.id]: job, + }; + + job.frames.get(frame) + .then(({ height: imHeight, width: imWidth }: { height: number; width: number }) => { + const key = `${taskID}_${frame}`; + + if (result) { + const bin = window.atob(result.blob); + const uint8Array = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) { + uint8Array[i] = bin.charCodeAt(i); + } + const float32Arr = new Float32Array(uint8Array.buffer); + plugin.data.embeddings.set(key, new Tensor('float32', float32Arr, [1, 256, 64, 64])); + } + + const modelScale = { + width: imWidth, + height: imHeight, + scale: getModelScale(imWidth, imHeight), + }; + + const composedClicks = [...pos_points, ...neg_points].map(([x, y], index) => ({ + clickType: index < pos_points.length ? 1 : 0 as 0 | 1, + x, + y, + })); + + const isLowResMaskSuitable = JSON + .stringify(composedClicks.slice(0, -1)) === JSON.stringify(plugin.data.lastClicks); + const feeds = modelData({ + clicks: composedClicks, + tensor: plugin.data.embeddings.get(key) as Tensor, + modelScale, + maskInput: isLowResMaskSuitable ? plugin.data.lowResMasks.get(key) || null : null, + }); + + function toMatImage(input: number[], width: number, height: number): number[][] { + const image = Array(height).fill(0); + for (let i = 0; i < image.length; i++) { + image[i] = Array(width).fill(0); + } + + for (let i = 0; i < input.length; i++) { + const row = Math.floor(i / width); + const col = i % width; + image[row][col] = input[i] > 0 ? 255 : 0; + } + + return image; + } + + function onnxToImage(input: any, width: number, height: number): number[][] { + return toMatImage(input, width, height); + } + + plugin.data.worker.postMessage({ + action: WorkerAction.DECODE, + payload: feeds, + }); + + plugin.data.worker.onmessage = ((e) => { + if (e.data.action !== WorkerAction.DECODE) { + const error = 'Caught unexpected action response from worker: ' + + `${e.data.action}, while "${WorkerAction.DECODE}" was expected`; + reject(new Error(error)); + } + + if (!e.data.error) { + const { + masks, lowResMasks, xtl, ytl, xbr, ybr, + } = e.data.payload; + const imageData = onnxToImage(masks.data, masks.dims[3], masks.dims[2]); + plugin.data.lowResMasks.set(key, lowResMasks); + plugin.data.lastClicks = composedClicks; + + resolve({ + mask: imageData, + bounds: [xtl, ytl, xbr, ybr], + }); + } else { + reject(new Error(`Decoder error. ${e.data.error}`)); + } + }); + + plugin.data.worker.onerror = ((error) => { + reject(error); + }); + }); + }); }, }, }, }, data: { + initialized: false, core: null, + worker: new Worker(new URL('./inference.worker', import.meta.url)), jobs: {}, modelID: 'pth-facebookresearch-sam-vit-h', modelURL: '/assets/decoder.onnx', embeddings: new LRUCache({ - // float32 tensor [256, 64, 64] is 4 MB, max 512 MB - max: 128, + // float32 tensor [256, 64, 64] is 4 MB, max 128 MB + max: 32, updateAgeOnGet: true, updateAgeOnHas: true, }), lowResMasks: new LRUCache({ - // float32 tensor [1, 256, 256] is 0.25 MB, max 32 MB - max: 128, + // float32 tensor [1, 256, 256] is 0.25 MB, max 8 MB + max: 32, updateAgeOnGet: true, updateAgeOnHas: true, }), - session: null, + lastClicks: [], }, callbacks: { onStatusChange: null, @@ -292,9 +329,6 @@ const samPlugin: SAMPlugin = { const builder: ComponentBuilder = ({ core }) => { samPlugin.data.core = core; core.plugins.register(samPlugin); - InferenceSession.create(samPlugin.data.modelURL).then((session) => { - samPlugin.data.session = session; - }); return { name: samPlugin.name, diff --git a/cvat-ui/plugins/sam/src/ts/inference.worker.ts b/cvat-ui/plugins/sam/src/ts/inference.worker.ts new file mode 100644 index 000000000000..ff3927efaf87 --- /dev/null +++ b/cvat-ui/plugins/sam/src/ts/inference.worker.ts @@ -0,0 +1,90 @@ +// Copyright (C) 2024 CVAT.ai Corporation +// +// SPDX-License-Identifier: MIT + +import { InferenceSession, env, Tensor } from 'onnxruntime-web'; + +let decoder: InferenceSession | null = null; + +env.wasm.wasmPaths = '/assets/'; + +export enum WorkerAction { + INIT = 'init', + DECODE = 'decode', +} + +export interface InitBody { + decoderURL: string; +} + +export interface DecodeBody { + image_embeddings: Tensor; + point_coords: Tensor; + point_labels: Tensor; + orig_im_size: Tensor; + mask_input: Tensor; + has_mask_input: Tensor; + readonly [name: string]: Tensor; +} + +export interface WorkerOutput { + action: WorkerAction; + error?: string; +} + +export interface WorkerInput { + action: WorkerAction; + payload: InitBody | DecodeBody; +} + +const errorToMessage = (error: unknown): string => { + if (error instanceof Error) { + return error.message; + } + if (typeof error === 'string') { + return error; + } + + console.error(error); + return 'Unknown error, please check console'; +}; + +// eslint-disable-next-line no-restricted-globals +if ((self as any).importScripts) { + onmessage = (e: MessageEvent) => { + if (e.data.action === WorkerAction.INIT) { + if (decoder) { + return; + } + + const body = e.data.payload as InitBody; + InferenceSession.create(body.decoderURL).then((decoderSession) => { + decoder = decoderSession; + postMessage({ action: WorkerAction.INIT }); + }).catch((error: unknown) => { + postMessage({ action: WorkerAction.INIT, error: errorToMessage(error) }); + }); + } else if (!decoder) { + postMessage({ + action: e.data.action, + error: 'Worker was not initialized', + }); + } else if (e.data.action === WorkerAction.DECODE) { + decoder.run((e.data.payload as DecodeBody)).then((results) => { + postMessage({ + action: WorkerAction.DECODE, + payload: { + masks: results.masks, + lowResMasks: results.low_res_masks, + xtl: Number(results.xtl.data[0]), + ytl: Number(results.ytl.data[0]), + xbr: Number(results.xbr.data[0]), + ybr: Number(results.ybr.data[0]), + }, + }); + }).catch((error: unknown) => { + postMessage({ action: WorkerAction.DECODE, error: errorToMessage(error) }); + }); + } + }; +} diff --git a/cvat/__init__.py b/cvat/__init__.py index 950b50a7a444..c7a73f70341b 100644 --- a/cvat/__init__.py +++ b/cvat/__init__.py @@ -4,6 +4,6 @@ from cvat.utils.version import get_version -VERSION = (2, 11, 1, 'final', 0) +VERSION = (2, 11, 2, 'final', 0) __version__ = get_version(VERSION) diff --git a/cvat/apps/analytics_report/report/create.py b/cvat/apps/analytics_report/report/create.py index 3a8da9dca55a..fdf44e6d666d 100644 --- a/cvat/apps/analytics_report/report/create.py +++ b/cvat/apps/analytics_report/report/create.py @@ -66,20 +66,20 @@ def _get_scheduler(self): def _get_queue(self): return django_rq.get_queue(settings.CVAT_QUEUES.ANALYTICS_REPORTS.value) - def _make_queue_job_prefix(self, obj) -> str: + def _make_queue_job_id_base(self, obj) -> str: if isinstance(obj, Task): - return f"{self._QUEUE_JOB_PREFIX_TASK}{obj.id}-" + return f"{self._QUEUE_JOB_PREFIX_TASK}{obj.id}" else: - return f"{self._QUEUE_JOB_PREFIX_PROJECT}{obj.id}-" + return f"{self._QUEUE_JOB_PREFIX_PROJECT}{obj.id}" def _make_custom_analytics_check_job_id(self) -> str: return uuid4().hex - def _make_initial_queue_job_id(self, obj) -> str: - return f"{self._make_queue_job_prefix(obj)}initial" + def _make_queue_job_id(self, obj, start_time: timezone.datetime) -> str: + return f"{self._make_queue_job_id_base(obj)}-{start_time.timestamp()}" - def _make_regular_queue_job_id(self, obj, start_time: timezone.datetime) -> str: - return f"{self._make_queue_job_prefix(obj)}{start_time.timestamp()}" + def _make_autoupdate_blocker_key(self, obj) -> str: + return f"cvat:analytics:autoupdate-blocker:{self._make_queue_job_id_base(obj)}" @classmethod def _get_last_report_time(cls, obj): @@ -90,39 +90,6 @@ def _get_last_report_time(cls, obj): except ObjectDoesNotExist: return None - def _find_next_job_id(self, existing_job_ids, obj, *, now) -> str: - job_id_prefix = self._make_queue_job_prefix(obj) - - def _get_timestamp(job_id: str) -> timezone.datetime: - job_timestamp = job_id.split(job_id_prefix, maxsplit=1)[-1] - if job_timestamp == "initial": - return timezone.datetime.min.replace(tzinfo=timezone.utc) - else: - return timezone.datetime.fromtimestamp(float(job_timestamp), tz=timezone.utc) - - max_job_id = max( - (j for j in existing_job_ids if j.startswith(job_id_prefix)), - key=_get_timestamp, - default=None, - ) - max_timestamp = _get_timestamp(max_job_id) if max_job_id else None - - last_update_time = self._get_last_report_time(obj) - if last_update_time is None: - # Report has never been computed, is queued, or is being computed - queue_job_id = self._make_initial_queue_job_id(obj) - elif max_timestamp is not None and now < max_timestamp: - # Reuse the existing next job - queue_job_id = max_job_id - else: - # Add an updating job in the queue in the next time frame - delay = self._get_analytics_check_job_delay() - intervals = max(1, 1 + (now - last_update_time) // delay) - next_update_time = last_update_time + delay * intervals - queue_job_id = self._make_regular_queue_job_id(obj, next_update_time) - - return queue_job_id - class AnalyticsReportsNotAvailable(Exception): pass @@ -133,9 +100,6 @@ def schedule_analytics_report_autoupdate_job(self, *, job=None, task=None, proje delay = self._get_analytics_check_job_delay() next_job_time = now.utcnow() + delay - scheduler = self._get_scheduler() - existing_job_ids = set(j.id for j in scheduler.get_jobs(until=next_job_time)) - target_obj = None cvat_project_id = None cvat_task_id = None @@ -157,9 +121,19 @@ def schedule_analytics_report_autoupdate_job(self, *, job=None, task=None, proje target_obj = project cvat_project_id = project.id - queue_job_id = self._find_next_job_id(existing_job_ids, target_obj, now=now) - if queue_job_id not in existing_job_ids: - scheduler.enqueue_at( + with django_rq.get_connection(settings.CVAT_QUEUES.ANALYTICS_REPORTS.value) as connection: + # The blocker key is used to avoid scheduling a report update job + # for every single change. The first time this method is called + # for a given object, we schedule the job and create a blocker + # that expires at the same time as the job is supposed to start. + # Until the blocker expires, we don't schedule any more jobs. + blocker_key = self._make_autoupdate_blocker_key(target_obj) + if connection.exists(blocker_key): + return + + queue_job_id = self._make_queue_job_id(target_obj, next_job_time) + + self._get_scheduler().enqueue_at( next_job_time, self._check_analytics_report, cvat_task_id=cvat_task_id, @@ -167,6 +141,8 @@ def schedule_analytics_report_autoupdate_job(self, *, job=None, task=None, proje job_id=queue_job_id, ) + connection.set(blocker_key, queue_job_id, exat=next_job_time) + def schedule_analytics_check_job(self, *, job=None, task=None, project=None, user_id): rq_id = self._make_custom_analytics_check_job_id() diff --git a/cvat/apps/dataset_manager/task.py b/cvat/apps/dataset_manager/task.py index 9b195c26bd03..b6dcb906e36f 100644 --- a/cvat/apps/dataset_manager/task.py +++ b/cvat/apps/dataset_manager/task.py @@ -391,43 +391,51 @@ def _set_updated_date(self): self.db_job.segment.task.touch() self.db_job.touch() - def _save_to_db(self, data): + @staticmethod + def _data_is_empty(data): + return not (data["tags"] or data["shapes"] or data["tracks"]) + + def _create(self, data): self.reset() self._save_tags_to_db(data["tags"]) self._save_shapes_to_db(data["shapes"]) self._save_tracks_to_db(data["tracks"]) - return self.ir_data.tags or self.ir_data.shapes or self.ir_data.tracks - - def _create(self, data): - if self._save_to_db(data): - self._set_updated_date() - def create(self, data): self._create(data) handle_annotations_change(self.db_job, self.data, "create") + if not self._data_is_empty(self.data): + self._set_updated_date() + def put(self, data): deleted_data = self._delete() handle_annotations_change(self.db_job, deleted_data, "delete") + + deleted_data_is_empty = self._data_is_empty(deleted_data) + self._create(data) handle_annotations_change(self.db_job, self.data, "create") + if not deleted_data_is_empty or not self._data_is_empty(self.data): + self._set_updated_date() def update(self, data): self._delete(data) self._create(data) handle_annotations_change(self.db_job, self.data, "update") + if not self._data_is_empty(self.data): + self._set_updated_date() + def _delete(self, data=None): - deleted_shapes = 0 deleted_data = {} if data is None: self.init_from_db() deleted_data = self.data - deleted_shapes += self.db_job.labeledimage_set.all().delete()[0] - deleted_shapes += self.db_job.labeledshape_set.all().delete()[0] - deleted_shapes += self.db_job.labeledtrack_set.all().delete()[0] + self.db_job.labeledimage_set.all().delete() + self.db_job.labeledshape_set.all().delete() + self.db_job.labeledtrack_set.all().delete() else: labeledimage_ids = [image["id"] for image in data["tags"]] labeledshape_ids = [shape["id"] for shape in data["shapes"]] @@ -446,9 +454,9 @@ def _delete(self, data=None): self.ir_data.shapes = data['shapes'] self.ir_data.tracks = data['tracks'] - deleted_shapes += labeledimage_set.delete()[0] - deleted_shapes += labeledshape_set.delete()[0] - deleted_shapes += labeledtrack_set.delete()[0] + labeledimage_set.delete() + labeledshape_set.delete() + labeledtrack_set.delete() deleted_data = { "tags": data["tags"], @@ -456,15 +464,15 @@ def _delete(self, data=None): "tracks": data["tracks"], } - if deleted_shapes: - self._set_updated_date() - return deleted_data def delete(self, data=None): deleted_data = self._delete(data) handle_annotations_change(self.db_job, deleted_data, "delete") + if not self._data_is_empty(deleted_data): + self._set_updated_date() + @staticmethod def _extend_attributes(attributeval_set, default_attribute_values): shape_attribute_specs_set = set(attr.spec_id for attr in attributeval_set) diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index 2ea7f7c431d6..13c515c44c44 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -22,6 +22,7 @@ from random import shuffle from cvat.apps.engine.utils import rotate_image from cvat.apps.engine.models import DimensionType, SortingMethod +from rest_framework.exceptions import ValidationError # fixes: "OSError:broken data stream" when executing line 72 while loading images downloaded from the web # see: https://stackoverflow.com/questions/42462431/oserror-broken-data-stream-when-reading-image-file @@ -722,6 +723,7 @@ def save_as_chunk( class Mpeg4ChunkWriter(IChunkWriter): FORMAT = 'mp4' + MAX_MBS_PER_FRAME = 36864 def __init__(self, quality=67): # translate inversed range [1:100] to [0:51] @@ -752,6 +754,12 @@ def _add_video_stream(self, container, w, h, rate, options): if w % 2: w += 1 + # libopenh264 has 4K limitations, https://github.com/opencv/cvat/issues/7425 + if h * w > (self.MAX_MBS_PER_FRAME << 8): + raise ValidationError( + 'The video codec being used does not support such high video resolution, refer https://github.com/opencv/cvat/issues/7425' + ) + video_stream = container.add_stream(self._codec_name, rate=rate) video_stream.pix_fmt = "yuv420p" video_stream.width = w diff --git a/cvat/apps/events/signals.py b/cvat/apps/events/signals.py index 31cdd27f9c51..25d320c35e1d 100644 --- a/cvat/apps/events/signals.py +++ b/cvat/apps/events/signals.py @@ -7,6 +7,7 @@ from django.core.exceptions import ObjectDoesNotExist from cvat.apps.engine.models import ( + TimestampedModel, Project, Task, Job, @@ -30,7 +31,19 @@ @receiver(pre_save, sender=Issue, dispatch_uid="issue:update_receiver") @receiver(pre_save, sender=Comment, dispatch_uid="comment:update_receiver") @receiver(pre_save, sender=Label, dispatch_uid="label:update_receiver") -def resource_update(sender, instance, **kwargs): +def resource_update(sender, *, instance, update_fields, **kwargs): + if ( + isinstance(instance, TimestampedModel) + and update_fields and list(update_fields) == ["updated_date"] + ): + # This is an optimization for the common case where only the date is bumped + # (see `TimestampedModel.touch`). Since the actual update of the field will + # be performed _after_ this signal is sent (in `DateTimeField.pre_save`), + # and no other fields are updated, there is guaranteed to be no difference + # between the old and current states of the instance. Therefore, no events + # will need be logged, so we can just exit immediately. + return + resource_name = instance.__class__.__name__.lower() try: diff --git a/cvat/apps/webhooks/signals.py b/cvat/apps/webhooks/signals.py index 0d34950cf6ff..3e17e8f3d8f6 100644 --- a/cvat/apps/webhooks/signals.py +++ b/cvat/apps/webhooks/signals.py @@ -136,25 +136,32 @@ def get_sender(instance): @receiver(pre_save, sender=Invitation, dispatch_uid=__name__ + ":invitation:pre_save") @receiver(pre_save, sender=Membership, dispatch_uid=__name__ + ":membership:pre_save") def pre_save_resource_event(sender, instance, **kwargs): - try: - old_instance = sender.objects.get(pk=instance.pk) - except ObjectDoesNotExist: - return + instance._webhooks_selected_webhooks = [] - old_serializer = get_serializer(instance=old_instance) - serializer = get_serializer(instance=instance) - diff = get_instance_diff(old_data=old_serializer.data, data=serializer.data) + if instance.pk is None: + created = True + else: + try: + old_instance = sender.objects.get(pk=instance.pk) + created = False + except ObjectDoesNotExist: + created = True - if not diff: - return + resource_name = instance.__class__.__name__.lower() - before_update = { - attr: value["old_value"] - for attr, value in diff.items() - } + event_type = event_name("create" if created else "update", resource_name) + if event_type not in map(lambda a: a[0], EventTypeChoice.choices()): + return - instance._before_update = before_update + instance._webhooks_selected_webhooks = select_webhooks(instance, event_type) + if not instance._webhooks_selected_webhooks: + return + if created: + instance._webhooks_old_data = None + else: + old_serializer = get_serializer(instance=old_instance) + instance._webhooks_old_data = old_serializer.data @receiver(post_save, sender=Project, dispatch_uid=__name__ + ":project:post_save") @receiver(post_save, sender=Task, dispatch_uid=__name__ + ":task:post_save") @@ -164,31 +171,38 @@ def pre_save_resource_event(sender, instance, **kwargs): @receiver(post_save, sender=Organization, dispatch_uid=__name__ + ":organization:post_save") @receiver(post_save, sender=Invitation, dispatch_uid=__name__ + ":invitation:post_save") @receiver(post_save, sender=Membership, dispatch_uid=__name__ + ":membership:post_save") -def post_save_resource_event(sender, instance, created, **kwargs): - resource_name = instance.__class__.__name__.lower() +def post_save_resource_event(sender, instance, **kwargs): + selected_webhooks = instance._webhooks_selected_webhooks + del instance._webhooks_selected_webhooks - event_type = event_name("create" if created else "update", resource_name) - if event_type not in map(lambda a: a[0], EventTypeChoice.choices()): + if not selected_webhooks: return - filtered_webhooks = select_webhooks(instance, event_type) - if not filtered_webhooks: - return + old_data = instance._webhooks_old_data + del instance._webhooks_old_data + + created = old_data is None + + resource_name = instance.__class__.__name__.lower() + event_type = event_name("create" if created else "update", resource_name) + + serializer = get_serializer(instance=instance) data = { "event": event_type, - resource_name: get_serializer(instance=instance).data, + resource_name: serializer.data, "sender": get_sender(instance), } if not created: - if before_update := getattr(instance, "_before_update", None): - data["before_update"] = before_update - else: - return + if diff := get_instance_diff(old_data=old_data, data=serializer.data): + data["before_update"] = { + attr: value["old_value"] + for attr, value in diff.items() + } transaction.on_commit( - lambda: batch_add_to_queue(filtered_webhooks, data), + lambda: batch_add_to_queue(selected_webhooks, data), robust=True, ) diff --git a/cvat/schema.yml b/cvat/schema.yml index 739321276d9f..4a6edd1fe288 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: CVAT REST API - version: 2.11.1 + version: 2.11.2 description: REST API for Computer Vision Annotation Tool (CVAT) termsOfService: https://www.google.com/policies/terms/ contact: diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 4ea0fd38afdb..84e89e572c65 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -105,7 +105,6 @@ def generate_secret_key(): 'health_check', 'health_check.cache', 'health_check.db', - 'health_check.contrib.migrations', 'health_check.contrib.psutil', 'cvat.apps.iam', 'cvat.apps.dataset_manager', @@ -287,7 +286,7 @@ class CVAT_QUEUES(Enum): 'HOST': redis_inmem_host, 'PORT': redis_inmem_port, 'DB': 0, - 'PASSWORD': urllib.parse.quote(redis_inmem_password), + 'PASSWORD': redis_inmem_password, } RQ_QUEUES = { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index e145f17b19fc..11d044b136c5 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -101,6 +101,10 @@ services: socks_proxy: dockerfile: Dockerfile.ui + cvat_clickhouse: + ports: + - '8123:8123' + cvat_opa: ports: - '8181:8181' @@ -112,3 +116,7 @@ services: cvat_redis_ondisk: ports: - '6666:6666' + + cvat_vector: + ports: + - '8282:80' diff --git a/docker-compose.yml b/docker-compose.yml index b30680566b09..d33630b75d7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -70,7 +70,7 @@ services: cvat_server: container_name: cvat_server - image: cvat/server:${CVAT_VERSION:-v2.11.1} + image: cvat/server:${CVAT_VERSION:-v2.11.2} restart: always depends_on: <<: *backend-deps @@ -105,7 +105,7 @@ services: cvat_utils: container_name: cvat_utils - image: cvat/server:${CVAT_VERSION:-v2.11.1} + image: cvat/server:${CVAT_VERSION:-v2.11.2} restart: always depends_on: *backend-deps environment: @@ -122,7 +122,7 @@ services: cvat_worker_import: container_name: cvat_worker_import - image: cvat/server:${CVAT_VERSION:-v2.11.1} + image: cvat/server:${CVAT_VERSION:-v2.11.2} restart: always depends_on: *backend-deps environment: @@ -138,7 +138,7 @@ services: cvat_worker_export: container_name: cvat_worker_export - image: cvat/server:${CVAT_VERSION:-v2.11.1} + image: cvat/server:${CVAT_VERSION:-v2.11.2} restart: always depends_on: *backend-deps environment: @@ -154,7 +154,7 @@ services: cvat_worker_annotation: container_name: cvat_worker_annotation - image: cvat/server:${CVAT_VERSION:-v2.11.1} + image: cvat/server:${CVAT_VERSION:-v2.11.2} restart: always depends_on: *backend-deps environment: @@ -170,7 +170,7 @@ services: cvat_worker_webhooks: container_name: cvat_worker_webhooks - image: cvat/server:${CVAT_VERSION:-v2.11.1} + image: cvat/server:${CVAT_VERSION:-v2.11.2} restart: always depends_on: *backend-deps environment: @@ -186,7 +186,7 @@ services: cvat_worker_quality_reports: container_name: cvat_worker_quality_reports - image: cvat/server:${CVAT_VERSION:-v2.11.1} + image: cvat/server:${CVAT_VERSION:-v2.11.2} restart: always depends_on: *backend-deps environment: @@ -202,7 +202,7 @@ services: cvat_worker_analytics_reports: container_name: cvat_worker_analytics_reports - image: cvat/server:${CVAT_VERSION:-v2.11.1} + image: cvat/server:${CVAT_VERSION:-v2.11.2} restart: always depends_on: *backend-deps environment: @@ -218,7 +218,7 @@ services: cvat_ui: container_name: cvat_ui - image: cvat/ui:${CVAT_VERSION:-v2.11.1} + image: cvat/ui:${CVAT_VERSION:-v2.11.2} restart: always depends_on: - cvat_server diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index 83875c4a324a..d99c4cfd8e86 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -113,7 +113,7 @@ cvat: additionalVolumeMounts: [] replicas: 1 image: cvat/server - tag: v2.11.1 + tag: v2.11.2 imagePullPolicy: Always permissionFix: enabled: true @@ -137,7 +137,7 @@ cvat: frontend: replicas: 1 image: cvat/ui - tag: v2.11.1 + tag: v2.11.2 imagePullPolicy: Always labels: {} # test: test diff --git a/site/content/en/docs/getting_started/overview.md b/site/content/en/docs/getting_started/overview.md index 81d6cae45b82..16052001d809 100644 --- a/site/content/en/docs/getting_started/overview.md +++ b/site/content/en/docs/getting_started/overview.md @@ -170,7 +170,6 @@ product support or are an integral part of our ecosystem. | ----------------------------------------------------------------------------------- | --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | [**Human Protocol**](https://hmt.ai) | Cloud and Self-hosted | Incorporates CVAT to augment annotation services within the Human Protocol framework, enhancing its capabilities in data labeling. | | [**FiftyOne**](https://fiftyone.ai) | Cloud and Self-hosted | An open-source tool for dataset management and model analysis in computer vision, FiftyOne is [closely integrated](https://voxel51.com/docs/fiftyone/integrations/cvat.html) with CVAT to enhance annotation capabilities and label refinement. | -| [**Toloka**](https://toloka.ai) | Cloud | Utilizes CVAT for crowdsourced data labeling and annotation services, enriching Toloka's diverse task handling capabilities. For more information, see [**Integration with Toloka**](/docs/integration/toloka/). | | [**Hugging Face**](https://huggingface.co/) & [**Roboflow**](https://roboflow.com/) | Cloud | In CVAT Cloud, models from Hugging Face and Roboflow can be added to enhance computer vision tasks. For more information, see [**Integration with Hugging Face and Roboflow**](https://www.cvat.ai/post/integrating-hugging-face-and-roboflow-models) | diff --git a/site/content/en/docs/integration/toloka.md b/site/content/en/docs/integration/toloka.md deleted file mode 100644 index d681eada2d53..000000000000 --- a/site/content/en/docs/integration/toloka.md +++ /dev/null @@ -1,288 +0,0 @@ ---- -title: 'Toloka' -linkTitle: 'Toloka' -weight: 2 ---- - -To have your dataset annotated through Toloka, simply establish -a project in CVAT set the pricing, and let Toloka annotators -take care of the annotations for you. - -> **Note:** Integration with Toloka.ai is available in **beta** version. - -See: - -- [Glossary](#glossary) -- [Preconditions](#preconditions) -- [Creating Toloka project](#creating-toloka-project) -- [Adding tasks and jobs to the Toloka project](#adding-tasks-and-jobs-to-the-toloka-project) -- [Adding instructions for annotators to the Toloka project](#adding-instructions-for-annotators-to-the-toloka-project) -- [Toloka pool setup](#toloka-pool-setup) -- [Changing Toloka pool](#changing-toloka-pool) -- [Reviewing annotated jobs](#reviewing-annotated-jobs) - - [Accepting job](#accepting-job) - - [Rejecting job](#rejecting-job) -- [Moving Toloka pool to archive](#moving-toloka-pool-to-archive) -- [Moving Toloka project to archive](#moving-toloka-project-to-archive) -- [Resource sync between CVAT and Toloka](#resource-sync-between-cvat-and-toloka) - - [Acceptance/Rejection synchronization](#acceptancerejection-synchronization) - -## Glossary - -This page contains several terms used to describe interactions between systems and actors. -Refer to the table below for clarity on how we define and use them. - - - -| Term | Explanation | -| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Toloka | Toloka is a crowdsourcing platform that allows users to assign tasks to a broad group of participants, often termed "crowd workers". In the context of this article, when we mention Toloka, we are specifically referring to one of its UI interfaces. | -| CVAT | CVAT is a tool designed for annotating video and image data for computer vision tasks. In this article's context, when we reference CVAT, we mean one of its UI interfaces. | -| Requester | An individual who establishes an annotation project within CVAT determines the price, creates tasks and jobs within the project, and then releases it on Toloka. | -| Toloker | A person who annotates the Requester's dataset. | - - - -## Preconditions - -Actor: Requester. - -Requester must have a [CVAT account](/docs/manual/basics/registration/) -and [Toloka Requester account](https://platform.toloka.ai/auth?role=REQUESTER&retpath=%2Fsignup%2Frequester). - -To get access to the feature in CVAT, send request to [CVAT Support](mailto:support@cvat.ai) - -## Creating Toloka project - -The requester can set up a project within CVAT and subsequently -connect it to Toloka, making it accessible for annotations by Tolokers. - -To initiate your Toloka project, proceed with the following steps: - -1. [Log in to CVAT](/docs/manual/basics/registration/#account-access) - and [initiate a new Project](/docs/manual/advanced/projects/). -
Here you can can setup user guide which will be shown on Toloka platform, - see [Adding instructions for annotators to the Toloka project](#adding-instructions-for-annotators-to-the-toloka-project). -2. Navigate to the project page and select **Actions** > **Setup crowdsourcing project**. - - ![Setting Up Toloka Project](/images/toloka01.png) - -3. Fill in the following fields in the **Setup crowdsourcing project** form: - - ![Toloka Project Configuration](/images/toloka02.png) - - - **Provider**: Choose Toloka as your provider. - - **Environment**: Select either **Sandbox** for a testing environment or **Production** for a live environment. - - **API key**: Enter the [Requester API key](https://toloka.ai/docs/toloka-kit/registration/). - - **Project description** (Optional): Provide a brief description of your project. - -4. Click **Submit**. - A pop-up indicating **Toloka Project Created** will appear, along with the **Update project** form. - - ![Project Update Form](/images/toloka03.png) - - - ![Open Project in Toloka](/images/toloka04.png) will take you to the published project in Toloka. - - ![Open Project in CVAT](/images/toloka05.png) will take you back to the project in CVAT. - -In CVAT, all projects related to Toloka will be labeled as **Toloka**. - -![Toloka Label on Project](/images/toloka06.png) - -The status indicator changes based on the -state of the project: - -- Green - active -- Dark grey - archived - - -## Adding tasks and jobs to the Toloka project - -To add tasks to the Toloka project, -see [**Create annotation task**](/docs/manual/basics/create_an_annotation_task/#create-a-task). - -On step 2 of the **Create annotation task** procedure, -from the drop-down list, select your Toloka project. - -![Toloka Add Task](/images/toloka07.jpg) - -## Adding instructions for annotators to the Toloka project - -To add instructions for annotators to the Toloka project, -see [**Adding specification to Project**](/docs/manual/advanced/specification/#adding-specification-to-project) -documentation or [**Adding instructions**](https://www.youtube.com/embed/hAN9UGRvwOk) -video tutorial. - -## Toloka pool setup - -After creating a task and its associated jobs in -CVAT, you'll need to configure a Toloka pool, -specifying all task requirements and setting the price for task completion. - -To set up the Toloka pool, do the following: - -1. Open Toloka task, go to **Actions** > **Setup Toloka pool**. - - ![Set up Toloka pool](/images/toloka08.jpg) - -2. In the **Create crowdsourcing task** form, fill in the following fields: - - ![Set up Toloka pool](/images/toloka09.jpg) - - - **Price per job**: Specify the payment amount for completing - one job within the project. - - **Use project description**: Switch this toggle if you want - to use the overarching project description for individual tasks. - - **Description**: Provide details about the pool. - This field is visible only when the **Use project description** toggle is off. - - **Sensible content**: Switch this toggle if your dataset - contains images intended for an adult audience. - - **Accept solutions automatically**: Enable this if you - wish for completed jobs to be automatically accepted. - - **Close pool after completion, sec**: The interval during which the pool will - remain open from the moment all tasks are completed. - Minimum — 0, maximum — 259200 seconds (three days). - - **Time per task suite, sec**: Enter the time limit, in seconds, within which each job must be completed. - The Toloker will see the deadline in the task information on the main Toloka page - and also in CVAT interface. Uncompleted tasks are redistributed to other Tolokers. - - **Days for manual review by requester**: Specify the Review period in days — - the number of days for the review (from 1 to 21 days from the task completion date). - The Toloker will see the deadline in the task information on the main Toloka page. - - **Audience**: Add rules to make jobs available only to Tolokers who meet certain criteria. - For example, you might require Tolokers to be proficient in English and have higher education. - These rules operate based on filter principles. For more information, - see [Toloks Filters](https://toloka.ai/docs/guide/filters/#:~:text=You%20can%20use%20filters%20to,faster%20and%20spend%20less%20money.) - documentation, [CVAT Filters](/docs/manual/advanced/filter/) documentation - or [CVAT Filters](https://www.youtube.com/watch?v=lj6KLIFn24A) video tutorial. - - ![Toloka Rules](/images/toloka10.jpg) - -3. Click **Submit**. You will see the **Toloka task was created** pop-up and the **Update pool** form. - - ![Update pool](/images/toloka11.jpg) - - - ![Open pool in Toloka](/images/toloka04.png) opens pool in Toloka. - - ![Open task in CVAT](/images/toloka05.png) opens task in CVAT. - -4. Open the CVAT task that was published to Toloka, go to **Actions** > **Start Toloka pool**. -
Project, that you created will now be visible to Tolokers.
- - ![Toloka Project](/images/toloka12.jpg) - - -Pools status indicator has the following states: - -- Green - open for annotating -- Light gray - closed -- Dark grey - archived - - -## Changing Toloka pool - -To change started Toloka pool, you need to stop it first. - -1. Open Toloka task, **Actions** > **Stop Toloka pool**. -2. Implement changes. -3. Open Toloka task, go to **Actions** > **Start Toloka pool**. - -## Reviewing annotated jobs - -In case the pool you've created are not in the **Accept solutions automatically** -mode, you will need to manually review and accept them -within time limits that were defined in the Toloka pool settings. - -To approve or reject the job, use the **Accept** and **Reject** buttons. - -![Toloka Project](/images/toloka13.jpg) - -### Accepting job - -To accept the annotated job, do the following: - -1. Go to the Toloka task and open the job. -2. Review the result of annotation and in case all is fine, on the top menu, - click **Accept**. -3. Optionally, you may add comment. - - ![Toloka Project](/images/toloka14.jpg) - -4. Click **OK**. - -### Rejecting job - -> Note, that Toloker can open dispute and appeal the rejected -> job on the Toloka platform. - -To reject the annotated job, do the following: - -1. Go to the Toloka task and open the job. -
On the top menu, you will see **Accept** and **Reject** buttons. - ![Toloka Project](/images/toloka13.jpg) - -2. Review the result of the annotation and in case something is wrong, - on the top menu, click **Reject**. -3. Add comment why this work was rejected - - ![Toloka Project](/images/toloka15.jpg) - -4. Click **OK**. - -After you reject the job, the menu bar will change and only the **Accept** -button will be active. - -Rejected job can be accepted later. - -## Moving Toloka pool to archive - -After annotation is complete, you can move -the Toloka pools to archive without -archiving the whole Project. - -> **Note**, that to archive pool, all jobs within task -> must be in the Complete state. - -> **Note**, that pool must accepted and -> without active assignments on the Toloka -> side. - -> Keep in mind, that if you -> **Rejected** the job, it will not become -> unassigned immediately, to give -> Toloker time to open a dispute. - -To archive complete jobs, do the following: - -1. Open Toloka task, and go to **Actions**. -2. (Optional) If the task is ongoing, select **Stop Toloka pool**. -3. Select **Archive Toloka pool** -4. In the pop-up click **OK**. - -## Moving Toloka project to archive - -After annotation is completed, you can move the Toloka project to the archive. - -> Note that all jobs must be complete. -> Tasks must not have active assignments or assignments that are being disputed. -> All project pools must be closed/archived. - -1. Open Toloka project, go to **Actions** > **Archive Toloka project**. - - ![Toloka Project](/images/toloka16.jpg) - -2. In the pop-up, click **Yes**. - -## Resource sync between CVAT and Toloka - -There are two types of synchronization between CVAT and Toloka: - -- **Explicit synchronization**: Triggered manually by the requester - by clicking the **Sync Toloka project**/**Sync Toloka pool** button within the CVAT interface. -- **Implicit Synchronization**: Occurs automatically at predetermined intervals. - Resources that have been requested by users will be synchronized - without any manual intervention. - -### Acceptance/Rejection synchronization - -In addition to project and pool synchronization, it is essential to synchronize -the status of assignments. If a requester accepts or rejects an assignment -through Toloka's client interface, this action automatically -synchronizes with CVAT to ensure that the data remains -current and consistent across both platforms. diff --git a/site/content/en/docs/manual/advanced/specification.md b/site/content/en/docs/manual/advanced/specification.md index 9d7249ced187..1498a2d120bf 100644 --- a/site/content/en/docs/manual/advanced/specification.md +++ b/site/content/en/docs/manual/advanced/specification.md @@ -63,18 +63,35 @@ To add specification to the **Task**, do the following: ## Access to specification for annotators -To open specification, do the following: -1. Open the job to see the annotation interface. -2. In the top right corner, click **Guide button**(![Guide Icon](/images/guide_icon.jpg)). +The specification is opened automatically when the job has `new annotation` state. +It means, that it will open when the assigned user begins working on the first +job within a **Project** or **Task**. + +The specifications will not automatically reopen +if the user moves to another job within the same **Project** or **Task**. -The specification is opened automatically once for a user when the job has `new annotation` state. +If a **Project** or **Task** is reassigned to another annotator, the specifications will +automatically be shown when the annotator opens the first job but will +not reappear for subsequent jobs. -Additionally, you may tell CVAT interface to open the specification, by adding a dedicated query parameter to link: +To enable the option for specifications to always open automatically, +append the `?openGuide` parameter to the end of the job URL you share with the annotator: -` + +``` /tasks//jobs/?openGuide -` +``` +For example: + +``` +https://app.cvat.ai/tasks/taskID/jobs/jobID?openGuide +``` + +To open specification manually, do the following: + +1. Open the job to see the annotation interface. +2. In the top right corner, click **Guide button**(![Guide Icon](/images/guide_icon.jpg)). ## Markdown editor guide @@ -100,7 +117,7 @@ You can write in raw markdown or use the toolbar on the top of the editor. | 6 | Add a single line of code. | | 7 | Add a block of code. | | 8 | Add a comment. The comment is only visible to Guide editors and remains invisible to annotators. | -| 9 | Add a picture. To use this option, first, upload the picture to an external resource and then add the link in the editor. Alternatively, you can drag and drop a picture into the editor, which will upload it to the CVAT server and add it to the specification. | +| 9 | Add a picture. To use this option, first, upload the picture to an external resource and then add the link in the editor. Alternatively, you can drag and drop a picture into the editor, which will upload it to the CVAT server and add it to the specification. | | 10 | Add a list: bullet list, numbered list, and checklist. | | 11 | Hide the editor pane: options to hide the right pane, show both panes or hide the left pane. | | 12 | Enable full-screen mode. | diff --git a/site/content/en/images/image128.jpg b/site/content/en/images/image128.jpg index 99dda6cba16f..f72fcb838488 100644 Binary files a/site/content/en/images/image128.jpg and b/site/content/en/images/image128.jpg differ diff --git a/tests/cypress/e2e/features/masks_basics.js b/tests/cypress/e2e/features/masks_basics.js index 04bf6212ceac..28b80e696520 100644 --- a/tests/cypress/e2e/features/masks_basics.js +++ b/tests/cypress/e2e/features/masks_basics.js @@ -163,6 +163,7 @@ context('Manipulations with masks', { scrollBehavior: false }, () => { describe('Tests to make sure that empty masks cannot be created', () => { beforeEach(() => { cy.removeAnnotations(); + cy.saveJob('PUT'); }); function checkEraseTools(baseTool = '.cvat-brush-tools-brush', disabled = true) { diff --git a/tests/cypress/e2e/features/single_object_annotation.js b/tests/cypress/e2e/features/single_object_annotation.js index 61efc6f7045f..d53a27529a7f 100644 --- a/tests/cypress/e2e/features/single_object_annotation.js +++ b/tests/cypress/e2e/features/single_object_annotation.js @@ -149,7 +149,7 @@ context('Single object annotation mode', { scrollBehavior: false }, () => { describe('Tests basic features of single shape annotation mode', () => { afterEach(() => { cy.removeAnnotations(); - cy.saveJob(); + cy.saveJob('PUT'); }); it('Check basic single shape annotation pipeline for polygon', () => {