diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f415aa19180..ba874209df24 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added intelligent scissors blocking feature () - Support cloud storage status () - Support cloud storage preview () +- cvat-core: support cloud storages () ### Changed diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 15be69a6b42f..697ef85b445c 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.15.0", + "version": "3.16.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index 229b9bec6ae8..d5abde0a8443 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.15.0", + "version": "3.16.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-core/src/api-implementation.js b/cvat-core/src/api-implementation.js index 787c4303f90e..0301fdec6031 100644 --- a/cvat-core/src/api-implementation.js +++ b/cvat-core/src/api-implementation.js @@ -16,13 +16,20 @@ camelToSnake, } = require('./common'); - const { TaskStatus, TaskMode, DimensionType } = require('./enums'); + const { + TaskStatus, + TaskMode, + DimensionType, + CloudStorageProviderType, + CloudStorageCredentialsType, + } = require('./enums'); const User = require('./user'); const { AnnotationFormats } = require('./annotation-formats'); const { ArgumentError } = require('./exceptions'); const { Task } = require('./session'); const { Project } = require('./project'); + const { CloudStorage } = require('./cloud-storage'); function implementAPI(cvat) { cvat.plugins.list.implementation = PluginRegistry.list; @@ -262,6 +269,49 @@ cvat.projects.searchNames.implementation = async (search, limit) => serverProxy.projects.searchNames(search, limit); + cvat.cloudStorages.get.implementation = async (filter) => { + checkFilter(filter, { + page: isInteger, + displayName: isString, + resourceName: isString, + description: isString, + id: isInteger, + owner: isString, + search: isString, + providerType: isEnum.bind(CloudStorageProviderType), + credentialsType: isEnum.bind(CloudStorageCredentialsType), + }); + + checkExclusiveFields(filter, ['id', 'search'], ['page']); + + const searchParams = new URLSearchParams(); + for (const field of [ + 'displayName', + 'credentialsType', + 'providerType', + 'owner', + 'search', + 'id', + 'page', + 'description', + ]) { + if (Object.prototype.hasOwnProperty.call(filter, field)) { + searchParams.set(camelToSnake(field), filter[field]); + } + } + + if (Object.prototype.hasOwnProperty.call(filter, 'resourceName')) { + searchParams.set('resource', filter.resourceName); + } + + const cloudStoragesData = await serverProxy.cloudStorages.get(searchParams.toString()); + const cloudStorages = cloudStoragesData.map((cloudStorage) => new CloudStorage(cloudStorage)); + + cloudStorages.count = cloudStoragesData.count; + + return cloudStorages; + }; + return cvat; } diff --git a/cvat-core/src/api.js b/cvat-core/src/api.js index d517b444caa3..e50cdbc8f8cb 100644 --- a/cvat-core/src/api.js +++ b/cvat-core/src/api.js @@ -22,6 +22,8 @@ function build() { const { Attribute, Label } = require('./labels'); const MLModel = require('./ml-model'); const { FrameData } = require('./frames'); + const { CloudStorage } = require('./cloud-storage'); + const enums = require('./enums'); @@ -748,6 +750,41 @@ function build() { PluginError, ServerError, }, + /** + * Namespace is used for getting cloud storages + * @namespace cloudStorages + * @memberof module:API.cvat + */ + cloudStorages: { + /** + * @typedef {Object} CloudStorageFilter + * @property {string} displayName Check if displayName contains this value + * @property {string} resourceName Check if resourceName contains this value + * @property {module:API.cvat.enums.ProviderType} providerType Check if providerType equal this value + * @property {integer} id Check if id equals this value + * @property {integer} page Get specific page + * (default REST API returns 20 clouds storages per request. + * In order to get more, it is need to specify next page) + * @property {string} owner Check if an owner name contains this value + * @property {string} search Combined search of contains among all the fields + * @global + */ + + /** + * Method returns a list of cloud storages corresponding to a filter + * @method get + * @async + * @memberof module:API.cvat.cloudStorages + * @param {CloudStorageFilter} [filter={}] cloud storage filter + * @returns {module:API.cvat.classes.CloudStorage[]} + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ServerError} + */ + async get(filter = {}) { + const result = await PluginRegistry.apiWrapper(cvat.cloudStorages.get, filter); + return result; + }, + }, /** * Namespace is used for access to classes * @namespace classes @@ -768,6 +805,7 @@ function build() { Issue, Review, FrameData, + CloudStorage, }, }; @@ -780,6 +818,7 @@ function build() { cvat.lambda = Object.freeze(cvat.lambda); cvat.client = Object.freeze(cvat.client); cvat.enums = Object.freeze(cvat.enums); + cvat.cloudStorages = Object.freeze(cvat.cloudStorages); const implementAPI = require('./api-implementation'); diff --git a/cvat-core/src/cloud-storage.js b/cvat-core/src/cloud-storage.js new file mode 100644 index 000000000000..4fd8bd3a96e5 --- /dev/null +++ b/cvat-core/src/cloud-storage.js @@ -0,0 +1,520 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +(() => { + const PluginRegistry = require('./plugins'); + const serverProxy = require('./server-proxy'); + const { isBrowser, isNode } = require('browser-or-node'); + const { ArgumentError } = require('./exceptions'); + const { CloudStorageCredentialsType, CloudStorageProviderType } = require('./enums'); + + /** + * Class representing a cloud storage + * @memberof module:API.cvat.classes + */ + class CloudStorage { + // TODO: add storage availability status (avaliable/unavaliable) + constructor(initialData) { + const data = { + id: undefined, + display_name: undefined, + description: undefined, + credentials_type: undefined, + provider_type: undefined, + resource: undefined, + account_name: undefined, + key: undefined, + secret_key: undefined, + session_token: undefined, + specific_attributes: undefined, + owner: undefined, + created_date: undefined, + updated_date: undefined, + manifest_path: undefined, + manifests: undefined, + }; + + for (const property in data) { + if (Object.prototype.hasOwnProperty.call(data, property) && property in initialData) { + data[property] = initialData[property]; + } + } + + Object.defineProperties( + this, + Object.freeze({ + /** + * @name id + * @type {integer} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + */ + id: { + get: () => data.id, + }, + /** + * Storage name + * @name displayName + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + displayName: { + get: () => data.display_name, + set: (value) => { + if (typeof value !== 'string') { + throw new ArgumentError(`Value must be string. ${typeof value} was found`); + } else if (!value.trim().length) { + throw new ArgumentError('Value must not be empty string'); + } + data.display_name = value; + }, + }, + /** + * Storage description + * @name description + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + description: { + get: () => data.description, + set: (value) => { + if (typeof value !== 'string') { + throw new ArgumentError('Value must be string'); + } + data.description = value; + }, + }, + /** + * Azure account name + * @name accountName + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + accountName: { + get: () => data.account_name, + set: (value) => { + if (typeof value === 'string') { + if (value.trim().length) { + data.account_name = value; + } else { + throw new ArgumentError('Value must not be empty'); + } + } else { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } + }, + }, + /** + * AWS access key id + * @name accessKey + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + accessKey: { + get: () => data.key, + set: (value) => { + if (typeof value === 'string') { + if (value.trim().length) { + data.key = value; + } else { + throw new ArgumentError('Value must not be empty'); + } + } else { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } + }, + }, + /** + * AWS secret key + * @name secretKey + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + secretKey: { + get: () => data.secret_key, + set: (value) => { + if (typeof value === 'string') { + if (value.trim().length) { + data.secret_key = value; + } else { + throw new ArgumentError('Value must not be empty'); + } + } else { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } + }, + }, + /** + * Session token + * @name token + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + token: { + get: () => data.session_token, + set: (value) => { + if (typeof value === 'string') { + if (value.trim().length) { + data.session_token = value; + } else { + throw new ArgumentError('Value must not be empty'); + } + } else { + throw new ArgumentError(`Value must be a string. ${typeof value} was found`); + } + }, + }, + /** + * Unique resource name + * @name resourceName + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + resourceName: { + get: () => data.resource, + set: (value) => { + if (typeof value !== 'string') { + throw new ArgumentError(`Value must be string. ${typeof value} was found`); + } else if (!value.trim().length) { + throw new ArgumentError('Value must not be empty'); + } + data.resource = value; + }, + }, + /** + * @name manifestPath + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + manifestPath: { + get: () => data.manifest_path, + set: (value) => { + if (typeof value === 'string') { + data.manifest_path = value; + } else { + throw new ArgumentError('Value must be a string'); + } + }, + }, + /** + * @name providerType + * @type {module:API.cvat.enums.ProviderType} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + providerType: { + get: () => data.provider_type, + set: (key) => { + if (key !== undefined && !!CloudStorageProviderType[key]) { + data.provider_type = CloudStorageProviderType[key]; + } else { + throw new ArgumentError('Value must be one CloudStorageProviderType keys'); + } + }, + }, + /** + * @name credentialsType + * @type {module:API.cvat.enums.CloudStorageCredentialsType} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + credentialsType: { + get: () => data.credentials_type, + set: (key) => { + if (key !== undefined && !!CloudStorageCredentialsType[key]) { + data.credentials_type = CloudStorageCredentialsType[key]; + } else { + throw new ArgumentError('Value must be one CloudStorageCredentialsType keys'); + } + }, + }, + /** + * @name specificAttributes + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + specificAttributes: { + get: () => data.specific_attributes, + set: (attributesValue) => { + if (typeof attributesValue === 'string') { + const attrValues = new URLSearchParams( + Array.from(new URLSearchParams(attributesValue).entries()).filter( + ([key, value]) => !!key && !!value, + ), + ).toString(); + if (!attrValues) { + throw new ArgumentError('Value must match the key1=value1&key2=value2'); + } + data.specific_attributes = attributesValue; + } else { + throw new ArgumentError('Value must be a string'); + } + }, + }, + /** + * Instance of a user who has created the cloud storage + * @name owner + * @type {module:API.cvat.classes.User} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + */ + owner: { + get: () => data.owner, + }, + /** + * @name createdDate + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + */ + createdDate: { + get: () => data.created_date, + }, + /** + * @name updatedDate + * @type {string} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + */ + updatedDate: { + get: () => data.updated_date, + }, + /** + * @name manifests + * @type {string[]} + * @memberof module:API.cvat.classes.CloudStorage + * @instance + * @throws {module:API.cvat.exceptions.ArgumentError} + */ + manifests: { + get: () => data.manifests, + set: (manifests) => { + if (Array.isArray(manifests)) { + for (const elem of manifests) { + if (typeof elem !== 'string') { + throw new ArgumentError('Each element of the manifests array must be a string'); + } + } + data.manifests = manifests; + } else { + throw new ArgumentError('Value must be an array'); + } + }, + }, + }), + ); + } + + /** + * Method updates data of a created cloud storage or creates new cloud storage + * @method save + * @returns {module:API.cvat.classes.CloudStorage} + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async save() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.save); + return result; + } + + /** + * Method deletes a cloud storage from a server + * @method delete + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async delete() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.delete); + return result; + } + + /** + * Method returns cloud storage content + * @method getContent + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async getContent() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getContent); + return result; + } + + /** + * Method returns the cloud storage preview + * @method getPreview + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async getPreview() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getPreview); + return result; + } + + /** + * Method returns cloud storage status + * @method getStatus + * @memberof module:API.cvat.classes.CloudStorage + * @readonly + * @instance + * @async + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ + async getStatus() { + const result = await PluginRegistry.apiWrapper.call(this, CloudStorage.prototype.getStatus); + return result; + } + } + + CloudStorage.prototype.save.implementation = async function () { + function prepareOptionalFields(cloudStorageInstance) { + const data = {}; + if (cloudStorageInstance.description) { + data.description = cloudStorageInstance.description; + } + + if (cloudStorageInstance.accountName) { + data.account_name = cloudStorageInstance.accountName; + } + + if (cloudStorageInstance.accessKey) { + data.key = cloudStorageInstance.accessKey; + } + + if (cloudStorageInstance.secretKey) { + data.secret_key = cloudStorageInstance.secretKey; + } + + if (cloudStorageInstance.token) { + data.session_token = cloudStorageInstance.token; + } + + if (cloudStorageInstance.specificAttributes) { + data.specific_attributes = cloudStorageInstance.specificAttributes; + } + return data; + } + // update + if (typeof this.id !== 'undefined') { + // providr_type and recource should not change; + // send to the server only the values that have changed + const initialData = {}; + if (this.displayName) { + initialData.display_name = this.displayName; + } + if (this.credentialsType) { + initialData.credentials_type = this.credentialsType; + } + + if (this.manifests) { + initialData.manifests = this.manifests; + } + + const cloudStorageData = { + ...initialData, + ...prepareOptionalFields(this), + }; + + await serverProxy.cloudStorages.update(this.id, cloudStorageData); + return this; + } + + // create + const initialData = { + display_name: this.displayName, + credentials_type: this.credentialsType, + provider_type: this.providerType, + resource: this.resourceName, + manifests: this.manifests, + }; + + const cloudStorageData = { + ...initialData, + ...prepareOptionalFields(this), + }; + + const cloudStorage = await serverProxy.cloudStorages.create(cloudStorageData); + return new CloudStorage(cloudStorage); + }; + + CloudStorage.prototype.delete.implementation = async function () { + const result = await serverProxy.cloudStorages.delete(this.id); + return result; + }; + + CloudStorage.prototype.getContent.implementation = async function () { + const result = await serverProxy.cloudStorages.getContent(this.id, this.manifestPath); + return result; + }; + + CloudStorage.prototype.getPreview.implementation = async function getPreview() { + return new Promise((resolve, reject) => { + serverProxy.cloudStorages + .getPreview(this.id) + .then((result) => { + if (isNode) { + resolve(global.Buffer.from(result, 'binary').toString('base64')); + } else if (isBrowser) { + const reader = new FileReader(); + reader.onload = () => { + resolve(reader.result); + }; + reader.readAsDataURL(result); + } + }) + .catch((error) => { + reject(error); + }); + }); + }; + + CloudStorage.prototype.getStatus.implementation = async function () { + const result = await serverProxy.cloudStorages.getStatus(this.id); + return result; + }; + + module.exports = { + CloudStorage, + }; +})(); diff --git a/cvat-core/src/enums.js b/cvat-core/src/enums.js index 4ce6d80c08c6..c8ecab559650 100644 --- a/cvat-core/src/enums.js +++ b/cvat-core/src/enums.js @@ -1,4 +1,4 @@ -// Copyright (C) 2019-2020 Intel Corporation +// Copyright (C) 2019-2021 Intel Corporation // // SPDX-License-Identifier: MIT @@ -333,6 +333,36 @@ '#733380', ]; + /** + * Types of cloud storage providers + * @enum {string} + * @name CloudStorageProviderType + * @memberof module:API.cvat.enums + * @property {string} AWS_S3 'AWS_S3_BUCKET' + * @property {string} AZURE 'AZURE_CONTAINER' + * @readonly + */ + const CloudStorageProviderType = Object.freeze({ + AWS_S3_BUCKET: 'AWS_S3_BUCKET', + AZURE_CONTAINER: 'AZURE_CONTAINER', + }); + + /** + * Types of cloud storage credentials + * @enum {string} + * @name CloudStorageCredentialsType + * @memberof module:API.cvat.enums + * @property {string} KEY_SECRET_KEY_PAIR 'KEY_SECRET_KEY_PAIR' + * @property {string} ACCOUNT_NAME_TOKEN_PAIR 'ACCOUNT_NAME_TOKEN_PAIR' + * @property {string} ANONYMOUS_ACCESS 'ANONYMOUS_ACCESS' + * @readonly + */ + const CloudStorageCredentialsType = Object.freeze({ + KEY_SECRET_KEY_PAIR: 'KEY_SECRET_KEY_PAIR', + ACCOUNT_NAME_TOKEN_PAIR: 'ACCOUNT_NAME_TOKEN_PAIR', + ANONYMOUS_ACCESS: 'ANONYMOUS_ACCESS', + }); + module.exports = { ShareFileType, TaskStatus, @@ -348,5 +378,7 @@ colors, Source, DimensionType, + CloudStorageProviderType, + CloudStorageCredentialsType, }; })(); diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index 3d914ce9db7a..cdc98940f0c3 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -1145,9 +1145,7 @@ const closureId = Date.now(); predictAnnotations.latestRequest.id = closureId; - const predicate = () => ( - !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId - ); + const predicate = () => !predictAnnotations.latestRequest.fetching || predictAnnotations.latestRequest.id !== closureId; if (predictAnnotations.latestRequest.fetching) { waitFor(5, predicate).then(() => { if (predictAnnotations.latestRequest.id !== closureId) { @@ -1181,6 +1179,121 @@ } } + async function createCloudStorage(storageDetail) { + const { backendAPI } = config; + + try { + const response = await Axios.post(`${backendAPI}/cloudstorages`, JSON.stringify(storageDetail), { + proxy: config.proxy, + headers: { + 'Content-Type': 'application/json', + }, + }); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } + } + + async function updateCloudStorage(id, cloudStorageData) { + const { backendAPI } = config; + + try { + await Axios.patch(`${backendAPI}/cloudstorages/${id}`, JSON.stringify(cloudStorageData), { + proxy: config.proxy, + headers: { + 'Content-Type': 'application/json', + }, + }); + } catch (errorData) { + throw generateError(errorData); + } + } + + async function getCloudStorages(filter = '') { + const { backendAPI } = config; + + let response = null; + try { + response = await Axios.get(`${backendAPI}/cloudstorages?page_size=12&${filter}`, { + proxy: config.proxy, + }); + } catch (errorData) { + throw generateError(errorData); + } + + response.data.results.count = response.data.count; + return response.data.results; + } + + async function getCloudStorageContent(id, manifestPath) { + const { backendAPI } = config; + + let response = null; + try { + const url = `${backendAPI}/cloudstorages/${id}/content${ + manifestPath ? `?manifest_path=${manifestPath}` : '' + }`; + response = await Axios.get(url, { + proxy: config.proxy, + }); + } catch (errorData) { + throw generateError(errorData); + } + + return response.data; + } + + async function getCloudStoragePreview(id) { + const { backendAPI } = config; + + let response = null; + try { + const url = `${backendAPI}/cloudstorages/${id}/preview`; + response = await workerAxios.get(url, { + proxy: config.proxy, + responseType: 'arraybuffer', + }); + } catch (errorData) { + throw generateError({ + ...errorData, + message: '', + response: { + ...errorData.response, + data: String.fromCharCode.apply(null, new Uint8Array(errorData.response.data)), + }, + }); + } + + return new Blob([new Uint8Array(response)]); + } + + async function getCloudStorageStatus(id) { + const { backendAPI } = config; + + let response = null; + try { + const url = `${backendAPI}/cloudstorages/${id}/status`; + response = await Axios.get(url, { + proxy: config.proxy, + }); + } catch (errorData) { + throw generateError(errorData); + } + + return response.data; + } + + async function deleteCloudStorage(id) { + const { backendAPI } = config; + + try { + await Axios.delete(`${backendAPI}/cloudstorages/${id}`); + } catch (errorData) { + throw generateError(errorData); + } + } + Object.defineProperties( this, Object.freeze({ @@ -1310,6 +1423,19 @@ }), writable: false, }, + + cloudStorages: { + value: Object.freeze({ + get: getCloudStorages, + getContent: getCloudStorageContent, + getPreview: getCloudStoragePreview, + getStatus: getCloudStorageStatus, + create: createCloudStorage, + delete: deleteCloudStorage, + update: updateCloudStorage, + }), + writable: false, + }, }), ); } diff --git a/cvat-core/src/session.js b/cvat-core/src/session.js index c5366ea1d5f5..eaba48bc078f 100644 --- a/cvat-core/src/session.js +++ b/cvat-core/src/session.js @@ -1012,6 +1012,7 @@ use_cache: undefined, copy_data: undefined, dimension: undefined, + cloud_storage_id: undefined, }; const updatedFields = new FieldUpdateTrigger({ @@ -1373,7 +1374,7 @@ get: () => [...data.jobs], }, /** - * List of files from shared resource + * List of files from shared resource or list of cloud storage files * @name serverFiles * @type {string[]} * @memberof module:API.cvat.classes.Task @@ -1535,6 +1536,15 @@ */ get: () => data.dimension, }, + /** + * @name cloudStorageId + * @type {integer|null} + * @memberof module:API.cvat.classes.Task + * @instance + */ + cloudStorageId: { + get: () => data.cloud_storage_id, + }, _internalData: { get: () => data, }, @@ -2062,6 +2072,9 @@ if (typeof this.copyData !== 'undefined') { taskDataSpec.copy_data = this.copyData; } + if (typeof this.cloudStorageId !== 'undefined') { + taskDataSpec.cloud_storage_id = this.cloudStorageId; + } const task = await serverProxy.tasks.createTask(taskSpec, taskDataSpec, onUpdate); return new Task(task); diff --git a/cvat-core/tests/api/cloud-storages.js b/cvat-core/tests/api/cloud-storages.js new file mode 100644 index 000000000000..7b5d1bf4ed0e --- /dev/null +++ b/cvat-core/tests/api/cloud-storages.js @@ -0,0 +1,178 @@ +// Copyright (C) 2021 Intel Corporation +// +// SPDX-License-Identifier: MIT + +// Setup mock for a server +jest.mock('../../src/server-proxy', () => { + const mock = require('../mocks/server-proxy.mock'); + return mock; +}); + +// Initialize api +window.cvat = require('../../src/api'); + +const { CloudStorage } = require('../../src/cloud-storage'); +const { cloudStoragesDummyData } = require('../mocks/dummy-data.mock'); + +describe('Feature: get cloud storages', () => { + test('get all cloud storages', async () => { + const result = await window.cvat.cloudStorages.get(); + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(cloudStoragesDummyData.count); + for (const item of result) { + expect(item).toBeInstanceOf(CloudStorage); + } + }); + + test('get cloud storage by id', async () => { + const result = await window.cvat.cloudStorages.get({ + id: 1, + }); + const cloudStorage = result[0]; + + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(1); + expect(cloudStorage).toBeInstanceOf(CloudStorage); + expect(cloudStorage.id).toBe(1); + expect(cloudStorage.providerType).toBe('AWS_S3_BUCKET'); + expect(cloudStorage.credentialsType).toBe('KEY_SECRET_KEY_PAIR'); + expect(cloudStorage.resourceName).toBe('bucket'); + expect(cloudStorage.displayName).toBe('Demonstration bucket'); + expect(cloudStorage.manifests).toHaveLength(1); + expect(cloudStorage.manifests[0]).toBe('manifest.jsonl'); + expect(cloudStorage.specificAttributes).toBe(''); + expect(cloudStorage.description).toBe('It is first bucket'); + }); + + test('get a cloud storage by an unknown id', async () => { + const result = await window.cvat.cloudStorages.get({ + id: 10, + }); + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(0); + }); + + test('get a cloud storage by an invalid id', async () => { + expect( + window.cvat.cloudStorages.get({ + id: '1', + }), + ).rejects.toThrow(window.cvat.exceptions.ArgumentError); + }); + + test('get cloud storages by filters', async () => { + const filters = new Map([ + ['providerType', 'AWS_S3_BUCKET'], + ['resourceName', 'bucket'], + ['displayName', 'Demonstration bucket'], + ['credentialsType', 'KEY_SECRET_KEY_PAIR'], + ['description', 'It is first bucket'], + ]); + + const result = await window.cvat.cloudStorages.get(Object.fromEntries(filters)); + + const [cloudStorage] = result; + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(1); + expect(cloudStorage).toBeInstanceOf(CloudStorage); + expect(cloudStorage.id).toBe(1); + filters.forEach((value, key) => { + expect(cloudStorage[key]).toBe(value); + }); + }); + + test('get cloud storage by invalid filters', async () => { + expect( + window.cvat.cloudStorages.get({ + unknown: '5', + }), + ).rejects.toThrow(window.cvat.exceptions.ArgumentError); + }); +}); + +describe('Feature: create a cloud storage', () => { + test('create new cloud storage without an id', async () => { + const cloudStorage = new window.cvat.classes.CloudStorage({ + display_name: 'new cloud storage', + provider_type: 'AZURE_CONTAINER', + resource: 'newcontainer', + credentials_type: 'ACCOUNT_NAME_TOKEN_PAIR', + account_name: 'accountname', + session_token: 'x'.repeat(135), + manifests: ['manifest.jsonl'], + }); + + const result = await cloudStorage.save(); + expect(typeof result.id).toBe('number'); + }); +}); + +describe('Feature: update a cloud storage', () => { + test('update cloud storage with some new field values', async () => { + const newValues = new Map([ + ['displayName', 'new display name'], + ['credentialsType', 'ANONYMOUS_ACCESS'], + ['description', 'new description'], + ['specificAttributes', 'region=eu-west-1'], + ]); + + let result = await window.cvat.cloudStorages.get({ + id: 1, + }); + + let [cloudStorage] = result; + + for (const [key, value] of newValues) { + cloudStorage[key] = value; + } + + cloudStorage.save(); + + result = await window.cvat.cloudStorages.get({ + id: 1, + }); + [cloudStorage] = result; + + newValues.forEach((value, key) => { + expect(cloudStorage[key]).toBe(value); + }); + }); + + test('Update manifests in a cloud storage', async () => { + const newManifests = [ + 'sub1/manifest.jsonl', + 'sub2/manifest.jsonl', + ]; + + let result = await window.cvat.cloudStorages.get({ + id: 1, + }); + let [cloudStorage] = result; + + cloudStorage.manifests = newManifests; + cloudStorage.save(); + + result = await window.cvat.cloudStorages.get({ + id: 1, + }); + [cloudStorage] = result; + + expect(cloudStorage.manifests).toEqual(newManifests); + }); +}); + +describe('Feature: delete a cloud storage', () => { + test('delete a cloud storage', async () => { + let result = await window.cvat.cloudStorages.get({ + id: 2, + }); + + await result[0].delete(); + result = await window.cvat.cloudStorages.get({ + id: 2, + }); + + expect(Array.isArray(result)).toBeTruthy(); + expect(result).toHaveLength(0); + }); +}); diff --git a/cvat-core/tests/mocks/dummy-data.mock.js b/cvat-core/tests/mocks/dummy-data.mock.js index 17e886ca5e34..7b67d00ea890 100644 --- a/cvat-core/tests/mocks/dummy-data.mock.js +++ b/cvat-core/tests/mocks/dummy-data.mock.js @@ -2547,6 +2547,56 @@ const frameMetaDummyData = { }, }; +const cloudStoragesDummyData = { + count: 2, + next: null, + previous: null, + results: [ + { + id: 2, + owner: { + url: 'http://localhost:7000/api/v1/users/1', + id: 1, + username: 'maya', + first_name: '', + last_name: '' + }, + manifests: [ + 'manifest.jsonl' + ], + provider_type: 'AZURE_CONTAINER', + resource: 'container', + display_name: 'Demonstration container', + created_date: '2021-09-01T09:29:47.094244Z', + updated_date: '2021-09-01T09:29:47.103264Z', + credentials_type: 'ACCOUNT_NAME_TOKEN_PAIR', + specific_attributes: '', + description: 'It is first container' + }, + { + id: 1, + owner: { + url: 'http://localhost:7000/api/v1/users/1', + id: 1, + username: 'maya', + first_name: '', + last_name: '' + }, + manifests: [ + 'manifest.jsonl' + ], + provider_type: 'AWS_S3_BUCKET', + resource: 'bucket', + display_name: 'Demonstration bucket', + created_date: '2021-08-31T09:03:09.350817Z', + updated_date: '2021-08-31T15:16:21.394773Z', + credentials_type: 'KEY_SECRET_KEY_PAIR', + specific_attributes: '', + description: 'It is first bucket' + } + ] +}; + module.exports = { tasksDummyData, projectsDummyData, @@ -2557,4 +2607,5 @@ module.exports = { jobAnnotationsDummyData, frameMetaDummyData, formatsDummyData, + cloudStoragesDummyData, }; diff --git a/cvat-core/tests/mocks/server-proxy.mock.js b/cvat-core/tests/mocks/server-proxy.mock.js index a5111756cd97..7c9e2e155731 100644 --- a/cvat-core/tests/mocks/server-proxy.mock.js +++ b/cvat-core/tests/mocks/server-proxy.mock.js @@ -12,6 +12,7 @@ const { taskAnnotationsDummyData, jobAnnotationsDummyData, frameMetaDummyData, + cloudStoragesDummyData, } = require('./dummy-data.mock'); function QueryStringToJSON(query, ignoreList = []) { @@ -318,6 +319,63 @@ class ServerProxy { return null; } + async function getCloudStorages(filter = '') { + const queries = QueryStringToJSON(filter); + const result = cloudStoragesDummyData.results.filter((item) => { + for (const key in queries) { + if (Object.prototype.hasOwnProperty.call(queries, key)) { + if (queries[key] !== item[key]) { + return false; + } + } + } + return true; + }); + return result; + } + + async function updateCloudStorage(id, cloudStorageData) { + const cloudStorage = cloudStoragesDummyData.results.find((item) => item.id === id); + if (cloudStorage) { + for (const prop in cloudStorageData) { + if ( + Object.prototype.hasOwnProperty.call(cloudStorageData, prop) + && Object.prototype.hasOwnProperty.call(cloudStorage, prop) + ) { + cloudStorage[prop] = cloudStorageData[prop]; + } + } + } + } + + async function createCloudStorage(cloudStorageData) { + const id = Math.max(...cloudStoragesDummyData.results.map((item) => item.id)) + 1; + cloudStoragesDummyData.results.push({ + id, + provider_type: cloudStorageData.provider_type, + resource: cloudStorageData.resource, + display_name: cloudStorageData.display_name, + credentials_type: cloudStorageData.credentials_type, + specific_attributes: cloudStorageData.specific_attributes, + description: cloudStorageData.description, + owner: 1, + created_date: '2021-09-01T09:29:47.094244+03:00', + updated_date: '2021-09-01T09:29:47.103264+03:00', + }); + + const result = await getCloudStorages(`?id=${id}`); + return result[0]; + } + + async function deleteCloudStorage(id) { + const cloudStorages = cloudStoragesDummyData.results; + const cloudStorageId = cloudStorages.findIndex((item) => item.id === id); + if (cloudStorageId !== -1) { + cloudStorages.splice(cloudStorageId); + } + } + + Object.defineProperties( this, Object.freeze({ @@ -384,6 +442,16 @@ class ServerProxy { getAnnotations, }, }, + + cloudStorages: { + value: Object.freeze({ + get: getCloudStorages, + update: updateCloudStorage, + create: createCloudStorage, + delete: deleteCloudStorage, + }), + writable: false, + }, }), ); }