Merge pull request #2377 from openvinotoolkit/mk/share_without_copying_

Share without copying & mount cloud storages

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Manual review pipeline: issues/comments/workspace (<https://github.com/openvinotoolkit/cvat/pull/2357>)
 - Added basic projects implementation (<https://github.com/openvinotoolkit/cvat/pull/2255>)
+- Added documentation on how to mount cloud starage(AWS S3 bucket, Azure container, Google Drive) as FUSE (<https://github.com/openvinotoolkit/cvat/pull/2377>)
+- Added ability to work with share files without copying inside (<https://github.com/openvinotoolkit/cvat/pull/2377>)
 
 ### Changed
 

cvat-canvas/README.md

@@ -188,6 +188,7 @@ Standard JS events are used.
     - canvas.roiselected => {points: number[]}
     - canvas.resizeshape => {id: number}
     - canvas.contextmenu => { mouseEvent: MouseEvent, objectState: ObjectState,  pointID: number }
+    - canvas.error => { exception: Error }
 ```
 
 ### WEB

cvat-canvas/src/typescript/canvasModel.ts

@@ -133,6 +133,7 @@ export enum UpdateReasons {
     DRAG_CANVAS = 'drag_canvas',
     ZOOM_CANVAS = 'zoom_canvas',
     CONFIG_UPDATED = 'config_updated',
+    DATA_FAILED = 'data_failed',
 }
 
 export enum Mode {
@@ -168,6 +169,7 @@ export interface CanvasModel {
     readonly selected: any;
     geometry: Geometry;
     mode: Mode;
+    exception: Error | null;
 
     zoom(x: number, y: number, direction: number): void;
     move(topOffset: number, leftOffset: number): void;
@@ -224,6 +226,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel {
         splitData: SplitData;
         selected: any;
         mode: Mode;
+        exception: Error | null;
     };
 
     public constructor() {
@@ -284,6 +287,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel {
             },
             selected: null,
             mode: Mode.IDLE,
+            exception: null,
         };
     }
 
@@ -411,6 +415,8 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel {
                 this.notify(UpdateReasons.OBJECTS_UPDATED);
             })
             .catch((exception: any): void => {
+                this.data.exception = exception;
+                this.notify(UpdateReasons.DATA_FAILED);
                 throw exception;
             });
     }
@@ -743,4 +749,7 @@ export class CanvasModelImpl extends MasterImpl implements CanvasModel {
     public get mode(): Mode {
         return this.data.mode;
     }
+    public get exception(): Error {
+        return this.data.exception;
+    }
 }

cvat-canvas/src/typescript/canvasView.ts

@@ -1343,6 +1343,14 @@ export class CanvasViewImpl implements CanvasView, Listener {
             this.mode = Mode.IDLE;
             this.canvas.style.cursor = '';
         }
+        else if (reason === UpdateReasons.DATA_FAILED) {
+            const event: CustomEvent = new CustomEvent('canvas.error', {
+                detail: {
+                    exception: model.exception,
+                },
+            });
+            this.canvas.dispatchEvent(event);
+        }
 
         if (model.imageBitmap && [UpdateReasons.IMAGE_CHANGED, UpdateReasons.OBJECTS_UPDATED].includes(reason)) {
             this.redrawBitmap();

cvat-core/src/download.worker.js

@@ -20,7 +20,9 @@ onmessage = (e) => {
         .catch((error) => {
             postMessage({
                 id: e.data.id,
-                error,
+                error: error,
+                status: error.response.status,
+                responseData: error.response.data,
                 isSuccess: false,
             });
         });

cvat-core/src/server-proxy.js

@@ -31,7 +31,13 @@
                     if (e.data.isSuccess) {
                         requests[e.data.id].resolve(e.data.responseData);
                     } else {
-                        requests[e.data.id].reject(e.data.error);
+                        requests[e.data.id].reject({
+                            error: e.data.error,
+                            response: {
+                                status: e.data.status,
+                                data: e.data.responseData,
+                            },
+                        });
                     }
 
                     delete requests[e.data.id];
@@ -725,7 +731,14 @@
                         },
                     );
                 } catch (errorData) {
-                    throw generateError(errorData);
+                    throw generateError({
+                        ...errorData,
+                        message: '',
+                        response: {
+                            ...errorData.response,
+                            data: String.fromCharCode.apply(null, new Uint8Array(errorData.response.data)),
+                        },
+                    });
                 }
 
                 return response;

cvat-core/src/session.js

@@ -973,6 +973,7 @@
                 data_original_chunk_type: undefined,
                 use_zip_chunks: undefined,
                 use_cache: undefined,
+                copy_data: undefined,
             };
 
             let updatedFields = {
@@ -1239,6 +1240,22 @@
                             data.use_cache = useCache;
                         },
                     },
+                    /**
+                     * @name copyData
+                     * @type {boolean}
+                     * @memberof module:API.cvat.classes.Task
+                     * @instance
+                     * @throws {module:API.cvat.exceptions.ArgumentError}
+                     */
+                    copyData: {
+                        get: () => data.copy_data,
+                        set: (copyData) => {
+                            if (typeof copyData !== 'boolean') {
+                                throw new ArgumentError('Value must be a boolean');
+                            }
+                            data.copy_data = copyData;
+                        },
+                    },
                     /**
                      * After task has been created value can be appended only.
                      * @name labels
@@ -1908,6 +1925,9 @@
         if (typeof this.dataChunkSize !== 'undefined') {
             taskDataSpec.chunk_size = this.dataChunkSize;
         }
+        if (typeof this.copyData !== 'undefined') {
+            taskDataSpec.copy_data = this.copyData;
+        }
 
         const task = await serverProxy.tasks.createTask(taskSpec, taskDataSpec, onUpdate);
         return new Task(task);

cvat-ui/src/actions/annotation-actions.ts

@@ -185,6 +185,7 @@ export enum AnnotationActionTypes {
     SAVE_LOGS_FAILED = 'SAVE_LOGS_FAILED',
     INTERACT_WITH_CANVAS = 'INTERACT_WITH_CANVAS',
     SET_AI_TOOLS_REF = 'SET_AI_TOOLS_REF',
+    GET_DATA_FAILED = 'GET_DATA_FAILED',
     SWITCH_REQUEST_REVIEW_DIALOG = 'SWITCH_REQUEST_REVIEW_DIALOG',
     SWITCH_SUBMIT_REVIEW_DIALOG = 'SWITCH_SUBMIT_REVIEW_DIALOG',
     SET_FORCE_EXIT_ANNOTATION_PAGE_FLAG = 'SET_FORCE_EXIT_ANNOTATION_PAGE_FLAG',
@@ -218,6 +219,15 @@ export function changeWorkspace(workspace: Workspace): AnyAction {
     };
 }
 
+export function getDataFailed(error: any): AnyAction {
+    return {
+        type: AnnotationActionTypes.GET_DATA_FAILED,
+        payload: {
+            error,
+        },
+    };
+}
+
 export function addZLayer(): AnyAction {
     return {
         type: AnnotationActionTypes.ADD_Z_LAYER,
@@ -913,7 +923,16 @@ export function getJobAsync(tid: number, jid: number, initialFrame: number, init
             const frameData = await job.frames.get(frameNumber);
             // call first getting of frame data before rendering interface
             // to load and decode first chunk
-            await frameData.data();
+            try {
+                await frameData.data();
+            } catch (error) {
+                dispatch({
+                    type: AnnotationActionTypes.GET_DATA_FAILED,
+                    payload: {
+                        error,
+                    },
+                });
+            }
             const states = await job.annotations.get(frameNumber, showAllInterpolationTracks, filters);
             const issues = await job.issues();
             const reviews = await job.reviews();

cvat-ui/src/actions/tasks-actions.ts

@@ -394,6 +394,9 @@ export function createTaskAsync(data: any): ThunkAction<Promise<void>, {}, {}, A
         if (data.advanced.dataChunkSize) {
             description.data_chunk_size = data.advanced.dataChunkSize;
         }
+        if (data.advanced.copyData) {
+            description.copy_data = data.advanced.copyData;
+        }
 
         const taskInstance = new cvat.classes.Task(description);
         taskInstance.clientFiles = data.files.local;

cvat-ui/src/components/annotation-page/canvas/canvas-wrapper.tsx

@@ -90,6 +90,7 @@ interface Props {
     onSwitchGrid(enabled: boolean): void;
     onSwitchAutomaticBordering(enabled: boolean): void;
     onFetchAnnotation(): void;
+    onGetDataFailed(error: any): void;
     onStartIssue(position: number[]): void;
 }
 
@@ -322,10 +323,17 @@ export default class CanvasWrapperComponent extends React.PureComponent<Props> {
         canvasInstance.html().removeEventListener('canvas.splitted', this.onCanvasTrackSplitted);
 
         canvasInstance.html().removeEventListener('canvas.contextmenu', this.onCanvasPointContextMenu);
+        canvasInstance.html().removeEventListener('canvas.error', this.onCanvasErrorOccurrence);
 
         window.removeEventListener('resize', this.fitCanvas);
     }
 
+    private onCanvasErrorOccurrence = (event: any): void => {
+        const { exception } = event.detail;
+        const { onGetDataFailed } = this.props;
+        onGetDataFailed(exception);
+    };
+
     private onCanvasShapeDrawn = (event: any): void => {
         const {
             jobInstance, activeLabelID, activeObjectType, frame, onShapeDrawn, onCreateAnnotations,
@@ -745,6 +753,7 @@ export default class CanvasWrapperComponent extends React.PureComponent<Props> {
         canvasInstance.html().addEventListener('canvas.splitted', this.onCanvasTrackSplitted);
 
         canvasInstance.html().addEventListener('canvas.contextmenu', this.onCanvasPointContextMenu);
+        canvasInstance.html().addEventListener('canvas.error', this.onCanvasErrorOccurrence);
     }
 
     public render(): JSX.Element {

cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx

@@ -26,11 +26,13 @@ export interface AdvancedConfiguration {
     useZipChunks: boolean;
     dataChunkSize?: number;
     useCache: boolean;
+    copyData?: boolean;
 }
 
 type Props = FormComponentProps & {
     onSubmit(values: AdvancedConfiguration): void;
     installedGit: boolean;
+    activeFileManagerTab: string;
 };
 
 function isPositiveInteger(_: any, value: any, callback: any): void {
@@ -114,6 +116,26 @@ class AdvancedConfigurationForm extends React.PureComponent<Props> {
         form.resetFields();
     }
 
+    renderCopyDataChechbox(): JSX.Element {
+        const { form } = this.props;
+        return (
+            <Row>
+                <Col>
+                    <Form.Item help='If you have a low data transfer rate over the network you can copy data into CVAT to speed up work'>
+                        {form.getFieldDecorator('copyData', {
+                            initialValue: false,
+                            valuePropName: 'checked',
+                        })(
+                            <Checkbox>
+                                <Text className='cvat-text-color'>Copy data into CVAT</Text>
+                            </Checkbox>,
+                        )}
+                    </Form.Item>
+                </Col>
+            </Row>
+        );
+    }
+
     private renderImageQuality(): JSX.Element {
         const { form } = this.props;
 
@@ -386,10 +408,12 @@ class AdvancedConfigurationForm extends React.PureComponent<Props> {
     }
 
     public render(): JSX.Element {
-        const { installedGit } = this.props;
-
+        const { installedGit, activeFileManagerTab } = this.props;
         return (
             <Form>
+
+                {activeFileManagerTab === 'share' ? this.renderCopyDataChechbox() : null}
+
                 <Row>
                     <Col>{this.renderUzeZipChunks()}</Col>
                 </Row>

cvat-ui/src/components/create-task-page/create-task-content.tsx

@@ -25,6 +25,7 @@ export interface CreateTaskData {
     advanced: AdvancedConfiguration;
     labels: any[];
     files: Files;
+    activeFileManagerTab: string;
 }
 
 interface Props {
@@ -53,6 +54,7 @@ const defaultState = {
         share: [],
         remote: [],
     },
+    activeFileManagerTab: 'local',
 };
 
 class CreateTaskContent extends React.PureComponent<Props & RouteComponentProps, State> {
@@ -132,6 +134,14 @@ class CreateTaskContent extends React.PureComponent<Props & RouteComponentProps,
         });
     };
 
+    private changeFileManagerTab = (key: string): void => {
+        const values = this.state;
+        this.setState({
+            ...values,
+            activeFileManagerTab: key
+        });
+    };
+
     private handleSubmitClick = (): void => {
         if (!this.validateLabelsOrProject()) {
             notification.error({
@@ -238,6 +248,7 @@ class CreateTaskContent extends React.PureComponent<Props & RouteComponentProps,
                 <Text type='danger'>* </Text>
                 <Text className='cvat-text-color'>Select files:</Text>
                 <ConnectedFileManager
+                    onChangeActiveKey={this.changeFileManagerTab}
                     ref={(container: any): void => {
                         this.fileManagerContainer = container;
                     }}
@@ -255,6 +266,7 @@ class CreateTaskContent extends React.PureComponent<Props & RouteComponentProps,
                     <Collapse.Panel key='1' header={<Text className='cvat-title'>Advanced configuration</Text>}>
                         <AdvancedConfigurationForm
                             installedGit={installedGit}
+                            activeFileManagerTab={this.state.activeFileManagerTab}
                             wrappedComponentRef={(component: any): void => {
                                 this.advancedConfigurationComponent = component;
                             }}

cvat-ui/src/components/file-manager/file-manager.tsx

@@ -31,6 +31,7 @@ interface Props {
     withRemote: boolean;
     treeData: TreeNodeNormal[];
     onLoadData: (key: string, success: () => void, failure: () => void) => void;
+    onChangeActiveKey(key: string): void;
 }
 
 export default class FileManager extends React.PureComponent<Props, State> {
@@ -215,7 +216,7 @@ export default class FileManager extends React.PureComponent<Props, State> {
     }
 
     public render(): JSX.Element {
-        const { withRemote } = this.props;
+        const { withRemote, onChangeActiveKey } = this.props;
         const { active } = this.state;
 
         return (
@@ -224,11 +225,12 @@ export default class FileManager extends React.PureComponent<Props, State> {
                     type='card'
                     activeKey={active}
                     tabBarGutter={5}
-                    onChange={(activeKey: string): void =>
+                    onChange={(activeKey: string): void => {
+                        onChangeActiveKey(activeKey);
                         this.setState({
                             active: activeKey as any,
-                        })
-                    }
+                        });
+                    }}
                 >
                     {this.renderLocalSelector()}
                     {this.renderShareSelector()}