From d497bb6a90dd3c9625c3c8e8f73278019e21983d Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Fri, 29 Sep 2023 10:03:49 +0300 Subject: [PATCH] Remove the Git repository synchronization functionality (#6904) This functionality has accumulated significant technical debt: * Most importantly, it does not use the current authorization system, rendering it accessible only for admin users. * It doesn't follow the regular API conventions and is not visible in the API schema. This necessitates a special code in the SDK. * The initialization code in `base.py` is not safe when multiple instances of the server starts at the same time (each instance may end up generating its own key). The team has decided that the cost of fixing these issues outweighs the benefit of the functionality, so remove it. --- .vscode/launch.json | 16 - CHANGELOG.md | 3 +- Dockerfile | 10 +- cvat-cli/src/cvat_cli/cli.py | 4 - cvat-cli/src/cvat_cli/parser.py | 17 - cvat-sdk/cvat_sdk/core/client.py | 10 - cvat-sdk/cvat_sdk/core/git.py | 59 -- cvat-sdk/cvat_sdk/core/proxies/tasks.py | 12 - cvat-ui/package.json | 2 +- cvat-ui/src/actions/tasks-actions.ts | 17 +- .../advanced-configuration-form.tsx | 105 +--- .../create-task-page/create-task-content.tsx | 6 - .../create-task-page/create-task-page.tsx | 63 +-- cvat-ui/src/components/header/header.tsx | 2 - cvat-ui/src/components/task-page/details.tsx | 214 +------ cvat-ui/src/components/task-page/styles.scss | 35 -- .../create-task-page/create-task-page.tsx | 17 +- cvat-ui/src/reducers/index.ts | 1 - cvat-ui/src/reducers/plugins-reducer.ts | 6 - cvat-ui/src/utils/git-utils.ts | 217 -------- cvat-ui/src/utils/validation-patterns.ts | 9 +- cvat-ui/webpack.config.js | 2 +- cvat/apps/dataset_repo/README.md | 19 +- cvat/apps/dataset_repo/apps.py | 9 - cvat/apps/dataset_repo/dataset_repo.py | 525 ------------------ cvat/apps/dataset_repo/management/__init__.py | 3 - .../management/commands/__init__.py | 3 - .../management/commands/update_git_states.py | 21 - .../migrations/0002_auto_20190123_1305.py | 3 +- .../migrations/0007_delete_gitdata.py | 15 + cvat/apps/dataset_repo/models.py | 26 - cvat/apps/dataset_repo/tests.py | 516 ----------------- cvat/apps/dataset_repo/urls.py | 17 - cvat/apps/dataset_repo/views.py | 149 ----- cvat/apps/engine/views.py | 3 +- cvat/requirements/base.in | 1 - cvat/requirements/base.txt | 8 +- cvat/settings/base.py | 53 -- cvat/urls.py | 3 - docker-compose.yml | 2 +- helm-chart/values.yaml | 5 - .../administration/advanced/backup_guide.md | 2 +- site/content/en/docs/api_sdk/cli/_index.md | 7 - site/content/en/docs/faq.md | 2 +- .../manual/advanced/task-synchronization.md | 25 - .../basics/create_an_annotation_task.md | 2 - .../en/docs/manual/basics/task-details.md | 10 - ssh/README.md | 25 - supervisord/server.conf | 7 +- supervisord/utils.conf | 15 +- supervisord/worker.annotation.conf | 7 +- supervisord/worker.export.conf | 7 +- supervisord/worker.import.conf | 7 +- supervisord/worker.quality_reports.conf | 7 +- supervisord/worker.webhooks.conf | 2 +- ..._try_create_task_incorrect_dataset_repo.js | 62 --- tests/docker-compose.test_servers.yml | 14 - tests/git_server/entrypoint.sh | 13 - tests/python/sdk/test_tasks.py | 35 -- 59 files changed, 55 insertions(+), 2402 deletions(-) delete mode 100644 cvat-sdk/cvat_sdk/core/git.py delete mode 100644 cvat-ui/src/utils/git-utils.ts delete mode 100644 cvat/apps/dataset_repo/apps.py delete mode 100644 cvat/apps/dataset_repo/dataset_repo.py delete mode 100644 cvat/apps/dataset_repo/management/__init__.py delete mode 100644 cvat/apps/dataset_repo/management/commands/__init__.py delete mode 100644 cvat/apps/dataset_repo/management/commands/update_git_states.py create mode 100644 cvat/apps/dataset_repo/migrations/0007_delete_gitdata.py delete mode 100644 cvat/apps/dataset_repo/tests.py delete mode 100644 cvat/apps/dataset_repo/urls.py delete mode 100644 cvat/apps/dataset_repo/views.py delete mode 100644 site/content/en/docs/manual/advanced/task-synchronization.md delete mode 100644 ssh/README.md delete mode 100644 tests/cypress/e2e/actions_tasks2/case_76_try_create_task_incorrect_dataset_repo.js delete mode 100755 tests/git_server/entrypoint.sh diff --git a/.vscode/launch.json b/.vscode/launch.json index 4d0f09c55257..f2dfd13e5cf3 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -339,22 +339,6 @@ "env": {}, "console": "internalConsole" }, - { - "name": "server: git", - "type": "python", - "request": "launch", - "justMyCode": false, - "stopOnEntry": false, - "python": "${command:python.interpreterPath}", - "program": "${workspaceRoot}/manage.py", - "args": [ - "update_git_states" - ], - "django": true, - "cwd": "${workspaceFolder}", - "env": {}, - "console": "internalConsole" - }, { "name": "server: RQ - cleaning", "type": "python", diff --git a/CHANGELOG.md b/CHANGELOG.md index adc25267b522..8b12654da4da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed -- TDB +- Functionality for synchronizing a task with a Git repository + () ### Fixed diff --git a/Dockerfile b/Dockerfile index 77c9e5edc36b..f96bedd4888b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -113,7 +113,6 @@ RUN apt-get update && \ ca-certificates \ curl \ git \ - git-lfs \ libgeos-c1v5 \ libgl1 \ libgomp1 \ @@ -126,7 +125,6 @@ RUN apt-get update && \ python3 \ python3-distutils \ python3-venv \ - ssh \ supervisor \ tzdata \ && ln -fs /usr/share/zoneinfo/${TZ} /etc/localtime && \ @@ -140,12 +138,7 @@ COPY --from=build-smokescreen /tmp/smokescreen /usr/local/bin/smokescreen # Add a non-root user ENV USER=${USER} ENV HOME /home/${USER} -RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} && \ - if [ -z ${socks_proxy} ]; then \ - echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30\"" >> ${HOME}/.bashrc; \ - else \ - echo export "GIT_SSH_COMMAND=\"ssh -o StrictHostKeyChecking=no -o ConnectTimeout=30 -o ProxyCommand='nc -X 5 -x ${socks_proxy} %h %p'\"" >> ${HOME}/.bashrc; \ - fi +RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER} ARG CLAM_AV="no" RUN if [ "$CLAM_AV" = "yes" ]; then \ @@ -184,7 +177,6 @@ RUN if [ "${CVAT_DEBUG_ENABLED}" = 'yes' ]; then \ COPY cvat/nginx.conf /etc/nginx/nginx.conf COPY --chown=${USER} components /tmp/components COPY --chown=${USER} supervisord/ ${HOME}/supervisord -COPY --chown=${USER} ssh ${HOME}/.ssh COPY --chown=${USER} wait-for-it.sh manage.py backend_entrypoint.sh ${HOME}/ COPY --chown=${USER} utils/ ${HOME}/utils COPY --chown=${USER} cvat/ ${HOME}/cvat diff --git a/cvat-cli/src/cvat_cli/cli.py b/cvat-cli/src/cvat_cli/cli.py index 114e5bed8945..e7945b18bb2e 100644 --- a/cvat-cli/src/cvat_cli/cli.py +++ b/cvat-cli/src/cvat_cli/cli.py @@ -43,8 +43,6 @@ def tasks_create( annotation_path: str = "", annotation_format: str = "CVAT XML 1.1", status_check_period: int = 2, - dataset_repository_url: str = "", - lfs: bool = False, **kwargs, ) -> None: """ @@ -68,8 +66,6 @@ def tasks_create( annotation_path=annotation_path, annotation_format=annotation_format, status_check_period=status_check_period, - dataset_repository_url=dataset_repository_url, - use_lfs=lfs, pbar=DeferredTqdmProgressReporter(), ) print("Created task id", task.id) diff --git a/cvat-cli/src/cvat_cli/parser.py b/cvat-cli/src/cvat_cli/parser.py index f03a52f9b41a..83f033e2930c 100644 --- a/cvat-cli/src/cvat_cli/parser.py +++ b/cvat-cli/src/cvat_cli/parser.py @@ -193,17 +193,6 @@ def make_cmdline_parser() -> argparse.ArgumentParser: """ ), ) - task_create_parser.add_argument( - "--dataset_repository_url", - default="", - type=str, - help=textwrap.dedent( - """\ - git repository to store annotations e.g. - https://github.com/user/repos [annotation/] - """ - ), - ) task_create_parser.add_argument( "--frame_step", default=None, @@ -232,12 +221,6 @@ def make_cmdline_parser() -> argparse.ArgumentParser: type=parse_label_arg, help="string or file containing JSON labels specification", ) - task_create_parser.add_argument( - "--lfs", - default=False, - action="store_true", - help="using lfs for dataset repository (default: %(default)s)", - ) task_create_parser.add_argument( "--project_id", default=None, type=int, help="project ID if project exists" ) diff --git a/cvat-sdk/cvat_sdk/core/client.py b/cvat-sdk/cvat_sdk/core/client.py index 5b767f7ed232..554027282b35 100644 --- a/cvat-sdk/cvat_sdk/core/client.py +++ b/cvat-sdk/cvat_sdk/core/client.py @@ -322,16 +322,6 @@ class CVAT_API_V2: def __init__(self, host: str): self.host = host.rstrip("/") self.base = self.host + "/api/" - self.git = self.host + "/git/repository/" - - def git_create(self, task_id: int) -> str: - return self.git + f"create/{task_id}" - - def git_check(self, rq_id: int) -> str: - return self.git + f"check/{rq_id}" - - def git_get(self, task_id: int) -> str: - return self.git + f"get/{task_id}" def make_endpoint_url( self, diff --git a/cvat-sdk/cvat_sdk/core/git.py b/cvat-sdk/cvat_sdk/core/git.py deleted file mode 100644 index 8347e6ca5097..000000000000 --- a/cvat-sdk/cvat_sdk/core/git.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (C) 2022 CVAT.ai Corporation -# -# SPDX-License-Identifier: MIT - -from __future__ import annotations - -import json -from time import sleep -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from cvat_sdk.core.client import Client - - -def create_git_repo( - client: Client, - *, - task_id: int, - repo_url: str, - status_check_period: int = None, - use_lfs: bool = True, -): - if status_check_period is None: - status_check_period = client.config.status_check_period - - common_headers = client.api_client.get_common_headers() - - response = client.api_client.rest_client.POST( - client.api_map.git_create(task_id), - post_params={"path": repo_url, "lfs": use_lfs, "tid": task_id}, - headers=common_headers, - ) - response_json = json.loads(response.data) - rq_id = response_json["rq_id"] - client.logger.info(f"Create RQ ID: {rq_id}") - - client.logger.debug("Awaiting a dataset repository to be created for the task %s...", task_id) - check_url = client.api_map.git_check(rq_id) - status = None - while status != "finished": - sleep(status_check_period) - response = client.api_client.rest_client.GET(check_url, headers=common_headers) - response_json = json.loads(response.data) - status = response_json["status"] - if status == "failed" or status == "unknown": - client.logger.error( - "Dataset repository creation request for task %s failed" "with status %s.", - task_id, - status, - ) - break - - client.logger.debug( - "Awaiting a dataset repository to be created for the task %s. Response status: %s", - task_id, - status, - ) - - client.logger.debug("Dataset repository creation completed with status: %s.", status) diff --git a/cvat-sdk/cvat_sdk/core/proxies/tasks.py b/cvat-sdk/cvat_sdk/core/proxies/tasks.py index 29e84dc3545d..0bebcc6507d4 100644 --- a/cvat-sdk/cvat_sdk/core/proxies/tasks.py +++ b/cvat-sdk/cvat_sdk/core/proxies/tasks.py @@ -16,7 +16,6 @@ from PIL import Image from cvat_sdk.api_client import apis, exceptions, models -from cvat_sdk.core import git from cvat_sdk.core.downloading import Downloader from cvat_sdk.core.helpers import get_paginated_collection from cvat_sdk.core.progress import ProgressReporter @@ -349,8 +348,6 @@ def create_from_data( annotation_path: str = "", annotation_format: str = "CVAT XML 1.1", status_check_period: int = None, - dataset_repository_url: str = "", - use_lfs: bool = False, pbar: Optional[ProgressReporter] = None, ) -> Task: """ @@ -381,15 +378,6 @@ def create_from_data( if annotation_path: task.import_annotations(annotation_format, annotation_path, pbar=pbar) - if dataset_repository_url: - git.create_git_repo( - self._client, - task_id=task.id, - repo_url=dataset_repository_url, - status_check_period=status_check_period, - use_lfs=use_lfs, - ) - task.fetch() return task diff --git a/cvat-ui/package.json b/cvat-ui/package.json index a42d3f25bee8..b24ee60adba6 100644 --- a/cvat-ui/package.json +++ b/cvat-ui/package.json @@ -1,6 +1,6 @@ { "name": "cvat-ui", - "version": "1.56.2", + "version": "1.57.0", "description": "CVAT single-page application", "main": "src/index.tsx", "scripts": { diff --git a/cvat-ui/src/actions/tasks-actions.ts b/cvat-ui/src/actions/tasks-actions.ts index e5f9e5edcaf8..57b948dc47c7 100644 --- a/cvat-ui/src/actions/tasks-actions.ts +++ b/cvat-ui/src/actions/tasks-actions.ts @@ -257,27 +257,12 @@ ThunkAction, {}, {}, AnyAction> { taskInstance.serverFiles = data.files.share.concat(data.files.cloudStorage); taskInstance.remoteFiles = data.files.remote; - if (data.advanced.repository) { - const [gitPlugin] = (await cvat.plugins.list()).filter((plugin: any): boolean => plugin.name === 'Git'); - - if (gitPlugin) { - gitPlugin.callbacks.onStatusChange = (status: string): void => { - onProgress?.(status); - }; - gitPlugin.data.task = taskInstance; - gitPlugin.data.repos = data.advanced.repository; - gitPlugin.data.format = data.advanced.format; - gitPlugin.data.lfs = data.advanced.lfs; - } - } - try { const savedTask = await taskInstance.save((status: RQStatus, progress: number, message: string): void => { if (status === RQStatus.UNKNOWN) { onProgress?.(`${message} ${progress ? `${Math.floor(progress * 100)}%` : ''}`); } else if ([RQStatus.QUEUED, RQStatus.STARTED].includes(status)) { - const helperMessage = data.advanced.repository ? - 'Do not leave the page' : 'You may close the window.'; + const helperMessage = 'You may close the window.'; onProgress?.(`${message} ${progress ? `${Math.floor(progress * 100)}%` : ''}. ${helperMessage}`); } else { onProgress?.(`${status}: ${message}`); diff --git a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx index 30ce8a552db7..73d6faabca5e 100644 --- a/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx +++ b/cvat-ui/src/components/create-task-page/advanced-configuration-form.tsx @@ -1,5 +1,5 @@ // Copyright (C) 2020-2022 Intel Corporation -// Copyright (C) 2022 CVAT.ai Corporation +// Copyright (C) 2022-2023 CVAT.ai Corporation // // SPDX-License-Identifier: MIT @@ -7,7 +7,6 @@ import React, { RefObject } from 'react'; import { Row, Col } from 'antd/lib/grid'; import { PercentageOutlined, QuestionCircleOutlined } from '@ant-design/icons'; import Input from 'antd/lib/input'; -import Select from 'antd/lib/select'; import Space from 'antd/lib/space'; import Switch from 'antd/lib/switch'; import Tooltip from 'antd/lib/tooltip'; @@ -26,8 +25,6 @@ import { getCore, Storage, StorageData } from 'cvat-core-wrapper'; const core = getCore(); -const { Option } = Select; - export enum SortingMethod { LEXICOGRAPHICAL = 'lexicographical', NATURAL = 'natural', @@ -43,9 +40,6 @@ export interface AdvancedConfiguration { startFrame?: number; stopFrame?: number; frameFilter?: string; - lfs: boolean; - format?: string, - repository?: string; useZipChunks: boolean; dataChunkSize?: number; useCache: boolean; @@ -59,7 +53,6 @@ export interface AdvancedConfiguration { const initialValues: AdvancedConfiguration = { imageQuality: 70, - lfs: false, useZipChunks: true, useCache: true, copyData: false, @@ -83,12 +76,10 @@ interface Props { onChangeUseProjectTargetStorage(value: boolean): void; onChangeSourceStorageLocation: (value: StorageLocation) => void; onChangeTargetStorageLocation: (value: StorageLocation) => void; - installedGit: boolean; projectId: number | null; useProjectSourceStorage: boolean; useProjectTargetStorage: boolean; activeFileManagerTab: string; - dumpers: []; sourceStorageLocation: StorageLocation; targetStorageLocation: StorageLocation; } @@ -101,23 +92,6 @@ function validateURL(_: RuleObject, value: string): Promise { return Promise.resolve(); } -function validateRepositoryPath(_: RuleObject, value: string): Promise { - if (value && !patterns.validatePath.pattern.test(value)) { - return Promise.reject(new Error('Repository path is not a valid path')); - } - - return Promise.resolve(); -} - -function validateRepository(_: RuleObject, value: string): Promise<[void, void]> | Promise { - if (value) { - const [url, path] = value.split(/\s+/); - return Promise.all([validateURL(_, url), validateRepositoryPath(_, path)]); - } - - return Promise.resolve(); -} - const isInteger = ({ min, max }: { min?: number; max?: number }) => ( _: RuleObject, value?: number | string, @@ -352,79 +326,6 @@ class AdvancedConfigurationForm extends React.PureComponent { ); } - private renderGitLFSBox(): JSX.Element { - return ( - - - - - Use LFS (Large File Support): - - - - - ); - } - - private renderGitRepositoryURL(): JSX.Element { - return ( - - - - ); - } - - private renderGitFormat(): JSX.Element { - const { dumpers } = this.props; - return ( - - - - ); - } - - private renderGit(): JSX.Element { - return ( - <> - - {this.renderGitRepositoryURL()} - - - {this.renderGitFormat()} - - - {this.renderGitLFSBox()} - - - - ); - } - private renderBugTracker(): JSX.Element { return ( { } public render(): JSX.Element { - const { installedGit, activeFileManagerTab } = this.props; + const { activeFileManagerTab } = this.props; return (
@@ -584,8 +485,6 @@ class AdvancedConfigurationForm extends React.PureComponent { {this.renderChunkSize()} - {installedGit ? this.renderGit() : null} - {this.renderBugTracker()} diff --git a/cvat-ui/src/components/create-task-page/create-task-content.tsx b/cvat-ui/src/components/create-task-page/create-task-content.tsx index d69cafbffe65..362e32ca31be 100644 --- a/cvat-ui/src/components/create-task-page/create-task-content.tsx +++ b/cvat-ui/src/components/create-task-page/create-task-content.tsx @@ -50,8 +50,6 @@ enum SupportedShareTypes { interface Props { onCreate: (data: CreateTaskData, onProgress?: (status: string, progress?: number) => void) => Promise; projectId: number | null; - installedGit: boolean; - dumpers:[]; many: boolean; } @@ -71,7 +69,6 @@ const defaultState: State = { }, subset: '', advanced: { - lfs: false, useZipChunks: true, useCache: true, sortingMethod: SortingMethod.LEXICOGRAPHICAL, @@ -815,7 +812,6 @@ class CreateTaskContent extends React.PureComponent Advanced configuration}> void) => Promise; - installedGit: boolean; - dumpers: [] } export default function CreateTaskPage(props: Props): JSX.Element { - const { - onCreate, installedGit, dumpers, - } = props; + const { onCreate } = props; const location = useLocation(); - const [error, setError] = useState(''); let projectId = null; const params = new URLSearchParams(location.search); @@ -34,58 +26,13 @@ export default function CreateTaskPage(props: Props): JSX.Element { projectId = +(params.get('projectId') as string); } const many = params.get('many') === 'true'; - const handleCreate: typeof onCreate = (...onCreateParams) => onCreate(...onCreateParams) - .catch((err) => { - setError(err.toString()); - throw err; - }); - - useEffect(() => { - if (error) { - let errorCopy = error; - const sshKeys: string[] = []; - while (errorCopy.length) { - const startIndex = errorCopy.search(/'ssh/); - if (startIndex === -1) break; - let sshKey = errorCopy.slice(startIndex + 1); - const stopIndex = sshKey.search(/'/); - sshKey = sshKey.slice(0, stopIndex); - sshKeys.push(sshKey); - errorCopy = errorCopy.slice(stopIndex + 1); - } - - if (sshKeys.length) { - Modal.error({ - width: 800, - title: 'Could not clone the repository', - className: 'cvat-create-task-clone-repository-fail', - content: ( - <> - - Please make sure it exists and you have access - - - Consider adding the following public ssh keys to git: - -