From f2c84a2653fc601c90bcb84154f0fe7a6be937df Mon Sep 17 00:00:00 2001 From: Dmitry Agapov Date: Fri, 25 Sep 2020 15:06:51 +0300 Subject: [PATCH] Nuclio as a plugin in CVAT, improved system to check installed plugins (#2192) * allow to run cvat without nuclio * fix new line * fix comments * Updated core version * refactoring * minor refactoring, fixed eslint issues, added documentation to cvat-core, updated ui version, updated changelog * move plugins to serverViewSet Co-authored-by: Boris Sekachev --- CHANGELOG.md | 1 + .../analytics/docker-compose.analytics.yml | 1 + components/serverless/README.md | 7 ++ .../serverless/docker-compose.serverless.yml | 28 +++++++ cvat-core/package-lock.json | 2 +- cvat-core/package.json | 2 +- cvat-core/src/api-implementation.js | 5 ++ cvat-core/src/api.js | 54 ++++++++----- cvat-core/src/server-proxy.js | 13 ++++ cvat-ui/src/actions/plugins-actions.ts | 52 ++++++------- cvat-ui/src/components/cvat-app.tsx | 12 ++- cvat-ui/src/components/header/header.tsx | 75 ++++++++++--------- cvat-ui/src/index.tsx | 17 ++--- cvat-ui/src/reducers/interfaces.ts | 18 ++++- cvat-ui/src/reducers/plugins-reducer.ts | 12 ++- cvat-ui/src/reducers/root-reducer.ts | 1 + cvat-ui/src/utils/plugin-checker.ts | 29 ------- cvat/apps/engine/serializers.py | 5 ++ cvat/apps/engine/views.py | 22 +++++- cvat/apps/lambda_manager/urls.py | 8 +- cvat/urls.py | 3 +- docker-compose.yml | 19 ----- 22 files changed, 225 insertions(+), 161 deletions(-) create mode 100644 components/serverless/README.md create mode 100644 components/serverless/docker-compose.serverless.yml delete mode 100644 cvat-ui/src/utils/plugin-checker.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index aa9a696b0378..04e9db7abc8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - UI models (like DEXTR) were redesigned to be more interactive () - Used Ubuntu:20.04 as a base image for CVAT Dockerfile () - Right colors of label tags in label mapping when a user runs automatic detection () +- Nuclio became an optional component of CVAT () - A key to remove a point from a polyshape [Ctrl => Alt] () ### Deprecated diff --git a/components/analytics/docker-compose.analytics.yml b/components/analytics/docker-compose.analytics.yml index 5edcd5056d24..b097b087dae4 100644 --- a/components/analytics/docker-compose.analytics.yml +++ b/components/analytics/docker-compose.analytics.yml @@ -63,6 +63,7 @@ services: DJANGO_LOG_SERVER_PORT: 5000 DJANGO_LOG_VIEWER_HOST: kibana DJANGO_LOG_VIEWER_PORT: 5601 + CVAT_ANALYTICS: 1 no_proxy: kibana,logstash,nuclio,${no_proxy} volumes: diff --git a/components/serverless/README.md b/components/serverless/README.md new file mode 100644 index 000000000000..03edc6e803db --- /dev/null +++ b/components/serverless/README.md @@ -0,0 +1,7 @@ +## Serverless for Computer Vision Annotation Tool (CVAT) + +### Run docker container +```bash +# From project root directory +docker-compose -f docker-compose.yml -f components/serverless/docker-compose.serverless.yml up -d +``` diff --git a/components/serverless/docker-compose.serverless.yml b/components/serverless/docker-compose.serverless.yml new file mode 100644 index 000000000000..7663785b12e3 --- /dev/null +++ b/components/serverless/docker-compose.serverless.yml @@ -0,0 +1,28 @@ +version: '2.3' +services: + serverless: + container_name: nuclio + image: quay.io/nuclio/dashboard:1.4.8-amd64 + restart: always + networks: + default: + aliases: + - nuclio + volumes: + - /tmp:/tmp + - /var/run/docker.sock:/var/run/docker.sock + environment: + http_proxy: + https_proxy: + no_proxy: 172.28.0.1,${no_proxy} + NUCLIO_CHECK_FUNCTION_CONTAINERS_HEALTHINESS: "true" + ports: + - "8070:8070" + + cvat: + environment: + CVAT_SERVERLESS: 1 + no_proxy: kibana,logstash,nuclio,${no_proxy} + +volumes: + cvat_events: diff --git a/cvat-core/package-lock.json b/cvat-core/package-lock.json index 43cc3a58e092..0da9bfcfd463 100644 --- a/cvat-core/package-lock.json +++ b/cvat-core/package-lock.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.7.1", + "version": "3.8.0", "lockfileVersion": 1, "requires": true, "dependencies": { diff --git a/cvat-core/package.json b/cvat-core/package.json index 4735233dc81c..c106c78cb886 100644 --- a/cvat-core/package.json +++ b/cvat-core/package.json @@ -1,6 +1,6 @@ { "name": "cvat-core", - "version": "3.7.1", + "version": "3.8.0", "description": "Part of Computer Vision Tool which presents an interface for client-side integration", "main": "babel.config.js", "scripts": { diff --git a/cvat-core/src/api-implementation.js b/cvat-core/src/api-implementation.js index 39c0d9112cb0..c2a429abff5b 100644 --- a/cvat-core/src/api-implementation.js +++ b/cvat-core/src/api-implementation.js @@ -223,6 +223,11 @@ return tasks; }; + cvat.server.installedApps.implementation = async () => { + const result = await serverProxy.server.installedApps(); + return result; + }; + return cvat; } diff --git a/cvat-core/src/api.js b/cvat-core/src/api.js index 1b70911839b2..68cb46bd38bd 100644 --- a/cvat-core/src/api.js +++ b/cvat-core/src/api.js @@ -136,7 +136,6 @@ function build() { return result; }, /** - * Method allows to register on a server * @method register * @async @@ -272,6 +271,20 @@ function build() { .apiWrapper(cvat.server.request, url, data); return result; }, + + /** + * Method returns apps that are installed on the server + * @method installedApps + * @async + * @memberof module:API.cvat.server + * @returns {Object} map {installedApp: boolean} + * @throws {module:API.cvat.exceptions.PluginError} + * @throws {module:API.cvat.exceptions.ServerError} + */ + async installedApps() { + const result = await PluginRegistry.apiWrapper(cvat.server.installedApps); + return result; + }, }, /** * Namespace is used for getting tasks @@ -470,34 +483,35 @@ function build() { return result; }, /** - * Install plugin to CVAT - * @method register - * @async - * @memberof module:API.cvat.plugins - * @param {Plugin} [plugin] plugin for registration - * @throws {module:API.cvat.exceptions.PluginError} - */ + * Install plugin to CVAT + * @method register + * @async + * @memberof module:API.cvat.plugins + * @param {Plugin} [plugin] plugin for registration + * @throws {module:API.cvat.exceptions.PluginError} + */ async register(plugin) { const result = await PluginRegistry .apiWrapper(cvat.plugins.register, plugin); return result; }, }, + /** - * Namespace is used for serverless functions management (mainly related with DL models) - * @namespace lambda - * @memberof module:API.cvat - */ + * Namespace is used for serverless functions management (mainly related with DL models) + * @namespace lambda + * @memberof module:API.cvat + */ lambda: { /** - * Method returns list of available serverless models - * @method list - * @async - * @memberof module:API.cvat.lambda - * @returns {module:API.cvat.classes.MLModel[]} - * @throws {module:API.cvat.exceptions.ServerError} - * @throws {module:API.cvat.exceptions.PluginError} - */ + * Method returns list of available serverless models + * @method list + * @async + * @memberof module:API.cvat.lambda + * @returns {module:API.cvat.classes.MLModel[]} + * @throws {module:API.cvat.exceptions.ServerError} + * @throws {module:API.cvat.exceptions.PluginError} + */ async list() { const result = await PluginRegistry .apiWrapper(cvat.lambda.list); diff --git a/cvat-core/src/server-proxy.js b/cvat-core/src/server-proxy.js index fa817e70550a..5fcbe15f7add 100644 --- a/cvat-core/src/server-proxy.js +++ b/cvat-core/src/server-proxy.js @@ -812,6 +812,18 @@ } } + async function installedApps() { + const { backendAPI } = config; + try { + const response = await Axios.get(`${backendAPI}/server/plugins`, { + proxy: config.proxy, + }); + return response.data; + } catch (errorData) { + throw generateError(errorData); + } + } + Object.defineProperties(this, Object.freeze({ server: { value: Object.freeze({ @@ -828,6 +840,7 @@ register, request: serverRequest, userAgreements, + installedApps, }), writable: false, }, diff --git a/cvat-ui/src/actions/plugins-actions.ts b/cvat-ui/src/actions/plugins-actions.ts index c1c739f6aa36..fcdf88cc8ebb 100644 --- a/cvat-ui/src/actions/plugins-actions.ts +++ b/cvat-ui/src/actions/plugins-actions.ts @@ -3,43 +3,35 @@ // SPDX-License-Identifier: MIT import { ActionUnion, createAction, ThunkAction } from 'utils/redux'; -import { SupportedPlugins } from 'reducers/interfaces'; -import PluginChecker from 'utils/plugin-checker'; +import { PluginsList } from 'reducers/interfaces'; +import getCore from '../cvat-core-wrapper'; + +const core = getCore(); export enum PluginsActionTypes { - CHECK_PLUGINS = 'CHECK_PLUGINS', - CHECKED_ALL_PLUGINS = 'CHECKED_ALL_PLUGINS', + GET_PLUGINS = 'GET_PLUGINS', + GET_PLUGINS_SUCCESS = 'GET_PLUGINS_SUCCESS', + GET_PLUGINS_FAILED = 'GET_PLUGINS_FAILED', } -type PluginObjects = Record; - const pluginActions = { - checkPlugins: () => createAction(PluginsActionTypes.CHECK_PLUGINS), - checkedAllPlugins: (list: PluginObjects) => ( - createAction(PluginsActionTypes.CHECKED_ALL_PLUGINS, { - list, - }) + checkPlugins: () => createAction(PluginsActionTypes.GET_PLUGINS), + checkPluginsSuccess: (list: PluginsList) => createAction( + PluginsActionTypes.GET_PLUGINS_SUCCESS, { list }, + ), + checkPluginsFailed: (error: any) => createAction( + PluginsActionTypes.GET_PLUGINS_FAILED, { error }, ), }; export type PluginActions = ActionUnion; -export function checkPluginsAsync(): ThunkAction { - return async (dispatch): Promise => { - dispatch(pluginActions.checkPlugins()); - const plugins: PluginObjects = { - ANALYTICS: false, - GIT_INTEGRATION: false, - }; - - const promises: Promise[] = [ - // check must return true/false with no exceptions - PluginChecker.check(SupportedPlugins.ANALYTICS), - PluginChecker.check(SupportedPlugins.GIT_INTEGRATION), - ]; - - const values = await Promise.all(promises); - [plugins.ANALYTICS, plugins.GIT_INTEGRATION] = values; - dispatch(pluginActions.checkedAllPlugins(plugins)); - }; -} +export const getPluginsAsync = (): ThunkAction => async (dispatch): Promise => { + dispatch(pluginActions.checkPlugins()); + try { + const list: PluginsList = await core.server.installedApps(); + dispatch(pluginActions.checkPluginsSuccess(list)); + } catch (error) { + dispatch(pluginActions.checkPluginsFailed(error)); + } +}; diff --git a/cvat-ui/src/components/cvat-app.tsx b/cvat-ui/src/components/cvat-app.tsx index c32717c73dfa..cd928f8b8345 100644 --- a/cvat-ui/src/components/cvat-app.tsx +++ b/cvat-ui/src/components/cvat-app.tsx @@ -65,6 +65,7 @@ interface CVATAppProps { authActionsInitialized: boolean; notifications: NotificationsState; user: any; + isModelPluginActive: boolean; } class CVATApplication extends React.PureComponent { @@ -115,6 +116,7 @@ class CVATApplication extends React.PureComponent - + { isModelPluginActive + && } diff --git a/cvat-ui/src/components/header/header.tsx b/cvat-ui/src/components/header/header.tsx index 379e7c70daa3..ed7fbc4f6132 100644 --- a/cvat-ui/src/components/header/header.tsx +++ b/cvat-ui/src/components/header/header.tsx @@ -22,7 +22,7 @@ import { CVATLogo, AccountIcon } from 'icons'; import ChangePasswordDialog from 'components/change-password-modal/change-password-modal'; import { switchSettingsDialog as switchSettingsDialogAction } from 'actions/settings-actions'; import { logoutAsync, authActions } from 'actions/auth-actions'; -import { SupportedPlugins, CombinedState } from 'reducers/interfaces'; +import { CombinedState } from 'reducers/interfaces'; import SettingsModal from './settings-modal/settings-modal'; const core = getCore(); @@ -53,8 +53,10 @@ interface StateToProps { changePasswordDialogShown: boolean; changePasswordFetching: boolean; logoutFetching: boolean; - installedAnalytics: boolean; renderChangePasswordItem: boolean; + isAnalyticsPluginActive: boolean; + isModelsPluginActive: boolean; + isGitPluginActive: boolean; } interface DispatchToProps { @@ -111,8 +113,10 @@ function mapStateToProps(state: CombinedState): StateToProps { changePasswordDialogShown, changePasswordFetching, logoutFetching, - installedAnalytics: list[SupportedPlugins.ANALYTICS], renderChangePasswordItem, + isAnalyticsPluginActive: list.ANALYTICS, + isModelsPluginActive: list.MODELS, + isGitPluginActive: list.GIT_INTEGRATION, }; } @@ -132,7 +136,6 @@ function HeaderContainer(props: Props): JSX.Element { const { user, tool, - installedAnalytics, logoutFetching, changePasswordFetching, settingsDialogShown, @@ -141,6 +144,8 @@ function HeaderContainer(props: Props): JSX.Element { switchSettingsDialog, switchChangePasswordDialog, renderChangePasswordItem, + isAnalyticsPluginActive, + isModelsPluginActive, } = props; const { @@ -276,38 +281,40 @@ function HeaderContainer(props: Props): JSX.Element { > Tasks - - { installedAnalytics - && ( - + )} + {isAnalyticsPluginActive && ( + - )} + } + > + Analytics + + )}