From ae549a07c97e28f5c7d6aa6b6574f783834ae29a Mon Sep 17 00:00:00 2001 From: "Irina V. Kuzmina" Date: Mon, 9 Sep 2024 17:17:19 +0300 Subject: [PATCH 1/3] QL: Replace the use of convertConnectionType with another function in registry --- src/server/index.ts | 4 +- .../plugins/ql/module/js/build-graph.ts | 6 ++- .../charts/plugins/ql/module/js/index.ts | 3 +- .../plugins/ql/module/url/build-sources.ts | 4 +- .../charts/plugins/ql/module/url/index.ts | 3 +- .../charts/plugins/ql/module/url/types.ts | 2 + .../charts/plugins/ql/ql-chart-runner.ts | 10 ++++- src/server/modes/charts/plugins/ql/types.ts | 5 +++ .../ql/utils/__tests__/misc-helpers.test.ts | 3 ++ .../charts/plugins/ql/utils/connection.ts | 13 ++++++- .../charts/plugins/ql/utils/misc-helpers.ts | 39 ++++++++++++------- src/server/modes/charts/plugins/ql/worker.ts | 13 +++++++ src/server/registry/index.ts | 19 ++++----- 13 files changed, 91 insertions(+), 33 deletions(-) create mode 100644 src/server/modes/charts/plugins/ql/types.ts diff --git a/src/server/index.ts b/src/server/index.ts index f64b256565..1e4d9dd8a7 100644 --- a/src/server/index.ts +++ b/src/server/index.ts @@ -8,14 +8,14 @@ import {getAppEndpointsConfig} from '../shared/endpoints'; import {appEnv} from './app-env'; import {getOpensourceLayoutConfig} from './components/layout/opensource-layout-config'; import authZitadel from './middlewares/auth-zitadel'; -import {convertConnectionType} from './modes/charts/plugins/ql/utils/connection'; +import {getConnectorToQlConnectionTypeMap} from './modes/charts/plugins/ql/utils/connection'; import initOpensourceApp from './modes/opensource/app'; import {nodekit} from './nodekit'; import {registry} from './registry'; import {registerAppPlugins} from './registry/utils/register-app-plugins'; registry.registerGetLayoutConfig(getOpensourceLayoutConfig); -registry.registerConvertConnectorTypeToQLConnectionType(convertConnectionType); +registry.setupQLConnectionTypeMap(getConnectorToQlConnectionTypeMap()); registerAppPlugins(); diff --git a/src/server/modes/charts/plugins/ql/module/js/build-graph.ts b/src/server/modes/charts/plugins/ql/module/js/build-graph.ts index b153c3a725..819fac7b98 100644 --- a/src/server/modes/charts/plugins/ql/module/js/build-graph.ts +++ b/src/server/modes/charts/plugins/ql/module/js/build-graph.ts @@ -21,6 +21,7 @@ import {mapQlConfigToLatestVersion} from '../../../../../../../shared/modules/co import prepareSingleResult from '../../../datalens/js/helpers/misc/prepare-single-result'; import {extractColorPalettesFromData} from '../../../helpers/color-palettes'; import {getFieldList} from '../../../helpers/misc'; +import type {QLConnectionTypeMap} from '../../utils/connection'; import { doesQueryContainOrderBy, getColumnsAndRows, @@ -46,10 +47,12 @@ type BuildGraphArgs = { ChartEditor: IChartEditor; features: FeatureConfig; palettes: Record; + qlConnectionTypeMap: QLConnectionTypeMap; }; // eslint-disable-next-line complexity -export function buildGraph({shared, ChartEditor, features, palettes}: BuildGraphArgs) { +export function buildGraph(args: BuildGraphArgs) { + const {shared, ChartEditor, features, palettes, qlConnectionTypeMap} = args; const data = ChartEditor.getLoadedData(); log('LOADED DATA:', data); @@ -67,6 +70,7 @@ export function buildGraph({shared, ChartEditor, features, palettes}: BuildGraph queries: config.queries, connectionType: config.connection.type, data: loadedData, + qlConnectionTypeMap, }); if ( diff --git a/src/server/modes/charts/plugins/ql/module/js/index.ts b/src/server/modes/charts/plugins/ql/module/js/index.ts index e660cbddac..c1b987c762 100644 --- a/src/server/modes/charts/plugins/ql/module/js/index.ts +++ b/src/server/modes/charts/plugins/ql/module/js/index.ts @@ -9,6 +9,7 @@ export default ({shared, ChartEditor}: {shared: QlConfig; ChartEditor: IChartEdi const features = getServerFeatures(app.nodekit.ctx); const {getAvailablePalettesMap} = registry.common.functions.getAll(); const palettes = getAvailablePalettesMap(); + const qlConnectionTypeMap = registry.getQLConnectionTypeMap(); - return buildGraph({shared, ChartEditor, features, palettes}); + return buildGraph({shared, ChartEditor, features, palettes, qlConnectionTypeMap}); }; diff --git a/src/server/modes/charts/plugins/ql/module/url/build-sources.ts b/src/server/modes/charts/plugins/ql/module/url/build-sources.ts index 87343cf0a8..8a1f928747 100644 --- a/src/server/modes/charts/plugins/ql/module/url/build-sources.ts +++ b/src/server/modes/charts/plugins/ql/module/url/build-sources.ts @@ -39,7 +39,7 @@ const prepareDefaultDate = (date: string) => { }; export function buildSources(args: BuildSourcesArgs) { - const {shared, ChartEditor, palettes} = args; + const {shared, ChartEditor, palettes, qlConnectionTypeMap} = args; const config = mapQlConfigToLatestVersion(shared, {i18n: ChartEditor.getTranslation}); const urlParams = ChartEditor.getParams(); @@ -190,6 +190,7 @@ export function buildSources(args: BuildSourcesArgs) { params: localParams, paramsDescription: localParamsDescription, + qlConnectionTypeMap, }); sources[`ql_${i}`] = source; @@ -208,6 +209,7 @@ export function buildSources(args: BuildSourcesArgs) { params, paramsDescription: config.params, + qlConnectionTypeMap, }), }; } diff --git a/src/server/modes/charts/plugins/ql/module/url/index.ts b/src/server/modes/charts/plugins/ql/module/url/index.ts index 77d4ab1c12..a61428efcb 100644 --- a/src/server/modes/charts/plugins/ql/module/url/index.ts +++ b/src/server/modes/charts/plugins/ql/module/url/index.ts @@ -6,6 +6,7 @@ import {buildSources} from './build-sources'; export default function ({shared, ChartEditor}: {shared: QlConfig; ChartEditor: IChartEditor}) { const {getAvailablePalettesMap} = registry.common.functions.getAll(); const palettes = getAvailablePalettesMap(); + const qlConnectionTypeMap = registry.getQLConnectionTypeMap(); - return buildSources({shared, ChartEditor, palettes}); + return buildSources({shared, ChartEditor, palettes, qlConnectionTypeMap}); } diff --git a/src/server/modes/charts/plugins/ql/module/url/types.ts b/src/server/modes/charts/plugins/ql/module/url/types.ts index c7f6c636b5..0de535bf35 100644 --- a/src/server/modes/charts/plugins/ql/module/url/types.ts +++ b/src/server/modes/charts/plugins/ql/module/url/types.ts @@ -1,7 +1,9 @@ import type {IChartEditor, Palette, QlConfig} from '../../../../../../../shared'; +import type {QLConnectionTypeMap} from '../../utils/connection'; export type BuildSourcesArgs = { shared: QlConfig; ChartEditor: IChartEditor; palettes: Record; + qlConnectionTypeMap: QLConnectionTypeMap; }; diff --git a/src/server/modes/charts/plugins/ql/ql-chart-runner.ts b/src/server/modes/charts/plugins/ql/ql-chart-runner.ts index bf9c7388f9..f8c6354b56 100644 --- a/src/server/modes/charts/plugins/ql/ql-chart-runner.ts +++ b/src/server/modes/charts/plugins/ql/ql-chart-runner.ts @@ -11,12 +11,20 @@ import type {WizardWorker} from '../../../../components/charts-engine/components import type {RunnerHandler, RunnerHandlerProps} from '../../../../components/charts-engine/runners'; import {runChart} from '../../../../components/charts-engine/runners/chart'; import {runWorkerChart} from '../../../../components/charts-engine/runners/worker'; +import {registry} from '../../../../registry'; + +import type {QLAdditionalData} from './types'; let wizardWorkersPool: Pool | null = null; async function getQlWorker(): Promise> { if (wizardWorkersPool === null) { const scriptPath = path.resolve(__dirname, './worker'); - wizardWorkersPool = workerpool.pool(scriptPath); + const additionalData: QLAdditionalData = { + qlConnectionTypeMap: registry.getQLConnectionTypeMap(), + }; + wizardWorkersPool = workerpool.pool(scriptPath, { + workerThreadOpts: {workerData: additionalData}, + }); } return wizardWorkersPool.proxy(); diff --git a/src/server/modes/charts/plugins/ql/types.ts b/src/server/modes/charts/plugins/ql/types.ts new file mode 100644 index 0000000000..f19c3ce3b9 --- /dev/null +++ b/src/server/modes/charts/plugins/ql/types.ts @@ -0,0 +1,5 @@ +import type {QLConnectionTypeMap} from './utils/connection'; + +export type QLAdditionalData = { + qlConnectionTypeMap: QLConnectionTypeMap; +}; diff --git a/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts b/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts index a57053946d..33fd794174 100644 --- a/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts +++ b/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts @@ -12,6 +12,7 @@ const mockedBuildSourceArgsSet = { paramsDescription: [ {type: 'string', name: 'years', defaultValue: ['1990', '1995'], overridenValue: '1995'}, ], + qlConnectionTypeMap: {}, }; const expectedBuildSourceResultSet = { @@ -32,6 +33,7 @@ const mockedBuildSourceArgsSingle = { paramsDescription: [ {type: 'string', name: 'years', defaultValue: ['1990', '1995'], overridenValue: '1995'}, ], + qlConnectionTypeMap: {}, }; const expectedBuildSourceResultSingle = { @@ -52,6 +54,7 @@ const mockedBuildSourceArgsPrewrapped = { paramsDescription: [ {type: 'string', name: 'years', defaultValue: ['1990', '1995'], overridenValue: '1995'}, ], + qlConnectionTypeMap: {}, }; const expectedBuildSourceResultPrewrapped = { diff --git a/src/server/modes/charts/plugins/ql/utils/connection.ts b/src/server/modes/charts/plugins/ql/utils/connection.ts index 52ea6f4e1f..22836adf29 100644 --- a/src/server/modes/charts/plugins/ql/utils/connection.ts +++ b/src/server/modes/charts/plugins/ql/utils/connection.ts @@ -19,8 +19,17 @@ export const CONNECTOR_TYPE_TO_QL_CONNECTION_TYPE_MAP: Record; + +export function convertConnectionType( + connectionTypeMap: QLConnectionTypeMap, + connectionType: string, +) { + const mappedConnectionType = connectionTypeMap[connectionType]; if (!mappedConnectionType) { throw new Error('Unsupported connection type'); diff --git a/src/server/modes/charts/plugins/ql/utils/misc-helpers.ts b/src/server/modes/charts/plugins/ql/utils/misc-helpers.ts index a0b77dbb52..06b3e3ec48 100644 --- a/src/server/modes/charts/plugins/ql/utils/misc-helpers.ts +++ b/src/server/modes/charts/plugins/ql/utils/misc-helpers.ts @@ -28,8 +28,9 @@ import type { QlConfigResultEntryMetadataDataColumnOrGroup, QlConfigResultEntryMetadataDataGroup, } from '../../../../../../shared/types/config/ql'; -import {registry} from '../../../../../registry'; +import type {QLConnectionTypeMap} from './connection'; +import {convertConnectionType} from './connection'; import {LINEAR_VISUALIZATIONS, LOG_INFO, LOG_TIMING, QUERY_TITLE} from './constants'; export { @@ -389,17 +390,18 @@ export function buildSource({ query, params, paramsDescription, + qlConnectionTypeMap, }: { id: string; connectionType: string; query: string; params: StringParams; paramsDescription: QlConfigParam[]; + qlConnectionTypeMap: QLConnectionTypeMap; }) { - const convertConnectionType = registry.getConvertConnectorTypeToQLConnectionType(); let sqlQuery = query; - const datalensQLConnectionType = convertConnectionType(connectionType); + const datalensQLConnectionType = convertConnectionType(qlConnectionTypeMap, connectionType); const QLRequestParams: Record = {}; if ( @@ -506,16 +508,16 @@ export function getRows(data: QLResult, field = 'sql'): string[][] { return rows; } -export function getColumns( - data: QLResult, - connectionType: string, - field = 'sql', -): QlConfigResultEntryMetadataDataColumn[] | null { +export function getColumns(args: { + data: QLResult; + connectionType: string; + field?: string; + qlConnectionTypeMap: QLConnectionTypeMap; +}): QlConfigResultEntryMetadataDataColumn[] | null { + const {data, connectionType, field = 'sql', qlConnectionTypeMap} = args; const row = data[field].find((entry: QLResultEntry) => entry.event === 'metadata'); - const convertConnectionType = registry.getConvertConnectorTypeToQLConnectionType(); - - const datalensQLConnectionType = convertConnectionType(connectionType); + const datalensQLConnectionType = convertConnectionType(qlConnectionTypeMap, connectionType); if (row) { const metadataRow = row as QLResultEntryMetadata; @@ -553,12 +555,14 @@ export function getColumnsAndRows({ queries, connectionType, data, + qlConnectionTypeMap, }: { chartType: QLChartType; ChartEditor: IChartEditor; queries: QLConfigQuery[]; connectionType: string; data: {[key: string]: any}; + qlConnectionTypeMap: QLConnectionTypeMap; }) { let columns: QlConfigResultEntryMetadataDataColumn[] | null = []; @@ -572,7 +576,12 @@ export function getColumnsAndRows({ let localColumns: QlConfigResultEntryMetadataDataColumn[] = []; try { - const parsedColumns = getColumns(data, connectionType, `ql_${i}`); + const parsedColumns = getColumns({ + data, + connectionType, + field: `ql_${i}`, + qlConnectionTypeMap, + }); if (parsedColumns === null) { return; @@ -661,7 +670,11 @@ export function getColumnsAndRows({ } } else { try { - columns = getColumns(data, connectionType || ConnectorType.Clickhouse); + columns = getColumns({ + data, + connectionType: connectionType || ConnectorType.Clickhouse, + qlConnectionTypeMap, + }); if (columns !== null) { rows = getRows(data); diff --git a/src/server/modes/charts/plugins/ql/worker.ts b/src/server/modes/charts/plugins/ql/worker.ts index 6219f6b72a..3cf1f43ce0 100644 --- a/src/server/modes/charts/plugins/ql/worker.ts +++ b/src/server/modes/charts/plugins/ql/worker.ts @@ -1,3 +1,5 @@ +import {workerData} from 'worker_threads'; + import workerPool from 'workerpool'; import type {QlConfig, QlExtendedConfig} from '../../../../../shared'; @@ -17,8 +19,15 @@ import {getChartApiContext} from '../../../../components/charts-engine/component import {createI18nInstance} from '../../../../utils/language'; import qlModule from './module/private-module'; +import type {QLAdditionalData} from './types'; import {identifyParams} from './utils/identify-params'; +function getQLAdditionalData(): QLAdditionalData { + return { + qlConnectionTypeMap: workerData?.qlConnectionTypeMap ?? {}, + }; +} + const worker: WizardWorker = { buildParams: async (args: BuildParamsArgs) => { const {shared, userLang} = args; @@ -48,11 +57,13 @@ const worker: WizardWorker = { const console = new Console({}); qlModule.setConsole(console); + const {qlConnectionTypeMap} = getQLAdditionalData(); return { exports: qlModule.buildSources({ shared: shared as QlConfig, ChartEditor: context.ChartEditor, palettes, + qlConnectionTypeMap, }), runtimeMetadata: context.__runtimeMetadata, logs: console.getLogs(), @@ -144,11 +155,13 @@ const worker: WizardWorker = { const console = new Console({}); qlModule.setConsole(console); + const {qlConnectionTypeMap} = getQLAdditionalData(); const result = qlModule.buildGraph({ shared: shared as QlConfig, ChartEditor: context.ChartEditor, palettes, features, + qlConnectionTypeMap, }); return { diff --git a/src/server/registry/index.ts b/src/server/registry/index.ts index 98fef68ea8..b911bdda20 100644 --- a/src/server/registry/index.ts +++ b/src/server/registry/index.ts @@ -5,9 +5,9 @@ import getGatewayControllers from '@gravity-ui/gateway'; import type {AppContext} from '@gravity-ui/nodekit'; import type {ChartsEngine} from '../components/charts-engine'; -import {convertConnectionType} from '../modes/charts/plugins/ql/utils/connection'; +import type {QLConnectionTypeMap} from '../modes/charts/plugins/ql/utils/connection'; +import {getConnectorToQlConnectionTypeMap} from '../modes/charts/plugins/ql/utils/connection'; import type {GetLayoutConfig} from '../types/app-layout'; -import type {ConvertConnectorTypeToQLConnectionType} from '../types/connections'; import type {XlsxConverterFn} from '../types/xlsxConverter'; import commonRegistry from './units/common'; @@ -24,7 +24,7 @@ let gateway: ReturnType; let getLayoutConfig: GetLayoutConfig | undefined; let yfmPlugins: MarkdownItPluginCb[]; let getXlsxConverter: XlsxConverterFn | undefined; -let convertConnectorTypeToQLConnectionType: ConvertConnectorTypeToQLConnectionType | undefined; +let qLConnectionTypeMap: QLConnectionTypeMap | undefined; export const registry = { common: commonRegistry, @@ -109,15 +109,12 @@ export const registry = { getXlsxConverter() { return getXlsxConverter; }, - registerConvertConnectorTypeToQLConnectionType(fn: ConvertConnectorTypeToQLConnectionType) { - if (!convertConnectorTypeToQLConnectionType) { - convertConnectorTypeToQLConnectionType = fn; + setupQLConnectionTypeMap(map: QLConnectionTypeMap) { + if (!qLConnectionTypeMap) { + qLConnectionTypeMap = map; } }, - getConvertConnectorTypeToQLConnectionType() { - if (!convertConnectorTypeToQLConnectionType) { - return convertConnectionType; - } - return convertConnectorTypeToQLConnectionType; + getQLConnectionTypeMap() { + return qLConnectionTypeMap ?? getConnectorToQlConnectionTypeMap(); }, }; From 7474beb3dc77a8f370e91599ff4e44113900e361 Mon Sep 17 00:00:00 2001 From: "Irina V. Kuzmina" Date: Mon, 9 Sep 2024 17:37:28 +0300 Subject: [PATCH 2/3] fix test --- .../ql/utils/__tests__/misc-helpers.test.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts b/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts index 33fd794174..4b0ba6bdc9 100644 --- a/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts +++ b/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts @@ -4,15 +4,19 @@ import {buildSource, doesQueryContainOrderBy, iterateThroughVisibleQueries} from const MOCK_ID = 'MOCK_ID'; -const mockedBuildSourceArgsSet = { +const commonBuildSourceArgsSet = { id: MOCK_ID, connectionType: 'postgres', + qlConnectionTypeMap: {postgres: 'postgres'}, +}; + +const mockedBuildSourceArgsSet = { + ...commonBuildSourceArgsSet, query: 'select built_year, iznos from public.sample where built_year in {{years}} limit 10', params: {years: '1995'}, paramsDescription: [ {type: 'string', name: 'years', defaultValue: ['1990', '1995'], overridenValue: '1995'}, ], - qlConnectionTypeMap: {}, }; const expectedBuildSourceResultSet = { @@ -26,14 +30,12 @@ const expectedBuildSourceResultSet = { }; const mockedBuildSourceArgsSingle = { - id: MOCK_ID, - connectionType: 'postgres', + ...commonBuildSourceArgsSet, query: 'select built_year, iznos from public.sample where built_year = {{years}} limit 10', params: {years: '1995'}, paramsDescription: [ {type: 'string', name: 'years', defaultValue: ['1990', '1995'], overridenValue: '1995'}, ], - qlConnectionTypeMap: {}, }; const expectedBuildSourceResultSingle = { @@ -47,14 +49,12 @@ const expectedBuildSourceResultSingle = { }; const mockedBuildSourceArgsPrewrapped = { - id: MOCK_ID, - connectionType: 'postgres', + ...commonBuildSourceArgsSet, query: 'select built_year, iznos from public.sample where built_year in ({{years}}) limit 10', params: {years: '1995'}, paramsDescription: [ {type: 'string', name: 'years', defaultValue: ['1990', '1995'], overridenValue: '1995'}, ], - qlConnectionTypeMap: {}, }; const expectedBuildSourceResultPrewrapped = { From 7d5478776792db68481d40c791e1cb7c205f438d Mon Sep 17 00:00:00 2001 From: "Irina V. Kuzmina" Date: Mon, 9 Sep 2024 17:43:47 +0300 Subject: [PATCH 3/3] fix --- .../charts/plugins/ql/utils/__tests__/misc-helpers.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts b/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts index 4b0ba6bdc9..b9462ca833 100644 --- a/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts +++ b/src/server/modes/charts/plugins/ql/utils/__tests__/misc-helpers.test.ts @@ -1,4 +1,5 @@ import type {QLConfigQuery} from '../../../../../../../shared'; +import {ConnectorType} from '../../../../../../../shared'; import {convertConnectionType} from '../connection'; import {buildSource, doesQueryContainOrderBy, iterateThroughVisibleQueries} from '../misc-helpers'; @@ -7,7 +8,7 @@ const MOCK_ID = 'MOCK_ID'; const commonBuildSourceArgsSet = { id: MOCK_ID, connectionType: 'postgres', - qlConnectionTypeMap: {postgres: 'postgres'}, + qlConnectionTypeMap: {postgres: ConnectorType.Postgres}, }; const mockedBuildSourceArgsSet = {