diff --git a/.changeset/clean-toes-learn.md b/.changeset/clean-toes-learn.md new file mode 100644 index 00000000000..d78bbd6f642 --- /dev/null +++ b/.changeset/clean-toes-learn.md @@ -0,0 +1,3 @@ +--- +'@finos/legend-graph': patch +--- diff --git a/.changeset/kind-rabbits-draw.md b/.changeset/kind-rabbits-draw.md new file mode 100644 index 00000000000..457d23317cc --- /dev/null +++ b/.changeset/kind-rabbits-draw.md @@ -0,0 +1,6 @@ +--- +'@finos/legend-extension-dsl-data-space': minor +'@finos/legend-application-query': minor +--- + +Use PMCD returned by mapping analysis to build minimal graph for query diff --git a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts index 1e90584b975..c54479278cd 100644 --- a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts +++ b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts @@ -15,7 +15,6 @@ */ import { LegendApplicationPlugin } from '@finos/legend-application'; -import type { Query } from '@finos/legend-graph'; import type { QueryBuilderState } from '@finos/legend-query-builder'; import type React from 'react'; import type { LegendQueryPluginManager } from '../application/LegendQueryPluginManager.js'; @@ -52,10 +51,13 @@ export type QuerySetupActionConfiguration = { }; export type ExistingQueryEditorStateBuilder = ( - query: Query, editorStore: ExistingQueryEditorStore, ) => Promise; +export type ExistingQueryGraphBuilderChecker = ( + editorStore: ExistingQueryEditorStore, +) => boolean | undefined; + export type QueryEditorActionConfiguration = { key: string; renderer: ( @@ -98,6 +100,11 @@ export abstract class LegendQueryApplicationPlugin extends LegendApplicationPlug */ getExtraExistingQueryEditorStateBuilders?(): ExistingQueryEditorStateBuilder[]; + /** + * Get the list of existing query graph builder checkers + */ + getExtraExistingQueryGraphBuilderCheckers?(): ExistingQueryGraphBuilderChecker[]; + /** * Get the list of query editor action renderer configurations. */ diff --git a/packages/legend-application-query/src/stores/QueryEditorStore.ts b/packages/legend-application-query/src/stores/QueryEditorStore.ts index 1e0416e09e1..c055e4bc70e 100644 --- a/packages/legend-application-query/src/stores/QueryEditorStore.ts +++ b/packages/legend-application-query/src/stores/QueryEditorStore.ts @@ -381,6 +381,10 @@ export abstract class QueryEditorStore { // do nothing } + requiresGraphBuilding(): boolean { + return true; + } + async buildQueryForPersistence( query: Query, rawLambda: RawLambda, @@ -485,7 +489,9 @@ export abstract class QueryEditorStore { ); yield this.setUpEditorState(); - yield flowResult(this.buildGraph()); + if (this.requiresGraphBuilding()) { + yield flowResult(this.buildGraph()); + } this.queryBuilderState = (yield this.initializeQueryBuilderState( stopWatch, )) as QueryBuilderState; @@ -991,9 +997,24 @@ export class ExistingQueryEditorStore extends QueryEditorStore { ); } - async initializeQueryBuilderState( - stopWatch: StopWatch, - ): Promise { + override requiresGraphBuilding(): boolean { + const existingQueryGraphBuilderCheckers = + this.applicationStore.pluginManager + .getApplicationPlugins() + .flatMap( + (plugin) => + plugin.getExtraExistingQueryGraphBuilderCheckers?.() ?? [], + ); + for (const checker of existingQueryGraphBuilderCheckers) { + const isGraphBuildingRequired = checker(this); + if (isGraphBuildingRequired !== undefined) { + return isGraphBuildingRequired; + } + } + return true; + } + + async setupQuery(): Promise { const query = await this.graphManagerState.graphManager.getQuery( this.queryId, this.graphManagerState.graph, @@ -1003,6 +1024,11 @@ export class ExistingQueryEditorStore extends QueryEditorStore { this.applicationStore.userDataService, query.id, ); + } + + async initializeQueryBuilderState( + stopWatch: StopWatch, + ): Promise { let queryBuilderState: QueryBuilderState | undefined; const existingQueryEditorStateBuilders = this.applicationStore.pluginManager .getApplicationPlugins() @@ -1010,12 +1036,15 @@ export class ExistingQueryEditorStore extends QueryEditorStore { (plugin) => plugin.getExtraExistingQueryEditorStateBuilders?.() ?? [], ); for (const builder of existingQueryEditorStateBuilders) { - queryBuilderState = await builder(query, this); + queryBuilderState = await builder(this); if (queryBuilderState) { break; } } + await this.setupQuery(); + const query = guaranteeNonNullable(this.query); + // if no extension found, fall back to basic `class -> mapping -> runtime` mode queryBuilderState = queryBuilderState ?? diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx index c969f796eec..6e142a080fa 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx @@ -17,7 +17,6 @@ import packageJson from '../../../package.json' assert { type: 'json' }; import { type QuerySetupActionConfiguration, - type ExistingQueryEditorStateBuilder, type ExistingQueryEditorStore, LegendQueryApplicationPlugin, generateExistingQueryEditorRoute, @@ -25,6 +24,8 @@ import { LegendQueryEventHelper, createViewProjectHandler, createViewSDLCProjectHandler, + type ExistingQueryEditorStateBuilder, + type ExistingQueryGraphBuilderChecker, } from '@finos/legend-application-query'; import { SquareIcon } from '@finos/legend-art'; import { @@ -38,22 +39,38 @@ import { } from '../../__lib__/query/DSL_DataSpace_LegendQueryNavigation.js'; import { DataSpaceQueryCreator } from './DataSpaceQueryCreator.js'; import { createQueryDataSpaceTaggedValue } from '../../stores/query/DataSpaceQueryCreatorStore.js'; -import { Query, isValidFullPath } from '@finos/legend-graph'; +import { + Query, + isValidFullPath, + GRAPH_MANAGER_EVENT, + type V1_PureModelContextData, + createGraphBuilderReport, +} from '@finos/legend-graph'; import { QUERY_PROFILE_PATH, QUERY_PROFILE_TAG_DATA_SPACE, } from '../../graph/DSL_DataSpace_MetaModelConst.js'; import { - DataSpaceQueryBuilderState, DataSpaceProjectInfo, + DataSpaceQueryBuilderState, } from '../../stores/query/DataSpaceQueryBuilderState.js'; import type { DataSpaceInfo } from '../../stores/query/DataSpaceInfo.js'; import { getOwnDataSpace } from '../../graph-manager/DSL_DataSpace_GraphManagerHelper.js'; -import { assertErrorThrown, LogEvent, uuid } from '@finos/legend-shared'; +import { + assertErrorThrown, + guaranteeNonNullable, + isString, + LogEvent, + StopWatch, + uuid, +} from '@finos/legend-shared'; import type { QueryBuilderState } from '@finos/legend-query-builder'; import { DataSpaceQuerySetup } from './DataSpaceQuerySetup.js'; import { DSL_DataSpace_getGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.js'; -import { StoreProjectData } from '@finos/legend-server-depot'; +import { + retrieveProjectEntitiesWithDependencies, + StoreProjectData, +} from '@finos/legend-server-depot'; import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js'; export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryApplicationPlugin { @@ -101,185 +118,278 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli override getExtraExistingQueryEditorStateBuilders(): ExistingQueryEditorStateBuilder[] { return [ async ( - query: Query, editorStore: ExistingQueryEditorStore, ): Promise => { - const dataSpaceTaggedValue = query.taggedValues?.find( + const dataSpaceTaggedValue = editorStore.lightQuery?.taggedValues?.find( (taggedValue) => taggedValue.profile === QUERY_PROFILE_PATH && taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && isValidFullPath(taggedValue.value), ); - + let isLightGraphEnabled = true; if (dataSpaceTaggedValue) { const dataSpacePath = dataSpaceTaggedValue.value; - const dataSpace = getOwnDataSpace( - dataSpacePath, - editorStore.graphManagerState.graph, - ); - const mapping = query.mapping.value; - const matchingExecutionContext = dataSpace.executionContexts.find( - (ec) => ec.mapping.value === mapping, - ); - if (!matchingExecutionContext) { - // if a matching execution context is not found, it means this query is not - // properly created from a data space, therefore, we cannot support this case - return undefined; - } + const mappingPath = editorStore.lightQuery?.mapping; let dataSpaceAnalysisResult; - try { - const project = StoreProjectData.serialization.fromJson( - await editorStore.depotServerClient.getProject( - query.groupId, - query.artifactId, - ), - ); - dataSpaceAnalysisResult = - await DSL_DataSpace_getGraphManagerExtension( - editorStore.graphManagerState.graphManager, - ).retrieveDataSpaceAnalysisFromCache(() => - retrieveAnalyticsResultCache( - project, - query.versionId, - dataSpace.path, - editorStore.depotServerClient, + let dataSpaceAnalysisResultMetaModel; + const { groupId, artifactId, versionId } = + editorStore.getProjectInfo(); + if (dataSpacePath && isString(mappingPath)) { + try { + editorStore.initState.setMessage( + 'Fetching dataspace analysis result', + ); + const project = StoreProjectData.serialization.fromJson( + await editorStore.depotServerClient.getProject( + groupId, + artifactId, ), ); - } catch { - // do nothing - } - const projectInfo = new DataSpaceProjectInfo( - query.groupId, - query.artifactId, - query.versionId, - createViewProjectHandler(editorStore.applicationStore), - createViewSDLCProjectHandler( + dataSpaceAnalysisResult = + await DSL_DataSpace_getGraphManagerExtension( + editorStore.graphManagerState.graphManager, + ).analyzeDataSpaceCoverage( + dataSpacePath, + () => + retrieveProjectEntitiesWithDependencies( + project, + versionId, + editorStore.depotServerClient, + ), + () => + retrieveAnalyticsResultCache( + project, + versionId, + dataSpacePath, + editorStore.depotServerClient, + ), + ); + const stopWatch = new StopWatch(); + // initialize system + stopWatch.record(); + await editorStore.graphManagerState.initializeSystem(); + stopWatch.record( + GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS, + ); + + // build graph + let pmcd: V1_PureModelContextData | undefined = undefined; + if (dataSpaceAnalysisResult) { + const mappingModelCoverageAnalysisResult = + dataSpaceAnalysisResult?.executionContexts.find( + (value) => value.mapping === mappingPath, + )?.mappingModelCoverageAnalysisResult; + pmcd = mappingModelCoverageAnalysisResult?.model; + } + const graph_buildReport = createGraphBuilderReport(); + dataSpaceAnalysisResultMetaModel = + await DSL_DataSpace_getGraphManagerExtension( + editorStore.graphManagerState.graphManager, + ).buildDataSpaceAnalytics( + dataSpaceAnalysisResult, + editorStore.graphManagerState.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + graph_buildReport, + editorStore.graphManagerState.graph, + pmcd, + editorStore.getProjectInfo(), + ); + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + editorStore.applicationStore.timeService.finalizeTimingsRecord( + stopWatch, + ), + dependencies: dependency_buildReport, + dependenciesCount: + editorStore.graphManagerState.graph.dependencyManager + .numberOfDependencies, + graph: graph_buildReport, + }; + editorStore.logBuildGraphMetrics(graphBuilderReportData); + + editorStore.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } catch (error) { + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + isLightGraphEnabled = false; + editorStore.graphManagerState.graph = + editorStore.graphManagerState.createNewGraph(); + await editorStore.buildGraph(); + } + + const dataSpace = getOwnDataSpace( + dataSpacePath, + editorStore.graphManagerState.graph, + ); + const mapping = + editorStore.graphManagerState.graph.getMapping(mappingPath); + const matchingExecutionContext = dataSpace.executionContexts.find( + (ec) => ec.mapping.value === mapping, + ); + if (!matchingExecutionContext) { + // if a matching execution context is not found, it means this query is not + // properly created from a data space, therefore, we cannot support this case + return undefined; + } + await editorStore.setupQuery(); + const query = guaranteeNonNullable(editorStore.query); + const projectInfo = new DataSpaceProjectInfo( + query.groupId, + query.artifactId, + query.versionId, + createViewProjectHandler(editorStore.applicationStore), + createViewSDLCProjectHandler( + editorStore.applicationStore, + editorStore.depotServerClient, + ), + ); + const dataSpaceQueryBuilderState = new DataSpaceQueryBuilderState( editorStore.applicationStore, + editorStore.graphManagerState, editorStore.depotServerClient, - ), - ); - const dataSpaceQueryBuilderState = new DataSpaceQueryBuilderState( - editorStore.applicationStore, - editorStore.graphManagerState, - editorStore.depotServerClient, - dataSpace, - matchingExecutionContext, - (dataSpaceInfo: DataSpaceInfo) => { - if (dataSpaceInfo.defaultExecutionContext) { - const createQuery = async (): Promise => { - // prepare the new query to save - const _query = new Query(); - _query.name = query.name; - _query.id = query.id; - _query.groupId = query.groupId; - _query.artifactId = query.artifactId; - _query.versionId = query.versionId; - _query.mapping = query.mapping; - _query.runtime = query.runtime; - _query.taggedValues = [ - createQueryDataSpaceTaggedValue(dataSpaceInfo.path), - ].concat( - (query.taggedValues ?? []).filter( - (taggedValue) => taggedValue !== dataSpaceTaggedValue, - ), - ); - _query.stereotypes = query.stereotypes; - _query.content = query.content; - _query.owner = query.owner; - _query.lastUpdatedAt = query.lastUpdatedAt; + dataSpace, + matchingExecutionContext, + isLightGraphEnabled, + (dataSpaceInfo: DataSpaceInfo) => { + if (dataSpaceInfo.defaultExecutionContext) { + const createQuery = async (): Promise => { + // prepare the new query to save + const _query = new Query(); + _query.name = query.name; + _query.id = query.id; + _query.groupId = query.groupId; + _query.artifactId = query.artifactId; + _query.versionId = query.versionId; + _query.mapping = query.mapping; + _query.runtime = query.runtime; + _query.taggedValues = [ + createQueryDataSpaceTaggedValue(dataSpaceInfo.path), + ].concat( + (query.taggedValues ?? []).filter( + (taggedValue) => taggedValue !== dataSpaceTaggedValue, + ), + ); + _query.stereotypes = query.stereotypes; + _query.content = query.content; + _query.owner = query.owner; + _query.lastUpdatedAt = query.lastUpdatedAt; - try { - if (!query.isCurrentUserQuery) { - _query.id = uuid(); - const newQuery = - await editorStore.graphManagerState.graphManager.createQuery( + try { + if (!query.isCurrentUserQuery) { + _query.id = uuid(); + const newQuery = + await editorStore.graphManagerState.graphManager.createQuery( + _query, + editorStore.graphManagerState.graph, + ); + editorStore.applicationStore.notificationService.notifySuccess( + `Successfully created query!`, + ); + LegendQueryEventHelper.notify_QueryCreateSucceeded( + editorStore.applicationStore.eventService, + { queryId: newQuery.id }, + ); + editorStore.applicationStore.navigationService.navigator.goToLocation( + generateExistingQueryEditorRoute(newQuery.id), + ); + } else { + await editorStore.graphManagerState.graphManager.updateQuery( _query, editorStore.graphManagerState.graph, ); - editorStore.applicationStore.notificationService.notifySuccess( - `Successfully created query!`, - ); - LegendQueryEventHelper.notify_QueryCreateSucceeded( - editorStore.applicationStore.eventService, - { queryId: newQuery.id }, - ); - editorStore.applicationStore.navigationService.navigator.goToLocation( - generateExistingQueryEditorRoute(newQuery.id), - ); - } else { - await editorStore.graphManagerState.graphManager.updateQuery( - _query, - editorStore.graphManagerState.graph, + editorStore.applicationStore.notificationService.notifySuccess( + `Successfully updated query!`, + ); + editorStore.applicationStore.navigationService.navigator.reload(); + } + } catch (error) { + assertErrorThrown(error); + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, ); - editorStore.applicationStore.notificationService.notifySuccess( - `Successfully updated query!`, + editorStore.applicationStore.notificationService.notifyError( + error, ); - editorStore.applicationStore.navigationService.navigator.reload(); } - } catch (error) { - assertErrorThrown(error); - editorStore.applicationStore.logService.error( - LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), - error, - ); - editorStore.applicationStore.notificationService.notifyError( - error, - ); - } - }; + }; - editorStore.applicationStore.alertService.setActionAlertInfo({ - message: `To change the data space associated with this query, you need to ${ - query.isCurrentUserQuery - ? 'update the query' - : 'create a new query' - } to proceed`, - type: ActionAlertType.CAUTION, - actions: [ - { - label: query.isCurrentUserQuery - ? 'Update query' - : 'Create new query', - type: ActionAlertActionType.PROCEED_WITH_CAUTION, - handler: () => { - createQuery().catch( - editorStore.applicationStore.alertUnhandledError, - ); + editorStore.applicationStore.alertService.setActionAlertInfo({ + message: `To change the data space associated with this query, you need to ${ + query.isCurrentUserQuery + ? 'update the query' + : 'create a new query' + } to proceed`, + type: ActionAlertType.CAUTION, + actions: [ + { + label: query.isCurrentUserQuery + ? 'Update query' + : 'Create new query', + type: ActionAlertActionType.PROCEED_WITH_CAUTION, + handler: () => { + createQuery().catch( + editorStore.applicationStore.alertUnhandledError, + ); + }, }, - }, - { - label: 'Abort', - type: ActionAlertActionType.PROCEED, - default: true, - }, - ], - }); - } else { - editorStore.applicationStore.notificationService.notifyWarning( - `Can't switch data space: default execution context not specified`, - ); - } - }, - true, - dataSpaceAnalysisResult, - undefined, - undefined, - undefined, - projectInfo, - editorStore.applicationStore.config.options.queryBuilderConfig, - ); - const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResult?.executionContextsIndex.get( - matchingExecutionContext.name, - )?.mappingModelCoverageAnalysisResult; - if (mappingModelCoverageAnalysisResult) { - dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult = - mappingModelCoverageAnalysisResult; + { + label: 'Abort', + type: ActionAlertActionType.PROCEED, + default: true, + }, + ], + }); + } else { + editorStore.applicationStore.notificationService.notifyWarning( + `Can't switch data space: default execution context not specified`, + ); + } + }, + true, + dataSpaceAnalysisResultMetaModel, + undefined, + undefined, + undefined, + projectInfo, + ); + const mappingModelCoverageAnalysisResult = + dataSpaceAnalysisResultMetaModel?.executionContextsIndex.get( + matchingExecutionContext.name, + )?.mappingModelCoverageAnalysisResult; + if (mappingModelCoverageAnalysisResult) { + dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult = + mappingModelCoverageAnalysisResult; + } + return dataSpaceQueryBuilderState; } - return dataSpaceQueryBuilderState; } return undefined; }, ]; } + + override getExtraExistingQueryGraphBuilderCheckers(): ExistingQueryGraphBuilderChecker[] { + return [ + (editorStore: ExistingQueryEditorStore): boolean | undefined => { + const dataSpaceTaggedValue = editorStore.lightQuery?.taggedValues?.find( + (taggedValue) => + taggedValue.profile === QUERY_PROFILE_PATH && + taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && + isValidFullPath(taggedValue.value), + ); + if (dataSpaceTaggedValue) { + return false; + } + return true; + }, + ]; + } } diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx index 4a1f702b2d5..04713bf8315 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx @@ -44,11 +44,16 @@ import { getMappingCompatibleRuntimes, PackageableElementExplicitReference, RuntimePointer, + type PackageableRuntime, } from '@finos/legend-graph'; import type { DataSpaceInfo } from '../../stores/query/DataSpaceInfo.js'; import { generateGAVCoordinates } from '@finos/legend-storage'; import { useEffect, useMemo, useState } from 'react'; -import { debounce, guaranteeType } from '@finos/legend-shared'; +import { + debounce, + guaranteeType, + guaranteeNonNullable, +} from '@finos/legend-shared'; import { flowResult } from 'mobx'; import { DataSpace, @@ -57,6 +62,7 @@ import { import { DataSpaceIcon } from '../DSL_DataSpace_Icon.js'; import { DataSpaceAdvancedSearchModal } from './DataSpaceAdvancedSearchModal.js'; import type { EditorStore } from '@finos/legend-application-studio'; +import { useQueryEditorStore } from '@finos/legend-application-query'; export type DataSpaceOption = { label: string; @@ -105,6 +111,25 @@ export const formatDataSpaceOptionLabel = ( ); +const resolveExecutionContextRuntimes = ( + queryBuilderState: DataSpaceQueryBuilderState, +): PackageableRuntime[] => { + if (queryBuilderState.dataSpaceAnalysisResult) { + const executionContext = Array.from( + queryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).find( + (e) => + e.mapping.path === + queryBuilderState.executionContext.mapping.value.path, + ); + return guaranteeNonNullable(executionContext).compatibleRuntimes; + } + return getMappingCompatibleRuntimes( + queryBuilderState.executionContext.mapping.value, + queryBuilderState.graphManagerState.usableRuntimes, + ); +}; + type ExecutionContextOption = { label: string; value: DataSpaceExecutionContext; @@ -129,6 +154,7 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( (props: { queryBuilderState: DataSpaceQueryBuilderState }) => { const { queryBuilderState } = props; const applicationStore = useApplicationStore(); + const editorStore = useQueryEditorStore(); const [dataSpaceSearchText, setDataSpaceSearchText] = useState(''); // data space @@ -187,15 +213,15 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( return; } queryBuilderState.setExecutionContext(option.value); - queryBuilderState.propagateExecutionContextChange(option.value); + queryBuilderState.propagateExecutionContextChange( + option.value, + editorStore, + ); queryBuilderState.onExecutionContextChange?.(option.value); }; // runtime - const runtimeOptions = getMappingCompatibleRuntimes( - queryBuilderState.executionContext.mapping.value, - queryBuilderState.graphManagerState.usableRuntimes, - ) + const runtimeOptions = resolveExecutionContextRuntimes(queryBuilderState) .map( (rt) => new RuntimePointer(PackageableElementExplicitReference.create(rt)), @@ -225,11 +251,7 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( }); // class - const classes = resolveUsableDataSpaceClasses( - queryBuilderState.dataSpace, - queryBuilderState.executionContext.mapping.value, - queryBuilderState.graphManagerState, - ); + const classes = resolveUsableDataSpaceClasses(queryBuilderState); useEffect(() => { flowResult(queryBuilderState.loadDataSpaces('')).catch( @@ -407,6 +429,7 @@ export const queryDataSpace = async ( editorStore.depotServerClient, dataSpace, dataSpace.defaultExecutionContext, + false, (dataSpaceInfo: DataSpaceInfo) => { queryBuilderState.dataSpace = guaranteeType( queryBuilderState.graphManagerState.graph.getElement( diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts index 48a9d179922..a8333234336 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts @@ -17,14 +17,19 @@ import { type AbstractPureGraphManager, AbstractPureGraphManagerExtension, + type PureProtocolProcessorPlugin, + type V1_PureModelContextData, + type PureModel, + type GraphManagerOperationReport, } from '@finos/legend-graph'; -import type { Entity } from '@finos/legend-storage'; +import type { Entity, ProjectGAVCoordinates } from '@finos/legend-storage'; import { guaranteeNonNullable, type ActionState, type PlainObject, } from '@finos/legend-shared'; import type { DataSpaceAnalysisResult } from '../../action/analytics/DataSpaceAnalysis.js'; +import type { V1_DataSpaceAnalysisResult } from './v1/engine/analytics/V1_DataSpaceAnalysis.js'; export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPureGraphManagerExtension { abstract analyzeDataSpace( @@ -34,10 +39,23 @@ export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPu actionState?: ActionState, ): Promise; - abstract retrieveDataSpaceAnalysisFromCache( - cacheRetriever: () => Promise>, + abstract analyzeDataSpaceCoverage( + dataSpacePath: string, + entitiesRetriever: () => Promise, + cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise; + ): Promise; + + abstract buildDataSpaceAnalytics( + analytics: + | PlainObject + | V1_DataSpaceAnalysisResult, + plugins: PureProtocolProcessorPlugin[], + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + pmcd?: V1_PureModelContextData | undefined, + projectInfo?: ProjectGAVCoordinates, + ): Promise; } export const DSL_DataSpace_getGraphManagerExtension = ( diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts index 03b3a2f59bf..8f67084c1a4 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts @@ -31,8 +31,11 @@ import { V1_buildDatasetSpecification, type PureProtocolProcessorPlugin, V1_buildModelCoverageAnalysisResult, + type V1_PureModelContextData, + type GraphManagerOperationReport, + LegendSDLC, } from '@finos/legend-graph'; -import type { Entity } from '@finos/legend-storage'; +import type { Entity, ProjectGAVCoordinates } from '@finos/legend-storage'; import { ActionState, assertErrorThrown, @@ -73,7 +76,7 @@ import { } from '../../../action/analytics/DataSpaceAnalysis.js'; import { DSL_DataSpace_PureGraphManagerExtension } from '../DSL_DataSpace_PureGraphManagerExtension.js'; import { - type V1_DataSpaceAnalysisResult, + V1_DataSpaceAnalysisResult, V1_DataSpaceAssociationDocumentationEntry, V1_DataSpaceClassDocumentationEntry, V1_DataSpaceEnumerationDocumentationEntry, @@ -83,6 +86,7 @@ import { V1_DataSpaceMultiExecutionServiceExecutableInfo, } from './engine/analytics/V1_DataSpaceAnalysis.js'; import { getDiagram } from '@finos/legend-extension-dsl-diagram/graph'; +import { resolveVersion } from '@finos/legend-server-depot'; const ANALYZE_DATA_SPACE_TRACE = 'analyze data space'; const TEMPORARY__TDS_SAMPLE_VALUES__DELIMETER = '-- e.g.'; @@ -141,20 +145,54 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu ); } - async retrieveDataSpaceAnalysisFromCache( - cacheRetriever: () => Promise>, + async analyzeDataSpaceCoverage( + dataSpacePath: string, + entitiesRetriever: () => Promise, + cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise { - const cacheResult = await this.fetchDataSpaceAnalysisFromCache( - cacheRetriever, - actionState, - ); - return cacheResult - ? this.buildDataSpaceAnalytics( - cacheResult, - this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), - ) + ): Promise { + const cacheResult = cacheRetriever + ? await this.fetchDataSpaceAnalysisFromCache(cacheRetriever, actionState) : undefined; + const engineClient = this.graphManager.engine.getEngineServerClient(); + let analysisResult: PlainObject; + if ( + cacheResult && + V1_deserializeDataSpaceAnalysisResult( + cacheResult, + this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + ).executionContexts.every( + (e) => e.mappingModelCoverageAnalysisResult.model !== undefined, + ) + ) { + analysisResult = cacheResult; + } else { + actionState?.setMessage('Fetching project entities and dependencies...'); + const entities = await entitiesRetriever(); + actionState?.setMessage('Analyzing data space...'); + analysisResult = await engineClient.postWithTracing< + PlainObject + >( + engineClient.getTraceData(ANALYZE_DATA_SPACE_TRACE), + `${engineClient._pure()}/analytics/dataSpace/coverage`, + { + clientVersion: V1_PureGraphManager.DEV_PROTOCOL_VERSION, + dataSpace: dataSpacePath, + model: { + _type: V1_PureModelContextType.DATA, + elements: entities.map((entity) => entity.content), + }, + }, + {}, + undefined, + undefined, + { enableCompression: true }, + ); + } + return V1_deserializeDataSpaceAnalysisResult( + analysisResult, + this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + ); } private async fetchDataSpaceAnalysisFromCache( @@ -177,11 +215,25 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu return cacheResult; } - private async buildDataSpaceAnalytics( - json: PlainObject, + async buildDataSpaceAnalytics( + analytics: + | PlainObject + | V1_DataSpaceAnalysisResult, plugins: PureProtocolProcessorPlugin[], + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + pmcd?: V1_PureModelContextData | undefined, + projectInfo?: ProjectGAVCoordinates, ): Promise { - const analysisResult = V1_deserializeDataSpaceAnalysisResult(json, plugins); + let analysisResult: V1_DataSpaceAnalysisResult; + if (analytics instanceof V1_DataSpaceAnalysisResult) { + analysisResult = analytics; + } else { + analysisResult = V1_deserializeDataSpaceAnalysisResult( + analytics, + plugins, + ); + } const result = new DataSpaceAnalysisResult(); result.name = analysisResult.name; result.package = analysisResult.package; @@ -228,25 +280,32 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu // NOTE: we will relax the check and not throw here for unknown support info type } - // create an empty graph - const extensionElementClasses = this.graphManager.pluginManager - .getPureGraphPlugins() - .flatMap((plugin) => plugin.getExtraPureGraphExtensionClasses?.() ?? []); - const systemModel = new SystemModel(extensionElementClasses); - const coreModel = new CoreModel(extensionElementClasses); - await this.graphManager.buildSystem( - coreModel, - systemModel, - ActionState.create(), - {}, - ); - systemModel.initializeAutoImports(); - const graph = new PureModel( - coreModel, - systemModel, - this.graphManager.pluginManager.getPureGraphPlugins(), - ); - + let graphEntities; + let graph: PureModel; + if (pureGraph) { + graph = pureGraph; + } else { + // create an empty graph + const extensionElementClasses = this.graphManager.pluginManager + .getPureGraphPlugins() + .flatMap( + (plugin) => plugin.getExtraPureGraphExtensionClasses?.() ?? [], + ); + const systemModel = new SystemModel(extensionElementClasses); + const coreModel = new CoreModel(extensionElementClasses); + await this.graphManager.buildSystem( + coreModel, + systemModel, + ActionState.create(), + {}, + ); + systemModel.initializeAutoImports(); + graph = new PureModel( + coreModel, + systemModel, + this.graphManager.pluginManager.getPureGraphPlugins(), + ); + } // Create dummy mappings and runtimes // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should // simplify the model here or potentially refactor the backend analytics endpoint to return these as model @@ -275,56 +334,79 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu runtime.runtimeValue = new V1_EngineRuntime(); return runtime; }); - - // prepare the model context data - const graphEntities = analysisResult.model.elements - // NOTE: this is a temporary hack to fix a problem with data space analytics - // where the classes for properties are not properly surveyed - // We need to wait for the actual fix in backend to be merged and released - // See https://github.com/finos/legend-engine/pull/836 - .concat( - uniq( - analysisResult.model.elements.flatMap((element) => { - if (element instanceof V1_Class) { - return element.derivedProperties - .map((prop) => prop.returnType) - .concat(element.properties.map((prop) => prop.type)); - } - return []; - }), - ) - // make sure to not include types already returned by the analysis - .filter( - (path) => - !analysisResult.model.elements - .map((el) => el.path) - .includes(path), + if (pmcd && projectInfo) { + graphEntities = pmcd.elements + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter((el) => !graph.getNullableElement(el.path, false)) + .map((el) => this.graphManager.elementProtocolToEntity(el)); + await this.graphManager.buildGraph( + graph, + graphEntities, + ActionState.create(), + { + origin: new LegendSDLC( + projectInfo.groupId, + projectInfo.artifactId, + resolveVersion(projectInfo.versionId), + ), + }, + graphReport, + true, + ); + } else { + // prepare the model context data + graphEntities = analysisResult.model.elements + // NOTE: this is a temporary hack to fix a problem with data space analytics + // where the classes for properties are not properly surveyed + // We need to wait for the actual fix in backend to be merged and released + // See https://github.com/finos/legend-engine/pull/836 + .concat( + uniq( + analysisResult.model.elements.flatMap((element) => { + if (element instanceof V1_Class) { + return element.derivedProperties + .map((prop) => prop.returnType) + .concat(element.properties.map((prop) => prop.type)); + } + return []; + }), ) - .map((path) => { - const [pkgPath, name] = resolvePackagePathAndElementName(path); - if (!pkgPath) { - // exclude package-less elements (i.e. primitive types) - return undefined; - } - const _class = new V1_Class(); - _class.name = name; - _class.package = pkgPath; - return _class; - }) - .filter(isNonNullable), - ) - .concat(mappingModels) - .concat(runtimeModels) - // NOTE: if an element could be found in the graph already it means it comes from system - // so we could rid of it - .filter((el) => !graph.getNullableElement(el.path, false)) - .map((el) => this.graphManager.elementProtocolToEntity(el)); + // make sure to not include types already returned by the analysis + .filter( + (path) => + !analysisResult.model.elements + .map((el) => el.path) + .includes(path), + ) + .map((path) => { + const [pkgPath, name] = resolvePackagePathAndElementName(path); + if (!pkgPath) { + // exclude package-less elements (i.e. primitive types) + return undefined; + } + const _class = new V1_Class(); + _class.name = name; + _class.package = pkgPath; + return _class; + }) + .filter(isNonNullable), + ) + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter((el) => !graph.getNullableElement(el.path, false)) + .map((el) => this.graphManager.elementProtocolToEntity(el)); - await this.graphManager.buildGraph( - graph, - graphEntities, - ActionState.create(), - ); + await this.graphManager.buildGraph( + graph, + graphEntities, + ActionState.create(), + ); + } // execution context result.executionContextsIndex = new Map< @@ -344,6 +426,7 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu contextAnalysisResult.mappingModelCoverageAnalysisResult = V1_buildModelCoverageAnalysisResult( context.mappingModelCoverageAnalysisResult, + this.graphManager, contextAnalysisResult.mapping, ); contextAnalysisResult.compatibleRuntimes = context.compatibleRuntimes.map( diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts index 6a1a189e1a3..7d2c366c20c 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts @@ -29,12 +29,20 @@ import { RuntimePointer, type Runtime, Class, - type Mapping, getDescendantsOfPackage, Package, + createGraphBuilderReport, + GRAPH_MANAGER_EVENT, + LegendSDLC, + V1_EngineRuntime, + V1_Mapping, + V1_PackageableRuntime, + V1_PureGraphManager, + resolvePackagePathAndElementName, } from '@finos/legend-graph'; import { DepotScope, + resolveVersion, SNAPSHOT_VERSION_ALIAS, type DepotServerClient, type StoredEntity, @@ -46,6 +54,10 @@ import { getNullableFirstEntry, filterByType, uniq, + StopWatch, + LogEvent, + guaranteeNonNullable, + guaranteeType, } from '@finos/legend-shared'; import { action, flow, makeObservable, observable } from 'mobx'; import { renderDataSpaceQueryBuilderSetupPanelContent } from '../../components/query/DataSpaceQueryBuilder.js'; @@ -57,12 +69,14 @@ import { DATA_SPACE_ELEMENT_CLASSIFIER_PATH } from '../../graph-manager/protocol import { type DataSpaceInfo, extractDataSpaceInfo } from './DataSpaceInfo.js'; import { DataSpaceAdvancedSearchState } from './DataSpaceAdvancedSearchState.js'; import type { DataSpaceAnalysisResult } from '../../graph-manager/action/analytics/DataSpaceAnalysis.js'; +import type { QueryEditorStore } from '@finos/legend-application-query'; export const resolveUsableDataSpaceClasses = ( - dataSpace: DataSpace, - mapping: Mapping, - graphManagerState: GraphManagerState, + queryBuilderState: DataSpaceQueryBuilderState, ): Class[] => { + const dataSpace = queryBuilderState.dataSpace; + const mapping = queryBuilderState.executionContext.mapping.value; + const graphManagerState = queryBuilderState.graphManagerState; if (dataSpace.elements?.length) { const dataSpaceElements = dataSpace.elements.map((ep) => ep.element.value); return uniq([ @@ -73,6 +87,19 @@ export const resolveUsableDataSpaceClasses = ( .flat() .filter(filterByType(Class)), ]); + } else if ( + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult + ) { + const compatibleClassPaths = + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult.mappedEntities.map( + (e) => e.classPath, + ); + const uniqueCompatibleClasses = compatibleClassPaths.filter( + (val, index) => compatibleClassPaths.indexOf(val) === index, + ); + return graphManagerState.graph.classes.filter((c) => + uniqueCompatibleClasses.includes(c.path), + ); } return getMappingCompatibleClasses(mapping, graphManagerState.usableClasses); }; @@ -138,6 +165,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { dataSpaces: DataSpaceInfo[] = []; showRuntimeSelector = false; advancedSearchState?: DataSpaceAdvancedSearchState | undefined; + isLightGraphEnabled!: boolean; constructor( applicationStore: GenericLegendApplicationStore, @@ -145,6 +173,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { depotServerClient: DepotServerClient, dataSpace: DataSpace, executionContext: DataSpaceExecutionContext, + isLightGraphEnabled: boolean, onDataSpaceChange: (val: DataSpaceInfo) => void, isAdvancedDataSpaceSearchEnabled: boolean, dataSpaceAnalysisResult?: DataSpaceAnalysisResult | undefined, @@ -161,11 +190,13 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { makeObservable(this, { dataSpaces: observable, executionContext: observable, + isLightGraphEnabled: observable, showRuntimeSelector: observable, advancedSearchState: observable, showAdvancedSearchPanel: action, hideAdvancedSearchPanel: action, setExecutionContext: action, + setIsLightGraphEnabled: action, setShowRuntimeSelector: action, loadDataSpaces: flow, }); @@ -174,6 +205,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { this.dataSpace = dataSpace; this.executionContext = executionContext; this.projectInfo = projectInfo; + this.isLightGraphEnabled = isLightGraphEnabled; this.onDataSpaceChange = onDataSpaceChange; this.onExecutionContextChange = onExecutionContextChange; this.onRuntimeChange = onRuntimeChange; @@ -188,6 +220,10 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { : 'query-builder__setup__data-space'; } + setIsLightGraphEnabled(val: boolean): void { + this.isLightGraphEnabled = val; + } + showAdvancedSearchPanel(): void { if (this.projectInfo && this.isAdvancedDataSpaceSearchEnabled) { this.advancedSearchState = new DataSpaceAdvancedSearchState( @@ -275,26 +311,125 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { * - If no class is chosen, try to choose a compatible class * - If the chosen class is compatible with the new selected execution context mapping, do nothing, otherwise, try to choose a compatible class */ - propagateExecutionContextChange( + async propagateExecutionContextChange( executionContext: DataSpaceExecutionContext, - ): void { + editorStore?: QueryEditorStore, + isGraphBuildingNotRequired?: boolean, + ): Promise { const mapping = executionContext.mapping.value; - this.changeMapping(mapping); + let compatibleClasses; const mappingModelCoverageAnalysisResult = this.dataSpaceAnalysisResult?.executionContextsIndex.get( executionContext.name, )?.mappingModelCoverageAnalysisResult; - if (mappingModelCoverageAnalysisResult) { + if (this.dataSpaceAnalysisResult && mappingModelCoverageAnalysisResult) { + if (!isGraphBuildingNotRequired && editorStore) { + const stopWatch = new StopWatch(); + const graph = this.graphManagerState.createNewGraph(); + + const graph_buildReport = createGraphBuilderReport(); + // Create dummy mappings and runtimes + // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should + // simplify the model here or potentially refactor the backend analytics endpoint to return these as model + const mappingModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).map((context) => context.mapping), + ).map((m) => { + const _mapping = new V1_Mapping(); + const [packagePath, name] = resolvePackagePathAndElementName(m.path); + _mapping.package = packagePath; + _mapping.name = name; + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(_mapping); + }); + const runtimeModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ) + .map((context) => context.defaultRuntime) + .concat( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).flatMap((val) => val.compatibleRuntimes), + ), + ).map((r) => { + const runtime = new V1_PackageableRuntime(); + const [packagePath, name] = resolvePackagePathAndElementName(r.path); + runtime.package = packagePath; + runtime.name = name; + runtime.runtimeValue = new V1_EngineRuntime(); + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(runtime); + }); + const graphEntities = guaranteeNonNullable( + mappingModelCoverageAnalysisResult.entities, + ) + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter( + (el) => + !graph.getNullableElement(el.path, false) && + !el.path.startsWith('meta::'), + ); + await this.graphManagerState.graphManager.buildGraph( + graph, + graphEntities, + ActionState.create(), + { + origin: new LegendSDLC( + guaranteeNonNullable(this.projectInfo).groupId, + guaranteeNonNullable(this.projectInfo).artifactId, + resolveVersion(guaranteeNonNullable(this.projectInfo).versionId), + ), + }, + graph_buildReport, + true, + ); + this.graphManagerState.graph = graph; + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + this.applicationStore.timeService.finalizeTimingsRecord(stopWatch), + dependencies: dependency_buildReport, + dependenciesCount: + this.graphManagerState.graph.dependencyManager.numberOfDependencies, + graph: graph_buildReport, + }; + editorStore.logBuildGraphMetrics(graphBuilderReportData); + + this.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } + const compatibleClassPaths = + mappingModelCoverageAnalysisResult.mappedEntities.map( + (e) => e.classPath, + ); + const uniqueCompatibleClasses = compatibleClassPaths.filter( + (val, index) => compatibleClassPaths.indexOf(val) === index, + ); + compatibleClasses = this.graphManagerState.graph.classes.filter((c) => + uniqueCompatibleClasses.includes(c.path), + ); this.explorerState.mappingModelCoverageAnalysisResult = mappingModelCoverageAnalysisResult; + } else { + compatibleClasses = resolveUsableDataSpaceClasses(this); } + this.changeMapping(mapping); + this.changeRuntime(new RuntimePointer(executionContext.defaultRuntime)); - const compatibleClasses = resolveUsableDataSpaceClasses( - this.dataSpace, - mapping, - this.graphManagerState, - ); // if there is no chosen class or the chosen one is not compatible // with the mapping then pick a compatible class if possible if (!this.class || !compatibleClasses.includes(this.class)) { @@ -303,5 +438,6 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { this.changeClass(possibleNewClass); } } + this.explorerState.refreshTreeData(); } } diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts index 579d752d36c..4291e2e224e 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts @@ -23,6 +23,9 @@ import { type Runtime, type Class, type RawLambda, + type V1_PureModelContextData, + createGraphBuilderReport, + GRAPH_MANAGER_EVENT, } from '@finos/legend-graph'; import { QueryEditorStore, @@ -30,14 +33,18 @@ import { type LegendQueryApplicationStore, createViewProjectHandler, createViewSDLCProjectHandler, + LEGEND_QUERY_APP_EVENT, } from '@finos/legend-application-query'; import { type DepotServerClient, StoreProjectData, + retrieveProjectEntitiesWithDependencies, } from '@finos/legend-server-depot'; import { guaranteeNonNullable, guaranteeType, + LogEvent, + StopWatch, uuid, } from '@finos/legend-shared'; import { @@ -117,7 +124,97 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { }; } + override requiresGraphBuilding(): boolean { + return false; + } + async initializeQueryBuilderState(): Promise { + let dataSpaceAnalysisResult; + let dataSpaceAnalysisResultMetaModel; + let isLightGraphEnabled = true; + try { + const stopWatch = new StopWatch(); + const project = StoreProjectData.serialization.fromJson( + await this.depotServerClient.getProject(this.groupId, this.artifactId), + ); + this.initState.setMessage('Fetching dataspace analysis result'); + dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( + this.graphManagerState.graphManager, + ).analyzeDataSpaceCoverage( + this.dataSpacePath, + () => + retrieveProjectEntitiesWithDependencies( + project, + this.versionId, + this.depotServerClient, + ), + () => + retrieveAnalyticsResultCache( + project, + this.versionId, + this.dataSpacePath, + this.depotServerClient, + ), + ); + const mappingPath = dataSpaceAnalysisResult?.executionContexts.find( + (e) => e.name === this.executionContext, + )?.mapping; + if (dataSpaceAnalysisResult && mappingPath) { + // initialize system + stopWatch.record(); + await this.graphManagerState.initializeSystem(); + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS); + + // build graph + let pmcd: V1_PureModelContextData | undefined = undefined; + if (dataSpaceAnalysisResult) { + const mappingModelCoverageAnalysisResult = + dataSpaceAnalysisResult?.executionContexts.find( + (value) => value.mapping === mappingPath, + )?.mappingModelCoverageAnalysisResult; + pmcd = mappingModelCoverageAnalysisResult?.model; + } + const graph_buildReport = createGraphBuilderReport(); + dataSpaceAnalysisResultMetaModel = + await DSL_DataSpace_getGraphManagerExtension( + this.graphManagerState.graphManager, + ).buildDataSpaceAnalytics( + dataSpaceAnalysisResult, + this.graphManagerState.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + graph_buildReport, + this.graphManagerState.graph, + pmcd, + this.getProjectInfo(), + ); + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + this.applicationStore.timeService.finalizeTimingsRecord(stopWatch), + dependencies: dependency_buildReport, + dependenciesCount: + this.graphManagerState.graph.dependencyManager.numberOfDependencies, + graph: graph_buildReport, + }; + this.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } else { + isLightGraphEnabled = false; + this.graphManagerState.graph = this.graphManagerState.createNewGraph(); + await this.buildGraph(); + } + } catch (error) { + this.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + this.graphManagerState.graph = this.graphManagerState.createNewGraph(); + isLightGraphEnabled = false; + await this.buildGraph(); + } const dataSpace = getDataSpace( this.dataSpacePath, this.graphManagerState.graph, @@ -128,24 +225,6 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { ), `Can't find execution context '${this.executionContext}'`, ); - let dataSpaceAnalysisResult; - try { - const project = StoreProjectData.serialization.fromJson( - await this.depotServerClient.getProject(this.groupId, this.artifactId), - ); - dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( - this.graphManagerState.graphManager, - ).retrieveDataSpaceAnalysisFromCache(() => - retrieveAnalyticsResultCache( - project, - this.versionId, - dataSpace.path, - this.depotServerClient, - ), - ); - } catch { - // do nothing - } const projectInfo = new DataSpaceProjectInfo( this.groupId, this.artifactId, @@ -162,6 +241,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { this.depotServerClient, dataSpace, executionContext, + isLightGraphEnabled, (dataSpaceInfo: DataSpaceInfo) => { if (dataSpaceInfo.defaultExecutionContext) { this.applicationStore.navigationService.navigator.goToLocation( @@ -182,7 +262,7 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { } }, true, - dataSpaceAnalysisResult, + dataSpaceAnalysisResultMetaModel, (ec: DataSpaceExecutionContext) => { // runtime should already be set const runtimePointer = guaranteeType( @@ -245,7 +325,11 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { projectInfo, ); queryBuilderState.setExecutionContext(executionContext); - queryBuilderState.propagateExecutionContextChange(executionContext); + queryBuilderState.propagateExecutionContextChange( + executionContext, + this, + true, + ); // set runtime if already chosen if (this.runtimePath) { diff --git a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts index eb881be5988..0f041ba5912 100644 --- a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts @@ -262,6 +262,7 @@ export abstract class AbstractPureGraphManager { buildState: ActionState, options?: GraphBuilderOptions, report?: GraphManagerOperationReport, + buildRequiredGraph?: boolean | undefined, ): Promise; /** diff --git a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts index 38808ac627e..2857f19bf29 100644 --- a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import type { Entity } from '@finos/legend-storage'; import type { Mapping } from '../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js'; export type RawMappingModelCoverageAnalysisResult = object; @@ -22,10 +23,12 @@ export class MappedEntity { readonly __PROPERTIES_INDEX = new Map(); path: string; + classPath: string; properties: MappedProperty[]; - constructor(path: string, properties: MappedProperty[]) { + constructor(path: string, classPath: string, properties: MappedProperty[]) { this.path = path; + this.classPath = classPath; this.properties = properties; properties.forEach((property) => this.__PROPERTIES_INDEX.set(property.name, property), @@ -73,12 +76,18 @@ export class MappingModelCoverageAnalysisResult { readonly mapping: Mapping; mappedEntities: MappedEntity[]; + entities?: Entity[] | undefined; - constructor(mappedEntities: MappedEntity[], mapping: Mapping) { + constructor( + mappedEntities: MappedEntity[], + mapping: Mapping, + entities?: Entity[] | undefined, + ) { this.mappedEntities = mappedEntities; this.mapping = mapping; mappedEntities.forEach((entity) => this.__ENTITIES_INDEX.set(entity.path, entity), ); + this.entities = entities; } } diff --git a/packages/legend-graph/src/graph-manager/action/query/Query.ts b/packages/legend-graph/src/graph-manager/action/query/Query.ts index a1681136189..29c88c22115 100644 --- a/packages/legend-graph/src/graph-manager/action/query/Query.ts +++ b/packages/legend-graph/src/graph-manager/action/query/Query.ts @@ -67,6 +67,8 @@ export class LightQuery { artifactId!: string; owner?: string | undefined; lastUpdatedAt?: number | undefined; + mapping?: string | PackageableElementReference | undefined; + taggedValues?: QueryTaggedValue[] | undefined; isCurrentUserQuery = false; } @@ -80,6 +82,8 @@ export const toLightQuery = (query: Query): LightQuery => { lightQuery.versionId = query.versionId; lightQuery.owner = query.owner; lightQuery.isCurrentUserQuery = query.isCurrentUserQuery; + lightQuery.mapping = query.mapping.value.path; + lightQuery.taggedValues = query.taggedValues; return lightQuery; }; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts index 3ce03a77569..3f83e1d4765 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts @@ -802,6 +802,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { buildState: ActionState, options?: GraphBuilderOptions, _report?: GraphManagerOperationReport, + buildRequiredGraph?: boolean | undefined, ): Promise { const stopWatch = new StopWatch(); const report = _report ?? createGraphBuilderReport(); @@ -834,14 +835,25 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { ]; // build - await this.buildGraphFromInputs( - graph, - buildInputs, - report, - stopWatch, - buildState, - options, - ); + if (!buildRequiredGraph) { + await this.buildGraphFromInputs( + graph, + buildInputs, + report, + stopWatch, + buildState, + options, + ); + } else { + await this.buildRequiredGraphFromInputs( + graph, + buildInputs, + report, + stopWatch, + buildState, + options, + ); + } /** * For now, we delete the section index. We are able to read both resolved and unresolved element paths @@ -1128,6 +1140,32 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { ); } + private async buildRequiredGraphFromInputs( + graph: PureModel, + inputs: V1_PureGraphBuilderInput[], + report: GraphManagerOperationReport, + stopWatch: StopWatch, + graphBuilderState: ActionState, + options?: GraphBuilderOptions, + ): Promise { + // index + graphBuilderState.setMessage( + `Indexing ${report.elementCount.total} elements...`, + ); + await this.initializeAndIndexElements(graph, inputs, options); + stopWatch.record(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS); + // build types + graphBuilderState.setMessage(`Building domain models...`); + await this.buildTypes(graph, inputs, options); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS, + ); + + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS, + ); + } + private async buildLightGraphFromInputs( graph: PureModel, inputs: V1_PureGraphBuilderInput[], @@ -2889,6 +2927,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { : this.getFullGraphModelData(graph); return V1_buildModelCoverageAnalysisResult( await this.engine.analyzeMappingModelCoverage(input), + this, mapping, ); } @@ -2902,6 +2941,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { V1_MappingModelCoverageAnalysisResult, input as PlainObject, ), + this, mapping, ); } diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts index b341b2f8f46..b120b63841f 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts @@ -97,10 +97,29 @@ export const V1_buildLightQuery = ( protocol.artifactId, `Query 'artifactId' field is missing`, ); + metamodel.mapping = protocol.mapping; metamodel.owner = protocol.owner; metamodel.lastUpdatedAt = protocol.lastUpdatedAt; metamodel.isCurrentUserQuery = currentUserId !== undefined && protocol.owner === currentUserId; + // NOTE: we don't properly process tagged values and stereotypes for query + // because these profiles/tags/stereotypes can come from external systems. + metamodel.taggedValues = protocol.taggedValues?.map((taggedValueProtocol) => { + const taggedValue = new QueryTaggedValue(); + taggedValue.profile = guaranteeNonEmptyString( + taggedValueProtocol.tag.profile, + `Tagged value 'tag.profile' field is missing or empty`, + ); + taggedValue.tag = guaranteeNonEmptyString( + taggedValueProtocol.tag.value, + `Tagged value 'tag.value' field is missing or empty`, + ); + taggedValue.value = guaranteeNonEmptyString( + taggedValueProtocol.value, + `Tagged value 'value' field is missing or empty`, + ); + return taggedValue; + }); return metamodel; }; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts index 52228332c6a..cf3d1d36458 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts @@ -37,7 +37,13 @@ import { MappingModelCoverageAnalysisResult, } from '../../../../../../graph-manager/action/analytics/MappingModelCoverageAnalysis.js'; import type { V1_PureModelContext } from '../../model/context/V1_PureModelContext.js'; -import { V1_pureModelContextPropSchema } from '../../transformation/pureProtocol/V1_PureProtocolSerialization.js'; +import { + V1_deserializePureModelContextData, + V1_pureModelContextPropSchema, + V1_serializePureModelContextData, +} from '../../transformation/pureProtocol/V1_PureProtocolSerialization.js'; +import type { V1_PureModelContextData } from '../../model/context/V1_PureModelContextData.js'; +import type { V1_PureGraphManager } from '../../V1_PureGraphManager.js'; enum V1_MappedPropertyType { ENUM = 'enum', @@ -103,11 +109,13 @@ const V1_deserializeMappedProperty = ( class V1_MappedEntity { path!: string; + classPath!: string; properties: V1_MappedProperty[] = []; static readonly serialization = new SerializationFactory( createModelSchema(V1_MappedEntity, { path: primitive(), + classPath: primitive(), properties: list( custom( (prop) => V1_serializeMappedProperty(prop), @@ -134,12 +142,19 @@ export class V1_MappingModelCoverageAnalysisInput { export class V1_MappingModelCoverageAnalysisResult { mappedEntities: V1_MappedEntity[] = []; + model?: V1_PureModelContextData | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_MappingModelCoverageAnalysisResult, { mappedEntities: list( usingModelSchema(V1_MappedEntity.serialization.schema), ), + model: optional( + custom( + (val) => V1_serializePureModelContextData(val), + (val) => V1_deserializePureModelContextData(val), + ), + ), }), ); } @@ -158,14 +173,21 @@ const buildMappedProperty = (protocol: V1_MappedProperty): MappedProperty => const buildMappedEntity = (protocol: V1_MappedEntity): MappedEntity => new MappedEntity( protocol.path, + protocol.classPath, protocol.properties.map((p) => buildMappedProperty(p)), ); export const V1_buildModelCoverageAnalysisResult = ( protocol: V1_MappingModelCoverageAnalysisResult, + graphManager: V1_PureGraphManager, mapping: Mapping, -): MappingModelCoverageAnalysisResult => - new MappingModelCoverageAnalysisResult( +): MappingModelCoverageAnalysisResult => { + const entities = protocol.model?.elements.map((el) => + graphManager.elementProtocolToEntity(el), + ); + return new MappingModelCoverageAnalysisResult( protocol.mappedEntities.map((p) => buildMappedEntity(p)), mapping, + entities, ); +}; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts index 3d2e9bbd284..94e21fc17ff 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts @@ -87,6 +87,8 @@ export class V1_LightQuery { artifactId!: string; versionId!: string; lastUpdatedAt?: number | undefined; + mapping?: string | undefined; + taggedValues?: V1_TaggedValue[] | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_Query, { @@ -94,8 +96,10 @@ export class V1_LightQuery { id: primitive(), groupId: primitive(), lastUpdatedAt: optional(primitive()), + mapping: optional(primitive()), name: primitive(), owner: optional(primitive()), + taggedValues: optional(list(usingModelSchema(V1_taggedValueModelSchema))), versionId: primitive(), }), {