From d300f8bfb1f48641e9580da7f7d3477dd583bb83 Mon Sep 17 00:00:00 2001 From: Kevin Mattutat Date: Mon, 11 Oct 2021 13:13:27 +0200 Subject: [PATCH] feat(elasticsearch-plugin): Extend config with customScriptFields Relates to #1143 --- .../e2e/elasticsearch-plugin.e2e-spec.ts | 34 +++++++++ .../src/build-elastic-body.spec.ts | 23 ++++++ .../src/build-elastic-body.ts | 41 ++++++++++- .../src/custom-script-fields.resolver.ts | 26 +++++++ .../src/elasticsearch.service.ts | 70 ++++++++++++++++++- .../src/graphql-schema-extensions.ts | 47 ++++++++++++- .../src/indexer.controller.ts | 7 +- packages/elasticsearch-plugin/src/options.ts | 39 ++++++++++- packages/elasticsearch-plugin/src/plugin.ts | 19 ++++- packages/elasticsearch-plugin/src/types.ts | 37 ++++++++++ 10 files changed, 333 insertions(+), 10 deletions(-) create mode 100644 packages/elasticsearch-plugin/src/custom-script-fields.resolver.ts diff --git a/packages/elasticsearch-plugin/e2e/elasticsearch-plugin.e2e-spec.ts b/packages/elasticsearch-plugin/e2e/elasticsearch-plugin.e2e-spec.ts index fe4514d2a0..97654722db 100644 --- a/packages/elasticsearch-plugin/e2e/elasticsearch-plugin.e2e-spec.ts +++ b/packages/elasticsearch-plugin/e2e/elasticsearch-plugin.e2e-spec.ts @@ -139,6 +139,17 @@ describe('Elasticsearch plugin', () => { }, }, }, + searchConfig: { + scriptFields: { + answerDouble: { + graphQlType: 'Int!', + environment: 'product', + scriptFn: input => ({ + script: `doc['answer'].value * 2`, + }), + }, + }, + }, }), DefaultJobQueuePlugin, ], @@ -1330,6 +1341,29 @@ describe('Elasticsearch plugin', () => { }); }); }); + + describe('scriptFields', () => { + it('script mapping', async () => { + const query = `{ + search(input: { take: 1, groupByProduct: true, sort: { name: ASC } }) { + items { + productVariantName + customScriptFields { + answerDouble + } + } + } + }`; + const { search } = await shopClient.query(gql(query)); + + expect(search.items[0]).toEqual({ + productVariantName: 'Bonsai Tree', + customScriptFields: { + answerDouble: 84, + }, + }); + }); + }); }); export const SEARCH_PRODUCTS = gql` diff --git a/packages/elasticsearch-plugin/src/build-elastic-body.spec.ts b/packages/elasticsearch-plugin/src/build-elastic-body.spec.ts index 06eea9ff05..a75b4c7498 100644 --- a/packages/elasticsearch-plugin/src/build-elastic-body.spec.ts +++ b/packages/elasticsearch-plugin/src/build-elastic-body.spec.ts @@ -495,6 +495,29 @@ describe('buildElasticBody()', () => { }); }); + it('scriptFields option', () => { + const config: DeepRequired = { + ...searchConfig, + ...{ + scriptFields: { + test: { + graphQlType: 'String', + environment: 'both', + scriptFn: input => ({ + script: `doc['property'].dummyScript(${input.term})`, + }), + }, + }, + }, + }; + const result = buildElasticBody({ term: 'test' }, config, CHANNEL_ID, LanguageCode.en); + expect(result.script_fields).toEqual({ + test: { + script: `doc['property'].dummyScript(test)`, + }, + }); + }); + describe('price ranges', () => { it('not grouped by product', () => { const result = buildElasticBody( diff --git a/packages/elasticsearch-plugin/src/build-elastic-body.ts b/packages/elasticsearch-plugin/src/build-elastic-body.ts index 9be028d555..2342e53bad 100644 --- a/packages/elasticsearch-plugin/src/build-elastic-body.ts +++ b/packages/elasticsearch-plugin/src/build-elastic-body.ts @@ -2,7 +2,7 @@ import { LanguageCode, LogicalOperator, PriceRange, SortOrder } from '@vendure/c import { DeepRequired, ID, UserInputError } from '@vendure/core'; import { SearchConfig } from './options'; -import { ElasticSearchInput, SearchRequestBody } from './types'; +import { CustomScriptMapping, ElasticSearchInput, SearchRequestBody } from './types'; /** * Given a SearchInput object, returns the corresponding Elasticsearch body. @@ -91,7 +91,6 @@ export function buildElasticBody( ensureBoolFilterExists(query); query.bool.filter.push({ term: { enabled: true } }); } - if (priceRange) { ensureBoolFilterExists(query); query.bool.filter = query.bool.filter.concat(createPriceFilters(priceRange, false)); @@ -122,6 +121,11 @@ export function buildElasticBody( sortArray.push({ [priceField]: { order: sort.price === SortOrder.ASC ? 'asc' : 'desc' } }); } } + const scriptFields: any | undefined = createScriptFields( + searchConfig.scriptFields, + input, + groupByProduct, + ); const body: SearchRequestBody = { query: searchConfig.mapQuery @@ -131,6 +135,12 @@ export function buildElasticBody( from: skip || 0, size: take || 10, track_total_hits: searchConfig.totalItemsMaxSize, + ...(scriptFields !== undefined + ? { + _source: true, + script_fields: scriptFields, + } + : undefined), }; if (groupByProduct) { body.collapse = { field: `productId` }; @@ -144,6 +154,33 @@ function ensureBoolFilterExists(query: { bool: { filter?: any } }) { } } +function createScriptFields( + scriptFields: { [fieldName: string]: CustomScriptMapping<[ElasticSearchInput]> }, + input: ElasticSearchInput, + groupByProduct?: boolean, +): any | undefined { + if (scriptFields) { + const fields = Object.keys(scriptFields); + if (fields.length) { + const result: any = {}; + for (const name of fields) { + const scriptField = scriptFields[name]; + if (scriptField.environment === 'product' && groupByProduct === true) { + (result as any)[name] = scriptField.scriptFn(input); + } + if (scriptField.environment === 'variant' && groupByProduct === false) { + (result as any)[name] = scriptField.scriptFn(input); + } + if (scriptField.environment === 'both' || scriptField.environment === undefined) { + (result as any)[name] = scriptField.scriptFn(input); + } + } + return result; + } + } + return undefined; +} + function createPriceFilters(range: PriceRange, withTax: boolean): any[] { const withTaxFix = withTax ? 'WithTax' : ''; return [ diff --git a/packages/elasticsearch-plugin/src/custom-script-fields.resolver.ts b/packages/elasticsearch-plugin/src/custom-script-fields.resolver.ts new file mode 100644 index 0000000000..d5d5fe335d --- /dev/null +++ b/packages/elasticsearch-plugin/src/custom-script-fields.resolver.ts @@ -0,0 +1,26 @@ +import { Inject } from '@nestjs/common'; +import { ResolveField, Resolver } from '@nestjs/graphql'; +import { DeepRequired } from '@vendure/common/lib/shared-types'; + +import { ELASTIC_SEARCH_OPTIONS } from './constants'; +import { ElasticsearchOptions } from './options'; + +/** + * This resolver is only required if scriptFields are defined for both products and product variants. + * This particular configuration will result in a union type for the + * `SearchResult.customScriptFields` GraphQL field. + */ +@Resolver('CustomScriptFields') +export class CustomScriptFieldsResolver { + constructor(@Inject(ELASTIC_SEARCH_OPTIONS) private options: DeepRequired) {} + + @ResolveField() + __resolveType(value: any): string { + const productScriptFields = Object.entries(this.options.searchConfig?.scriptFields || {}) + .filter(([, scriptField]) => scriptField.environment !== 'variant') + .map(([k]) => k); + return Object.keys(value).every(k => productScriptFields.includes(k)) + ? 'CustomProductScriptFields' + : 'CustomProductVariantScriptFields'; + } +} diff --git a/packages/elasticsearch-plugin/src/elasticsearch.service.ts b/packages/elasticsearch-plugin/src/elasticsearch.service.ts index 4b54d7952b..cbdf8fa7c4 100644 --- a/packages/elasticsearch-plugin/src/elasticsearch.service.ts +++ b/packages/elasticsearch-plugin/src/elasticsearch.service.ts @@ -23,6 +23,8 @@ import { createIndices, getClient } from './indexing-utils'; import { ElasticsearchOptions } from './options'; import { CustomMapping, + CustomScriptEnvironment, + CustomScriptMapping, ElasticSearchInput, ElasticSearchResponse, ElasticSearchResult, @@ -194,7 +196,17 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { totalItems, }; } catch (e) { - Logger.error(e.message, loggerCtx, e.stack); + if (e.meta.body.error.type && e.meta.body.error.type === 'search_phase_execution_exception') { + // Log runtime error of the script exception instead of stacktrace + Logger.error( + e.message, + loggerCtx, + JSON.stringify(e.meta.body.error.root_cause || [], null, 2), + ); + Logger.verbose(JSON.stringify(e.meta.body.error.failed_shards || [], null, 2), loggerCtx); + } else { + Logger.error(e.message, loggerCtx, e.stack); + } throw e; } } else { @@ -208,7 +220,17 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { totalItems: body.hits.total ? body.hits.total.value : 0, }; } catch (e) { - Logger.error(e.message, loggerCtx, e.stack); + if (e.meta.body.error.type && e.meta.body.error.type === 'search_phase_execution_exception') { + // Log runtime error of the script exception instead of stacktrace + Logger.error( + e.message, + loggerCtx, + JSON.stringify(e.meta.body.error.root_cause || [], null, 2), + ); + Logger.verbose(JSON.stringify(e.meta.body.error.failed_shards || [], null, 2), loggerCtx); + } else { + Logger.error(e.message, loggerCtx, e.stack); + } throw e; } } @@ -460,6 +482,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { private mapVariantToSearchResult(hit: SearchHit): ElasticSearchResult { const source = hit._source; + const fields = hit.fields; const { productAsset, productVariantAsset } = this.getSearchResultAssets(source); const result = { ...source, @@ -480,11 +503,18 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { this.options.customProductVariantMappings, false, ); + ElasticsearchService.addScriptMappings( + result, + fields, + this.options.searchConfig?.scriptFields, + 'variant', + ); return result; } private mapProductToSearchResult(hit: SearchHit): ElasticSearchResult { const source = hit._source; + const fields = hit.fields; const { productAsset, productVariantAsset } = this.getSearchResultAssets(source); const result = { ...source, @@ -513,6 +543,12 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { score: hit._score || 0, }; ElasticsearchService.addCustomMappings(result, source, this.options.customProductMappings, true); + ElasticsearchService.addScriptMappings( + result, + fields, + this.options.searchConfig?.scriptFields, + 'product', + ); return result; } @@ -555,4 +591,34 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { } return result; } + + private static addScriptMappings( + result: any, + fields: any, + mappings: { [fieldName: string]: CustomScriptMapping }, + environment: CustomScriptEnvironment, + ): any { + const customMappings = Object.keys(mappings || {}); + if (customMappings.length) { + const customScriptFieldsResult: any = {}; + for (const name of customMappings) { + const env = mappings[name].environment; + if (env === environment || env === 'both') { + const fieldVal = (fields as any)[name] || undefined; + if (Array.isArray(fieldVal)) { + if (fieldVal.length === 1) { + customScriptFieldsResult[name] = fieldVal[0]; + } + if (fieldVal.length > 1) { + customScriptFieldsResult[name] = JSON.stringify(fieldVal); + } + } else { + customScriptFieldsResult[name] = fieldVal; + } + } + } + (result as any).customScriptFields = customScriptFieldsResult; + } + return result; + } } diff --git a/packages/elasticsearch-plugin/src/graphql-schema-extensions.ts b/packages/elasticsearch-plugin/src/graphql-schema-extensions.ts index 1d2869c563..771dd1cd03 100644 --- a/packages/elasticsearch-plugin/src/graphql-schema-extensions.ts +++ b/packages/elasticsearch-plugin/src/graphql-schema-extensions.ts @@ -44,8 +44,53 @@ export function generateSchemaExtensions(options: ElasticsearchOptions): Documen function generateCustomMappingTypes(options: ElasticsearchOptions): DocumentNode | undefined { const productMappings = Object.entries(options.customProductMappings || {}); const variantMappings = Object.entries(options.customProductVariantMappings || {}); + const scriptProductFields = Object.entries(options.searchConfig?.scriptFields || {}).filter( + ([, scriptField]) => scriptField.environment !== 'variant', + ); + const scriptVariantFields = Object.entries(options.searchConfig?.scriptFields || {}).filter( + ([, scriptField]) => scriptField.environment !== 'product', + ); + let sdl = ``; + + if (scriptProductFields.length || scriptVariantFields.length) { + if (scriptProductFields.length) { + sdl += ` + type CustomProductScriptFields { + ${scriptProductFields.map(([name, def]) => `${name}: ${def.graphQlType}`)} + } + `; + } + if (scriptVariantFields.length) { + sdl += ` + type CustomProductVariantScriptFields { + ${scriptVariantFields.map(([name, def]) => `${name}: ${def.graphQlType}`)} + } + `; + } + if (scriptProductFields.length && scriptVariantFields.length) { + sdl += ` + union CustomScriptFields = CustomProductScriptFields | CustomProductVariantScriptFields + + extend type SearchResult { + customScriptFields: CustomScriptFields! + } + `; + } else if (scriptProductFields.length) { + sdl += ` + extend type SearchResult { + customScriptFields: CustomProductScriptFields! + } + `; + } else if (scriptVariantFields.length) { + sdl += ` + extend type SearchResult { + customScriptFields: CustomProductVariantScriptFields! + } + `; + } + } + if (productMappings.length || variantMappings.length) { - let sdl = ``; if (productMappings.length) { sdl += ` type CustomProductMappings { diff --git a/packages/elasticsearch-plugin/src/indexer.controller.ts b/packages/elasticsearch-plugin/src/indexer.controller.ts index 7d9af38910..130df94f0e 100644 --- a/packages/elasticsearch-plugin/src/indexer.controller.ts +++ b/packages/elasticsearch-plugin/src/indexer.controller.ts @@ -837,7 +837,7 @@ export class ElasticsearchIndexerController implements OnModuleInit, OnModuleDes const productTranslation = this.getTranslation(product, languageCode); const productAsset = product.featuredAsset; - return { + const item: VariantIndexItem = { channelId: ctx.channelId, languageCode, productVariantId: 0, @@ -875,6 +875,11 @@ export class ElasticsearchIndexerController implements OnModuleInit, OnModuleDes inStock: false, productInStock: false, }; + const customMappings = Object.entries(this.options.customProductMappings); + for (const [name, def] of customMappings) { + item[name] = def.valueFn(product, [], languageCode); + } + return item; } private getTranslation( diff --git a/packages/elasticsearch-plugin/src/options.ts b/packages/elasticsearch-plugin/src/options.ts index 922015780d..ca6e66fbec 100644 --- a/packages/elasticsearch-plugin/src/options.ts +++ b/packages/elasticsearch-plugin/src/options.ts @@ -2,7 +2,7 @@ import { ClientOptions } from '@elastic/elasticsearch'; import { DeepRequired, EntityRelationPaths, ID, LanguageCode, Product, ProductVariant } from '@vendure/core'; import deepmerge from 'deepmerge'; -import { CustomMapping, ElasticSearchInput } from './types'; +import { CustomMapping, CustomScriptMapping, ElasticSearchInput } from './types'; /** * @description @@ -396,6 +396,42 @@ export interface SearchConfig { channelId: ID, enabledOnly: boolean, ) => any; + /** + * @description + * Sets `script_fields` inside the elasticsearch body which allows returning a script evaluation for each hit + * @since 1.2.4 + * @example + * ```TypeScript + * indexMappingProperties: { + * location: { + * type: 'geo_point', // contains function arcDistance + * }, + * }, + * customProductMappings: { + * location: { + * graphQlType: 'String', + * valueFn: (product: Product) => { + * const custom = product.customFields.location; + * return `${custom.latitude},${location.longitude}`; + * }, + * } + * }, + * scriptFields: { + * distance: { + * graphQlType: 'Number' + * valFn: (input) => { + * // assuming SearchInput was extended with latitude and longitude + * const lat = input.latitude; + * const lon = input.longitude; + * return { + * script: `doc['location'].arcDistance(${lat}, ${lon})`, + * } + * } + * } + * } + * ``` + */ + scriptFields?: { [fieldName: string]: CustomScriptMapping<[ElasticSearchInput]> }; } /** @@ -465,6 +501,7 @@ export const defaultOptions: ElasticsearchRuntimeOptions = { }, priceRangeBucketInterval: 1000, mapQuery: query => query, + scriptFields: {}, }, customProductMappings: {}, customProductVariantMappings: {}, diff --git a/packages/elasticsearch-plugin/src/plugin.ts b/packages/elasticsearch-plugin/src/plugin.ts index 3ef5552d4a..51467f79e6 100644 --- a/packages/elasticsearch-plugin/src/plugin.ts +++ b/packages/elasticsearch-plugin/src/plugin.ts @@ -23,6 +23,7 @@ import { buffer, debounceTime, delay, filter, map } from 'rxjs/operators'; import { ELASTIC_SEARCH_OPTIONS, loggerCtx } from './constants'; import { CustomMappingsResolver } from './custom-mappings.resolver'; +import { CustomScriptFieldsResolver } from './custom-script-fields.resolver'; import { ElasticsearchIndexService } from './elasticsearch-index.service'; import { AdminElasticSearchResolver, @@ -220,9 +221,21 @@ import { ElasticsearchOptions, ElasticsearchRuntimeOptions, mergeWithDefaults } const requiresUnionResolver = 0 < Object.keys(options.customProductMappings || {}).length && 0 < Object.keys(options.customProductVariantMappings || {}).length; - return requiresUnionResolver - ? [ShopElasticSearchResolver, EntityElasticSearchResolver, CustomMappingsResolver] - : [ShopElasticSearchResolver, EntityElasticSearchResolver]; + const requiresUnionScriptResolver = + 0 < + Object.values(options.searchConfig.scriptFields || {}).filter( + field => field.environment !== 'product', + ).length && + 0 < + Object.values(options.searchConfig.scriptFields || {}).filter( + field => field.environment !== 'variant', + ).length; + return [ + ShopElasticSearchResolver, + EntityElasticSearchResolver, + ...(requiresUnionResolver ? [CustomMappingsResolver] : []), + ...(requiresUnionScriptResolver ? [CustomScriptFieldsResolver] : []), + ]; }, // `any` cast is there due to a strange error "Property '[Symbol.iterator]' is missing in type... URLSearchParams" // which looks like possibly a TS/definitions bug. diff --git a/packages/elasticsearch-plugin/src/types.ts b/packages/elasticsearch-plugin/src/types.ts index 9de39a8773..8c50c45561 100644 --- a/packages/elasticsearch-plugin/src/types.ts +++ b/packages/elasticsearch-plugin/src/types.ts @@ -102,6 +102,7 @@ export type SearchHit = { _score: number; _source: T; _type: string; + fields?: any; }; export type SearchRequestBody = { @@ -112,6 +113,8 @@ export type SearchRequestBody = { track_total_hits?: number | boolean; aggs?: any; collapse?: any; + _source?: boolean; + script_fields?: any; }; export type SearchResponseBody = { @@ -310,3 +313,37 @@ export type CustomMapping = | CustomBooleanMappingList | CustomBooleanMappingNullable | CustomBooleanMappingNullableList; + +export type CustomScriptEnvironment = 'product' | 'variant' | 'both'; +type CustomScriptMappingDefinition = { + graphQlType: T; + environment: CustomScriptEnvironment; + scriptFn: (...args: Args) => R; +}; + +type CustomScriptStringMapping = CustomScriptMappingDefinition; +type CustomScriptStringMappingNullable = CustomScriptMappingDefinition< + Args, + 'String', + any +>; +type CustomScriptIntMapping = CustomScriptMappingDefinition; +type CustomScriptIntMappingNullable = CustomScriptMappingDefinition; +type CustomScriptFloatMapping = CustomScriptMappingDefinition; +type CustomScriptFloatMappingNullable = CustomScriptMappingDefinition; +type CustomScriptBooleanMapping = CustomScriptMappingDefinition; +type CustomScriptBooleanMappingNullable = CustomScriptMappingDefinition< + Args, + 'Boolean', + any +>; + +export type CustomScriptMapping = + | CustomScriptStringMapping + | CustomScriptStringMappingNullable + | CustomScriptIntMapping + | CustomScriptIntMappingNullable + | CustomScriptFloatMapping + | CustomScriptFloatMappingNullable + | CustomScriptBooleanMapping + | CustomScriptBooleanMappingNullable;