From 4865c6edae97878d1a5e27b5e132c140635611b3 Mon Sep 17 00:00:00 2001 From: Kevin Mattutat Date: Mon, 11 Oct 2021 13:13:27 +0200 Subject: [PATCH] feat(elasticsearch): Extend search config with fields for script evaluation of every hit (#1143) --- .../e2e/elasticsearch-plugin.e2e-spec.ts | 42 +++++++++++-- .../src/build-elastic-body.spec.ts | 25 +++++++- .../src/build-elastic-body.ts | 40 ++++++++++++- .../src/custom-script-fields.resolver.ts | 26 ++++++++ .../src/elasticsearch.service.ts | 60 ++++++++++++++++++- .../src/graphql-schema-extensions.ts | 47 ++++++++++++++- .../src/indexer.controller.ts | 7 ++- packages/elasticsearch-plugin/src/options.ts | 39 +++++++++++- packages/elasticsearch-plugin/src/plugin.ts | 19 +++++- packages/elasticsearch-plugin/src/types.ts | 40 ++++++++++++- 10 files changed, 330 insertions(+), 15 deletions(-) create mode 100644 packages/elasticsearch-plugin/src/custom-script-fields.resolver.ts diff --git a/packages/elasticsearch-plugin/e2e/elasticsearch-plugin.e2e-spec.ts b/packages/elasticsearch-plugin/e2e/elasticsearch-plugin.e2e-spec.ts index 61bdb32169..081b04eb64 100644 --- a/packages/elasticsearch-plugin/e2e/elasticsearch-plugin.e2e-spec.ts +++ b/packages/elasticsearch-plugin/e2e/elasticsearch-plugin.e2e-spec.ts @@ -38,7 +38,7 @@ import { UpdateCollection, UpdateProduct, UpdateProductVariants, - UpdateTaxRate + UpdateTaxRate, } from '../../core/e2e/graphql/generated-e2e-admin-types'; import { SearchProductsShop } from '../../core/e2e/graphql/generated-e2e-shop-types'; import { @@ -130,6 +130,17 @@ describe('Elasticsearch plugin', () => { }, }, }, + searchConfig: { + scriptFields: { + answerDouble: { + graphQlType: 'Int!', + environment: 'product', + scriptFn: input => ({ + script: `doc['answer'].value * 2`, + }), + }, + }, + }, }), DefaultJobQueuePlugin, ], @@ -277,8 +288,8 @@ describe('Elasticsearch plugin', () => { }, ); expect(result.search.collections).toEqual([ - {collection: {id: 'T_2', name: 'Plants',},count: 3,}, - ]); + { collection: { id: 'T_2', name: 'Plants' }, count: 3 }, + ]); }); it('returns correct collections when grouped by product', async () => { @@ -291,7 +302,7 @@ describe('Elasticsearch plugin', () => { }, ); expect(result.search.collections).toEqual([ - {collection: {id: 'T_2', name: 'Plants',},count: 3,}, + { collection: { id: 'T_2', name: 'Plants' }, count: 3 }, ]); }); @@ -1242,6 +1253,29 @@ describe('Elasticsearch plugin', () => { }); }); }); + + describe('scriptFields', () => { + it('script mapping', async () => { + const query = `{ + search(input: { take: 1, groupByProduct: true, sort: { name: ASC } }) { + items { + productVariantName + customScriptFields { + answerDouble + } + } + } + }`; + const { search } = await shopClient.query(gql(query)); + + expect(search.items[0]).toEqual({ + productVariantName: 'Bonsai Tree', + customScriptFields: { + answerDouble: 84, + }, + }); + }); + }); }); export const SEARCH_PRODUCTS = gql` diff --git a/packages/elasticsearch-plugin/src/build-elastic-body.spec.ts b/packages/elasticsearch-plugin/src/build-elastic-body.spec.ts index 9166f8f19a..31d4e83c44 100644 --- a/packages/elasticsearch-plugin/src/build-elastic-body.spec.ts +++ b/packages/elasticsearch-plugin/src/build-elastic-body.spec.ts @@ -116,7 +116,7 @@ describe('buildElasticBody()', () => { it('facetValueFilters OR', () => { const result = buildElasticBody( - { facetValueFilters: [ { or: ['1', '2'] }] }, + { facetValueFilters: [{ or: ['1', '2'] }] }, searchConfig, CHANNEL_ID, LanguageCode.en, @@ -386,6 +386,29 @@ describe('buildElasticBody()', () => { }); }); + it('scriptFields option', () => { + const config: DeepRequired = { + ...searchConfig, + ...{ + scriptFields: { + test: { + graphQlType: 'String', + environment: 'both', + scriptFn: input => ({ + script: `doc['property'].dummyScript(${input.term})`, + }), + }, + }, + }, + }; + const result = buildElasticBody({ term: 'test' }, config, CHANNEL_ID, LanguageCode.en); + expect(result.script_fields).toEqual({ + test: { + script: `doc['property'].dummyScript(test)`, + }, + }); + }); + describe('price ranges', () => { it('not grouped by product', () => { const result = buildElasticBody( diff --git a/packages/elasticsearch-plugin/src/build-elastic-body.ts b/packages/elasticsearch-plugin/src/build-elastic-body.ts index a2084f22ce..01f19c5296 100644 --- a/packages/elasticsearch-plugin/src/build-elastic-body.ts +++ b/packages/elasticsearch-plugin/src/build-elastic-body.ts @@ -2,7 +2,7 @@ import { LanguageCode, LogicalOperator, PriceRange, SortOrder } from '@vendure/c import { DeepRequired, ID, UserInputError } from '@vendure/core'; import { SearchConfig } from './options'; -import { ElasticSearchInput, SearchRequestBody } from './types'; +import { CustomScriptMapping, ElasticSearchInput, SearchRequestBody } from './types'; /** * Given a SearchInput object, returns the corresponding Elasticsearch body. @@ -113,6 +113,11 @@ export function buildElasticBody( sortArray.push({ [priceField]: { order: sort.price === SortOrder.ASC ? 'asc' : 'desc' } }); } } + const scriptFields: any | undefined = createScriptFields( + searchConfig.scriptFields, + input, + groupByProduct, + ); return { query: searchConfig.mapQuery ? searchConfig.mapQuery(query, input, searchConfig, channelId, enabledOnly) @@ -121,6 +126,12 @@ export function buildElasticBody( from: skip || 0, size: take || 10, track_total_hits: searchConfig.totalItemsMaxSize, + ...(scriptFields !== undefined + ? { + _source: true, + script_fields: scriptFields, + } + : undefined), }; } @@ -130,6 +141,33 @@ function ensureBoolFilterExists(query: { bool: { filter?: any } }) { } } +function createScriptFields( + scriptFields: { [fieldName: string]: CustomScriptMapping<[ElasticSearchInput]> }, + input: ElasticSearchInput, + groupByProduct?: boolean, +): any | undefined { + if (scriptFields) { + const fields = Object.keys(scriptFields); + if (fields.length) { + const result: any = {}; + for (const name of fields) { + const scriptField = scriptFields[name]; + if (scriptField.environment === 'product' && groupByProduct === true) { + (result as any)[name] = scriptField.scriptFn(input); + } + if (scriptField.environment === 'variant' && groupByProduct === false) { + (result as any)[name] = scriptField.scriptFn(input); + } + if (scriptField.environment === 'both' || scriptField.environment === undefined) { + (result as any)[name] = scriptField.scriptFn(input); + } + } + return result; + } + } + return undefined; +} + function createPriceFilters(range: PriceRange, withTax: boolean, groupByProduct: boolean): any[] { const withTaxFix = withTax ? 'WithTax' : ''; if (groupByProduct) { diff --git a/packages/elasticsearch-plugin/src/custom-script-fields.resolver.ts b/packages/elasticsearch-plugin/src/custom-script-fields.resolver.ts new file mode 100644 index 0000000000..d5d5fe335d --- /dev/null +++ b/packages/elasticsearch-plugin/src/custom-script-fields.resolver.ts @@ -0,0 +1,26 @@ +import { Inject } from '@nestjs/common'; +import { ResolveField, Resolver } from '@nestjs/graphql'; +import { DeepRequired } from '@vendure/common/lib/shared-types'; + +import { ELASTIC_SEARCH_OPTIONS } from './constants'; +import { ElasticsearchOptions } from './options'; + +/** + * This resolver is only required if scriptFields are defined for both products and product variants. + * This particular configuration will result in a union type for the + * `SearchResult.customScriptFields` GraphQL field. + */ +@Resolver('CustomScriptFields') +export class CustomScriptFieldsResolver { + constructor(@Inject(ELASTIC_SEARCH_OPTIONS) private options: DeepRequired) {} + + @ResolveField() + __resolveType(value: any): string { + const productScriptFields = Object.entries(this.options.searchConfig?.scriptFields || {}) + .filter(([, scriptField]) => scriptField.environment !== 'variant') + .map(([k]) => k); + return Object.keys(value).every(k => productScriptFields.includes(k)) + ? 'CustomProductScriptFields' + : 'CustomProductVariantScriptFields'; + } +} diff --git a/packages/elasticsearch-plugin/src/elasticsearch.service.ts b/packages/elasticsearch-plugin/src/elasticsearch.service.ts index ffdc63eaa7..68bdb38b5c 100644 --- a/packages/elasticsearch-plugin/src/elasticsearch.service.ts +++ b/packages/elasticsearch-plugin/src/elasticsearch.service.ts @@ -23,6 +23,8 @@ import { createIndices, getClient } from './indexing-utils'; import { ElasticsearchOptions } from './options'; import { CustomMapping, + CustomScriptEnvironment, + CustomScriptMapping, ElasticSearchInput, ElasticSearchResponse, ProductIndexItem, @@ -195,7 +197,17 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { totalItems: body.hits.total ? body.hits.total.value : 0, }; } catch (e) { - Logger.error(e.message, loggerCtx, e.stack); + if (e.meta.body.error.type && e.meta.body.error.type === 'search_phase_execution_exception') { + // Log runtime error of the script exception instead of stacktrace + Logger.error( + e.message, + loggerCtx, + JSON.stringify(e.meta.body.error.root_cause || [], null, 2), + ); + Logger.verbose(JSON.stringify(e.meta.body.error.failed_shards || [], null, 2), loggerCtx); + } else { + Logger.error(e.message, loggerCtx, e.stack); + } throw e; } } else { @@ -209,7 +221,17 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { totalItems: body.hits.total ? body.hits.total.value : 0, }; } catch (e) { - Logger.error(e.message, loggerCtx, e.stack); + if (e.meta.body.error.type && e.meta.body.error.type === 'search_phase_execution_exception') { + // Log runtime error of the script exception instead of stacktrace + Logger.error( + e.message, + loggerCtx, + JSON.stringify(e.meta.body.error.root_cause || [], null, 2), + ); + Logger.verbose(JSON.stringify(e.meta.body.error.failed_shards || [], null, 2), loggerCtx); + } else { + Logger.error(e.message, loggerCtx, e.stack); + } throw e; } } @@ -409,6 +431,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { private mapVariantToSearchResult(hit: SearchHit): SearchResult { const source = hit._source; + const fields = hit.fields; const { productAsset, productVariantAsset } = this.getSearchResultAssets(source); const result = { ...source, @@ -424,11 +447,13 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { }; this.addCustomMappings(result, source, this.options.customProductVariantMappings); + this.addScriptMappings(result, fields, this.options.searchConfig?.scriptFields, 'variant'); return result; } private mapProductToSearchResult(hit: SearchHit): SearchResult { const source = hit._source; + const fields = hit.fields; const { productAsset, productVariantAsset } = this.getSearchResultAssets(source); const result = { ...source, @@ -455,6 +480,7 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { score: hit._score || 0, }; this.addCustomMappings(result, source, this.options.customProductMappings); + this.addScriptMappings(result, fields, this.options.searchConfig?.scriptFields, 'product'); return result; } @@ -494,4 +520,34 @@ export class ElasticsearchService implements OnModuleInit, OnModuleDestroy { } return result; } + + private addScriptMappings( + result: any, + fields: any, + mappings: { [fieldName: string]: CustomScriptMapping }, + environment: CustomScriptEnvironment, + ): any { + const customMappings = Object.keys(mappings || {}); + if (customMappings.length) { + const customScriptFieldsResult: any = {}; + for (const name of customMappings) { + const env = mappings[name].environment; + if (env === environment || env === 'both') { + const fieldVal = (fields as any)[name] || undefined; + if (Array.isArray(fieldVal)) { + if (fieldVal.length === 1) { + customScriptFieldsResult[name] = fieldVal[0]; + } + if (fieldVal.length > 1) { + customScriptFieldsResult[name] = JSON.stringify(fieldVal); + } + } else { + customScriptFieldsResult[name] = fieldVal; + } + } + } + (result as any).customScriptFields = customScriptFieldsResult; + } + return result; + } } diff --git a/packages/elasticsearch-plugin/src/graphql-schema-extensions.ts b/packages/elasticsearch-plugin/src/graphql-schema-extensions.ts index e295d6595f..2cfb44afb6 100644 --- a/packages/elasticsearch-plugin/src/graphql-schema-extensions.ts +++ b/packages/elasticsearch-plugin/src/graphql-schema-extensions.ts @@ -39,8 +39,53 @@ export function generateSchemaExtensions(options: ElasticsearchOptions): Documen function generateCustomMappingTypes(options: ElasticsearchOptions): DocumentNode | undefined { const productMappings = Object.entries(options.customProductMappings || {}); const variantMappings = Object.entries(options.customProductVariantMappings || {}); + const scriptProductFields = Object.entries(options.searchConfig?.scriptFields || {}).filter( + ([, scriptField]) => scriptField.environment !== 'variant', + ); + const scriptVariantFields = Object.entries(options.searchConfig?.scriptFields || {}).filter( + ([, scriptField]) => scriptField.environment !== 'product', + ); + let sdl = ``; + + if (scriptProductFields.length || scriptVariantFields.length) { + if (scriptProductFields.length) { + sdl += ` + type CustomProductScriptFields { + ${scriptProductFields.map(([name, def]) => `${name}: ${def.graphQlType}`)} + } + `; + } + if (scriptVariantFields.length) { + sdl += ` + type CustomProductVariantScriptFields { + ${scriptVariantFields.map(([name, def]) => `${name}: ${def.graphQlType}`)} + } + `; + } + if (scriptProductFields.length && scriptVariantFields.length) { + sdl += ` + union CustomScriptFields = CustomProductScriptFields | CustomProductVariantScriptFields + + extend type SearchResult { + customScriptFields: CustomScriptFields! + } + `; + } else if (scriptProductFields.length) { + sdl += ` + extend type SearchResult { + customScriptFields: CustomProductScriptFields! + } + `; + } else if (scriptVariantFields.length) { + sdl += ` + extend type SearchResult { + customScriptFields: CustomProductVariantScriptFields! + } + `; + } + } + if (productMappings.length || variantMappings.length) { - let sdl = ``; if (productMappings.length) { sdl += ` type CustomProductMappings { diff --git a/packages/elasticsearch-plugin/src/indexer.controller.ts b/packages/elasticsearch-plugin/src/indexer.controller.ts index bef99fd5eb..acef2b20fb 100644 --- a/packages/elasticsearch-plugin/src/indexer.controller.ts +++ b/packages/elasticsearch-plugin/src/indexer.controller.ts @@ -882,7 +882,7 @@ export class ElasticsearchIndexerController implements OnModuleInit, OnModuleDes languageCode: LanguageCode, ): ProductIndexItem { const productTranslation = this.getTranslation(product, ctx.languageCode); - return { + const item: ProductIndexItem = { channelId: ctx.channelId, languageCode, sku: '', @@ -910,6 +910,11 @@ export class ElasticsearchIndexerController implements OnModuleInit, OnModuleDes channelIds: [ctx.channelId], enabled: false, }; + const customMappings = Object.entries(this.options.customProductMappings); + for (const [name, def] of customMappings) { + item[name] = def.valueFn(product, [], languageCode); + } + return item; } private getTranslation( diff --git a/packages/elasticsearch-plugin/src/options.ts b/packages/elasticsearch-plugin/src/options.ts index df1ab4f705..c2a4b0f3a7 100644 --- a/packages/elasticsearch-plugin/src/options.ts +++ b/packages/elasticsearch-plugin/src/options.ts @@ -2,7 +2,7 @@ import { ClientOptions } from '@elastic/elasticsearch'; import { DeepRequired, ID, LanguageCode, Product, ProductVariant } from '@vendure/core'; import deepmerge from 'deepmerge'; -import { CustomMapping, ElasticSearchInput } from './types'; +import { CustomMapping, CustomScriptMapping, ElasticSearchInput } from './types'; /** * @description @@ -346,6 +346,42 @@ export interface SearchConfig { channelId: ID, enabledOnly: boolean, ) => any; + /** + * @description + * Sets `script_fields` inside the elasticsearch body which allows returning a script evaluation for each hit + * @since 1.2.4 + * @example + * ```TypeScript + * indexMappingProperties: { + * location: { + * type: 'geo_point', // contains function arcDistance + * }, + * }, + * customProductMappings: { + * location: { + * graphQlType: 'String', + * valueFn: (product: Product) => { + * const custom = product.customFields.location; + * return `${custom.latitude},${location.longitude}`; + * }, + * } + * }, + * scriptFields: { + * distance: { + * graphQlType: 'Number' + * valFn: (input) => { + * // assuming SearchInput was extended with latitude and longitude + * const lat = input.latitude; + * const lon = input.longitude; + * return { + * script: `doc['location'].arcDistance(${lat}, ${lon})`, + * } + * } + * } + * } + * ``` + */ + scriptFields?: { [fieldName: string]: CustomScriptMapping<[ElasticSearchInput]> }; } /** @@ -415,6 +451,7 @@ export const defaultOptions: ElasticsearchRuntimeOptions = { }, priceRangeBucketInterval: 1000, mapQuery: query => query, + scriptFields: {}, }, customProductMappings: {}, customProductVariantMappings: {}, diff --git a/packages/elasticsearch-plugin/src/plugin.ts b/packages/elasticsearch-plugin/src/plugin.ts index 356a05a54a..fa2131b06d 100644 --- a/packages/elasticsearch-plugin/src/plugin.ts +++ b/packages/elasticsearch-plugin/src/plugin.ts @@ -21,6 +21,7 @@ import { buffer, debounceTime, delay, filter, map } from 'rxjs/operators'; import { ELASTIC_SEARCH_OPTIONS, loggerCtx } from './constants'; import { CustomMappingsResolver } from './custom-mappings.resolver'; +import { CustomScriptFieldsResolver } from './custom-script-fields.resolver'; import { ElasticsearchIndexService } from './elasticsearch-index.service'; import { AdminElasticSearchResolver, @@ -210,9 +211,21 @@ import { ElasticsearchOptions, ElasticsearchRuntimeOptions, mergeWithDefaults } const requiresUnionResolver = 0 < Object.keys(options.customProductMappings || {}).length && 0 < Object.keys(options.customProductVariantMappings || {}).length; - return requiresUnionResolver - ? [ShopElasticSearchResolver, EntityElasticSearchResolver, CustomMappingsResolver] - : [ShopElasticSearchResolver, EntityElasticSearchResolver]; + const requiresUnionScriptResolver = + 0 < + Object.values(options.searchConfig.scriptFields || {}).filter( + field => field.environment !== 'product', + ).length && + 0 < + Object.values(options.searchConfig.scriptFields || {}).filter( + field => field.environment !== 'variant', + ).length; + return [ + ShopElasticSearchResolver, + EntityElasticSearchResolver, + ...(requiresUnionResolver ? [CustomMappingsResolver] : []), + ...(requiresUnionScriptResolver ? [CustomScriptFieldsResolver] : []), + ]; }, // `any` cast is there due to a strange error "Property '[Symbol.iterator]' is missing in type... URLSearchParams" // which looks like possibly a TS/definitions bug. diff --git a/packages/elasticsearch-plugin/src/types.ts b/packages/elasticsearch-plugin/src/types.ts index 0706a8181c..a458b4eacf 100644 --- a/packages/elasticsearch-plugin/src/types.ts +++ b/packages/elasticsearch-plugin/src/types.ts @@ -83,6 +83,7 @@ export type SearchHit = { _score: number; _source: T; _type: string; + fields?: any; }; export type SearchRequestBody = { @@ -92,6 +93,8 @@ export type SearchRequestBody = { size?: number; track_total_hits?: number | boolean; aggs?: any; + _source?: boolean; + script_fields?: any; }; export type SearchResponseBody = { @@ -191,7 +194,8 @@ export interface UpdateAssetMessageData { } type Maybe = T | undefined; -type CustomMappingDefinition = { +type CustomMappingTypes = 'String' | 'String!' | 'Int' | 'Int!' | 'Float' | 'Float!' | 'Boolean' | 'Boolean!'; +type CustomMappingDefinition = { graphQlType: T; valueFn: (...args: Args) => R; }; @@ -246,3 +250,37 @@ export type CustomMapping = | CustomFloatMappingNullable | CustomBooleanMapping | CustomBooleanMappingNullable; + +export type CustomScriptEnvironment = 'product' | 'variant' | 'both'; +type CustomScriptMappingDefinition = { + graphQlType: T; + environment: CustomScriptEnvironment; + scriptFn: (...args: Args) => R; +}; + +type CustomScriptStringMapping = CustomScriptMappingDefinition; +type CustomScriptStringMappingNullable = CustomScriptMappingDefinition< + Args, + 'String', + any +>; +type CustomScriptIntMapping = CustomScriptMappingDefinition; +type CustomScriptIntMappingNullable = CustomScriptMappingDefinition; +type CustomScriptFloatMapping = CustomScriptMappingDefinition; +type CustomScriptFloatMappingNullable = CustomScriptMappingDefinition; +type CustomScriptBooleanMapping = CustomScriptMappingDefinition; +type CustomScriptBooleanMappingNullable = CustomScriptMappingDefinition< + Args, + 'Boolean', + any +>; + +export type CustomScriptMapping = + | CustomScriptStringMapping + | CustomScriptStringMappingNullable + | CustomScriptIntMapping + | CustomScriptIntMappingNullable + | CustomScriptFloatMapping + | CustomScriptFloatMappingNullable + | CustomScriptBooleanMapping + | CustomScriptBooleanMappingNullable;