From 8a81d8c701a50e025788cfb67e49a8f84aa80b99 Mon Sep 17 00:00:00 2001 From: Alexia Toulmet Date: Wed, 9 Aug 2023 14:55:20 +0200 Subject: [PATCH] proto-beta: Tokenizer customization --- src/indexes.ts | 130 ++++++++++++++++++++++++++++ src/types/types.ts | 6 ++ tests/dictionary.test.ts | 124 ++++++++++++++++++++++++++ tests/non_separator_tokens.test.ts | 134 +++++++++++++++++++++++++++++ tests/separator_tokens.test.ts | 134 +++++++++++++++++++++++++++++ 5 files changed, 528 insertions(+) create mode 100644 tests/dictionary.test.ts create mode 100644 tests/non_separator_tokens.test.ts create mode 100644 tests/separator_tokens.test.ts diff --git a/src/indexes.ts b/src/indexes.ts index ea732da5b..39b338a7e 100644 --- a/src/indexes.ts +++ b/src/indexes.ts @@ -47,6 +47,9 @@ import { DocumentsDeletionQuery, SearchForFacetValuesParams, SearchForFacetValuesResponse, + SeparatorTokens, + NonSeparatorTokens, + Dictionary, } from './types' import { removeUndefinedFromObject } from './utils' import { HttpRequests } from './http-requests' @@ -1117,6 +1120,133 @@ class Index = Record> { return new EnqueuedTask(task) } + + /// + /// SEPARATOR TOKENS + /// + + /** + * Get the list of all separator tokens. + * + * @returns Promise containing array of separator tokens + */ + async getSeparatorTokens(): Promise { + const url = `indexes/${this.uid}/settings/separator-tokens` + return await this.httpRequest.get(url) + } + + /** + * Update the list of separator tokens. Overwrite the old list. + * + * @param separatorTokens - Array that contains separator tokens. + * @returns Promise containing an EnqueuedTask or null + */ + async updateSeparatorTokens( + separatorTokens: SeparatorTokens + ): Promise { + const url = `indexes/${this.uid}/settings/separator-tokens` + const task = await this.httpRequest.put(url, separatorTokens) + + return new EnqueuedTask(task) + } + + /** + * Reset the separator tokens list to its default value + * + * @returns Promise containing an EnqueuedTask + */ + async resetSeparatorTokens(): Promise { + const url = `indexes/${this.uid}/settings/separator-tokens` + const task = await this.httpRequest.delete(url) + + task.enqueuedAt = new Date(task.enqueuedAt) + + return task + } + + /// + /// NON-SEPARATOR TOKENS + /// + + /** + * Get the list of all non-separator tokens. + * + * @returns Promise containing array of non-separator tokens + */ + async getNonSeparatorTokens(): Promise { + const url = `indexes/${this.uid}/settings/non-separator-tokens` + return await this.httpRequest.get(url) + } + + /** + * Update the list of non-separator tokens. Overwrite the old list. + * + * @param nonSeparatorTokens - Array that contains non-separator tokens. + * @returns Promise containing an EnqueuedTask or null + */ + async updateNonSeparatorTokens( + nonSeparatorTokens: NonSeparatorTokens + ): Promise { + const url = `indexes/${this.uid}/settings/non-separator-tokens` + const task = await this.httpRequest.put(url, nonSeparatorTokens) + + return new EnqueuedTask(task) + } + + /** + * Reset the non-separator tokens list to its default value + * + * @returns Promise containing an EnqueuedTask + */ + async resetNonSeparatorTokens(): Promise { + const url = `indexes/${this.uid}/settings/non-separator-tokens` + const task = await this.httpRequest.delete(url) + + task.enqueuedAt = new Date(task.enqueuedAt) + + return task + } + + /// + /// DICTIONARY + /// + + /** + * Get the dictionary settings of a Meilisearch index. + * + * @returns Promise containing the dictionary settings + */ + async getDictionary(): Promise { + const url = `indexes/${this.uid}/settings/dictionary` + return await this.httpRequest.get(url) + } + + /** + * Update the the dictionary settings. Overwrite the old settings. + * + * @param dictionary - Array that contains the new dictionary settings. + * @returns Promise containing an EnqueuedTask or null + */ + async updateDictionary(dictionary: Dictionary): Promise { + const url = `indexes/${this.uid}/settings/dictionary` + const task = await this.httpRequest.put(url, dictionary) + + return new EnqueuedTask(task) + } + + /** + * Reset the dictionary settings to its default value + * + * @returns Promise containing an EnqueuedTask + */ + async resetDictionary(): Promise { + const url = `indexes/${this.uid}/settings/dictionary` + const task = await this.httpRequest.delete(url) + + task.enqueuedAt = new Date(task.enqueuedAt) + + return task + } } export { Index } diff --git a/src/types/types.ts b/src/types/types.ts index 56696b966..168c4a3dc 100644 --- a/src/types/types.ts +++ b/src/types/types.ts @@ -312,6 +312,9 @@ export type TypoTolerance = { twoTypos?: number | null } } | null +export type SeparatorTokens = string[] | null +export type NonSeparatorTokens = string[] | null +export type Dictionary = string[] | null export type FacetOrder = 'alpha' | 'count' @@ -336,6 +339,9 @@ export type Settings = { typoTolerance?: TypoTolerance faceting?: Faceting pagination?: PaginationSettings + separatorTokens?: SeparatorTokens + nonSeparatorTokens?: NonSeparatorTokens + dictionary?: Dictionary } /* diff --git a/tests/dictionary.test.ts b/tests/dictionary.test.ts new file mode 100644 index 000000000..a2f2e3a9f --- /dev/null +++ b/tests/dictionary.test.ts @@ -0,0 +1,124 @@ +import { EnqueuedTask } from '../src/enqueued-task' +import { + clearAllIndexes, + config, + BAD_HOST, + MeiliSearch, + getClient, + dataset, +} from './utils/meilisearch-test-utils' + +const index = { + uid: 'movies_test', +} + +jest.setTimeout(100 * 1000) + +afterAll(() => { + return clearAllIndexes(config) +}) + +describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( + 'Test on dictionary', + ({ permission }) => { + beforeEach(async () => { + const client = await getClient('Master') + const { taskUid } = await client.index(index.uid).addDocuments(dataset) + await client.waitForTask(taskUid) + }) + + test(`${permission} key: Get default dictionary`, async () => { + const client = await getClient(permission) + const response: string[] = await client.index(index.uid).getDictionary() + + expect(response).toEqual([]) + }) + + test(`${permission} key: Update dictionary`, async () => { + const client = await getClient(permission) + const newDictionary = ['J. K.', 'J. R. R.'] + const task: EnqueuedTask = await client + .index(index.uid) + .updateDictionary(newDictionary) + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client.index(index.uid).getDictionary() + + expect(response).toEqual(newDictionary) + }) + + test(`${permission} key: Update dictionary with null value`, async () => { + const client = await getClient(permission) + const newDictionary = null + const task: EnqueuedTask = await client + .index(index.uid) + .updateDictionary(newDictionary) + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client.index(index.uid).getDictionary() + + expect(response).toEqual([]) + }) + + test(`${permission} key: Reset dictionary`, async () => { + const client = await getClient(permission) + const task: EnqueuedTask = await client.index(index.uid).resetDictionary() + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client.index(index.uid).getDictionary() + + expect(response).toEqual([]) + }) + } +) + +describe.each([ + { host: BAD_HOST, trailing: false }, + { host: `${BAD_HOST}/api`, trailing: false }, + { host: `${BAD_HOST}/trailing/`, trailing: true }, +])('Tests on url construction', ({ host, trailing }) => { + test(`Test getDictionary route`, async () => { + const route = `indexes/${index.uid}/settings/dictionary` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).getDictionary() + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) + + test(`Test updateDictionary route`, async () => { + const route = `indexes/${index.uid}/settings/dictionary` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).updateDictionary([]) + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) + + test(`Test resetDictionary route`, async () => { + const route = `indexes/${index.uid}/settings/dictionary` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).resetDictionary() + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) +}) diff --git a/tests/non_separator_tokens.test.ts b/tests/non_separator_tokens.test.ts new file mode 100644 index 000000000..0037c2f30 --- /dev/null +++ b/tests/non_separator_tokens.test.ts @@ -0,0 +1,134 @@ +import { EnqueuedTask } from '../src/enqueued-task' +import { + clearAllIndexes, + config, + BAD_HOST, + MeiliSearch, + getClient, + dataset, +} from './utils/meilisearch-test-utils' + +const index = { + uid: 'movies_test', +} + +jest.setTimeout(100 * 1000) + +afterAll(() => { + return clearAllIndexes(config) +}) + +describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( + 'Test on non separator tokens', + ({ permission }) => { + beforeEach(async () => { + const client = await getClient('Master') + const { taskUid } = await client.index(index.uid).addDocuments(dataset) + await client.waitForTask(taskUid) + }) + + test(`${permission} key: Get default non separator tokens`, async () => { + const client = await getClient(permission) + const response: string[] = await client + .index(index.uid) + .getNonSeparatorTokens() + + expect(response).toEqual([]) + }) + + test(`${permission} key: Update non separator tokens`, async () => { + const client = await getClient(permission) + const newNonSeparatorTokens = ['&sep', '/', '|'] + const task: EnqueuedTask = await client + .index(index.uid) + .updateNonSeparatorTokens(newNonSeparatorTokens) + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client + .index(index.uid) + .getNonSeparatorTokens() + + expect(response).toEqual(newNonSeparatorTokens) + }) + + test(`${permission} key: Update non separator tokens with null value`, async () => { + const client = await getClient(permission) + const newNonSeparatorTokens = null + const task: EnqueuedTask = await client + .index(index.uid) + .updateNonSeparatorTokens(newNonSeparatorTokens) + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client + .index(index.uid) + .getNonSeparatorTokens() + + expect(response).toEqual([]) + }) + + test(`${permission} key: Reset NonSeparator tokens`, async () => { + const client = await getClient(permission) + const task: EnqueuedTask = await client + .index(index.uid) + .resetNonSeparatorTokens() + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client + .index(index.uid) + .getNonSeparatorTokens() + + expect(response).toEqual([]) + }) + } +) + +describe.each([ + { host: BAD_HOST, trailing: false }, + { host: `${BAD_HOST}/api`, trailing: false }, + { host: `${BAD_HOST}/trailing/`, trailing: true }, +])('Tests on url construction', ({ host, trailing }) => { + test(`Test getNonSeparatorTokens route`, async () => { + const route = `indexes/${index.uid}/settings/non-separator-tokens` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).getNonSeparatorTokens() + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) + + test(`Test updateNonSeparatorTokens route`, async () => { + const route = `indexes/${index.uid}/settings/non-separator-tokens` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).updateNonSeparatorTokens([]) + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) + + test(`Test resetNonSeparatorTokens route`, async () => { + const route = `indexes/${index.uid}/settings/non-separator-tokens` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).resetNonSeparatorTokens() + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) +}) diff --git a/tests/separator_tokens.test.ts b/tests/separator_tokens.test.ts new file mode 100644 index 000000000..ebb0f455b --- /dev/null +++ b/tests/separator_tokens.test.ts @@ -0,0 +1,134 @@ +import { EnqueuedTask } from '../src/enqueued-task' +import { + clearAllIndexes, + config, + BAD_HOST, + MeiliSearch, + getClient, + dataset, +} from './utils/meilisearch-test-utils' + +const index = { + uid: 'movies_test', +} + +jest.setTimeout(100 * 1000) + +afterAll(() => { + return clearAllIndexes(config) +}) + +describe.each([{ permission: 'Master' }, { permission: 'Admin' }])( + 'Test on separator tokens', + ({ permission }) => { + beforeEach(async () => { + const client = await getClient('Master') + const { taskUid } = await client.index(index.uid).addDocuments(dataset) + await client.waitForTask(taskUid) + }) + + test(`${permission} key: Get default separator tokens`, async () => { + const client = await getClient(permission) + const response: string[] = await client + .index(index.uid) + .getSeparatorTokens() + + expect(response).toEqual([]) + }) + + test(`${permission} key: Update separator tokens`, async () => { + const client = await getClient(permission) + const newSeparatorTokens = ['&sep', '/', '|'] + const task: EnqueuedTask = await client + .index(index.uid) + .updateSeparatorTokens(newSeparatorTokens) + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client + .index(index.uid) + .getSeparatorTokens() + + expect(response).toEqual(newSeparatorTokens) + }) + + test(`${permission} key: Update separator tokens with null value`, async () => { + const client = await getClient(permission) + const newSeparatorTokens = null + const task: EnqueuedTask = await client + .index(index.uid) + .updateSeparatorTokens(newSeparatorTokens) + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client + .index(index.uid) + .getSeparatorTokens() + + expect(response).toEqual([]) + }) + + test(`${permission} key: Reset separator tokens`, async () => { + const client = await getClient(permission) + const task: EnqueuedTask = await client + .index(index.uid) + .resetSeparatorTokens() + await client.index(index.uid).waitForTask(task.taskUid) + + const response: string[] = await client + .index(index.uid) + .getSeparatorTokens() + + expect(response).toEqual([]) + }) + } +) + +describe.each([ + { host: BAD_HOST, trailing: false }, + { host: `${BAD_HOST}/api`, trailing: false }, + { host: `${BAD_HOST}/trailing/`, trailing: true }, +])('Tests on url construction', ({ host, trailing }) => { + test(`Test getSeparatorTokens route`, async () => { + const route = `indexes/${index.uid}/settings/separator-tokens` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).getSeparatorTokens() + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) + + test(`Test updateSeparatorTokens route`, async () => { + const route = `indexes/${index.uid}/settings/separator-tokens` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).updateSeparatorTokens([]) + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) + + test(`Test resetSeparatorTokens route`, async () => { + const route = `indexes/${index.uid}/settings/separator-tokens` + const client = new MeiliSearch({ host }) + const strippedHost = trailing ? host.slice(0, -1) : host + await expect( + client.index(index.uid).resetSeparatorTokens() + ).rejects.toHaveProperty( + 'message', + `request to ${strippedHost}/${route} failed, reason: connect ECONNREFUSED ${BAD_HOST.replace( + 'http://', + '' + )}` + ) + }) +})