diff --git a/packages/migration/package.json b/packages/migration/package.json index af36cc12606..8e0f33f6566 100644 --- a/packages/migration/package.json +++ b/packages/migration/package.json @@ -18,6 +18,7 @@ "status:openhim": "migrate-mongo status -f ./src/migrate-mongo-config-openhim.js", "status:user-mgnt": "migrate-mongo status -f ./src/migrate-mongo-config-user-mgnt.js", "status:application-config": "migrate-mongo status -f ./src/migrate-mongo-config-application-config.js", + "reindex-search": "tsx src/reindex-search.ts", "precommit": "lint-staged", "test:compilation": "tsc --noEmit", "build": "rimraf build && tsc" @@ -36,6 +37,7 @@ "migrate-mongo": "^10.0.0", "minio": "^7.0.33", "mongoose": "^6.11.3", + "tsx": "^4.15.7", "uuid": "^3.2.1" }, "devDependencies": { diff --git a/packages/migration/run-migrations.sh b/packages/migration/run-migrations.sh index 44e86c2f498..5d479c6415b 100755 --- a/packages/migration/run-migrations.sh +++ b/packages/migration/run-migrations.sh @@ -37,3 +37,6 @@ yarn --cwd $SCRIPT_PATH migrate-mongo status --file $USER_MGNT_CONFIG # performance migration yarn --cwd $SCRIPT_PATH migrate-mongo up --file $PERFORMANCE_CONFIG yarn --cwd $SCRIPT_PATH migrate-mongo status --file $PERFORMANCE_CONFIG + +# search migration / reindex +yarn --cwd $SCRIPT_PATH reindex-search diff --git a/packages/migration/src/migrations/hearth/20240527130750-reindex-elasticsearch.ts b/packages/migration/src/migrations/hearth/20240527130750-reindex-elasticsearch.ts index fe9eec630ed..51b5ab0f286 100644 --- a/packages/migration/src/migrations/hearth/20240527130750-reindex-elasticsearch.ts +++ b/packages/migration/src/migrations/hearth/20240527130750-reindex-elasticsearch.ts @@ -10,7 +10,7 @@ */ import { Db, MongoClient } from 'mongodb' -import { reindex, client } from '../../utils/elasticsearch-helper.js' +import { client } from '../../utils/elasticsearch-helper.js' const ELASTICSEARCH_INDEX_NAME = 'ocrvs' @@ -63,7 +63,6 @@ const migrateIndexToAlias = async (timestamp: string) => { export const up = async (db: Db, _client: MongoClient) => { await createEmptyIndex() await migrateIndexToAlias('20240514125702-old-format') - await reindex('20240514125703') } export const down = async (db: Db, _client: MongoClient) => { diff --git a/packages/migration/src/reindex-search.ts b/packages/migration/src/reindex-search.ts new file mode 100644 index 00000000000..58a086c08a9 --- /dev/null +++ b/packages/migration/src/reindex-search.ts @@ -0,0 +1,70 @@ +/* + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. + * + * OpenCRVS is also distributed under the terms of the Civil Registration + * & Healthcare Disclaimer located at http://opencrvs.org/license. + * + * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. + */ + +const SEARCH_URL = process.env.SEARCH_URL || 'http://localhost:9090/' + +/** + * Streams MongoDB collections to ElasticSearch documents. Useful when the ElasticSearch schema changes. + */ +const triggerReindex = async () => { + const response = await fetch(new URL('reindex', SEARCH_URL), { + method: 'POST' + }) + + if (!response.ok) { + throw new Error( + `Problem reindexing ElasticSearch. Response: ${await response.text()}` + ) + } + + const data = await response.json() + return data.jobId +} + +/** + * Checks the status of the reindex, as it can take a while + */ +const checkReindexStatus = async (jobId: string) => { + const response = await fetch(new URL(`reindex/status/${jobId}`, SEARCH_URL), { + method: 'GET' + }) + + if (!response.ok) { + throw new Error( + `Problem checking reindex status from ElasticSearch. Response: ${await response.text()}` + ) + } + + const data = await response.json() + return data.status === 'completed' +} + +async function main() { + console.info(`Reindexing search...`) + const jobId = await triggerReindex() + await new Promise((resolve, reject) => { + const intervalId = setInterval(async () => { + try { + const isCompleted = await checkReindexStatus(jobId) + if (isCompleted) { + clearInterval(intervalId) + resolve() + } + } catch (error) { + clearInterval(intervalId) + reject(error) + } + }, 1000) + }) + console.info(`...done reindexing search with job id ${jobId}`) +} + +main() diff --git a/packages/migration/src/utils/elasticsearch-helper.ts b/packages/migration/src/utils/elasticsearch-helper.ts index 9d1cac39e8d..748e1cd78d0 100644 --- a/packages/migration/src/utils/elasticsearch-helper.ts +++ b/packages/migration/src/utils/elasticsearch-helper.ts @@ -11,7 +11,6 @@ import { Client } from '@elastic/elasticsearch' -const SEARCH_URL = process.env.SEARCH_URL || 'http://localhost:9090/' const ES_HOST = process.env.ES_HOST || 'localhost:9200' const ELASTICSEARCH_INDEX_NAME = 'ocrvs' @@ -107,61 +106,3 @@ export const searchCompositionByCriteria = async ( return null } } - -/** - * Streams MongoDB collections to ElasticSearch documents. Useful when the ElasticSearch schema changes. - */ -export const triggerReindex = async (timestamp: string) => { - const response = await fetch(new URL('reindex', SEARCH_URL), { - method: 'POST', - body: JSON.stringify({ timestamp }), - headers: { 'Content-Type': 'application/json' } - }) - - if (!response.ok) { - throw new Error( - `Problem reindexing ElasticSearch. Response: ${await response.text()}` - ) - } - - const data = await response.json() - return data.jobId -} - -/** - * Checks the status of the reindex, as it can take a while - */ -export const checkReindexStatus = async (jobId: string) => { - const response = await fetch(new URL(`reindex/status/${jobId}`, SEARCH_URL), { - method: 'GET' - }) - - if (!response.ok) { - throw new Error( - `Problem checking reindex status from ElasticSearch. Response: ${await response.text()}` - ) - } - - const data = await response.json() - return data.status === 'completed' -} - -export const reindex = async (timestamp: string) => { - console.info(`Reindexing ${timestamp}...`) - const jobId = await triggerReindex(timestamp) - await new Promise((resolve, reject) => { - const intervalId = setInterval(async () => { - try { - const isCompleted = await checkReindexStatus(jobId) - if (isCompleted) { - clearInterval(intervalId) - resolve() - } - } catch (error) { - clearInterval(intervalId) - reject(error) - } - }, 1000) - }) - console.info(`...done reindexing ${timestamp} (job id ${jobId})`) -} diff --git a/packages/search/src/config/routes.ts b/packages/search/src/config/routes.ts index 89b4a02c9b6..045ad5842f6 100644 --- a/packages/search/src/config/routes.ts +++ b/packages/search/src/config/routes.ts @@ -288,12 +288,7 @@ export const getRoutes = () => { handler: reindexHandler, config: { tags: ['api'], - auth: false, - validate: { - payload: Joi.object({ - timestamp: Joi.string() - }) - } + auth: false } }, { diff --git a/packages/search/src/features/reindex/handler.ts b/packages/search/src/features/reindex/handler.ts index b96c2f97d5c..af767fc7e2e 100644 --- a/packages/search/src/features/reindex/handler.ts +++ b/packages/search/src/features/reindex/handler.ts @@ -20,16 +20,15 @@ const indexingStatuses: Record< > = {} export async function reindexHandler( - request: Hapi.Request, + _request: Hapi.Request, h: Hapi.ResponseToolkit ) { - const { timestamp } = request.payload as { timestamp: string } const jobId = uuid() process.nextTick(async () => { try { indexingStatuses[jobId] = 'started' - await reindex(timestamp) + await reindex() await updateAliases() await prune() indexingStatuses[jobId] = 'completed' @@ -41,7 +40,7 @@ export async function reindexHandler( return h .response({ - message: `ElasticSearch reindexing started for timestamp ${timestamp}`, + message: `ElasticSearch reindexing started for job ${jobId}`, status: indexingStatuses[jobId] ?? 'accepted', jobId }) diff --git a/packages/search/src/features/reindex/reindex.ts b/packages/search/src/features/reindex/reindex.ts index 9555e844cf1..944daa2cb87 100644 --- a/packages/search/src/features/reindex/reindex.ts +++ b/packages/search/src/features/reindex/reindex.ts @@ -21,6 +21,7 @@ import { logger } from '@opencrvs/commons' import { getEventType } from '@search/utils/event' import { Transform } from 'stream' import { orderBy } from 'lodash' +import { format } from 'date-fns' const eventTransformers = { [EVENT_TYPE.BIRTH]: composeBirthDocument, @@ -28,13 +29,13 @@ const eventTransformers = { [EVENT_TYPE.MARRIAGE]: composeMarriageDocument } satisfies Record SearchDocument> -export const formatIndexName = (timestamp: string) => - `${OPENCRVS_INDEX_NAME}-${timestamp}` +export const formatIndexName = () => + `${OPENCRVS_INDEX_NAME}-${format(new Date(), 'yyyyMMddHHmmss')}` /** Streams the MongoDB records to ElasticSearch */ -export const reindex = async (timestamp: string) => { +export const reindex = async () => { const t1 = performance.now() - const index = formatIndexName(timestamp) + const index = formatIndexName() logger.info(`Reindexing to ${index}`) const stream = await streamAllRecords(true) diff --git a/yarn.lock b/yarn.lock index 334fc1e228e..9f629864d0a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -12780,7 +12780,7 @@ esbuild@^0.18.0, esbuild@^0.18.10: "@esbuild/win32-ia32" "0.18.20" "@esbuild/win32-x64" "0.18.20" -esbuild@^0.21.3: +esbuild@^0.21.3, esbuild@~0.21.4: version "0.21.5" resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.21.5.tgz#9ca301b120922959b766360d8ac830da0d02997d" integrity sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw== @@ -14220,6 +14220,13 @@ get-symbol-description@^1.0.2: es-errors "^1.3.0" get-intrinsic "^1.2.4" +get-tsconfig@^4.7.5: + version "4.7.5" + resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.7.5.tgz#5e012498579e9a6947511ed0cd403272c7acbbaf" + integrity sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw== + dependencies: + resolve-pkg-maps "^1.0.0" + get-value@^2.0.3, get-value@^2.0.6: version "2.0.6" resolved "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz" @@ -21869,6 +21876,11 @@ resolve-pathname@^3.0.0: resolved "https://registry.npmjs.org/resolve-pathname/-/resolve-pathname-3.0.0.tgz" integrity sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng== +resolve-pkg-maps@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f" + integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw== + resolve-url@^0.2.1: version "0.2.1" resolved "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz" @@ -23084,7 +23096,7 @@ string-similarity@^4.0.1: resolved "https://registry.npmjs.org/string-similarity/-/string-similarity-4.0.4.tgz" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -23102,6 +23114,15 @@ string-width@^1.0.1: is-fullwidth-code-point "^1.0.0" strip-ansi "^3.0.0" +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: + version "4.2.3" + resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + string-width@^2.1.1: version "2.1.1" resolved "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz" @@ -23261,7 +23282,7 @@ stringify-object@^3.2.2, stringify-object@^3.3.0: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -23289,6 +23310,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz" @@ -24189,6 +24217,16 @@ tsutils@^3.21.0: dependencies: tslib "^1.8.1" +tsx@^4.15.7: + version "4.15.7" + resolved "https://registry.yarnpkg.com/tsx/-/tsx-4.15.7.tgz#69d7499196a323507c4051d2ba10753edcc057e5" + integrity sha512-u3H0iSFDZM3za+VxkZ1kywdCeHCn+8/qHQS1MNoO2sONDgD95HlWtt8aB23OzeTmFP9IU4/8bZUdg58Uu5J4cg== + dependencies: + esbuild "~0.21.4" + get-tsconfig "^4.7.5" + optionalDependencies: + fsevents "~2.3.3" + tuf-js@^1.1.7: version "1.1.7" resolved "https://registry.yarnpkg.com/tuf-js/-/tuf-js-1.1.7.tgz#21b7ae92a9373015be77dfe0cb282a80ec3bbe43" @@ -25526,7 +25564,7 @@ workbox-window@7.1.0, workbox-window@^7.1.0: "@types/trusted-types" "^2.0.2" workbox-core "7.1.0" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -25561,6 +25599,15 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.0.1, wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz"