From 0adf8e962434e62f37f48feea5b024b7b66b61ca Mon Sep 17 00:00:00 2001 From: naftis Date: Wed, 26 Jun 2024 15:14:19 +0300 Subject: [PATCH] refactor: auto-generate timestamp when reindexing --- packages/migration/package.json | 2 +- .../migration/src/{utils => }/reindex-search.ts | 14 +++++--------- packages/search/src/config/routes.ts | 7 +------ packages/search/src/features/reindex/handler.ts | 7 +++---- packages/search/src/features/reindex/reindex.ts | 9 +++++---- yarn.lock | 16 +--------------- 6 files changed, 16 insertions(+), 39 deletions(-) rename packages/migration/src/{utils => }/reindex-search.ts (80%) diff --git a/packages/migration/package.json b/packages/migration/package.json index f697721d2c7..8e0f33f6566 100644 --- a/packages/migration/package.json +++ b/packages/migration/package.json @@ -18,7 +18,7 @@ "status:openhim": "migrate-mongo status -f ./src/migrate-mongo-config-openhim.js", "status:user-mgnt": "migrate-mongo status -f ./src/migrate-mongo-config-user-mgnt.js", "status:application-config": "migrate-mongo status -f ./src/migrate-mongo-config-application-config.js", - "reindex-search": "tsx src/utils/reindex-search.ts", + "reindex-search": "tsx src/reindex-search.ts", "precommit": "lint-staged", "test:compilation": "tsc --noEmit", "build": "rimraf build && tsc" diff --git a/packages/migration/src/utils/reindex-search.ts b/packages/migration/src/reindex-search.ts similarity index 80% rename from packages/migration/src/utils/reindex-search.ts rename to packages/migration/src/reindex-search.ts index 9fadc40dfd0..efdb2053277 100644 --- a/packages/migration/src/utils/reindex-search.ts +++ b/packages/migration/src/reindex-search.ts @@ -8,18 +8,15 @@ * * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. */ -import { format } from 'date-fns' const SEARCH_URL = process.env.SEARCH_URL || 'http://localhost:9090/' /** * Streams MongoDB collections to ElasticSearch documents. Useful when the ElasticSearch schema changes. */ -export const triggerReindex = async (timestamp: string) => { +export const triggerReindex = async () => { const response = await fetch(new URL('reindex', SEARCH_URL), { - method: 'POST', - body: JSON.stringify({ timestamp }), - headers: { 'Content-Type': 'application/json' } + method: 'POST' }) if (!response.ok) { @@ -51,9 +48,8 @@ export const checkReindexStatus = async (jobId: string) => { } async function main() { - const timestamp = format(new Date(), 'yyyyMMddHHmmss') - console.info(`Reindexing ${timestamp}...`) - const jobId = await triggerReindex(timestamp) + console.info(`Reindexing search...`) + const jobId = await triggerReindex() await new Promise((resolve, reject) => { const intervalId = setInterval(async () => { try { @@ -68,7 +64,7 @@ async function main() { } }, 1000) }) - console.info(`...done reindexing ${timestamp} (job id ${jobId})`) + console.info(`...done reindexing search with job id ${jobId}`) } main() diff --git a/packages/search/src/config/routes.ts b/packages/search/src/config/routes.ts index 89b4a02c9b6..045ad5842f6 100644 --- a/packages/search/src/config/routes.ts +++ b/packages/search/src/config/routes.ts @@ -288,12 +288,7 @@ export const getRoutes = () => { handler: reindexHandler, config: { tags: ['api'], - auth: false, - validate: { - payload: Joi.object({ - timestamp: Joi.string() - }) - } + auth: false } }, { diff --git a/packages/search/src/features/reindex/handler.ts b/packages/search/src/features/reindex/handler.ts index b96c2f97d5c..af767fc7e2e 100644 --- a/packages/search/src/features/reindex/handler.ts +++ b/packages/search/src/features/reindex/handler.ts @@ -20,16 +20,15 @@ const indexingStatuses: Record< > = {} export async function reindexHandler( - request: Hapi.Request, + _request: Hapi.Request, h: Hapi.ResponseToolkit ) { - const { timestamp } = request.payload as { timestamp: string } const jobId = uuid() process.nextTick(async () => { try { indexingStatuses[jobId] = 'started' - await reindex(timestamp) + await reindex() await updateAliases() await prune() indexingStatuses[jobId] = 'completed' @@ -41,7 +40,7 @@ export async function reindexHandler( return h .response({ - message: `ElasticSearch reindexing started for timestamp ${timestamp}`, + message: `ElasticSearch reindexing started for job ${jobId}`, status: indexingStatuses[jobId] ?? 'accepted', jobId }) diff --git a/packages/search/src/features/reindex/reindex.ts b/packages/search/src/features/reindex/reindex.ts index 9555e844cf1..944daa2cb87 100644 --- a/packages/search/src/features/reindex/reindex.ts +++ b/packages/search/src/features/reindex/reindex.ts @@ -21,6 +21,7 @@ import { logger } from '@opencrvs/commons' import { getEventType } from '@search/utils/event' import { Transform } from 'stream' import { orderBy } from 'lodash' +import { format } from 'date-fns' const eventTransformers = { [EVENT_TYPE.BIRTH]: composeBirthDocument, @@ -28,13 +29,13 @@ const eventTransformers = { [EVENT_TYPE.MARRIAGE]: composeMarriageDocument } satisfies Record SearchDocument> -export const formatIndexName = (timestamp: string) => - `${OPENCRVS_INDEX_NAME}-${timestamp}` +export const formatIndexName = () => + `${OPENCRVS_INDEX_NAME}-${format(new Date(), 'yyyyMMddHHmmss')}` /** Streams the MongoDB records to ElasticSearch */ -export const reindex = async (timestamp: string) => { +export const reindex = async () => { const t1 = performance.now() - const index = formatIndexName(timestamp) + const index = formatIndexName() logger.info(`Reindexing to ${index}`) const stream = await streamAllRecords(true) diff --git a/yarn.lock b/yarn.lock index 95ad3cfe972..9f629864d0a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7894,7 +7894,7 @@ dependencies: "@types/react" "*" -"@types/react@*", "@types/react@16 || 17 || 18", "@types/react@18.3.1", "@types/react@>=16": +"@types/react@*", "@types/react@16 || 17 || 18", "@types/react@18.3.1", "@types/react@>=16", "@types/react@^16": version "18.3.1" resolved "https://registry.yarnpkg.com/@types/react/-/react-18.3.1.tgz#fed43985caa834a2084d002e4771e15dfcbdbe8e" integrity sha512-V0kuGBX3+prX+DQ/7r2qsv1NsdfnCLnTgnRJ1pYnxykBhGMz+qj+box5lq7XsO5mtZsBqpjwwTu/7wszPfMBcw== @@ -7902,15 +7902,6 @@ "@types/prop-types" "*" csstype "^3.0.2" -"@types/react@^16": - version "16.14.60" - resolved "https://registry.yarnpkg.com/@types/react/-/react-16.14.60.tgz#f7ab62a329b82826f12d02bc8031d4ef4b5e0d81" - integrity sha512-wIFmnczGsTcgwCBeIYOuy2mdXEiKZ5znU/jNOnMZPQyCcIxauMGWlX0TNG4lZ7NxRKj7YUIZRneJQSSdB2jKgg== - dependencies: - "@types/prop-types" "*" - "@types/scheduler" "^0.16" - csstype "^3.0.2" - "@types/readdir-glob@*": version "1.1.3" resolved "https://registry.npmjs.org/@types/readdir-glob/-/readdir-glob-1.1.3.tgz" @@ -7979,11 +7970,6 @@ dependencies: htmlparser2 "^8.0.0" -"@types/scheduler@^0.16": - version "0.16.8" - resolved "https://registry.yarnpkg.com/@types/scheduler/-/scheduler-0.16.8.tgz#ce5ace04cfeabe7ef87c0091e50752e36707deff" - integrity sha512-WZLiwShhwLRmeV6zH+GkbOFT6Z6VklCItrDioxUnv+u4Ll+8vKeFySoFyK/0ctcRpOmwAicELfmys1sDc/Rw+A== - "@types/semver@^7.3.4": version "7.5.4" resolved "https://registry.npmjs.org/@types/semver/-/semver-7.5.4.tgz"