feat: reindex elasticsearch after running migrations

packages/migration/package.json

@@ -18,6 +18,7 @@
     "status:openhim": "migrate-mongo status -f ./src/migrate-mongo-config-openhim.js",
     "status:user-mgnt": "migrate-mongo status -f ./src/migrate-mongo-config-user-mgnt.js",
     "status:application-config": "migrate-mongo status -f ./src/migrate-mongo-config-application-config.js",
+    "reindex-search": "tsx src/reindex-search.ts",
     "precommit": "lint-staged",
     "test:compilation": "tsc --noEmit",
     "build": "rimraf build && tsc"
@@ -36,6 +37,7 @@
     "migrate-mongo": "^10.0.0",
     "minio": "^7.0.33",
     "mongoose": "^6.11.3",
+    "tsx": "^4.15.7",
     "uuid": "^3.2.1"
   },
   "devDependencies": {

packages/migration/run-migrations.sh

@@ -37,3 +37,6 @@ yarn --cwd $SCRIPT_PATH migrate-mongo status --file $USER_MGNT_CONFIG
 # performance migration
 yarn --cwd $SCRIPT_PATH migrate-mongo up --file $PERFORMANCE_CONFIG
 yarn --cwd $SCRIPT_PATH migrate-mongo status --file $PERFORMANCE_CONFIG
+
+# search migration / reindex
+yarn --cwd $SCRIPT_PATH reindex-search

packages/migration/src/migrations/hearth/20240527130750-reindex-elasticsearch.ts

@@ -10,7 +10,7 @@
  */
 
 import { Db, MongoClient } from 'mongodb'
-import { reindex, client } from '../../utils/elasticsearch-helper.js'
+import { client } from '../../utils/elasticsearch-helper.js'
 
 const ELASTICSEARCH_INDEX_NAME = 'ocrvs'
 
@@ -63,7 +63,6 @@ const migrateIndexToAlias = async (timestamp: string) => {
 export const up = async (db: Db, _client: MongoClient) => {
   await createEmptyIndex()
   await migrateIndexToAlias('20240514125702-old-format')
-  await reindex('20240514125703')
 }
 
 export const down = async (db: Db, _client: MongoClient) => {

packages/migration/src/reindex-search.ts

@@ -0,0 +1,70 @@
+/*
+ * This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at https://mozilla.org/MPL/2.0/.
+ *
+ * OpenCRVS is also distributed under the terms of the Civil Registration
+ * & Healthcare Disclaimer located at http://opencrvs.org/license.
+ *
+ * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS.
+ */
+
+const SEARCH_URL = process.env.SEARCH_URL || 'http://localhost:9090/'
+
+/**
+ * Streams MongoDB collections to ElasticSearch documents. Useful when the ElasticSearch schema changes.
+ */
+const triggerReindex = async () => {
+  const response = await fetch(new URL('reindex', SEARCH_URL), {
+    method: 'POST'
+  })
+
+  if (!response.ok) {
+    throw new Error(
+      `Problem reindexing ElasticSearch. Response: ${await response.text()}`
+    )
+  }
+
+  const data = await response.json()
+  return data.jobId
+}
+
+/**
+ * Checks the status of the reindex, as it can take a while
+ */
+const checkReindexStatus = async (jobId: string) => {
+  const response = await fetch(new URL(`reindex/status/${jobId}`, SEARCH_URL), {
+    method: 'GET'
+  })
+
+  if (!response.ok) {
+    throw new Error(
+      `Problem checking reindex status from ElasticSearch. Response: ${await response.text()}`
+    )
+  }
+
+  const data = await response.json()
+  return data.status === 'completed'
+}
+
+async function main() {
+  console.info(`Reindexing search...`)
+  const jobId = await triggerReindex()
+  await new Promise<void>((resolve, reject) => {
+    const intervalId = setInterval(async () => {
+      try {
+        const isCompleted = await checkReindexStatus(jobId)
+        if (isCompleted) {
+          clearInterval(intervalId)
+          resolve()
+        }
+      } catch (error) {
+        clearInterval(intervalId)
+        reject(error)
+      }
+    }, 1000)
+  })
+  console.info(`...done reindexing search with job id ${jobId}`)
+}
+
+main()

packages/migration/src/utils/elasticsearch-helper.ts

@@ -11,7 +11,6 @@
 
 import { Client } from '@elastic/elasticsearch'
 
-const SEARCH_URL = process.env.SEARCH_URL || 'http://localhost:9090/'
 const ES_HOST = process.env.ES_HOST || 'localhost:9200'
 const ELASTICSEARCH_INDEX_NAME = 'ocrvs'
 
@@ -107,61 +106,3 @@ export const searchCompositionByCriteria = async (
     return null
   }
 }
-
-/**
- * Streams MongoDB collections to ElasticSearch documents. Useful when the ElasticSearch schema changes.
- */
-export const triggerReindex = async (timestamp: string) => {
-  const response = await fetch(new URL('reindex', SEARCH_URL), {
-    method: 'POST',
-    body: JSON.stringify({ timestamp }),
-    headers: { 'Content-Type': 'application/json' }
-  })
-
-  if (!response.ok) {
-    throw new Error(
-      `Problem reindexing ElasticSearch. Response: ${await response.text()}`
-    )
-  }
-
-  const data = await response.json()
-  return data.jobId
-}
-
-/**
- * Checks the status of the reindex, as it can take a while
- */
-export const checkReindexStatus = async (jobId: string) => {
-  const response = await fetch(new URL(`reindex/status/${jobId}`, SEARCH_URL), {
-    method: 'GET'
-  })
-
-  if (!response.ok) {
-    throw new Error(
-      `Problem checking reindex status from ElasticSearch. Response: ${await response.text()}`
-    )
-  }
-
-  const data = await response.json()
-  return data.status === 'completed'
-}
-
-export const reindex = async (timestamp: string) => {
-  console.info(`Reindexing ${timestamp}...`)
-  const jobId = await triggerReindex(timestamp)
-  await new Promise<void>((resolve, reject) => {
-    const intervalId = setInterval(async () => {
-      try {
-        const isCompleted = await checkReindexStatus(jobId)
-        if (isCompleted) {
-          clearInterval(intervalId)
-          resolve()
-        }
-      } catch (error) {
-        clearInterval(intervalId)
-        reject(error)
-      }
-    }, 1000)
-  })
-  console.info(`...done reindexing ${timestamp} (job id ${jobId})`)
-}

packages/search/src/config/routes.ts

@@ -288,12 +288,7 @@ export const getRoutes = () => {
       handler: reindexHandler,
       config: {
         tags: ['api'],
-        auth: false,
-        validate: {
-          payload: Joi.object({
-            timestamp: Joi.string()
-          })
-        }
+        auth: false
       }
     },
     {

packages/search/src/features/reindex/handler.ts

@@ -20,16 +20,15 @@ const indexingStatuses: Record<
 > = {}
 
 export async function reindexHandler(
-  request: Hapi.Request,
+  _request: Hapi.Request,
   h: Hapi.ResponseToolkit
 ) {
-  const { timestamp } = request.payload as { timestamp: string }
   const jobId = uuid()
 
   process.nextTick(async () => {
     try {
       indexingStatuses[jobId] = 'started'
-      await reindex(timestamp)
+      await reindex()
       await updateAliases()
       await prune()
       indexingStatuses[jobId] = 'completed'
@@ -41,7 +40,7 @@ export async function reindexHandler(
 
   return h
     .response({
-      message: `ElasticSearch reindexing started for timestamp ${timestamp}`,
+      message: `ElasticSearch reindexing started for job ${jobId}`,
       status: indexingStatuses[jobId] ?? 'accepted',
       jobId
     })

packages/search/src/features/reindex/reindex.ts

@@ -21,20 +21,21 @@ import { logger } from '@opencrvs/commons'
 import { getEventType } from '@search/utils/event'
 import { Transform } from 'stream'
 import { orderBy } from 'lodash'
+import { format } from 'date-fns'
 
 const eventTransformers = {
   [EVENT_TYPE.BIRTH]: composeBirthDocument,
   [EVENT_TYPE.DEATH]: composeDeathDocument,
   [EVENT_TYPE.MARRIAGE]: composeMarriageDocument
 } satisfies Record<EVENT_TYPE, (record: ValidRecord) => SearchDocument>
 
-export const formatIndexName = (timestamp: string) =>
-  `${OPENCRVS_INDEX_NAME}-${timestamp}`
+export const formatIndexName = () =>
+  `${OPENCRVS_INDEX_NAME}-${format(new Date(), 'yyyyMMddHHmmss')}`
 
 /** Streams the MongoDB records to ElasticSearch */
-export const reindex = async (timestamp: string) => {
+export const reindex = async () => {
   const t1 = performance.now()
-  const index = formatIndexName(timestamp)
+  const index = formatIndexName()
   logger.info(`Reindexing to ${index}`)
 
   const stream = await streamAllRecords(true)

yarn.lock

@@ -12780,7 +12780,7 @@ esbuild@^0.18.0, esbuild@^0.18.10:
     "@esbuild/win32-ia32" "0.18.20"
     "@esbuild/win32-x64" "0.18.20"
 
-esbuild@^0.21.3:
+esbuild@^0.21.3, esbuild@~0.21.4:
   version "0.21.5"
   resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.21.5.tgz#9ca301b120922959b766360d8ac830da0d02997d"
   integrity sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==
@@ -14220,6 +14220,13 @@ get-symbol-description@^1.0.2:
     es-errors "^1.3.0"
     get-intrinsic "^1.2.4"
 
+get-tsconfig@^4.7.5:
+  version "4.7.5"
+  resolved "https://registry.yarnpkg.com/get-tsconfig/-/get-tsconfig-4.7.5.tgz#5e012498579e9a6947511ed0cd403272c7acbbaf"
+  integrity sha512-ZCuZCnlqNzjb4QprAzXKdpp/gh6KTxSJuw3IBsPnV/7fV4NxC9ckB+vPTt8w7fJA0TaSD7c55BR47JD6MEDyDw==
+  dependencies:
+    resolve-pkg-maps "^1.0.0"
+
 get-value@^2.0.3, get-value@^2.0.6:
   version "2.0.6"
   resolved "https://registry.npmjs.org/get-value/-/get-value-2.0.6.tgz"
@@ -21869,6 +21876,11 @@ resolve-pathname@^3.0.0:
   resolved "https://registry.npmjs.org/resolve-pathname/-/resolve-pathname-3.0.0.tgz"
   integrity sha512-C7rARubxI8bXFNB/hqcp/4iUeIXJhJZvFPFPiSPRnhU5UPxzMFIl+2E6yY6c4k9giDJAhtV+enfA+G89N6Csng==
 
+resolve-pkg-maps@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz#616b3dc2c57056b5588c31cdf4b3d64db133720f"
+  integrity sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==
+
 resolve-url@^0.2.1:
   version "0.2.1"
   resolved "https://registry.npmjs.org/resolve-url/-/resolve-url-0.2.1.tgz"
@@ -23084,7 +23096,7 @@ string-similarity@^4.0.1:
   resolved "https://registry.npmjs.org/string-similarity/-/string-similarity-4.0.4.tgz"
   integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==
 
-"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
+"string-width-cjs@npm:string-width@^4.2.0":
   version "4.2.3"
   resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -23102,6 +23114,15 @@ string-width@^1.0.1:
     is-fullwidth-code-point "^1.0.0"
     strip-ansi "^3.0.0"
 
+"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3:
+  version "4.2.3"
+  resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz"
+  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
+  dependencies:
+    emoji-regex "^8.0.0"
+    is-fullwidth-code-point "^3.0.0"
+    strip-ansi "^6.0.1"
+
 string-width@^2.1.1:
   version "2.1.1"
   resolved "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz"
@@ -23261,7 +23282,7 @@ stringify-object@^3.2.2, stringify-object@^3.3.0:
     is-obj "^1.0.1"
     is-regexp "^1.0.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
   version "6.0.1"
   resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -23289,6 +23310,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0:
   dependencies:
     ansi-regex "^4.1.0"
 
+strip-ansi@^6.0.0, strip-ansi@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz"
+  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
+  dependencies:
+    ansi-regex "^5.0.1"
+
 strip-ansi@^7.0.1, strip-ansi@^7.1.0:
   version "7.1.0"
   resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz"
@@ -24189,6 +24217,16 @@ tsutils@^3.21.0:
   dependencies:
     tslib "^1.8.1"
 
+tsx@^4.15.7:
+  version "4.15.7"
+  resolved "https://registry.yarnpkg.com/tsx/-/tsx-4.15.7.tgz#69d7499196a323507c4051d2ba10753edcc057e5"
+  integrity sha512-u3H0iSFDZM3za+VxkZ1kywdCeHCn+8/qHQS1MNoO2sONDgD95HlWtt8aB23OzeTmFP9IU4/8bZUdg58Uu5J4cg==
+  dependencies:
+    esbuild "~0.21.4"
+    get-tsconfig "^4.7.5"
+  optionalDependencies:
+    fsevents "~2.3.3"
+
 tuf-js@^1.1.7:
   version "1.1.7"
   resolved "https://registry.yarnpkg.com/tuf-js/-/tuf-js-1.1.7.tgz#21b7ae92a9373015be77dfe0cb282a80ec3bbe43"
@@ -25526,7 +25564,7 @@ [email protected], workbox-window@^7.1.0:
     "@types/trusted-types" "^2.0.2"
     workbox-core "7.1.0"
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
   version "7.0.0"
   resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -25561,6 +25599,15 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
+wrap-ansi@^7.0.0:
+  version "7.0.0"
+  resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz"
+  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
+  dependencies:
+    ansi-styles "^4.0.0"
+    string-width "^4.1.0"
+    strip-ansi "^6.0.0"
+
 wrap-ansi@^8.0.1, wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz"