feat(search-indexer): Move mapping logic to separate class

libs/cms/src/lib/search/cmsSync.module.ts

@@ -32,6 +32,7 @@ import { ManualChapterItemSyncService } from './importers/manualChapterItem.serv
 import { CustomPageSyncService } from './importers/customPage.service'
 import { GenericListItemSyncService } from './importers/genericListItem.service'
 import { TeamListSyncService } from './importers/teamList.service'
+import { MappingService } from './mapping.service'
 
 @Module({
   imports: [
@@ -69,6 +70,7 @@ import { TeamListSyncService } from './importers/teamList.service'
     CustomPageSyncService,
     GenericListItemSyncService,
     TeamListSyncService,
+    MappingService,
   ],
   exports: [CmsSyncService],
 })

libs/cms/src/lib/search/cmsSync.service.ts

@@ -8,34 +8,11 @@ import {
   SyncOptions,
   SyncResponse,
 } from '@island.is/content-search-indexer/types'
-import { ArticleSyncService } from './importers/article.service'
-import { SubArticleSyncService } from './importers/subArticle.service'
 import { ContentfulService } from './contentful.service'
-import { AnchorPageSyncService } from './importers/anchorPage.service'
-import { LifeEventPageSyncService } from './importers/lifeEventPage.service'
-import { ArticleCategorySyncService } from './importers/articleCategory.service'
-import { NewsSyncService } from './importers/news.service'
 import { Entry } from 'contentful'
 import { ElasticService, SearchInput } from '@island.is/content-search-toolkit'
-import { AdgerdirPageSyncService } from './importers/adgerdirPage'
-import { MenuSyncService } from './importers/menu.service'
-import { GroupedMenuSyncService } from './importers/groupedMenu.service'
 import { getElasticsearchIndex } from '@island.is/content-search-index-manager'
-import { OrganizationPageSyncService } from './importers/organizationPage.service'
-import { OrganizationSubpageSyncService } from './importers/organizationSubpage.service'
-import { FrontpageSyncService } from './importers/frontpage.service'
-import { SupportQNASyncService } from './importers/supportQNA.service'
-import { LinkSyncService } from './importers/link.service'
-import { ProjectPageSyncService } from './importers/projectPage.service'
-import { EnhancedAssetSyncService } from './importers/enhancedAsset.service'
-import { VacancySyncService } from './importers/vacancy.service'
-import { ServiceWebPageSyncService } from './importers/serviceWebPage.service'
-import { EventSyncService } from './importers/event.service'
-import { ManualSyncService } from './importers/manual.service'
-import { ManualChapterItemSyncService } from './importers/manualChapterItem.service'
-import { CustomPageSyncService } from './importers/customPage.service'
-import { GenericListItemSyncService } from './importers/genericListItem.service'
-import { TeamListSyncService } from './importers/teamList.service'
+import { MappingService } from './mapping.service'
 
 export interface PostSyncOptions {
   folderHash: string
@@ -56,62 +33,11 @@ export interface CmsSyncProvider<T, ProcessOutput = any> {
 
 @Injectable()
 export class CmsSyncService implements ContentSearchImporter<PostSyncOptions> {
-  private contentSyncProviders: CmsSyncProvider<any>[]
   constructor(
-    private readonly newsSyncService: NewsSyncService,
-    private readonly articleCategorySyncService: ArticleCategorySyncService,
-    private readonly articleSyncService: ArticleSyncService,
-    private readonly subArticleSyncService: SubArticleSyncService,
-    private readonly anchorPageSyncService: AnchorPageSyncService,
-    private readonly lifeEventPageSyncService: LifeEventPageSyncService,
-    private readonly adgerdirPageSyncService: AdgerdirPageSyncService,
     private readonly contentfulService: ContentfulService,
-    private readonly menuSyncService: MenuSyncService,
-    private readonly groupedMenuSyncService: GroupedMenuSyncService,
-    private readonly organizationPageSyncService: OrganizationPageSyncService,
-    private readonly organizationSubpageSyncService: OrganizationSubpageSyncService,
-    private readonly projectPageSyncService: ProjectPageSyncService,
-    private readonly frontpageSyncService: FrontpageSyncService,
-    private readonly supportQNASyncService: SupportQNASyncService,
-    private readonly linkSyncService: LinkSyncService,
-    private readonly enhancedAssetService: EnhancedAssetSyncService,
+    private readonly mappingService: MappingService,
     private readonly elasticService: ElasticService,
-    private readonly vacancyService: VacancySyncService,
-    private readonly serviceWebPageSyncService: ServiceWebPageSyncService,
-    private readonly eventSyncService: EventSyncService,
-    private readonly manualSyncService: ManualSyncService,
-    private readonly manualChapterItemSyncService: ManualChapterItemSyncService,
-    private readonly customPageSyncService: CustomPageSyncService,
-    private readonly genericListItemSyncService: GenericListItemSyncService,
-    private readonly teamListSyncService: TeamListSyncService,
-  ) {
-    this.contentSyncProviders = [
-      this.articleSyncService,
-      this.subArticleSyncService,
-      this.anchorPageSyncService,
-      this.lifeEventPageSyncService,
-      this.articleCategorySyncService,
-      this.newsSyncService,
-      this.adgerdirPageSyncService,
-      this.menuSyncService,
-      this.groupedMenuSyncService,
-      this.organizationPageSyncService,
-      this.organizationSubpageSyncService,
-      this.projectPageSyncService,
-      this.frontpageSyncService,
-      this.supportQNASyncService,
-      this.linkSyncService,
-      this.enhancedAssetService,
-      this.vacancyService,
-      this.serviceWebPageSyncService,
-      this.eventSyncService,
-      this.manualSyncService,
-      this.manualChapterItemSyncService,
-      this.customPageSyncService,
-      this.genericListItemSyncService,
-      this.teamListSyncService,
-    ]
-  }
+  ) {}
 
   private async getLastFolderHash(elasticIndex: string): Promise<string> {
     logger.info('Getting folder hash from index', {
@@ -228,12 +154,7 @@ export class CmsSyncService implements ContentSearchImporter<PostSyncOptions> {
     logger.info('Got sync data')
 
     // import data from all providers
-    const importableData = this.contentSyncProviders.map(
-      (contentSyncProvider) => {
-        const data = contentSyncProvider.processSyncData(items)
-        return contentSyncProvider.doMapping(data)
-      },
-    )
+    const importableData = this.mappingService.mapData(items)
 
     return {
       add: flatten(importableData),

libs/cms/src/lib/search/contentful.service.ts

@@ -24,6 +24,7 @@ import { Locale } from '@island.is/shared/types'
 import type { ApiResponse } from '@elastic/elasticsearch'
 import type { SearchResponse } from '@island.is/shared/types'
 import type { MappedData } from '@island.is/content-search-indexer/types'
+import { MappingService } from './mapping.service'
 
 type SyncCollection = ContentfulSyncCollection & {
   nextPageToken?: string
@@ -79,6 +80,7 @@ export class ContentfulService {
   constructor(
     private readonly elasticService: ElasticService,
     private readonly featureFlagService: FeatureFlagService,
+    private readonly mappingService: MappingService,
   ) {
     const params: CreateClientParams = {
       space: environment.contentful.space,
@@ -477,6 +479,9 @@ export class ContentfulService {
     const idsCopy = [...ids]
     let idsChunk = idsCopy.splice(-MAX_REQUEST_COUNT, MAX_REQUEST_COUNT)
 
+    let nestedProgress = idsChunk.length
+    const totalNested = ids.length
+
     while (idsChunk.length > 0) {
       const size = 100
       let page = 1
@@ -566,20 +571,42 @@ export class ContentfulService {
           (id) => !indexableEntries.some((entry) => entry.sys.id === id),
         )
 
+        let progress = 0
+        const total = rootEntryIds.length
+
         let chunkIds = rootEntryIds.splice(-chunkSize, chunkSize)
+        progress += chunkIds.length
 
         while (chunkIds.length > 0) {
           const items = await this.getContentfulData(chunkSize, {
             include: this.defaultIncludeDepth,
             'sys.id[in]': chunkIds.join(','),
             locale: this.contentfulLocaleMap[locale],
           })
-          indexableEntries.push(...items)
+
+          // import data from all providers
+          const importableData = this.mappingService.mapData(items)
+
+          await this.elasticService.bulk(elasticIndex, {
+            add: flatten(importableData),
+            remove: [],
+          })
+
+          logger.info(
+            `${progress}/${total} resolved root entries have been synced`,
+          )
+
           chunkIds = rootEntryIds.splice(-chunkSize, chunkSize)
+          progress += chunkIds.length
         }
       }
 
+      logger.info(
+        `${nestedProgress}/${totalNested} nested entries have been resolved`,
+      )
+
       idsChunk = idsCopy.splice(-MAX_REQUEST_COUNT, MAX_REQUEST_COUNT)
+      nestedProgress += idsChunk.length
     }
   }
 
@@ -643,7 +670,7 @@ export class ContentfulService {
         elasticIndex,
         locale,
         chunkSize,
-        indexableEntries, // This array is modified
+        indexableEntries,
       )
 
       logger.info(

libs/cms/src/lib/search/mapping.service.ts

@@ -0,0 +1,93 @@
+import { Injectable } from '@nestjs/common'
+
+import { ArticleSyncService } from './importers/article.service'
+import { SubArticleSyncService } from './importers/subArticle.service'
+import { AnchorPageSyncService } from './importers/anchorPage.service'
+import { LifeEventPageSyncService } from './importers/lifeEventPage.service'
+import { ArticleCategorySyncService } from './importers/articleCategory.service'
+import { NewsSyncService } from './importers/news.service'
+import { AdgerdirPageSyncService } from './importers/adgerdirPage'
+import { MenuSyncService } from './importers/menu.service'
+import { GroupedMenuSyncService } from './importers/groupedMenu.service'
+import { OrganizationPageSyncService } from './importers/organizationPage.service'
+import { OrganizationSubpageSyncService } from './importers/organizationSubpage.service'
+import { FrontpageSyncService } from './importers/frontpage.service'
+import { SupportQNASyncService } from './importers/supportQNA.service'
+import { LinkSyncService } from './importers/link.service'
+import { ProjectPageSyncService } from './importers/projectPage.service'
+import { EnhancedAssetSyncService } from './importers/enhancedAsset.service'
+import { VacancySyncService } from './importers/vacancy.service'
+import { ServiceWebPageSyncService } from './importers/serviceWebPage.service'
+import { EventSyncService } from './importers/event.service'
+import { ManualSyncService } from './importers/manual.service'
+import { ManualChapterItemSyncService } from './importers/manualChapterItem.service'
+import { CustomPageSyncService } from './importers/customPage.service'
+import { GenericListItemSyncService } from './importers/genericListItem.service'
+import { TeamListSyncService } from './importers/teamList.service'
+import type { CmsSyncProvider, processSyncDataInput } from './cmsSync.service'
+
+@Injectable()
+export class MappingService {
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private contentSyncProviders: CmsSyncProvider<any>[]
+  constructor(
+    private readonly newsSyncService: NewsSyncService,
+    private readonly articleCategorySyncService: ArticleCategorySyncService,
+    private readonly articleSyncService: ArticleSyncService,
+    private readonly subArticleSyncService: SubArticleSyncService,
+    private readonly anchorPageSyncService: AnchorPageSyncService,
+    private readonly lifeEventPageSyncService: LifeEventPageSyncService,
+    private readonly adgerdirPageSyncService: AdgerdirPageSyncService,
+    private readonly menuSyncService: MenuSyncService,
+    private readonly groupedMenuSyncService: GroupedMenuSyncService,
+    private readonly organizationPageSyncService: OrganizationPageSyncService,
+    private readonly organizationSubpageSyncService: OrganizationSubpageSyncService,
+    private readonly projectPageSyncService: ProjectPageSyncService,
+    private readonly frontpageSyncService: FrontpageSyncService,
+    private readonly supportQNASyncService: SupportQNASyncService,
+    private readonly linkSyncService: LinkSyncService,
+    private readonly enhancedAssetService: EnhancedAssetSyncService,
+    private readonly vacancyService: VacancySyncService,
+    private readonly serviceWebPageSyncService: ServiceWebPageSyncService,
+    private readonly eventSyncService: EventSyncService,
+    private readonly manualSyncService: ManualSyncService,
+    private readonly manualChapterItemSyncService: ManualChapterItemSyncService,
+    private readonly customPageSyncService: CustomPageSyncService,
+    private readonly genericListItemSyncService: GenericListItemSyncService,
+    private readonly teamListSyncService: TeamListSyncService,
+  ) {
+    this.contentSyncProviders = [
+      this.articleSyncService,
+      this.subArticleSyncService,
+      this.anchorPageSyncService,
+      this.lifeEventPageSyncService,
+      this.articleCategorySyncService,
+      this.newsSyncService,
+      this.adgerdirPageSyncService,
+      this.menuSyncService,
+      this.groupedMenuSyncService,
+      this.organizationPageSyncService,
+      this.organizationSubpageSyncService,
+      this.projectPageSyncService,
+      this.frontpageSyncService,
+      this.supportQNASyncService,
+      this.linkSyncService,
+      this.enhancedAssetService,
+      this.vacancyService,
+      this.serviceWebPageSyncService,
+      this.eventSyncService,
+      this.manualSyncService,
+      this.manualChapterItemSyncService,
+      this.customPageSyncService,
+      this.genericListItemSyncService,
+      this.teamListSyncService,
+    ]
+  }
+
+  mapData(entries: processSyncDataInput<unknown>) {
+    return this.contentSyncProviders.map((contentSyncProvider) => {
+      const data = contentSyncProvider.processSyncData(entries)
+      return contentSyncProvider.doMapping(data)
+    })
+  }
+}