diff --git a/libs/cms/src/lib/search/cmsSync.module.ts b/libs/cms/src/lib/search/cmsSync.module.ts index 261082084628..d6edead06251 100644 --- a/libs/cms/src/lib/search/cmsSync.module.ts +++ b/libs/cms/src/lib/search/cmsSync.module.ts @@ -32,6 +32,7 @@ import { ManualChapterItemSyncService } from './importers/manualChapterItem.serv import { CustomPageSyncService } from './importers/customPage.service' import { GenericListItemSyncService } from './importers/genericListItem.service' import { TeamListSyncService } from './importers/teamList.service' +import { MappingService } from './mapping.service' @Module({ imports: [ @@ -69,6 +70,7 @@ import { TeamListSyncService } from './importers/teamList.service' CustomPageSyncService, GenericListItemSyncService, TeamListSyncService, + MappingService, ], exports: [CmsSyncService], }) diff --git a/libs/cms/src/lib/search/cmsSync.service.ts b/libs/cms/src/lib/search/cmsSync.service.ts index b1593bf2c158..f11d3151192a 100644 --- a/libs/cms/src/lib/search/cmsSync.service.ts +++ b/libs/cms/src/lib/search/cmsSync.service.ts @@ -8,34 +8,11 @@ import { SyncOptions, SyncResponse, } from '@island.is/content-search-indexer/types' -import { ArticleSyncService } from './importers/article.service' -import { SubArticleSyncService } from './importers/subArticle.service' import { ContentfulService } from './contentful.service' -import { AnchorPageSyncService } from './importers/anchorPage.service' -import { LifeEventPageSyncService } from './importers/lifeEventPage.service' -import { ArticleCategorySyncService } from './importers/articleCategory.service' -import { NewsSyncService } from './importers/news.service' import { Entry } from 'contentful' import { ElasticService, SearchInput } from '@island.is/content-search-toolkit' -import { AdgerdirPageSyncService } from './importers/adgerdirPage' -import { MenuSyncService } from './importers/menu.service' -import { GroupedMenuSyncService } from './importers/groupedMenu.service' import { getElasticsearchIndex } from '@island.is/content-search-index-manager' -import { OrganizationPageSyncService } from './importers/organizationPage.service' -import { OrganizationSubpageSyncService } from './importers/organizationSubpage.service' -import { FrontpageSyncService } from './importers/frontpage.service' -import { SupportQNASyncService } from './importers/supportQNA.service' -import { LinkSyncService } from './importers/link.service' -import { ProjectPageSyncService } from './importers/projectPage.service' -import { EnhancedAssetSyncService } from './importers/enhancedAsset.service' -import { VacancySyncService } from './importers/vacancy.service' -import { ServiceWebPageSyncService } from './importers/serviceWebPage.service' -import { EventSyncService } from './importers/event.service' -import { ManualSyncService } from './importers/manual.service' -import { ManualChapterItemSyncService } from './importers/manualChapterItem.service' -import { CustomPageSyncService } from './importers/customPage.service' -import { GenericListItemSyncService } from './importers/genericListItem.service' -import { TeamListSyncService } from './importers/teamList.service' +import { MappingService } from './mapping.service' export interface PostSyncOptions { folderHash: string @@ -56,62 +33,11 @@ export interface CmsSyncProvider { @Injectable() export class CmsSyncService implements ContentSearchImporter { - private contentSyncProviders: CmsSyncProvider[] constructor( - private readonly newsSyncService: NewsSyncService, - private readonly articleCategorySyncService: ArticleCategorySyncService, - private readonly articleSyncService: ArticleSyncService, - private readonly subArticleSyncService: SubArticleSyncService, - private readonly anchorPageSyncService: AnchorPageSyncService, - private readonly lifeEventPageSyncService: LifeEventPageSyncService, - private readonly adgerdirPageSyncService: AdgerdirPageSyncService, private readonly contentfulService: ContentfulService, - private readonly menuSyncService: MenuSyncService, - private readonly groupedMenuSyncService: GroupedMenuSyncService, - private readonly organizationPageSyncService: OrganizationPageSyncService, - private readonly organizationSubpageSyncService: OrganizationSubpageSyncService, - private readonly projectPageSyncService: ProjectPageSyncService, - private readonly frontpageSyncService: FrontpageSyncService, - private readonly supportQNASyncService: SupportQNASyncService, - private readonly linkSyncService: LinkSyncService, - private readonly enhancedAssetService: EnhancedAssetSyncService, + private readonly mappingService: MappingService, private readonly elasticService: ElasticService, - private readonly vacancyService: VacancySyncService, - private readonly serviceWebPageSyncService: ServiceWebPageSyncService, - private readonly eventSyncService: EventSyncService, - private readonly manualSyncService: ManualSyncService, - private readonly manualChapterItemSyncService: ManualChapterItemSyncService, - private readonly customPageSyncService: CustomPageSyncService, - private readonly genericListItemSyncService: GenericListItemSyncService, - private readonly teamListSyncService: TeamListSyncService, - ) { - this.contentSyncProviders = [ - this.articleSyncService, - this.subArticleSyncService, - this.anchorPageSyncService, - this.lifeEventPageSyncService, - this.articleCategorySyncService, - this.newsSyncService, - this.adgerdirPageSyncService, - this.menuSyncService, - this.groupedMenuSyncService, - this.organizationPageSyncService, - this.organizationSubpageSyncService, - this.projectPageSyncService, - this.frontpageSyncService, - this.supportQNASyncService, - this.linkSyncService, - this.enhancedAssetService, - this.vacancyService, - this.serviceWebPageSyncService, - this.eventSyncService, - this.manualSyncService, - this.manualChapterItemSyncService, - this.customPageSyncService, - this.genericListItemSyncService, - this.teamListSyncService, - ] - } + ) {} private async getLastFolderHash(elasticIndex: string): Promise { logger.info('Getting folder hash from index', { @@ -228,12 +154,7 @@ export class CmsSyncService implements ContentSearchImporter { logger.info('Got sync data') // import data from all providers - const importableData = this.contentSyncProviders.map( - (contentSyncProvider) => { - const data = contentSyncProvider.processSyncData(items) - return contentSyncProvider.doMapping(data) - }, - ) + const importableData = this.mappingService.mapData(items) return { add: flatten(importableData), diff --git a/libs/cms/src/lib/search/contentful.service.ts b/libs/cms/src/lib/search/contentful.service.ts index 5d015de20bd9..878b0ca37a05 100644 --- a/libs/cms/src/lib/search/contentful.service.ts +++ b/libs/cms/src/lib/search/contentful.service.ts @@ -24,6 +24,7 @@ import { Locale } from '@island.is/shared/types' import type { ApiResponse } from '@elastic/elasticsearch' import type { SearchResponse } from '@island.is/shared/types' import type { MappedData } from '@island.is/content-search-indexer/types' +import { MappingService } from './mapping.service' type SyncCollection = ContentfulSyncCollection & { nextPageToken?: string @@ -79,6 +80,7 @@ export class ContentfulService { constructor( private readonly elasticService: ElasticService, private readonly featureFlagService: FeatureFlagService, + private readonly mappingService: MappingService, ) { const params: CreateClientParams = { space: environment.contentful.space, @@ -477,6 +479,9 @@ export class ContentfulService { const idsCopy = [...ids] let idsChunk = idsCopy.splice(-MAX_REQUEST_COUNT, MAX_REQUEST_COUNT) + let nestedProgress = idsChunk.length + const totalNested = ids.length + while (idsChunk.length > 0) { const size = 100 let page = 1 @@ -566,7 +571,11 @@ export class ContentfulService { (id) => !indexableEntries.some((entry) => entry.sys.id === id), ) + let progress = 0 + const total = rootEntryIds.length + let chunkIds = rootEntryIds.splice(-chunkSize, chunkSize) + progress += chunkIds.length while (chunkIds.length > 0) { const items = await this.getContentfulData(chunkSize, { @@ -574,12 +583,30 @@ export class ContentfulService { 'sys.id[in]': chunkIds.join(','), locale: this.contentfulLocaleMap[locale], }) - indexableEntries.push(...items) + + // import data from all providers + const importableData = this.mappingService.mapData(items) + + await this.elasticService.bulk(elasticIndex, { + add: flatten(importableData), + remove: [], + }) + + logger.info( + `${progress}/${total} resolved root entries have been synced`, + ) + chunkIds = rootEntryIds.splice(-chunkSize, chunkSize) + progress += chunkIds.length } } + logger.info( + `${nestedProgress}/${totalNested} nested entries have been resolved`, + ) + idsChunk = idsCopy.splice(-MAX_REQUEST_COUNT, MAX_REQUEST_COUNT) + nestedProgress += idsChunk.length } } @@ -643,7 +670,7 @@ export class ContentfulService { elasticIndex, locale, chunkSize, - indexableEntries, // This array is modified + indexableEntries, ) logger.info( diff --git a/libs/cms/src/lib/search/mapping.service.ts b/libs/cms/src/lib/search/mapping.service.ts new file mode 100644 index 000000000000..981675af8792 --- /dev/null +++ b/libs/cms/src/lib/search/mapping.service.ts @@ -0,0 +1,93 @@ +import { Injectable } from '@nestjs/common' + +import { ArticleSyncService } from './importers/article.service' +import { SubArticleSyncService } from './importers/subArticle.service' +import { AnchorPageSyncService } from './importers/anchorPage.service' +import { LifeEventPageSyncService } from './importers/lifeEventPage.service' +import { ArticleCategorySyncService } from './importers/articleCategory.service' +import { NewsSyncService } from './importers/news.service' +import { AdgerdirPageSyncService } from './importers/adgerdirPage' +import { MenuSyncService } from './importers/menu.service' +import { GroupedMenuSyncService } from './importers/groupedMenu.service' +import { OrganizationPageSyncService } from './importers/organizationPage.service' +import { OrganizationSubpageSyncService } from './importers/organizationSubpage.service' +import { FrontpageSyncService } from './importers/frontpage.service' +import { SupportQNASyncService } from './importers/supportQNA.service' +import { LinkSyncService } from './importers/link.service' +import { ProjectPageSyncService } from './importers/projectPage.service' +import { EnhancedAssetSyncService } from './importers/enhancedAsset.service' +import { VacancySyncService } from './importers/vacancy.service' +import { ServiceWebPageSyncService } from './importers/serviceWebPage.service' +import { EventSyncService } from './importers/event.service' +import { ManualSyncService } from './importers/manual.service' +import { ManualChapterItemSyncService } from './importers/manualChapterItem.service' +import { CustomPageSyncService } from './importers/customPage.service' +import { GenericListItemSyncService } from './importers/genericListItem.service' +import { TeamListSyncService } from './importers/teamList.service' +import type { CmsSyncProvider, processSyncDataInput } from './cmsSync.service' + +@Injectable() +export class MappingService { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + private contentSyncProviders: CmsSyncProvider[] + constructor( + private readonly newsSyncService: NewsSyncService, + private readonly articleCategorySyncService: ArticleCategorySyncService, + private readonly articleSyncService: ArticleSyncService, + private readonly subArticleSyncService: SubArticleSyncService, + private readonly anchorPageSyncService: AnchorPageSyncService, + private readonly lifeEventPageSyncService: LifeEventPageSyncService, + private readonly adgerdirPageSyncService: AdgerdirPageSyncService, + private readonly menuSyncService: MenuSyncService, + private readonly groupedMenuSyncService: GroupedMenuSyncService, + private readonly organizationPageSyncService: OrganizationPageSyncService, + private readonly organizationSubpageSyncService: OrganizationSubpageSyncService, + private readonly projectPageSyncService: ProjectPageSyncService, + private readonly frontpageSyncService: FrontpageSyncService, + private readonly supportQNASyncService: SupportQNASyncService, + private readonly linkSyncService: LinkSyncService, + private readonly enhancedAssetService: EnhancedAssetSyncService, + private readonly vacancyService: VacancySyncService, + private readonly serviceWebPageSyncService: ServiceWebPageSyncService, + private readonly eventSyncService: EventSyncService, + private readonly manualSyncService: ManualSyncService, + private readonly manualChapterItemSyncService: ManualChapterItemSyncService, + private readonly customPageSyncService: CustomPageSyncService, + private readonly genericListItemSyncService: GenericListItemSyncService, + private readonly teamListSyncService: TeamListSyncService, + ) { + this.contentSyncProviders = [ + this.articleSyncService, + this.subArticleSyncService, + this.anchorPageSyncService, + this.lifeEventPageSyncService, + this.articleCategorySyncService, + this.newsSyncService, + this.adgerdirPageSyncService, + this.menuSyncService, + this.groupedMenuSyncService, + this.organizationPageSyncService, + this.organizationSubpageSyncService, + this.projectPageSyncService, + this.frontpageSyncService, + this.supportQNASyncService, + this.linkSyncService, + this.enhancedAssetService, + this.vacancyService, + this.serviceWebPageSyncService, + this.eventSyncService, + this.manualSyncService, + this.manualChapterItemSyncService, + this.customPageSyncService, + this.genericListItemSyncService, + this.teamListSyncService, + ] + } + + mapData(entries: processSyncDataInput) { + return this.contentSyncProviders.map((contentSyncProvider) => { + const data = contentSyncProvider.processSyncData(entries) + return contentSyncProvider.doMapping(data) + }) + } +}