Skip to content

Commit

Permalink
feat(search-indexer): Move mapping logic to separate class (#16410)
Browse files Browse the repository at this point in the history
Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
  • Loading branch information
RunarVestmann and kodiakhq[bot] authored Oct 15, 2024
1 parent f560cd2 commit 4a0a50a
Show file tree
Hide file tree
Showing 4 changed files with 128 additions and 85 deletions.
2 changes: 2 additions & 0 deletions libs/cms/src/lib/search/cmsSync.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import { ManualChapterItemSyncService } from './importers/manualChapterItem.serv
import { CustomPageSyncService } from './importers/customPage.service'
import { GenericListItemSyncService } from './importers/genericListItem.service'
import { TeamListSyncService } from './importers/teamList.service'
import { MappingService } from './mapping.service'

@Module({
imports: [
Expand Down Expand Up @@ -69,6 +70,7 @@ import { TeamListSyncService } from './importers/teamList.service'
CustomPageSyncService,
GenericListItemSyncService,
TeamListSyncService,
MappingService,
],
exports: [CmsSyncService],
})
Expand Down
87 changes: 4 additions & 83 deletions libs/cms/src/lib/search/cmsSync.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,34 +8,11 @@ import {
SyncOptions,
SyncResponse,
} from '@island.is/content-search-indexer/types'
import { ArticleSyncService } from './importers/article.service'
import { SubArticleSyncService } from './importers/subArticle.service'
import { ContentfulService } from './contentful.service'
import { AnchorPageSyncService } from './importers/anchorPage.service'
import { LifeEventPageSyncService } from './importers/lifeEventPage.service'
import { ArticleCategorySyncService } from './importers/articleCategory.service'
import { NewsSyncService } from './importers/news.service'
import { Entry } from 'contentful'
import { ElasticService, SearchInput } from '@island.is/content-search-toolkit'
import { AdgerdirPageSyncService } from './importers/adgerdirPage'
import { MenuSyncService } from './importers/menu.service'
import { GroupedMenuSyncService } from './importers/groupedMenu.service'
import { getElasticsearchIndex } from '@island.is/content-search-index-manager'
import { OrganizationPageSyncService } from './importers/organizationPage.service'
import { OrganizationSubpageSyncService } from './importers/organizationSubpage.service'
import { FrontpageSyncService } from './importers/frontpage.service'
import { SupportQNASyncService } from './importers/supportQNA.service'
import { LinkSyncService } from './importers/link.service'
import { ProjectPageSyncService } from './importers/projectPage.service'
import { EnhancedAssetSyncService } from './importers/enhancedAsset.service'
import { VacancySyncService } from './importers/vacancy.service'
import { ServiceWebPageSyncService } from './importers/serviceWebPage.service'
import { EventSyncService } from './importers/event.service'
import { ManualSyncService } from './importers/manual.service'
import { ManualChapterItemSyncService } from './importers/manualChapterItem.service'
import { CustomPageSyncService } from './importers/customPage.service'
import { GenericListItemSyncService } from './importers/genericListItem.service'
import { TeamListSyncService } from './importers/teamList.service'
import { MappingService } from './mapping.service'

export interface PostSyncOptions {
folderHash: string
Expand All @@ -56,62 +33,11 @@ export interface CmsSyncProvider<T, ProcessOutput = any> {

@Injectable()
export class CmsSyncService implements ContentSearchImporter<PostSyncOptions> {
private contentSyncProviders: CmsSyncProvider<any>[]
constructor(
private readonly newsSyncService: NewsSyncService,
private readonly articleCategorySyncService: ArticleCategorySyncService,
private readonly articleSyncService: ArticleSyncService,
private readonly subArticleSyncService: SubArticleSyncService,
private readonly anchorPageSyncService: AnchorPageSyncService,
private readonly lifeEventPageSyncService: LifeEventPageSyncService,
private readonly adgerdirPageSyncService: AdgerdirPageSyncService,
private readonly contentfulService: ContentfulService,
private readonly menuSyncService: MenuSyncService,
private readonly groupedMenuSyncService: GroupedMenuSyncService,
private readonly organizationPageSyncService: OrganizationPageSyncService,
private readonly organizationSubpageSyncService: OrganizationSubpageSyncService,
private readonly projectPageSyncService: ProjectPageSyncService,
private readonly frontpageSyncService: FrontpageSyncService,
private readonly supportQNASyncService: SupportQNASyncService,
private readonly linkSyncService: LinkSyncService,
private readonly enhancedAssetService: EnhancedAssetSyncService,
private readonly mappingService: MappingService,
private readonly elasticService: ElasticService,
private readonly vacancyService: VacancySyncService,
private readonly serviceWebPageSyncService: ServiceWebPageSyncService,
private readonly eventSyncService: EventSyncService,
private readonly manualSyncService: ManualSyncService,
private readonly manualChapterItemSyncService: ManualChapterItemSyncService,
private readonly customPageSyncService: CustomPageSyncService,
private readonly genericListItemSyncService: GenericListItemSyncService,
private readonly teamListSyncService: TeamListSyncService,
) {
this.contentSyncProviders = [
this.articleSyncService,
this.subArticleSyncService,
this.anchorPageSyncService,
this.lifeEventPageSyncService,
this.articleCategorySyncService,
this.newsSyncService,
this.adgerdirPageSyncService,
this.menuSyncService,
this.groupedMenuSyncService,
this.organizationPageSyncService,
this.organizationSubpageSyncService,
this.projectPageSyncService,
this.frontpageSyncService,
this.supportQNASyncService,
this.linkSyncService,
this.enhancedAssetService,
this.vacancyService,
this.serviceWebPageSyncService,
this.eventSyncService,
this.manualSyncService,
this.manualChapterItemSyncService,
this.customPageSyncService,
this.genericListItemSyncService,
this.teamListSyncService,
]
}
) {}

private async getLastFolderHash(elasticIndex: string): Promise<string> {
logger.info('Getting folder hash from index', {
Expand Down Expand Up @@ -228,12 +154,7 @@ export class CmsSyncService implements ContentSearchImporter<PostSyncOptions> {
logger.info('Got sync data')

// import data from all providers
const importableData = this.contentSyncProviders.map(
(contentSyncProvider) => {
const data = contentSyncProvider.processSyncData(items)
return contentSyncProvider.doMapping(data)
},
)
const importableData = this.mappingService.mapData(items)

return {
add: flatten(importableData),
Expand Down
31 changes: 29 additions & 2 deletions libs/cms/src/lib/search/contentful.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import { Locale } from '@island.is/shared/types'
import type { ApiResponse } from '@elastic/elasticsearch'
import type { SearchResponse } from '@island.is/shared/types'
import type { MappedData } from '@island.is/content-search-indexer/types'
import { MappingService } from './mapping.service'

type SyncCollection = ContentfulSyncCollection & {
nextPageToken?: string
Expand Down Expand Up @@ -79,6 +80,7 @@ export class ContentfulService {
constructor(
private readonly elasticService: ElasticService,
private readonly featureFlagService: FeatureFlagService,
private readonly mappingService: MappingService,
) {
const params: CreateClientParams = {
space: environment.contentful.space,
Expand Down Expand Up @@ -477,6 +479,9 @@ export class ContentfulService {
const idsCopy = [...ids]
let idsChunk = idsCopy.splice(-MAX_REQUEST_COUNT, MAX_REQUEST_COUNT)

let nestedProgress = idsChunk.length
const totalNested = ids.length

while (idsChunk.length > 0) {
const size = 100
let page = 1
Expand Down Expand Up @@ -566,20 +571,42 @@ export class ContentfulService {
(id) => !indexableEntries.some((entry) => entry.sys.id === id),
)

let progress = 0
const total = rootEntryIds.length

let chunkIds = rootEntryIds.splice(-chunkSize, chunkSize)
progress += chunkIds.length

while (chunkIds.length > 0) {
const items = await this.getContentfulData(chunkSize, {
include: this.defaultIncludeDepth,
'sys.id[in]': chunkIds.join(','),
locale: this.contentfulLocaleMap[locale],
})
indexableEntries.push(...items)

// import data from all providers
const importableData = this.mappingService.mapData(items)

await this.elasticService.bulk(elasticIndex, {
add: flatten(importableData),
remove: [],
})

logger.info(
`${progress}/${total} resolved root entries have been synced`,
)

chunkIds = rootEntryIds.splice(-chunkSize, chunkSize)
progress += chunkIds.length
}
}

logger.info(
`${nestedProgress}/${totalNested} nested entries have been resolved`,
)

idsChunk = idsCopy.splice(-MAX_REQUEST_COUNT, MAX_REQUEST_COUNT)
nestedProgress += idsChunk.length
}
}

Expand Down Expand Up @@ -643,7 +670,7 @@ export class ContentfulService {
elasticIndex,
locale,
chunkSize,
indexableEntries, // This array is modified
indexableEntries,
)

logger.info(
Expand Down
93 changes: 93 additions & 0 deletions libs/cms/src/lib/search/mapping.service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
import { Injectable } from '@nestjs/common'

import { ArticleSyncService } from './importers/article.service'
import { SubArticleSyncService } from './importers/subArticle.service'
import { AnchorPageSyncService } from './importers/anchorPage.service'
import { LifeEventPageSyncService } from './importers/lifeEventPage.service'
import { ArticleCategorySyncService } from './importers/articleCategory.service'
import { NewsSyncService } from './importers/news.service'
import { AdgerdirPageSyncService } from './importers/adgerdirPage'
import { MenuSyncService } from './importers/menu.service'
import { GroupedMenuSyncService } from './importers/groupedMenu.service'
import { OrganizationPageSyncService } from './importers/organizationPage.service'
import { OrganizationSubpageSyncService } from './importers/organizationSubpage.service'
import { FrontpageSyncService } from './importers/frontpage.service'
import { SupportQNASyncService } from './importers/supportQNA.service'
import { LinkSyncService } from './importers/link.service'
import { ProjectPageSyncService } from './importers/projectPage.service'
import { EnhancedAssetSyncService } from './importers/enhancedAsset.service'
import { VacancySyncService } from './importers/vacancy.service'
import { ServiceWebPageSyncService } from './importers/serviceWebPage.service'
import { EventSyncService } from './importers/event.service'
import { ManualSyncService } from './importers/manual.service'
import { ManualChapterItemSyncService } from './importers/manualChapterItem.service'
import { CustomPageSyncService } from './importers/customPage.service'
import { GenericListItemSyncService } from './importers/genericListItem.service'
import { TeamListSyncService } from './importers/teamList.service'
import type { CmsSyncProvider, processSyncDataInput } from './cmsSync.service'

@Injectable()
export class MappingService {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
private contentSyncProviders: CmsSyncProvider<any>[]
constructor(
private readonly newsSyncService: NewsSyncService,
private readonly articleCategorySyncService: ArticleCategorySyncService,
private readonly articleSyncService: ArticleSyncService,
private readonly subArticleSyncService: SubArticleSyncService,
private readonly anchorPageSyncService: AnchorPageSyncService,
private readonly lifeEventPageSyncService: LifeEventPageSyncService,
private readonly adgerdirPageSyncService: AdgerdirPageSyncService,
private readonly menuSyncService: MenuSyncService,
private readonly groupedMenuSyncService: GroupedMenuSyncService,
private readonly organizationPageSyncService: OrganizationPageSyncService,
private readonly organizationSubpageSyncService: OrganizationSubpageSyncService,
private readonly projectPageSyncService: ProjectPageSyncService,
private readonly frontpageSyncService: FrontpageSyncService,
private readonly supportQNASyncService: SupportQNASyncService,
private readonly linkSyncService: LinkSyncService,
private readonly enhancedAssetService: EnhancedAssetSyncService,
private readonly vacancyService: VacancySyncService,
private readonly serviceWebPageSyncService: ServiceWebPageSyncService,
private readonly eventSyncService: EventSyncService,
private readonly manualSyncService: ManualSyncService,
private readonly manualChapterItemSyncService: ManualChapterItemSyncService,
private readonly customPageSyncService: CustomPageSyncService,
private readonly genericListItemSyncService: GenericListItemSyncService,
private readonly teamListSyncService: TeamListSyncService,
) {
this.contentSyncProviders = [
this.articleSyncService,
this.subArticleSyncService,
this.anchorPageSyncService,
this.lifeEventPageSyncService,
this.articleCategorySyncService,
this.newsSyncService,
this.adgerdirPageSyncService,
this.menuSyncService,
this.groupedMenuSyncService,
this.organizationPageSyncService,
this.organizationSubpageSyncService,
this.projectPageSyncService,
this.frontpageSyncService,
this.supportQNASyncService,
this.linkSyncService,
this.enhancedAssetService,
this.vacancyService,
this.serviceWebPageSyncService,
this.eventSyncService,
this.manualSyncService,
this.manualChapterItemSyncService,
this.customPageSyncService,
this.genericListItemSyncService,
this.teamListSyncService,
]
}

mapData(entries: processSyncDataInput<unknown>) {
return this.contentSyncProviders.map((contentSyncProvider) => {
const data = contentSyncProvider.processSyncData(entries)
return contentSyncProvider.doMapping(data)
})
}
}

0 comments on commit 4a0a50a

Please sign in to comment.