From 0729619e2938657536263707163b13804a58d3f8 Mon Sep 17 00:00:00 2001 From: Padmaja <52911293+padms@users.noreply.github.com> Date: Thu, 18 Aug 2022 19:36:15 +0530 Subject: [PATCH] =?UTF-8?q?=F0=9F=9A=A9=20Index=20local=20news=20(#933)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{ => common}/news/algolia.ts | 2 +- search/IndexSanityContent/index.ts | 2 + search/IndexSanityContent/localNews/index.ts | 30 +++ .../localNews/mapper.test.ts | 187 ++++++++++++++++++ search/IndexSanityContent/localNews/mapper.ts | 29 +++ search/IndexSanityContent/localNews/sanity.ts | 63 ++++++ search/IndexSanityContent/news/index.ts | 2 +- search/common/types.ts | 1 + 8 files changed, 314 insertions(+), 2 deletions(-) rename search/IndexSanityContent/{ => common}/news/algolia.ts (82%) create mode 100644 search/IndexSanityContent/localNews/index.ts create mode 100644 search/IndexSanityContent/localNews/mapper.test.ts create mode 100644 search/IndexSanityContent/localNews/mapper.ts create mode 100644 search/IndexSanityContent/localNews/sanity.ts diff --git a/search/IndexSanityContent/news/algolia.ts b/search/IndexSanityContent/common/news/algolia.ts similarity index 82% rename from search/IndexSanityContent/news/algolia.ts rename to search/IndexSanityContent/common/news/algolia.ts index dbf5afcde..08df263fb 100644 --- a/search/IndexSanityContent/news/algolia.ts +++ b/search/IndexSanityContent/common/news/algolia.ts @@ -3,7 +3,7 @@ import { Settings } from '@algolia/client-search' export const indexSettings: Settings = { searchableAttributes: ['pageTitle', 'text'], attributesToSnippet: ['text'], - attributesForFaceting: ['year', 'topicTags', 'countryTags'], + attributesForFaceting: ['year', 'topicTags', 'countryTags', 'localNewsTag'], attributeForDistinct: 'slug', distinct: 1, advancedSyntax: true, diff --git a/search/IndexSanityContent/index.ts b/search/IndexSanityContent/index.ts index f9d1222b1..d559a9955 100644 --- a/search/IndexSanityContent/index.ts +++ b/search/IndexSanityContent/index.ts @@ -7,6 +7,7 @@ import { indexNews } from './news' import { indexMagazine } from './magazine' import { languageFromIso, languageOrDefault } from '../common' import { pipe } from 'fp-ts/lib/function' +import { indexLocalNews } from './localNews' const timerTrigger: AzureFunction = async function (context: Context, req: HttpRequest): Promise { await new DotenvAzure().config({ @@ -21,6 +22,7 @@ const timerTrigger: AzureFunction = async function (context: Context, req: HttpR await indexTopic(language)().catch(logger.error) await indexNews(language)().catch(logger.error) await indexMagazine(language)().catch(logger.error) + await indexLocalNews(language)().catch(logger.error) } export default timerTrigger diff --git a/search/IndexSanityContent/localNews/index.ts b/search/IndexSanityContent/localNews/index.ts new file mode 100644 index 000000000..df094d479 --- /dev/null +++ b/search/IndexSanityContent/localNews/index.ts @@ -0,0 +1,30 @@ +import { flow, pipe } from 'fp-ts/lib/function' +import { flatten } from 'fp-ts/Array' +import { ap } from 'fp-ts/lib/Identity' +import * as E from 'fp-ts/lib/Either' +import * as T from 'fp-ts/lib/Task' +import * as TE from 'fp-ts/lib/TaskEither' +import { update, generateIndexName, Language, getSanityClient } from '../../common' +import { fetchData } from './sanity' +import { indexSettings } from '../common/news/algolia' +import { mapData } from './mapper' + +const indexIdentifier = 'NEWS' + +export const indexLocalNews = (language: Language) => { + const indexName = flow( + () => E.fromNullable('Use getEnvironment here after acceptance')('dev'), + E.map(generateIndexName(indexIdentifier)(language.isoCode)), + ) + const updateAlgolia = flow(indexName, E.map(flow(update, ap(indexSettings)))) + + return pipe( + getSanityClient(), + TE.fromEither, + TE.chainW(fetchData(language)), + TE.map((pages) => pipe(pages.map(mapData), flatten)), + TE.chainW((data) => pipe(updateAlgolia(), E.ap(E.of(data)), TE.fromEither)), + TE.flatten, + T.map(E.fold(console.error, console.log)), + ) +} diff --git a/search/IndexSanityContent/localNews/mapper.test.ts b/search/IndexSanityContent/localNews/mapper.test.ts new file mode 100644 index 000000000..3df0514e4 --- /dev/null +++ b/search/IndexSanityContent/localNews/mapper.test.ts @@ -0,0 +1,187 @@ +import { LocalNewsArticle } from './sanity' +import { NewsIndex } from '../../common' +import { mapData } from './mapper' + +describe('Local News', () => { + describe('mapper tests', () => { + describe('Vanilla version works', () => { + // Hei fremtids-Nils Magne, vi bør ikke ha en en-til-en type mot Sanity? + const newsArticle: LocalNewsArticle = { + title: 'title', + ingress: 'ingress', + slug: '/a/slug', + _id: 'id', + publishDateTime: '2021-11-26T07:00:00.000Z', + localNewsTag: 'Germany', + // tags: ['Oil', 'Norway'], + blocks: [ + { + blockKey: 'blockKey', + children: [ + { + childKey: 'childKey', + text: 'Some text', + }, + ], + }, + ], + } + + const sut = mapData + const result = sut(newsArticle) + + it('contains one entry', () => { + expect(result).toHaveLength(1) + }) + + it('entry looks as expected', () => { + expect(result[0]).toEqual({ + slug: '/a/slug', + objectID: 'id-blockKey-childKey', + pageTitle: 'title', + ingress: 'ingress', + type: 'localNews', + text: 'Some text', + publishDateTime: '2021-11-26T07:00:00.000Z', + year: 2021, + localNewsTag: 'Germany', + } as NewsIndex) + }) + }) + + describe('Multiple children works (advanced version)', () => { + const newsArticle: LocalNewsArticle & { year: number } = { + title: 'title', + ingress: 'ingress', + slug: '/a/slug', + _id: 'id', + publishDateTime: '2021-11-26T07:00:00.000Z', + year: 2021, + localNewsTag: 'Germany', + blocks: [ + { + blockKey: 'blockKey', + children: [ + { + childKey: 'childKey', + text: 'Some text', + }, + { + childKey: 'childKey2', + text: 'Some more text', + }, + ], + }, + { + blockKey: 'blockKey2', + children: [ + { + childKey: 'childKey3', + text: 'Some text 3', + }, + { + childKey: 'childKey4', + text: 'Some more text 4', + }, + ], + }, + ], + } + + const sut = mapData + const result = sut(newsArticle) + + it('contains one entry', () => { + expect(result).toHaveLength(4) + }) + + it('entry looks as expected', () => { + expect(result[0]).toEqual({ + slug: '/a/slug', + objectID: 'id-blockKey-childKey', + pageTitle: 'title', + ingress: 'ingress', + type: 'localNews', + text: 'Some text', + publishDateTime: '2021-11-26T07:00:00.000Z', + year: 2021, + localNewsTag: 'Germany', + } as NewsIndex) + expect(result[1]).toEqual({ + slug: '/a/slug', + objectID: 'id-blockKey-childKey2', + pageTitle: 'title', + ingress: 'ingress', + type: 'localNews', + text: 'Some more text', + publishDateTime: '2021-11-26T07:00:00.000Z', + year: 2021, + localNewsTag: 'Germany', + }) + expect(result[2]).toEqual({ + slug: '/a/slug', + objectID: 'id-blockKey2-childKey3', + pageTitle: 'title', + ingress: 'ingress', + type: 'localNews', + text: 'Some text 3', + publishDateTime: '2021-11-26T07:00:00.000Z', + year: 2021, + localNewsTag: 'Germany', + }) + expect(result[3]).toEqual({ + slug: '/a/slug', + objectID: 'id-blockKey2-childKey4', + pageTitle: 'title', + ingress: 'ingress', + type: 'localNews', + text: 'Some more text 4', + publishDateTime: '2021-11-26T07:00:00.000Z', + year: 2021, + localNewsTag: 'Germany', + }) + }) + }) + + describe('Empty children works', () => { + const newsArticle: LocalNewsArticle = { + title: 'title', + ingress: 'ingress', + slug: '/a/slug', + _id: 'id', + localNewsTag: 'UK', + blocks: [ + { + blockKey: 'blockKey', + children: [], + }, + ], + } + + const sut = mapData + const result = sut(newsArticle) + + it('Does not contain entries', () => { + expect(result).toHaveLength(0) + }) + }) + + describe('Empty blocks works', () => { + const newsArticle: LocalNewsArticle = { + title: 'title', + ingress: 'ingress', + slug: '/a/slug', + localNewsTag: 'USA', + _id: 'id', + blocks: [], + } + + const sut = mapData + const result = sut(newsArticle) + + it('Does not contain entries', () => { + expect(result).toHaveLength(0) + }) + }) + }) +}) diff --git a/search/IndexSanityContent/localNews/mapper.ts b/search/IndexSanityContent/localNews/mapper.ts new file mode 100644 index 000000000..54e8faaf1 --- /dev/null +++ b/search/IndexSanityContent/localNews/mapper.ts @@ -0,0 +1,29 @@ +import { pipe } from 'fp-ts/lib/function' +import * as A from 'fp-ts/lib/Array' +import { NewsIndex } from '../../common' +import type { LocalNewsArticle } from './sanity' + +type MapDataType = (article: LocalNewsArticle) => NewsIndex[] +export const mapData: MapDataType = (article) => { + const { publishDateTime, localNewsTag, title, ingress, slug } = article + // Hu hei hvor det går + const year = publishDateTime ? new Date(publishDateTime).getFullYear() : '' + return pipe( + A.bindTo('blocks')(article.blocks), + A.bind('children', ({ blocks }) => blocks.children), + A.map( + ({ blocks, children }) => + ({ + slug, + objectID: `${article._id}-${blocks.blockKey}-${children.childKey}`, + type: 'localNews', + pageTitle: title, + ingress, + text: children.text, + publishDateTime: publishDateTime, + localNewsTag, + year, + } as NewsIndex), + ), + ) +} diff --git a/search/IndexSanityContent/localNews/sanity.ts b/search/IndexSanityContent/localNews/sanity.ts new file mode 100644 index 000000000..1528f55e1 --- /dev/null +++ b/search/IndexSanityContent/localNews/sanity.ts @@ -0,0 +1,63 @@ +import * as E from 'fp-ts/lib/Either' +import * as TE from 'fp-ts/lib/TaskEither' +import { pipe } from 'fp-ts/lib/function' +import { SanityClient } from '@sanity/client' +import { Language } from '../../common' + +const publishDateTimeQuery = /* groq */ ` + select( + customPublicationDate == true => + publishDateTime, + coalesce(firstPublishedAt, _createdAt) + ) +` + +export const query = /* groq */ `*[_type == "localNews" && _lang == $lang && !(_id in path("drafts.**")) && excludeFromSearch != true] { + "slug": slug.current, + _id, + "title": title, + "ingress": ingress, + "type": _type, + "publishDateTime": ${publishDateTimeQuery}, + "localNewsTag": localNewsTag->[$lang], + "blocks": content[_type == "block"] { + "blockKey": _key, + "children": children[text match "*"] { + "childKey": _key, + "text": text + } + }, +} +` + +const getQueryParams = (language: Language) => ({ + lang: language.internalCode, +}) + +export type LocalNewsArticle = { + slug: string + title: string + ingress: string + // ISO 8601 + publishDateTime?: string + localNewsTag: string + blocks: { + blockKey: string + children: { + childKey: string + text: string + }[] + }[] + _id: string +} + +type FetchDataType = ( + query: string, +) => ( + getQueryparams: (language: Language) => Readonly>, +) => (language: Language) => (sanityClient: SanityClient) => TE.TaskEither + +const fetch: FetchDataType = (query) => (getQueryParams) => (language) => (sanityClient) => + pipe(TE.tryCatch(() => sanityClient.fetch(query, getQueryParams(language)), E.toError)) + +export const fetchData = fetch(query)(getQueryParams) diff --git a/search/IndexSanityContent/news/index.ts b/search/IndexSanityContent/news/index.ts index 1f07f659a..47921ae87 100644 --- a/search/IndexSanityContent/news/index.ts +++ b/search/IndexSanityContent/news/index.ts @@ -6,7 +6,7 @@ import * as T from 'fp-ts/lib/Task' import * as TE from 'fp-ts/lib/TaskEither' import { update, generateIndexName, getEnvironment, Language, getSanityClient } from '../../common' import { fetchData } from './sanity' -import { indexSettings } from './algolia' +import { indexSettings } from '../common/news/algolia' import { mapData } from './mapper' const indexIdentifier = 'NEWS' diff --git a/search/common/types.ts b/search/common/types.ts index f115fa7b5..c84b4d77c 100644 --- a/search/common/types.ts +++ b/search/common/types.ts @@ -57,6 +57,7 @@ export type NewsIndex = { topicTags: string[] countryTags: string[] thumbnailUrl: string + localNewsTag: string } export type MagazineIndex = {