From 83ecfb493347746e911e38b825e8be11425fa89d Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Thu, 30 May 2024 11:54:35 +0200 Subject: [PATCH 01/40] fix: glossary --- .../types/src/hasura/glossary.ts | 5 + shared/types/src/hasura/index.ts | 1 + .../src/glossary/__mocks__}/glossaryData.ts | 0 .../__tests__/addGlossaryContent.test.ts | 619 ++++++++++-------- .../utils/src/glossary/addGlossaryContent.ts | 24 + .../src}/glossary/explodeGlossaryTerms.ts | 3 +- shared/utils/src/glossary/index.ts | 1 + .../glossary/insertWebComponentGlossary.ts | 2 +- shared/utils/src/index.ts | 1 + .../src/controllers/glossary.ts | 38 ++ .../src/controllers/index.ts | 1 + .../src/ingester/cdtnDocuments.ts | 14 +- .../src/ingester/common/fetchGlossary.ts | 2 +- .../__tests__/addGlossaryToContent.test.ts | 9 +- .../contributions/addGlossaryToContent.ts | 5 +- .../src/ingester/contributions/generate.ts | 3 +- .../glossary/createGlossaryTransform.ts | 40 -- .../src/ingester/glossary/index.ts | 1 - .../src/ingester/glossary/types.ts | 4 - .../src/ingester/informations/generate.ts | 5 +- .../src/ingester/informations/markdown.ts | 11 +- .../src/services/contributions.ts | 75 +++ .../contribution/mapContributionToDocument.ts | 14 +- 23 files changed, 522 insertions(+), 356 deletions(-) rename targets/export-elasticsearch/src/ingester/types.ts => shared/types/src/hasura/glossary.ts (69%) rename {targets/export-elasticsearch/src/ingester/glossary/__mock__ => shared/utils/src/glossary/__mocks__}/glossaryData.ts (100%) rename targets/export-elasticsearch/src/ingester/glossary/__tests__/glossary.test.ts => shared/utils/src/glossary/__tests__/addGlossaryContent.test.ts (56%) create mode 100644 shared/utils/src/glossary/addGlossaryContent.ts rename {targets/export-elasticsearch/src/ingester => shared/utils/src}/glossary/explodeGlossaryTerms.ts (96%) create mode 100644 shared/utils/src/glossary/index.ts rename {targets/export-elasticsearch/src/ingester => shared/utils/src}/glossary/insertWebComponentGlossary.ts (93%) create mode 100644 targets/export-elasticsearch/src/controllers/glossary.ts delete mode 100644 targets/export-elasticsearch/src/ingester/glossary/createGlossaryTransform.ts delete mode 100644 targets/export-elasticsearch/src/ingester/glossary/index.ts delete mode 100644 targets/export-elasticsearch/src/ingester/glossary/types.ts create mode 100644 targets/export-elasticsearch/src/services/contributions.ts diff --git a/targets/export-elasticsearch/src/ingester/types.ts b/shared/types/src/hasura/glossary.ts similarity index 69% rename from targets/export-elasticsearch/src/ingester/types.ts rename to shared/types/src/hasura/glossary.ts index cba8515ad..924601214 100644 --- a/targets/export-elasticsearch/src/ingester/types.ts +++ b/shared/types/src/hasura/glossary.ts @@ -8,3 +8,8 @@ export interface Term { slug?: string; references?: string[]; } + +export type GlossaryTerms = { + definition: string | null; + pattern: RegExp; +}; diff --git a/shared/types/src/hasura/index.ts b/shared/types/src/hasura/index.ts index f96e7ce83..e9172aec0 100644 --- a/shared/types/src/hasura/index.ts +++ b/shared/types/src/hasura/index.ts @@ -12,3 +12,4 @@ export * from "./highlights"; export * from "./modeles-de-courrier"; export * from "./prequalified"; export * from "./themes"; +export * from "./glossary"; diff --git a/targets/export-elasticsearch/src/ingester/glossary/__mock__/glossaryData.ts b/shared/utils/src/glossary/__mocks__/glossaryData.ts similarity index 100% rename from targets/export-elasticsearch/src/ingester/glossary/__mock__/glossaryData.ts rename to shared/utils/src/glossary/__mocks__/glossaryData.ts diff --git a/targets/export-elasticsearch/src/ingester/glossary/__tests__/glossary.test.ts b/shared/utils/src/glossary/__tests__/addGlossaryContent.test.ts similarity index 56% rename from targets/export-elasticsearch/src/ingester/glossary/__tests__/glossary.test.ts rename to shared/utils/src/glossary/__tests__/addGlossaryContent.test.ts index f45cebadc..ffd454bcf 100644 --- a/targets/export-elasticsearch/src/ingester/glossary/__tests__/glossary.test.ts +++ b/shared/utils/src/glossary/__tests__/addGlossaryContent.test.ts @@ -1,24 +1,21 @@ -import { context } from "../../context"; -import { createGlossaryTransform } from ".."; -import { glossaryData } from "../__mock__/glossaryData"; +import { addGlossaryContent } from "../addGlossaryContent"; +import { glossaryData } from "../__mocks__/glossaryData"; describe("Glossary", () => { - describe("addGlossary", () => { - context.provide(); - const addGlossary = createGlossaryTransform(glossaryData); - test("should return a formatted html with web components tooltip", () => { + describe("addGlossaryContent", () => { + test("should return a formatted html with web components tooltip", async () => { const htmlContent = "

voici une convention collective et un web component mais aussi dispositions, ceci est un test

"; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `

voici une convention collective et un web component mais aussi dispositions, ceci est un test

` ); }); - test("should not replace html property for glossary word", () => { + test("should not replace html property for glossary word", async () => { const htmlContent = `test L'indemnité de fin de contrat n'est pas due dans les cas suivants `; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `test L'indemnité de fin de contrat n'est pas due dans les cas suivants @@ -26,12 +23,12 @@ describe("Glossary", () => { ); }); - test("should not add webcomponent tooltip in a summary tag", () => { + test("should not add webcomponent tooltip in a summary tag", async () => { const htmlContent = `test L'indemnité de fin de contrat n'est pas due dans les cas suivants `; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `test L'indemnité de fin de contrat n'est pas due dans les cas suivants @@ -39,37 +36,37 @@ describe("Glossary", () => { ); }); - test("should add webcomponent tooltip even if next tag is omitted", () => { + test("should add webcomponent tooltip even if next tag is omitted", async () => { const htmlContent = `

indemnité

test`; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `

indemnité

test` ); }); - test("should not add webcomponent tooltip in a summary tag with strong", () => { + test("should not add webcomponent tooltip in a summary tag with strong", async () => { const htmlContent = `L'indemnité de fin de contrat n'est pas due dans les cas suivants`; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `L'indemnité de fin de contrat n'est pas due dans les cas suivants` ); }); - test("should not add webcomponent tooltip in a summary tag with multiple strong", () => { + test("should not add webcomponent tooltip in a summary tag with multiple strong", async () => { const htmlContent = `L'indemnité de fin de contrat n'est pas due dans les cas suivants`; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `L'indemnité de fin de contrat n'est pas due dans les cas suivants` ); }); - test("should not add webcomponent tooltip in a summary tag with multiple strong", () => { + test("should not add webcomponent tooltip in a summary tag with multiple strong", async () => { const htmlContent = `Test L'indemnité de fin de contrat n'est pas due dans les cas suivants Test`; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `Test L'indemnité de fin de contrat n'est pas due dans les cas suivants Test` ); }); - test("should not add webcomponent tooltip in a summaries tag with multiple strong", () => { + test("should not add webcomponent tooltip in a summaries tag with multiple strong", async () => { const htmlContent = `L'indemnité de fin de contrat n'est pas due dans les cas suivantsL'indemnité doit être un tooltipL'indemnité de fin de contrat n'est pas due dans les cas suivants`; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `L'indemnité de fin de contrat n'est pas due dans les cas suivantsL'indemnité doit être un tooltipL'indemnité de fin de contrat n'est pas due dans les cas suivants` ); }); @@ -82,29 +79,34 @@ describe("Glossary", () => { ${"h4"} ${"h5"} ${"h6"} - `("should not add webcomponent tooltip in $heading", ({ heading }) => { - const markdown = `<${heading}>indemnité`; - expect(addGlossary(markdown)).toEqual(markdown); - }); + `( + "should not add webcomponent tooltip in $heading", + async ({ heading }) => { + const markdown = `<${heading}>indemnité`; + expect(await addGlossaryContent(glossaryData, markdown)).toEqual( + markdown + ); + } + ); - test('should not add webcomponent tooltip in a span tag with class "title"', () => { + test('should not add webcomponent tooltip in a span tag with class "title"', async () => { const htmlContent = `L'indemnité de fin de contrat n'est pas due dans les cas suivants`; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `L'indemnité de fin de contrat n'est pas due dans les cas suivants` ); }); - test('should not add webcomponent tooltip in a span tag with class "sub-title"', () => { + test('should not add webcomponent tooltip in a span tag with class "sub-title"', async () => { const htmlContent = `L'indemnité de fin de contrat n'est pas due dans les cas suivants`; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `L'indemnité de fin de contrat n'est pas due dans les cas suivants` ); }); - test("should not replace within tag attributes", () => { + test("should not replace within tag attributes", async () => { const htmlContent = '

voici une convention collective et un web component mais aussi dispositions, ceci est un test

'; - expect(addGlossary(htmlContent)).toEqual( + expect(await addGlossaryContent(glossaryData, htmlContent)).toEqual( `

voici une convention collective et un web component mais aussi dispositions, ceci est un test

` ); }); @@ -122,268 +124,315 @@ describe("Glossary", () => { ${`voici une convention collective et un web component mais aussi dispositions, ceci est un test`} | ${`voici une convention collective et un web component mais aussi dispositions, ceci est un test`} `( "should replace the string by adding webcomponent-tooltip in $markdownContent", - ({ markdownContent, result }: InputTest) => { - expect(addGlossary(markdownContent)).toEqual(result); + async ({ markdownContent, result }: InputTest) => { + expect(await addGlossaryContent(glossaryData, markdownContent)).toEqual( + result + ); } ); }); describe("test glossary replacements", () => { describe("test replace terms", () => { - test("should work on text without tags", () => { + test("should work on text without tags", async () => { const htmlContent = `word1`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word1", - term: "word1", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word1", + term: "word1", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `word1` ); }); - test("should work on text before tags", () => { + test("should work on text before tags", async () => { const htmlContent = `word1`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word1", - term: "word1", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word1", + term: "word1", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `word1` ); }); - test("should work on text after tags", () => { + test("should work on text after tags", async () => { const htmlContent = `word1`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word1", - term: "word1", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word1", + term: "word1", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `word1` ); }); - test("should work between tags", () => { + test("should work between tags", async () => { const htmlContent = `word1`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word1", - term: "word1", - variants: [], - }, - { - abbreviations: [], - definition: "word3", - term: "word3", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word1", + term: "word1", + variants: [], + }, + { + abbreviations: [], + definition: "word3", + term: "word3", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `word1` ); }); - test("should work between multiple tags", () => { + test("should work between multiple tags", async () => { const htmlContent = `word1`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word1", - term: "word1", - variants: [], - }, - { - abbreviations: [], - definition: "word3", - term: "word3", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word1", + term: "word1", + variants: [], + }, + { + abbreviations: [], + definition: "word3", + term: "word3", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `word1` ); }); - test("should not work in tag's parameter", () => { + test("should not work in tag's parameter", async () => { const htmlContent = ``; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word1", - term: "word1", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word1", + term: "word1", + variants: [], + }, + ], + htmlContent + ) ).toEqual(``); }); - test("should not work inside another webcomponent", () => { + test("should not work inside another webcomponent", async () => { const htmlContent = `word1 word2`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "word1 word2", - variants: [], - }, - { - abbreviations: [], - definition: "word", - term: "word1", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "word1 word2", + variants: [], + }, + { + abbreviations: [], + definition: "word", + term: "word1", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `word1 word2` ); }); - test("should work on plural", () => { + test("should work on plural", async () => { const htmlContent = `words`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "word", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "word", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `words` ); }); - test("should work on mulitple plurals", () => { + test("should work on mulitple plurals", async () => { const htmlContent = `words wards`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "word wards", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "word wards", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `words wards` ); }); - test("should not work on words containing piece of the term", () => { + test("should not work on words containing piece of the term", async () => { const htmlContent = `wordpress`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "word", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "word", + variants: [], + }, + ], + htmlContent + ) ).toEqual(`wordpress`); }); - test("should work on accentuated words", () => { + test("should work on accentuated words", async () => { const htmlContent = `wörd`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "wörd", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "wörd", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `wörd` ); }); - test("should keep initial term", () => { + test("should keep initial term", async () => { const htmlContent = `Word`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "word", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "word", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `Word` ); }); - test("should match capitals at start of word", () => { + test("should match capitals at start of word", async () => { const htmlContent = `Word Ward`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "word ward", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "word ward", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `Word Ward` ); }); - test("should not replace agreement if it is in definition", () => { + test("should not replace agreement if it is in definition", async () => { const htmlContent = `word`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "convention collective", - term: "word", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "convention collective", + term: "word", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `word` ); }); - test("should replace all terms", () => { + test("should replace all terms", async () => { const htmlContent = `word1 et word2 et Word1`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "définition de word1", - term: "word1", - variants: [], - }, - { - abbreviations: [], - definition: "définition de word2", - term: "word2", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "définition de word1", + term: "word1", + variants: [], + }, + { + abbreviations: [], + definition: "définition de word2", + term: "word2", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `word1 et word2 et Word1` ); @@ -391,74 +440,83 @@ describe("Glossary", () => { }); describe("test on agreements", () => { - test("should work as tooltip-cc with specific agreement terms", () => { + test("should work as tooltip-cc with specific agreement terms", async () => { const htmlContent = `convention collective`; - expect(createGlossaryTransform([])(htmlContent)).toEqual( + expect(await addGlossaryContent([], htmlContent)).toEqual( `convention collective` ); }); - test("should work as tooltip-cc with specific agreement terms in plurial", () => { + test("should work as tooltip-cc with specific agreement terms in plurial", async () => { const htmlContent = `conventions collectives`; - expect(createGlossaryTransform([])(htmlContent)).toEqual( + expect(await addGlossaryContent([], htmlContent)).toEqual( `conventions collectives` ); }); - test("should work as tooltip-cc with uppercase agreement terms", () => { + test("should work as tooltip-cc with uppercase agreement terms", async () => { const htmlContent = `Convention collective`; - expect(createGlossaryTransform([])(htmlContent)).toEqual( + expect(await addGlossaryContent([], htmlContent)).toEqual( `Convention collective` ); }); - test("should match multiple terms as tooltip-cc", () => { + test("should match multiple terms as tooltip-cc", async () => { const htmlContent = `Conventions collectives blabla accord de branche`; - expect(createGlossaryTransform([])(htmlContent)).toEqual( + expect(await addGlossaryContent([], htmlContent)).toEqual( `Conventions collectives blabla accord de branche` ); }); }); describe("test replace abbreviations", () => { - test("should match uppercase with abbreviations", () => { + test("should match uppercase with abbreviations", async () => { const htmlContent = `WW`; expect( - createGlossaryTransform([ - { - abbreviations: ["WW"], - definition: "word", - term: "word", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: ["WW"], + definition: "word", + term: "word", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `WW` ); }); - test("should not match lowercase with abbreviations", () => { + test("should not match lowercase with abbreviations", async () => { const htmlContent = `ww`; expect( - createGlossaryTransform([ - { - abbreviations: ["WW"], - definition: "word", - term: "word", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: ["WW"], + definition: "word", + term: "word", + variants: [], + }, + ], + htmlContent + ) ).toEqual(`ww`); }); - test("should match abbreviations between parenthesis", () => { + test("should match abbreviations between parenthesis", async () => { const htmlContent = `World web (WW) world`; expect( - createGlossaryTransform([ - { - abbreviations: ["WW"], - definition: "word", - term: "word", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: ["WW"], + definition: "word", + term: "word", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `World web (WW) world` ); @@ -466,80 +524,95 @@ describe("Glossary", () => { }); describe("test replace variants", () => { - test("should match a variant", () => { + test("should match a variant", async () => { const htmlContent = `ward`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "word", - variants: ["ward"], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "word", + variants: ["ward"], + }, + ], + htmlContent + ) ).toEqual( `ward` ); }); - test("should match a variant with plural", () => { + test("should match a variant with plural", async () => { const htmlContent = `wards`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "word", - term: "word", - variants: ["ward"], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "word", + term: "word", + variants: ["ward"], + }, + ], + htmlContent + ) ).toEqual( `wards` ); }); - test("should match with a special character", () => { + test("should match with a special character", async () => { const htmlContent = `entreprise lock-out suite à dépôt de bilan`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "Fermeture", - term: "Lock-out", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "Fermeture", + term: "Lock-out", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `entreprise lock-out suite à dépôt de bilan` ); }); - test("should match a term with an abbreviation in the same sentence", () => { + test("should match a term with an abbreviation in the same sentence", async () => { const htmlContent = `si le protocole d’accord préélectoral (PAP) en stipule autrement.`; expect( - createGlossaryTransform([ - { - abbreviations: ["PAP"], - definition: "protocole", - term: "Protocole d’accord préélectoral (PAP)", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: ["PAP"], + definition: "protocole", + term: "Protocole d’accord préélectoral (PAP)", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `si le protocole d’accord préélectoral (PAP) en stipule autrement.` ); }); - test("should not match a term from a link", () => { + test("should not match a term from a link", async () => { const htmlContent = `Une notice explicative précise les modalités de remplissage des procès-verbaux.`; expect( - createGlossaryTransform([ - { - abbreviations: [], - definition: "notice", - term: "Notice", - variants: [], - }, - ])(htmlContent) + await addGlossaryContent( + [ + { + abbreviations: [], + definition: "notice", + term: "Notice", + variants: [], + }, + ], + htmlContent + ) ).toEqual( `Une notice explicative précise les modalités de remplissage des procès-verbaux.` ); diff --git a/shared/utils/src/glossary/addGlossaryContent.ts b/shared/utils/src/glossary/addGlossaryContent.ts new file mode 100644 index 000000000..82ba74b15 --- /dev/null +++ b/shared/utils/src/glossary/addGlossaryContent.ts @@ -0,0 +1,24 @@ +import { explodeGlossaryTerms } from "./explodeGlossaryTerms"; +import { insertWebComponentGlossary } from "./insertWebComponentGlossary"; +import { Glossary } from "@socialgouv/cdtn-types"; + +export const addGlossaryContent = async ( + glossary: Glossary, + content: string +): Promise => { + glossary.sort((previous, next) => { + return next.term.length - previous.term.length; + }); + const glossaryTerms = explodeGlossaryTerms(glossary).map((item) => { + const definition = item.definition + ? encodeURIComponent(item.definition.replace(/'/g, "’")) + : null; + return { + ...item, + definition, + }; + }); + + if (!content) return ""; + return insertWebComponentGlossary(content, glossaryTerms); +}; diff --git a/targets/export-elasticsearch/src/ingester/glossary/explodeGlossaryTerms.ts b/shared/utils/src/glossary/explodeGlossaryTerms.ts similarity index 96% rename from targets/export-elasticsearch/src/ingester/glossary/explodeGlossaryTerms.ts rename to shared/utils/src/glossary/explodeGlossaryTerms.ts index 5742e4b01..9bbac238b 100644 --- a/targets/export-elasticsearch/src/ingester/glossary/explodeGlossaryTerms.ts +++ b/shared/utils/src/glossary/explodeGlossaryTerms.ts @@ -1,5 +1,4 @@ -import type { Glossary, Term } from "../types"; -import type { GlossaryTerms } from "./types"; +import { GlossaryTerms, Glossary, Term } from "@socialgouv/cdtn-types"; const conventionMatchers = "[Cc]onventions? [Cc]ollectives?|[Aa]ccords? de [Bb]ranches?|[Dd]ispositions? [Cc]onventionnelles?"; diff --git a/shared/utils/src/glossary/index.ts b/shared/utils/src/glossary/index.ts new file mode 100644 index 000000000..0279b15c6 --- /dev/null +++ b/shared/utils/src/glossary/index.ts @@ -0,0 +1 @@ +export * from "./addGlossaryContent"; diff --git a/targets/export-elasticsearch/src/ingester/glossary/insertWebComponentGlossary.ts b/shared/utils/src/glossary/insertWebComponentGlossary.ts similarity index 93% rename from targets/export-elasticsearch/src/ingester/glossary/insertWebComponentGlossary.ts rename to shared/utils/src/glossary/insertWebComponentGlossary.ts index 95db59774..ac1281678 100644 --- a/targets/export-elasticsearch/src/ingester/glossary/insertWebComponentGlossary.ts +++ b/shared/utils/src/glossary/insertWebComponentGlossary.ts @@ -1,4 +1,4 @@ -import type { GlossaryTerms } from "./types"; +import { GlossaryTerms } from "@socialgouv/cdtn-types"; export const insertWebComponentGlossary = ( initialContent: string, diff --git a/shared/utils/src/index.ts b/shared/utils/src/index.ts index 59aa26acc..a85e8b935 100644 --- a/shared/utils/src/index.ts +++ b/shared/utils/src/index.ts @@ -4,3 +4,4 @@ export * from "./gql-client"; export * from "./config"; export * from "./logger"; export * from "./dila-resolver"; +export * from "./glossary"; diff --git a/targets/export-elasticsearch/src/controllers/glossary.ts b/targets/export-elasticsearch/src/controllers/glossary.ts new file mode 100644 index 000000000..071fd3073 --- /dev/null +++ b/targets/export-elasticsearch/src/controllers/glossary.ts @@ -0,0 +1,38 @@ +import type { interfaces } from "inversify-express-utils"; +import { + controller, + httpPost, + request, + response, +} from "inversify-express-utils"; +import { Request, Response } from "express"; +import { getContentGlossary } from "../ingester/glossary/addGlossary"; +import { + editContentOfContribution, + fetchAllContributionsDocument, +} from "../services/contributions"; + +@controller("/glossary") +export class GlossaryController implements interfaces.Controller { + @httpPost("/") + index(@request() req: Request, @response() res: Response): Promise { + const content = (req.body as any).content; + if (typeof content !== "string") { + res.status(500).send("Invalid content"); + } + return getContentGlossary(content); + } + + @httpPost("/all") + async all(@request() req: Request, @response() res: Response): Promise { + const contributions = await fetchAllContributionsDocument(); + for (let i = 0; i < contributions.length; i++) { + const content = contributions[i].document.content; + const item = getContentGlossary(content); + await editContentOfContribution(contributions[i].cdtnId, { + ...contributions[i].document, + content: item, + }); + } + } +} diff --git a/targets/export-elasticsearch/src/controllers/index.ts b/targets/export-elasticsearch/src/controllers/index.ts index 7a3979305..242777881 100644 --- a/targets/export-elasticsearch/src/controllers/index.ts +++ b/targets/export-elasticsearch/src/controllers/index.ts @@ -1,2 +1,3 @@ export * from "./export"; export * from "./monitoring"; +export * from "./glossary"; diff --git a/targets/export-elasticsearch/src/ingester/cdtnDocuments.ts b/targets/export-elasticsearch/src/ingester/cdtnDocuments.ts index e52736e10..0aae3c21d 100644 --- a/targets/export-elasticsearch/src/ingester/cdtnDocuments.ts +++ b/targets/export-elasticsearch/src/ingester/cdtnDocuments.ts @@ -18,7 +18,6 @@ import { getDocumentBySourceWithRelation, } from "./common/fetchCdtnAdminDocuments"; import { splitArticle } from "./fichesTravailSplitter"; -import { createGlossaryTransform } from "./glossary"; import { getVersions } from "./versions"; import { generateContributions } from "./contributions"; import { generateAgreements } from "./agreements"; @@ -63,9 +62,6 @@ export async function cdtnDocumentsGen( const getBreadcrumbs = buildGetBreadcrumbs(themes); - const glossaryTerms = await getGlossary(); - const addGlossary = createGlossaryTransform(glossaryTerms); - logger.info("=== Courriers ==="); const modelesDeCourriers = await getDocumentBySource( SOURCES.LETTERS, @@ -137,7 +133,6 @@ export async function cdtnDocumentsGen( contributions, ccnData, ccnListWithHighlight, - addGlossary, getBreadcrumbs ); @@ -185,11 +180,13 @@ export async function cdtnDocumentsGen( delete section.text; return { ...section, - html: addGlossary(html), + html, }; }), })); - logger.info(`Mapped ${fichesMTWithGlossary.length} fiches travail with glossary`); + logger.info( + `Mapped ${fichesMTWithGlossary.length} fiches travail with glossary` + ); documentsCount = { ...documentsCount, [SOURCES.SHEET_MT_PAGE]: fichesMTWithGlossary.length, @@ -263,6 +260,7 @@ export async function cdtnDocumentsGen( await updateDocs(SOURCES.PREQUALIFIED, prequalified); logger.info("=== glossary ==="); + const glossaryTerms = await getGlossary(); documentsCount = { ...documentsCount, [SOURCES.GLOSSARY]: glossaryTerms.length, @@ -290,7 +288,7 @@ export async function cdtnDocumentsGen( const { documents: editorialContents, relatedIdsDocuments: relatedIdsEditorialDocuments, - } = await generateEditorialContents(documents, addGlossary); + } = await generateEditorialContents(documents); documentsCount = { ...documentsCount, [SOURCES.EDITORIAL_CONTENT]: editorialContents.length, diff --git a/targets/export-elasticsearch/src/ingester/common/fetchGlossary.ts b/targets/export-elasticsearch/src/ingester/common/fetchGlossary.ts index 1ad401754..301880abe 100644 --- a/targets/export-elasticsearch/src/ingester/common/fetchGlossary.ts +++ b/targets/export-elasticsearch/src/ingester/common/fetchGlossary.ts @@ -1,4 +1,4 @@ -import type { Glossary } from "../types"; +import { Glossary } from "@socialgouv/cdtn-types"; import { context } from "../context"; import { gqlClient } from "@shared/utils"; diff --git a/targets/export-elasticsearch/src/ingester/contributions/__tests__/addGlossaryToContent.test.ts b/targets/export-elasticsearch/src/ingester/contributions/__tests__/addGlossaryToContent.test.ts index caaed0d72..f66767ec9 100644 --- a/targets/export-elasticsearch/src/ingester/contributions/__tests__/addGlossaryToContent.test.ts +++ b/targets/export-elasticsearch/src/ingester/contributions/__tests__/addGlossaryToContent.test.ts @@ -6,26 +6,21 @@ describe("addGlossaryToContent", () => { ficheSpDescription: "Description", content: "Content", }; - const addGlossary = jest.fn(); - const result = addGlossaryToContent(content, addGlossary); + const result = addGlossaryToContent(content); expect(result).toEqual(content); - expect(addGlossary).not.toHaveBeenCalled(); }); it("should call addGlossary and return modified content if no ficheSpDescription", () => { const content = { content: "Content", }; - const addGlossary = jest.fn(() => "Modified content"); - const result = addGlossaryToContent(content, addGlossary); + const result = addGlossaryToContent(content); expect(result).toEqual({ content: "Modified content", }); - expect(addGlossary).toHaveBeenCalledWith("Content"); - expect(addGlossary).toHaveBeenCalledTimes(1); }); }); diff --git a/targets/export-elasticsearch/src/ingester/contributions/addGlossaryToContent.ts b/targets/export-elasticsearch/src/ingester/contributions/addGlossaryToContent.ts index a0df0cc2f..2e5714585 100644 --- a/targets/export-elasticsearch/src/ingester/contributions/addGlossaryToContent.ts +++ b/targets/export-elasticsearch/src/ingester/contributions/addGlossaryToContent.ts @@ -1,8 +1,7 @@ import { ContributionContent } from "@socialgouv/cdtn-types"; export function addGlossaryToContent( - content: ContributionContent, - addGlossary: (valueInHtml: string) => string + content: ContributionContent ): ContributionContent { if ( "ficheSpDescription" in content || @@ -11,7 +10,7 @@ export function addGlossaryToContent( return content; } else { return { - content: addGlossary(content.content), + content: content.content, }; } } diff --git a/targets/export-elasticsearch/src/ingester/contributions/generate.ts b/targets/export-elasticsearch/src/ingester/contributions/generate.ts index 043da3339..3d3a9fb6c 100644 --- a/targets/export-elasticsearch/src/ingester/contributions/generate.ts +++ b/targets/export-elasticsearch/src/ingester/contributions/generate.ts @@ -33,7 +33,6 @@ export async function generateContributions( contributions: DocumentElasticWithSource[], ccnData: DocumentElasticWithSource[], ccnListWithHighlight: Record, - addGlossary: (valueInHtml: string) => string, getBreadcrumbs: GetBreadcrumbsFn ): Promise { const breadcrumbsOfRootContributionsPerIndex = contributions.reduce( @@ -82,7 +81,7 @@ export async function generateContributions( generatedContributions.push({ ...contrib, ...generateMetadata(contrib), - ...addGlossaryToContent(content, addGlossary), + ...addGlossaryToContent(content), ...doc, breadcrumbs: contrib.breadcrumbs.length > 0 diff --git a/targets/export-elasticsearch/src/ingester/glossary/createGlossaryTransform.ts b/targets/export-elasticsearch/src/ingester/glossary/createGlossaryTransform.ts deleted file mode 100644 index 5fa3eef7a..000000000 --- a/targets/export-elasticsearch/src/ingester/glossary/createGlossaryTransform.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { context } from "../context"; -import type { Glossary } from "../types"; -import { explodeGlossaryTerms } from "./explodeGlossaryTerms"; -import { insertWebComponentGlossary } from "./insertWebComponentGlossary"; - -/** - * addGlossary is a heavy operation that is only needed while dumping for ES - */ - -export type AddGlossaryReturnFn = (content: string) => string; - -export const createGlossaryTransform = ( - glossary: Glossary -): AddGlossaryReturnFn => { - const DISABLE_GLOSSARY = context.get("disableGlossary") ?? false; - - glossary.sort((previous, next) => { - return next.term.length - previous.term.length; - }); - const glossaryTerms = explodeGlossaryTerms(glossary).map((item) => { - const definition = item.definition - ? encodeURIComponent(item.definition.replace(/'/g, "’")) - : null; - return { - ...item, - definition, - }; - }); - - function addGlossary(content: string): string { - if (DISABLE_GLOSSARY) { - return content; - } - if (!content) return ""; - - return insertWebComponentGlossary(content, glossaryTerms); - } - - return addGlossary; -}; diff --git a/targets/export-elasticsearch/src/ingester/glossary/index.ts b/targets/export-elasticsearch/src/ingester/glossary/index.ts deleted file mode 100644 index fd87a1f96..000000000 --- a/targets/export-elasticsearch/src/ingester/glossary/index.ts +++ /dev/null @@ -1 +0,0 @@ -export * from "./createGlossaryTransform"; diff --git a/targets/export-elasticsearch/src/ingester/glossary/types.ts b/targets/export-elasticsearch/src/ingester/glossary/types.ts deleted file mode 100644 index 73dae64a8..000000000 --- a/targets/export-elasticsearch/src/ingester/glossary/types.ts +++ /dev/null @@ -1,4 +0,0 @@ -export type GlossaryTerms = { - definition: string | null; - pattern: RegExp; -}; diff --git a/targets/export-elasticsearch/src/ingester/informations/generate.ts b/targets/export-elasticsearch/src/ingester/informations/generate.ts index daf8ae158..ea5b2d9d8 100644 --- a/targets/export-elasticsearch/src/ingester/informations/generate.ts +++ b/targets/export-elasticsearch/src/ingester/informations/generate.ts @@ -11,10 +11,9 @@ interface Return { } export const generateEditorialContents = ( - documents: DocumentElasticWithSource[], - addGlossary: (valueInHtml: string) => string + documents: DocumentElasticWithSource[] ): Return => { - const documentsMarkdownified = markdownTransform(addGlossary, documents); + const documentsMarkdownified = markdownTransform(documents); const relatedIdsDocuments = getRelatedIdsDocuments(documentsMarkdownified); return { documents: documentsMarkdownified, diff --git a/targets/export-elasticsearch/src/ingester/informations/markdown.ts b/targets/export-elasticsearch/src/ingester/informations/markdown.ts index 2ecb48574..999b3f977 100644 --- a/targets/export-elasticsearch/src/ingester/informations/markdown.ts +++ b/targets/export-elasticsearch/src/ingester/informations/markdown.ts @@ -8,8 +8,6 @@ import markdownToMardownAst from "remark-parse"; import markdownAstToHtmlAst from "remark-rehype"; import unified from "unified"; -import type { AddGlossaryReturnFn } from "../glossary"; - const htmlProcessor = unified() .use(markdownToMardownAst as any) .use(markdownAstToHtmlAst as any, { allowDangerousHtml: true }) @@ -17,7 +15,6 @@ const htmlProcessor = unified() .use(htmlAstStringify as any); export function markdownTransform( - addGlossary: AddGlossaryReturnFn, documents: DocumentElasticWithSource[] ): DocumentElasticWithSource[] { return documents.map(({ contents = [], ...rest }) => ({ @@ -27,16 +24,12 @@ export function markdownTransform( return { ...block, html: block.markdown - ? addGlossary( - htmlProcessor.processSync(block.markdown).contents as string - ) + ? (htmlProcessor.processSync(block.markdown).contents as string) : undefined, }; }); return content; }), - intro: addGlossary( - htmlProcessor.processSync(rest.intro).contents as string - ), + intro: htmlProcessor.processSync(rest.intro).contents as string, })); } diff --git a/targets/export-elasticsearch/src/services/contributions.ts b/targets/export-elasticsearch/src/services/contributions.ts new file mode 100644 index 000000000..de1fec19f --- /dev/null +++ b/targets/export-elasticsearch/src/services/contributions.ts @@ -0,0 +1,75 @@ +import { gqlClient, logger } from "@shared/utils"; + +const fetchContribs = ` +query fetchContribs() { + documents(where: {source: {_eq: "contributions"}, is_available: {_eq: true}, is_published: {_eq: true}}) { + slug + source + title + cdtnId: cdtn_id + document + } +} +`; + +interface HasuraReturn { + documents: ContributionTableDocument[]; +} + +export interface ContributionTableDocument { + cdtnId: string; + title: string; + slug: string; + document: any; + source: string; +} + +export async function fetchAllContributionsDocument(): Promise< + ContributionTableDocument[] +> { + const HASURA_GRAPHQL_ENDPOINT = "http://localhost:8080/v1/graphql"; + const HASURA_GRAPHQL_ENDPOINT_SECRET = "admin1"; + const res = await gqlClient({ + graphqlEndpoint: HASURA_GRAPHQL_ENDPOINT, + adminSecret: HASURA_GRAPHQL_ENDPOINT_SECRET, + }) + .query(fetchContribs, {}) + .toPromise(); + if (res.error) { + throw res.error; + } + if (!res.data?.documents.length) { + logger.error("No contributions found"); + return []; + } + return res.data.documents; +} + +const editContributionWithCdtnId = ` +mutation editContribution($cdtnId: String!, $document: jsonb) { + update_documents(where: {cdtn_id: {_eq: $cdtnId}, source: {_eq: "contributions"}}, _set: {document: $document}) { + affected_rows + } +} +`; + +export async function editContentOfContribution( + cdtnId: string, + document: any +): Promise { + const HASURA_GRAPHQL_ENDPOINT = "http://localhost:8080/v1/graphql"; + const HASURA_GRAPHQL_ENDPOINT_SECRET = "admin1"; + const res = await gqlClient({ + graphqlEndpoint: HASURA_GRAPHQL_ENDPOINT, + adminSecret: HASURA_GRAPHQL_ENDPOINT_SECRET, + }) + .mutation(editContributionWithCdtnId, { + cdtnId, + document, + }) + .toPromise(); + if (res.error) { + throw res.error; + } + return; +} diff --git a/targets/frontend/src/modules/contribution/mapContributionToDocument.ts b/targets/frontend/src/modules/contribution/mapContributionToDocument.ts index e6f314135..f2bbc60bc 100644 --- a/targets/frontend/src/modules/contribution/mapContributionToDocument.ts +++ b/targets/frontend/src/modules/contribution/mapContributionToDocument.ts @@ -14,11 +14,20 @@ async function getBaseDocument( questionId: string ) => Promise> ) { + const glossaryContent = await fetch("http://localhost:8787/glossary", { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + content: data.content, + }), + }).then((res) => res.json()); switch (data.content_type) { case "ANSWER": return { type: "content", - content: data.content, + content: glossaryContent, }; case "GENERIC_NO_CDT": return { @@ -85,7 +94,8 @@ export const mapContributionToDocument = async ( } as ContributionDocumentJson; return { - cdtn_id: document?.cdtn_id ?? generateCdtnId(SOURCES.CONTRIBUTIONS + data.id), + cdtn_id: + document?.cdtn_id ?? generateCdtnId(SOURCES.CONTRIBUTIONS + data.id), initial_id: data.id, source: SOURCES.CONTRIBUTIONS, meta_description: document?.meta_description ?? "", // la génération se fait à l'export car on a besoin du dernier contenu de la fiche sp From 0be4d5e6182f5bcaad16dc58874782fbe6c56254 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Thu, 30 May 2024 12:10:00 +0200 Subject: [PATCH 02/40] fix: glossary --- .../src/controllers/glossary.ts | 38 ---------- .../src/controllers/index.ts | 1 - ...test.ts => getContributionContent.test.ts} | 8 +- .../src/ingester/contributions/generate.ts | 4 +- ...ToContent.ts => getContributionContent.ts} | 2 +- .../src/services/contributions.ts | 75 ------------------- 6 files changed, 7 insertions(+), 121 deletions(-) delete mode 100644 targets/export-elasticsearch/src/controllers/glossary.ts rename targets/export-elasticsearch/src/ingester/contributions/__tests__/{addGlossaryToContent.test.ts => getContributionContent.test.ts} (67%) rename targets/export-elasticsearch/src/ingester/contributions/{addGlossaryToContent.ts => getContributionContent.ts} (88%) delete mode 100644 targets/export-elasticsearch/src/services/contributions.ts diff --git a/targets/export-elasticsearch/src/controllers/glossary.ts b/targets/export-elasticsearch/src/controllers/glossary.ts deleted file mode 100644 index 071fd3073..000000000 --- a/targets/export-elasticsearch/src/controllers/glossary.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { interfaces } from "inversify-express-utils"; -import { - controller, - httpPost, - request, - response, -} from "inversify-express-utils"; -import { Request, Response } from "express"; -import { getContentGlossary } from "../ingester/glossary/addGlossary"; -import { - editContentOfContribution, - fetchAllContributionsDocument, -} from "../services/contributions"; - -@controller("/glossary") -export class GlossaryController implements interfaces.Controller { - @httpPost("/") - index(@request() req: Request, @response() res: Response): Promise { - const content = (req.body as any).content; - if (typeof content !== "string") { - res.status(500).send("Invalid content"); - } - return getContentGlossary(content); - } - - @httpPost("/all") - async all(@request() req: Request, @response() res: Response): Promise { - const contributions = await fetchAllContributionsDocument(); - for (let i = 0; i < contributions.length; i++) { - const content = contributions[i].document.content; - const item = getContentGlossary(content); - await editContentOfContribution(contributions[i].cdtnId, { - ...contributions[i].document, - content: item, - }); - } - } -} diff --git a/targets/export-elasticsearch/src/controllers/index.ts b/targets/export-elasticsearch/src/controllers/index.ts index 242777881..7a3979305 100644 --- a/targets/export-elasticsearch/src/controllers/index.ts +++ b/targets/export-elasticsearch/src/controllers/index.ts @@ -1,3 +1,2 @@ export * from "./export"; export * from "./monitoring"; -export * from "./glossary"; diff --git a/targets/export-elasticsearch/src/ingester/contributions/__tests__/addGlossaryToContent.test.ts b/targets/export-elasticsearch/src/ingester/contributions/__tests__/getContributionContent.test.ts similarity index 67% rename from targets/export-elasticsearch/src/ingester/contributions/__tests__/addGlossaryToContent.test.ts rename to targets/export-elasticsearch/src/ingester/contributions/__tests__/getContributionContent.test.ts index f66767ec9..4f9685e61 100644 --- a/targets/export-elasticsearch/src/ingester/contributions/__tests__/addGlossaryToContent.test.ts +++ b/targets/export-elasticsearch/src/ingester/contributions/__tests__/getContributionContent.test.ts @@ -1,13 +1,13 @@ -import { addGlossaryToContent } from "../addGlossaryToContent"; +import { getContributionContent } from "../getContributionContent"; -describe("addGlossaryToContent", () => { +describe("getContributionContent", () => { it("should return original content if ficheSpDescription exists", () => { const content = { ficheSpDescription: "Description", content: "Content", }; - const result = addGlossaryToContent(content); + const result = getContributionContent(content); expect(result).toEqual(content); }); @@ -17,7 +17,7 @@ describe("addGlossaryToContent", () => { content: "Content", }; - const result = addGlossaryToContent(content); + const result = getContributionContent(content); expect(result).toEqual({ content: "Modified content", diff --git a/targets/export-elasticsearch/src/ingester/contributions/generate.ts b/targets/export-elasticsearch/src/ingester/contributions/generate.ts index 3d3a9fb6c..029d03194 100644 --- a/targets/export-elasticsearch/src/ingester/contributions/generate.ts +++ b/targets/export-elasticsearch/src/ingester/contributions/generate.ts @@ -16,7 +16,7 @@ import { fetchAgreementUnextended } from "./fetchCcUnextended"; import { getCcInfos } from "./getCcInfos"; import { generateContent } from "./generateContent"; import { GetBreadcrumbsFn } from "../breadcrumbs"; -import { addGlossaryToContent } from "./addGlossaryToContent"; +import { getContributionContent } from "./getContributionContent"; import { generateMessageBlock } from "./generateMessageBlock"; import { generateLinkedContent } from "./generateLinkedContent"; import pMap from "p-map"; @@ -81,7 +81,7 @@ export async function generateContributions( generatedContributions.push({ ...contrib, ...generateMetadata(contrib), - ...addGlossaryToContent(content), + ...getContributionContent(content), ...doc, breadcrumbs: contrib.breadcrumbs.length > 0 diff --git a/targets/export-elasticsearch/src/ingester/contributions/addGlossaryToContent.ts b/targets/export-elasticsearch/src/ingester/contributions/getContributionContent.ts similarity index 88% rename from targets/export-elasticsearch/src/ingester/contributions/addGlossaryToContent.ts rename to targets/export-elasticsearch/src/ingester/contributions/getContributionContent.ts index 2e5714585..b43bd17d9 100644 --- a/targets/export-elasticsearch/src/ingester/contributions/addGlossaryToContent.ts +++ b/targets/export-elasticsearch/src/ingester/contributions/getContributionContent.ts @@ -1,6 +1,6 @@ import { ContributionContent } from "@socialgouv/cdtn-types"; -export function addGlossaryToContent( +export function getContributionContent( content: ContributionContent ): ContributionContent { if ( diff --git a/targets/export-elasticsearch/src/services/contributions.ts b/targets/export-elasticsearch/src/services/contributions.ts deleted file mode 100644 index de1fec19f..000000000 --- a/targets/export-elasticsearch/src/services/contributions.ts +++ /dev/null @@ -1,75 +0,0 @@ -import { gqlClient, logger } from "@shared/utils"; - -const fetchContribs = ` -query fetchContribs() { - documents(where: {source: {_eq: "contributions"}, is_available: {_eq: true}, is_published: {_eq: true}}) { - slug - source - title - cdtnId: cdtn_id - document - } -} -`; - -interface HasuraReturn { - documents: ContributionTableDocument[]; -} - -export interface ContributionTableDocument { - cdtnId: string; - title: string; - slug: string; - document: any; - source: string; -} - -export async function fetchAllContributionsDocument(): Promise< - ContributionTableDocument[] -> { - const HASURA_GRAPHQL_ENDPOINT = "http://localhost:8080/v1/graphql"; - const HASURA_GRAPHQL_ENDPOINT_SECRET = "admin1"; - const res = await gqlClient({ - graphqlEndpoint: HASURA_GRAPHQL_ENDPOINT, - adminSecret: HASURA_GRAPHQL_ENDPOINT_SECRET, - }) - .query(fetchContribs, {}) - .toPromise(); - if (res.error) { - throw res.error; - } - if (!res.data?.documents.length) { - logger.error("No contributions found"); - return []; - } - return res.data.documents; -} - -const editContributionWithCdtnId = ` -mutation editContribution($cdtnId: String!, $document: jsonb) { - update_documents(where: {cdtn_id: {_eq: $cdtnId}, source: {_eq: "contributions"}}, _set: {document: $document}) { - affected_rows - } -} -`; - -export async function editContentOfContribution( - cdtnId: string, - document: any -): Promise { - const HASURA_GRAPHQL_ENDPOINT = "http://localhost:8080/v1/graphql"; - const HASURA_GRAPHQL_ENDPOINT_SECRET = "admin1"; - const res = await gqlClient({ - graphqlEndpoint: HASURA_GRAPHQL_ENDPOINT, - adminSecret: HASURA_GRAPHQL_ENDPOINT_SECRET, - }) - .mutation(editContributionWithCdtnId, { - cdtnId, - document, - }) - .toPromise(); - if (res.error) { - throw res.error; - } - return; -} From 1df3298fa932ad6248d99008809480e7a75a61e4 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Thu, 30 May 2024 15:12:21 +0200 Subject: [PATCH 03/40] fix: glossary --- shared/utils/package.json | 5 +++ .../utils/src/__tests__/dila-resolver.test.ts | 2 +- .../src/{array-utils.ts => arrayUtils.ts} | 0 .../src/{dila-resolver.ts => dilaResolver.ts} | 0 .../utils/src/glossary/addGlossaryContent.ts | 15 ++++++-- shared/utils/src/glossary/fetchGlossary.ts | 30 ++++++++++++++++ shared/utils/src/glossary/index.ts | 1 + .../utils/src/{gql-client.ts => gqlClient.ts} | 4 +-- .../src/{id-generator.ts => idGenerator.ts} | 0 shared/utils/src/index.ts | 8 ++--- shared/utils/src/markdownProcessor.ts | 11 ++++++ .../src/{url-generator.ts => urlGenerator.ts} | 0 targets/export-elasticsearch/package.json | 6 ---- .../src/ingester/cdtnDocuments.ts | 9 ++--- .../src/ingester/common/fetchGlossary.ts | 36 +++---------------- .../src/ingester/informations/generate.ts | 6 ++-- .../src/ingester/informations/markdown.ts | 35 ------------------ .../contribution/mapContributionToDocument.ts | 17 ++++----- 18 files changed, 83 insertions(+), 102 deletions(-) rename shared/utils/src/{array-utils.ts => arrayUtils.ts} (100%) rename shared/utils/src/{dila-resolver.ts => dilaResolver.ts} (100%) create mode 100644 shared/utils/src/glossary/fetchGlossary.ts rename shared/utils/src/{gql-client.ts => gqlClient.ts} (88%) rename shared/utils/src/{id-generator.ts => idGenerator.ts} (100%) create mode 100644 shared/utils/src/markdownProcessor.ts rename shared/utils/src/{url-generator.ts => urlGenerator.ts} (100%) delete mode 100644 targets/export-elasticsearch/src/ingester/informations/markdown.ts diff --git a/shared/utils/package.json b/shared/utils/package.json index eb40b8792..6cfaf7b02 100644 --- a/shared/utils/package.json +++ b/shared/utils/package.json @@ -8,6 +8,11 @@ "@urql/core": "^4.3.0", "graphql": "^16.8.1", "isomorphic-unfetch": "^4.0.2", + "rehype-raw": "^5.1.0", + "rehype-stringify": "^8.0.0", + "remark-parse": "^9.0.0", + "remark-rehype": "^8.1.0", + "unified": "^9.2.2", "uuid": "^9.0.1", "winston": "3.3.3", "xxhashjs": "^0.2.2" diff --git a/shared/utils/src/__tests__/dila-resolver.test.ts b/shared/utils/src/__tests__/dila-resolver.test.ts index 9bcd60f2d..0263af9d9 100644 --- a/shared/utils/src/__tests__/dila-resolver.test.ts +++ b/shared/utils/src/__tests__/dila-resolver.test.ts @@ -1,6 +1,6 @@ import type DilaApiClient from "@socialgouv/dila-api-client"; -import { createGetArticleReference, extractArticleId } from "../dila-resolver"; +import { createGetArticleReference, extractArticleId } from "../dilaResolver"; import getKaliArticlePayload from "./__mocks__/kaliArticle.json"; import getLegiArticlePayload from "./__mocks__/legiArticle.json"; diff --git a/shared/utils/src/array-utils.ts b/shared/utils/src/arrayUtils.ts similarity index 100% rename from shared/utils/src/array-utils.ts rename to shared/utils/src/arrayUtils.ts diff --git a/shared/utils/src/dila-resolver.ts b/shared/utils/src/dilaResolver.ts similarity index 100% rename from shared/utils/src/dila-resolver.ts rename to shared/utils/src/dilaResolver.ts diff --git a/shared/utils/src/glossary/addGlossaryContent.ts b/shared/utils/src/glossary/addGlossaryContent.ts index 82ba74b15..ea6bc86cc 100644 --- a/shared/utils/src/glossary/addGlossaryContent.ts +++ b/shared/utils/src/glossary/addGlossaryContent.ts @@ -1,11 +1,12 @@ +import { markdownProcessor } from "../markdownProcessor"; import { explodeGlossaryTerms } from "./explodeGlossaryTerms"; import { insertWebComponentGlossary } from "./insertWebComponentGlossary"; import { Glossary } from "@socialgouv/cdtn-types"; -export const addGlossaryContent = async ( +export const addGlossaryContent = ( glossary: Glossary, content: string -): Promise => { +): string => { glossary.sort((previous, next) => { return next.term.length - previous.term.length; }); @@ -22,3 +23,13 @@ export const addGlossaryContent = async ( if (!content) return ""; return insertWebComponentGlossary(content, glossaryTerms); }; + +export function addGlossaryContentToMarkdown( + glossary: Glossary, + markdown: string +): string { + return addGlossaryContent( + glossary, + markdownProcessor.processSync(markdown).contents as string + ); +} diff --git a/shared/utils/src/glossary/fetchGlossary.ts b/shared/utils/src/glossary/fetchGlossary.ts new file mode 100644 index 000000000..4b6b54640 --- /dev/null +++ b/shared/utils/src/glossary/fetchGlossary.ts @@ -0,0 +1,30 @@ +import { Glossary } from "@socialgouv/cdtn-types"; +import { gqlClient, gqlDefaultProps } from "../gqlClient"; + +const gqlGlossary = ` + query Glossary { + glossary { + term + abbreviations + definition + variants + references + slug + } + } +`; + +export async function fetchGlossary( + props = gqlDefaultProps +): Promise { + const result = await gqlClient(props) + .query<{ glossary: Glossary }>(gqlGlossary, {}) + .toPromise(); + if (result.error || !result.data) { + console.error(result.error); + throw new Error( + `Error fetching glossary => ${JSON.stringify(result.error)}` + ); + } + return result.data.glossary; +} diff --git a/shared/utils/src/glossary/index.ts b/shared/utils/src/glossary/index.ts index 0279b15c6..3673b2588 100644 --- a/shared/utils/src/glossary/index.ts +++ b/shared/utils/src/glossary/index.ts @@ -1 +1,2 @@ export * from "./addGlossaryContent"; +export * from "./fetchGlossary"; diff --git a/shared/utils/src/gql-client.ts b/shared/utils/src/gqlClient.ts similarity index 88% rename from shared/utils/src/gql-client.ts rename to shared/utils/src/gqlClient.ts index ab9c47b92..2304f4a80 100644 --- a/shared/utils/src/gql-client.ts +++ b/shared/utils/src/gqlClient.ts @@ -11,13 +11,13 @@ type GqlClientParameter = { adminSecret: string; }; -const defaultProps: GqlClientParameter = { +export const gqlDefaultProps: GqlClientParameter = { graphqlEndpoint: process.env.HASURA_GRAPHQL_ENDPOINT ?? "http://localhost:8080/v1/graphql", adminSecret: process.env.HASURA_GRAPHQL_ADMIN_SECRET ?? "admin1", }; -export const gqlClient = (props = defaultProps) => +export const gqlClient = (props = gqlDefaultProps) => createClient({ fetch, fetchOptions: { diff --git a/shared/utils/src/id-generator.ts b/shared/utils/src/idGenerator.ts similarity index 100% rename from shared/utils/src/id-generator.ts rename to shared/utils/src/idGenerator.ts diff --git a/shared/utils/src/index.ts b/shared/utils/src/index.ts index a85e8b935..27dd66642 100644 --- a/shared/utils/src/index.ts +++ b/shared/utils/src/index.ts @@ -1,7 +1,7 @@ -export * from "./id-generator"; -export * from "./url-generator"; -export * from "./gql-client"; +export * from "./idGenerator"; +export * from "./urlGenerator"; +export * from "./gqlClient"; export * from "./config"; export * from "./logger"; -export * from "./dila-resolver"; +export * from "./dilaResolver"; export * from "./glossary"; diff --git a/shared/utils/src/markdownProcessor.ts b/shared/utils/src/markdownProcessor.ts new file mode 100644 index 000000000..54e4914a9 --- /dev/null +++ b/shared/utils/src/markdownProcessor.ts @@ -0,0 +1,11 @@ +import htmlAstToAnotherHtmlAst from "rehype-raw"; +import htmlAstStringify from "rehype-stringify"; +import markdownToMardownAst from "remark-parse"; +import markdownAstToHtmlAst from "remark-rehype"; +import unified from "unified"; + +export const markdownProcessor = unified() + .use(markdownToMardownAst) + .use(markdownAstToHtmlAst, { allowDangerousHtml: true }) + .use(htmlAstToAnotherHtmlAst) + .use(htmlAstStringify); diff --git a/shared/utils/src/url-generator.ts b/shared/utils/src/urlGenerator.ts similarity index 100% rename from shared/utils/src/url-generator.ts rename to shared/utils/src/urlGenerator.ts diff --git a/targets/export-elasticsearch/package.json b/targets/export-elasticsearch/package.json index 265b3342d..f62084b5c 100644 --- a/targets/export-elasticsearch/package.json +++ b/targets/export-elasticsearch/package.json @@ -43,12 +43,6 @@ "p-map": "^4.0.0", "p-queue": "^6.6.2", "reflect-metadata": "^0.1.13", - "rehype-raw": "^5.1.0", - "rehype-stringify": "^8.0.0", - "remark-parse": "^9.0.0", - "remark-rehype": "^8.1.0", - "remark-stringify": "^9.0.1", - "unified": "^9.2.2", "zod": "^3.14.2" }, "devDependencies": { diff --git a/targets/export-elasticsearch/src/ingester/cdtnDocuments.ts b/targets/export-elasticsearch/src/ingester/cdtnDocuments.ts index 0aae3c21d..db8777e9c 100644 --- a/targets/export-elasticsearch/src/ingester/cdtnDocuments.ts +++ b/targets/export-elasticsearch/src/ingester/cdtnDocuments.ts @@ -21,13 +21,13 @@ import { splitArticle } from "./fichesTravailSplitter"; import { getVersions } from "./versions"; import { generateContributions } from "./contributions"; import { generateAgreements } from "./agreements"; -import { getGlossary } from "./common/fetchGlossary"; import { fetchThemes } from "./themes/fetchThemes"; import { updateExportEsStatusWithDocumentsCount } from "./exportStatus/updateExportEsStatusWithDocumentsCount"; import { generatePrequalified } from "./prequalified"; import { generateEditorialContents } from "./informations/generate"; import { populateRelatedDocuments } from "./common/populateRelatedDocuments"; import { mergeRelatedDocumentsToEditorialContents } from "./informations/mergeRelatedDocumentsToEditorialContents"; +import { getGlossary } from "./common/fetchGlossary"; /** * Find duplicate slugs @@ -175,13 +175,10 @@ export async function cdtnDocumentsGen( logger.info(`Fetched ${fichesMT.length} fiches travail`); const fichesMTWithGlossary = fichesMT.map(({ sections, ...infos }) => ({ ...infos, - sections: sections.map(({ html, ...section }: any) => { + sections: sections.map(({ ...section }: any) => { delete section.description; delete section.text; - return { - ...section, - html, - }; + return section; }), })); logger.info( diff --git a/targets/export-elasticsearch/src/ingester/common/fetchGlossary.ts b/targets/export-elasticsearch/src/ingester/common/fetchGlossary.ts index 301880abe..0dcfe2e32 100644 --- a/targets/export-elasticsearch/src/ingester/common/fetchGlossary.ts +++ b/targets/export-elasticsearch/src/ingester/common/fetchGlossary.ts @@ -1,37 +1,11 @@ -import { Glossary } from "@socialgouv/cdtn-types"; +import { fetchGlossary } from "@shared/utils"; import { context } from "../context"; -import { gqlClient } from "@shared/utils"; - -const gqlGlossary = ` -query Glossary { - glossary { - term - abbreviations - definition - variants - references - slug - } -} - -`; +import { Glossary } from "@socialgouv/cdtn-types"; -export async function getGlossary(): Promise { +export const getGlossary = async (): Promise => { const graphqlEndpoint: string = context.get("cdtnAdminEndpoint") || "http://localhost:8080/v1/graphql"; const adminSecret: string = context.get("cdtnAdminEndpointSecret") || "admin1"; - const result = await gqlClient({ - graphqlEndpoint, - adminSecret, - }) - .query<{ glossary: Glossary }>(gqlGlossary, {}) - .toPromise(); - if (result.error || !result.data) { - console.error(result.error); - throw new Error( - `error fetching kali blocks => ${JSON.stringify(result.error)}` - ); - } - return result.data.glossary; -} + return await fetchGlossary({ adminSecret, graphqlEndpoint }); +}; diff --git a/targets/export-elasticsearch/src/ingester/informations/generate.ts b/targets/export-elasticsearch/src/ingester/informations/generate.ts index ea5b2d9d8..8bb4e6447 100644 --- a/targets/export-elasticsearch/src/ingester/informations/generate.ts +++ b/targets/export-elasticsearch/src/ingester/informations/generate.ts @@ -2,7 +2,6 @@ import { DocumentElasticWithSource, EditorialContentDoc, } from "@socialgouv/cdtn-types"; -import { markdownTransform } from "./markdown"; import { getRelatedIdsDocuments } from "./getRelatedIdsDocuments"; interface Return { @@ -13,10 +12,9 @@ interface Return { export const generateEditorialContents = ( documents: DocumentElasticWithSource[] ): Return => { - const documentsMarkdownified = markdownTransform(documents); - const relatedIdsDocuments = getRelatedIdsDocuments(documentsMarkdownified); + const relatedIdsDocuments = getRelatedIdsDocuments(documents); return { - documents: documentsMarkdownified, + documents, relatedIdsDocuments, }; }; diff --git a/targets/export-elasticsearch/src/ingester/informations/markdown.ts b/targets/export-elasticsearch/src/ingester/informations/markdown.ts deleted file mode 100644 index 999b3f977..000000000 --- a/targets/export-elasticsearch/src/ingester/informations/markdown.ts +++ /dev/null @@ -1,35 +0,0 @@ -import type { - DocumentElasticWithSource, - EditorialContentDoc, -} from "@socialgouv/cdtn-types"; -import htmlAstToAnotherHtmlAst from "rehype-raw"; -import htmlAstStringify from "rehype-stringify"; -import markdownToMardownAst from "remark-parse"; -import markdownAstToHtmlAst from "remark-rehype"; -import unified from "unified"; - -const htmlProcessor = unified() - .use(markdownToMardownAst as any) - .use(markdownAstToHtmlAst as any, { allowDangerousHtml: true }) - .use(htmlAstToAnotherHtmlAst as any) - .use(htmlAstStringify as any); - -export function markdownTransform( - documents: DocumentElasticWithSource[] -): DocumentElasticWithSource[] { - return documents.map(({ contents = [], ...rest }) => ({ - ...rest, - contents: contents.map((content) => { - content.blocks = content.blocks.map((block: any) => { - return { - ...block, - html: block.markdown - ? (htmlProcessor.processSync(block.markdown).contents as string) - : undefined, - }; - }); - return content; - }), - intro: htmlProcessor.processSync(rest.intro).contents as string, - })); -} diff --git a/targets/frontend/src/modules/contribution/mapContributionToDocument.ts b/targets/frontend/src/modules/contribution/mapContributionToDocument.ts index f2bbc60bc..0a715ac7e 100644 --- a/targets/frontend/src/modules/contribution/mapContributionToDocument.ts +++ b/targets/frontend/src/modules/contribution/mapContributionToDocument.ts @@ -5,7 +5,11 @@ import { } from "@socialgouv/cdtn-types"; import { SOURCES } from "@socialgouv/cdtn-sources"; import { getReferences } from "./getReferences"; -import { generateCdtnId } from "@shared/utils"; +import { + generateCdtnId, + addGlossaryContent, + fetchGlossary, +} from "@shared/utils"; import { generateContributionSlug } from "./generateSlug"; async function getBaseDocument( @@ -14,20 +18,11 @@ async function getBaseDocument( questionId: string ) => Promise> ) { - const glossaryContent = await fetch("http://localhost:8787/glossary", { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - content: data.content, - }), - }).then((res) => res.json()); switch (data.content_type) { case "ANSWER": return { type: "content", - content: glossaryContent, + content: addGlossaryContent(await fetchGlossary(), data.content ?? ""), }; case "GENERIC_NO_CDT": return { From b052b356f812434bb65e3d00e61c19134ec9b837 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Thu, 30 May 2024 17:18:26 +0200 Subject: [PATCH 04/40] fix: glossary --- .../contribution/mapContributionToDocument.ts | 4 +- .../documents/api/documents.service.ts | 24 ++++++++-- .../modules/glossary/fetchDocumentBySource.ts | 37 ++++++++++++++ .../frontend/src/modules/glossary/index.ts | 7 +++ .../updateContributionsDocumentsGlossary.ts | 22 +++++++++ .../src/modules/glossary/updateDocument.ts | 26 ++++++++++ ...updateEditorialContentDocumentsGlossary.ts | 35 ++++++++++++++ .../src/transform/fiche-travail-emploi.ts | 3 ++ yarn.lock | 48 ++----------------- 9 files changed, 158 insertions(+), 48 deletions(-) create mode 100644 targets/frontend/src/modules/glossary/fetchDocumentBySource.ts create mode 100644 targets/frontend/src/modules/glossary/index.ts create mode 100644 targets/frontend/src/modules/glossary/updateContributionsDocumentsGlossary.ts create mode 100644 targets/frontend/src/modules/glossary/updateDocument.ts create mode 100644 targets/frontend/src/modules/glossary/updateEditorialContentDocumentsGlossary.ts diff --git a/targets/frontend/src/modules/contribution/mapContributionToDocument.ts b/targets/frontend/src/modules/contribution/mapContributionToDocument.ts index 0a715ac7e..185e245c6 100644 --- a/targets/frontend/src/modules/contribution/mapContributionToDocument.ts +++ b/targets/frontend/src/modules/contribution/mapContributionToDocument.ts @@ -18,11 +18,13 @@ async function getBaseDocument( questionId: string ) => Promise> ) { + const glossary = await fetchGlossary(); switch (data.content_type) { case "ANSWER": return { type: "content", - content: addGlossaryContent(await fetchGlossary(), data.content ?? ""), + content: data.content, + contentWithGlossary: addGlossaryContent(glossary, data.content ?? ""), }; case "GENERIC_NO_CDT": return { diff --git a/targets/frontend/src/modules/documents/api/documents.service.ts b/targets/frontend/src/modules/documents/api/documents.service.ts index 7fe475889..70226eb3b 100644 --- a/targets/frontend/src/modules/documents/api/documents.service.ts +++ b/targets/frontend/src/modules/documents/api/documents.service.ts @@ -2,7 +2,12 @@ import { DocumentsRepository } from "./documents.repository"; import { ConflictError, NotFoundError } from "src/lib/api/ApiErrors"; import { Information, InformationsRepository } from "src/modules/informations"; import { format, parseISO } from "date-fns"; -import { generateCdtnId, generateInitialId } from "@shared/utils"; +import { + addGlossaryContentToMarkdown, + fetchGlossary, + generateCdtnId, + generateInitialId, +} from "@shared/utils"; import slugify from "@socialgouv/cdtn-slugify"; import { ContributionRepository, @@ -37,10 +42,11 @@ export class DocumentsService { this.agreementRepository = agreementRepository; } - private mapInformationToDocument( + private async mapInformationToDocument( data: Information, document?: HasuraDocument - ): HasuraDocument { + ): Promise> { + const glossary = await fetchGlossary(); return { cdtn_id: document?.cdtn_id ?? generateCdtnId(data.title), initial_id: data.id ?? generateInitialId(), @@ -57,6 +63,10 @@ export class DocumentsService { ? format(new Date(data.updatedAt), "dd/MM/yyyy") : undefined, intro: data.intro, + introWithGlossary: addGlossaryContentToMarkdown( + glossary, + data.intro ?? "" + ), description: data.description, sectionDisplayMode: data.sectionDisplayMode, dismissalProcess: data.dismissalProcess, @@ -80,7 +90,13 @@ export class DocumentsService { ? { title: block.content, } - : { markdown: block.content }), + : { + markdown: block.content, + htmlWithGlossary: addGlossaryContentToMarkdown( + glossary, + block.content + ), + }), ...(block.type === "graphic" ? { size: block.file?.size, diff --git a/targets/frontend/src/modules/glossary/fetchDocumentBySource.ts b/targets/frontend/src/modules/glossary/fetchDocumentBySource.ts new file mode 100644 index 000000000..fde2f44d3 --- /dev/null +++ b/targets/frontend/src/modules/glossary/fetchDocumentBySource.ts @@ -0,0 +1,37 @@ +import { gqlClient, logger } from "@shared/utils"; +import { SourceRoute } from "@socialgouv/cdtn-sources"; +import { HasuraDocument } from "@socialgouv/cdtn-types"; + +const fetchDocumentQuery = ` +query fetchDocument() { + documents(where: {source: {_eq: $source}}) { + cdtn_id + document + } +} +`; + +interface HasuraReturn { + documents: Pick, "cdtn_id" | "document">[]; +} + +export interface Document { + cdtnId: string; + document: any; +} + +export async function fetchDocumentBySource( + source: SourceRoute +): Promise { + const res = await gqlClient() + .query(fetchDocumentQuery, { source }) + .toPromise(); + if (res.error) { + throw res.error; + } + if (!res.data?.documents.length) { + logger.error("No contributions found"); + return []; + } + return res.data.documents; +} diff --git a/targets/frontend/src/modules/glossary/index.ts b/targets/frontend/src/modules/glossary/index.ts new file mode 100644 index 000000000..8580a99f7 --- /dev/null +++ b/targets/frontend/src/modules/glossary/index.ts @@ -0,0 +1,7 @@ +import { updateContributionsDocumentsGlossary } from "./updateContributionsDocumentsGlossary"; +import { updateEditorialContentDocumentsGlossary } from "./updateEditorialContentDocumentsGlossary"; + +export const updateGlossary = async () => { + await updateContributionsDocumentsGlossary(); + await updateEditorialContentDocumentsGlossary(); +}; diff --git a/targets/frontend/src/modules/glossary/updateContributionsDocumentsGlossary.ts b/targets/frontend/src/modules/glossary/updateContributionsDocumentsGlossary.ts new file mode 100644 index 000000000..2639d163c --- /dev/null +++ b/targets/frontend/src/modules/glossary/updateContributionsDocumentsGlossary.ts @@ -0,0 +1,22 @@ +import { SOURCES } from "@socialgouv/cdtn-sources"; +import { fetchDocumentBySource } from "./fetchDocumentBySource"; +import { addGlossaryContent, fetchGlossary } from "@shared/utils"; +import { updateDocument } from "./updateDocument"; + +export const updateContributionsDocumentsGlossary = async () => { + const glossary = await fetchGlossary(); + const contributions = await fetchDocumentBySource(SOURCES.CONTRIBUTIONS); + for (const contribution of contributions) { + const document = contribution.document; + if (document.content_type === "ANSWER") { + const contentWithGlossary = addGlossaryContent( + glossary, + document.content ?? "" + ); + await updateDocument(contribution.cdtn_id, { + ...document, + contentWithGlossary, + }); + } + } +}; diff --git a/targets/frontend/src/modules/glossary/updateDocument.ts b/targets/frontend/src/modules/glossary/updateDocument.ts new file mode 100644 index 000000000..556a71e2a --- /dev/null +++ b/targets/frontend/src/modules/glossary/updateDocument.ts @@ -0,0 +1,26 @@ +import { gqlClient } from "@shared/utils"; + +const updateDocumentWithCdtnId = ` +mutation editDocument($cdtnId: String!, $document: jsonb) { + update_documents(where: {cdtn_id: {_eq: $cdtnId}, _set: {document: $document}) { + affected_rows + } +} +`; + +export async function updateDocument( + cdtnId: string, + document: any +): Promise { + const res = await gqlClient() + .mutation(updateDocumentWithCdtnId, { + cdtnId, + document, + }) + .toPromise(); + if (res.error) { + console.error(res.error); + throw res.error; + } + return; +} diff --git a/targets/frontend/src/modules/glossary/updateEditorialContentDocumentsGlossary.ts b/targets/frontend/src/modules/glossary/updateEditorialContentDocumentsGlossary.ts new file mode 100644 index 000000000..b4dd918a1 --- /dev/null +++ b/targets/frontend/src/modules/glossary/updateEditorialContentDocumentsGlossary.ts @@ -0,0 +1,35 @@ +import { SOURCES } from "@socialgouv/cdtn-sources"; +import { fetchDocumentBySource } from "./fetchDocumentBySource"; +import { addGlossaryContentToMarkdown, fetchGlossary } from "@shared/utils"; +import { updateDocument } from "./updateDocument"; + +export const updateEditorialContentDocumentsGlossary = async () => { + const glossary = await fetchGlossary(); + const editorialContents = await fetchDocumentBySource( + SOURCES.EDITORIAL_CONTENT + ); + for (const editorialContent of editorialContents) { + const document = editorialContent.document; + await updateDocument(editorialContent.cdtn_id, { + ...document, + introWithGlossary: addGlossaryContentToMarkdown( + glossary, + document.intro ?? "" + ), + blocks: document.blocks.map((block: any) => { + if ("markdown" in block) { + return { + ...block, + htmlWithGlossary: addGlossaryContentToMarkdown( + glossary, + block.markdown + ), + }; + } + return { + ...block, + }; + }), + }); + } +}; diff --git a/targets/ingester/src/transform/fiche-travail-emploi.ts b/targets/ingester/src/transform/fiche-travail-emploi.ts index c3c88f9b0..ef873e011 100644 --- a/targets/ingester/src/transform/fiche-travail-emploi.ts +++ b/targets/ingester/src/transform/fiche-travail-emploi.ts @@ -7,6 +7,7 @@ import { articleToReference, createReferenceResolver, } from "../lib/referenceResolver"; +import { addGlossaryContent, fetchGlossary } from "@shared/utils"; export default async function getFicheTravailEmploi(pkgName: string) { const [fichesMT, cdt] = await Promise.all([ @@ -15,6 +16,7 @@ export default async function getFicheTravailEmploi(pkgName: string) { `@socialgouv/legi-data/data/LEGITEXT000006072050.json` ), ]); + const glossary = await fetchGlossary(); const resolveCdtReference = createReferenceResolver(cdt); return fichesMT.map(({ pubId, sections, ...content }) => { return { @@ -23,6 +25,7 @@ export default async function getFicheTravailEmploi(pkgName: string) { is_searchable: true, sections: sections.map(({ references, ...section }) => ({ ...section, + htmlWithGlossary: addGlossaryContent(glossary, section.html), references: Object.keys(references).flatMap((key) => { if (key !== "LEGITEXT000006072050") { return []; diff --git a/yarn.lock b/yarn.lock index de2b70edc..6173efe3f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4802,7 +4802,12 @@ __metadata: graphql: ^16.8.1 isomorphic-unfetch: ^4.0.2 jest: ^29.7.0 + rehype-raw: ^5.1.0 + rehype-stringify: ^8.0.0 + remark-parse: ^9.0.0 + remark-rehype: ^8.1.0 typescript: ^5.4.3 + unified: ^9.2.2 uuid: ^9.0.1 winston: 3.3.3 xxhashjs: ^0.2.2 @@ -11499,16 +11504,10 @@ __metadata: p-map: ^4.0.0 p-queue: ^6.6.2 reflect-metadata: ^0.1.13 - rehype-raw: ^5.1.0 - rehype-stringify: ^8.0.0 - remark-parse: ^9.0.0 - remark-rehype: ^8.1.0 - remark-stringify: ^9.0.1 rimraf: 3.0.2 supertest: ^6.2.2 timekeeper: ^2.2.0 typescript: ^5.4.3 - unified: ^9.2.2 zod: ^3.14.2 languageName: unknown linkType: soft @@ -16460,13 +16459,6 @@ __metadata: languageName: node linkType: hard -"longest-streak@npm:^2.0.0": - version: 2.0.4 - resolution: "longest-streak@npm:2.0.4" - checksum: 28b8234a14963002c5c71035dee13a0a11e9e9d18ffa320fdc8796ed7437399204495702ed69cd2a7087b0af041a2a8b562829b7c1e2042e73a3374d1ecf6580 - languageName: node - linkType: hard - "loose-envify@npm:^1.0.0, loose-envify@npm:^1.1.0, loose-envify@npm:^1.4.0": version: 1.4.0 resolution: "loose-envify@npm:1.4.0" @@ -16763,20 +16755,6 @@ __metadata: languageName: node linkType: hard -"mdast-util-to-markdown@npm:^0.6.0": - version: 0.6.5 - resolution: "mdast-util-to-markdown@npm:0.6.5" - dependencies: - "@types/unist": ^2.0.0 - longest-streak: ^2.0.0 - mdast-util-to-string: ^2.0.0 - parse-entities: ^2.0.0 - repeat-string: ^1.0.0 - zwitch: ^1.0.0 - checksum: 7ebc47533bff6e8669f85ae124dc521ea570e9df41c0d9e4f0f43c19ef4a8c9928d741f3e4afa62fcca1927479b714582ff5fd684ef240d84ee5b75ab9d863cf - languageName: node - linkType: hard - "mdast-util-to-string@npm:^2.0.0": version: 2.0.0 resolution: "mdast-util-to-string@npm:2.0.0" @@ -20065,22 +20043,6 @@ __metadata: languageName: node linkType: hard -"remark-stringify@npm:^9.0.1": - version: 9.0.1 - resolution: "remark-stringify@npm:9.0.1" - dependencies: - mdast-util-to-markdown: ^0.6.0 - checksum: 93f46076f4d96ab1946d13e7dd43e83088480ac6b1dfe05a65e2c2f0e33d1f52a50175199b464a81803fc0f5b3bf182037665f89720b30515eba37bec4d63d56 - languageName: node - linkType: hard - -"repeat-string@npm:^1.0.0": - version: 1.6.1 - resolution: "repeat-string@npm:1.6.1" - checksum: 1b809fc6db97decdc68f5b12c4d1a671c8e3f65ec4a40c238bc5200e44e85bcc52a54f78268ab9c29fcf5fe4f1343e805420056d1f30fa9a9ee4c2d93e3cc6c0 - languageName: node - linkType: hard - "repeating@npm:^2.0.0": version: 2.0.1 resolution: "repeating@npm:2.0.1" From 8da55114ef7e2896fa34e2fb672dde9b1575bdc2 Mon Sep 17 00:00:00 2001 From: maxgfr <25312957+maxgfr@users.noreply.github.com> Date: Fri, 31 May 2024 17:31:27 +0200 Subject: [PATCH 05/40] fix: glossary --- .../modules/glossary/fetchDocumentBySource.ts | 2 +- .../frontend/src/modules/glossary/index.ts | 7 ++- .../updateContributionsDocumentsGlossary.ts | 13 ++-- .../src/modules/glossary/updateDocument.ts | 2 +- ...updateEditorialContentDocumentsGlossary.ts | 43 ++++++++----- targets/frontend/src/pages/api/glossary.ts | 27 +++++++++ .../pages/glossary/{index.js => index.tsx} | 60 +++++++++++++------ 7 files changed, 115 insertions(+), 39 deletions(-) create mode 100644 targets/frontend/src/pages/api/glossary.ts rename targets/frontend/src/pages/glossary/{index.js => index.tsx} (78%) diff --git a/targets/frontend/src/modules/glossary/fetchDocumentBySource.ts b/targets/frontend/src/modules/glossary/fetchDocumentBySource.ts index fde2f44d3..ce77ce26e 100644 --- a/targets/frontend/src/modules/glossary/fetchDocumentBySource.ts +++ b/targets/frontend/src/modules/glossary/fetchDocumentBySource.ts @@ -3,7 +3,7 @@ import { SourceRoute } from "@socialgouv/cdtn-sources"; import { HasuraDocument } from "@socialgouv/cdtn-types"; const fetchDocumentQuery = ` -query fetchDocument() { +query fetchDocument($source: String) { documents(where: {source: {_eq: $source}}) { cdtn_id document diff --git a/targets/frontend/src/modules/glossary/index.ts b/targets/frontend/src/modules/glossary/index.ts index 8580a99f7..5a7a6d1c3 100644 --- a/targets/frontend/src/modules/glossary/index.ts +++ b/targets/frontend/src/modules/glossary/index.ts @@ -1,7 +1,12 @@ +import { logger } from "@shared/utils"; import { updateContributionsDocumentsGlossary } from "./updateContributionsDocumentsGlossary"; import { updateEditorialContentDocumentsGlossary } from "./updateEditorialContentDocumentsGlossary"; export const updateGlossary = async () => { - await updateContributionsDocumentsGlossary(); + logger.info("=== Starting glossary update ==="); + logger.info("Updating editorial content documents glossary..."); await updateEditorialContentDocumentsGlossary(); + logger.info("Updating contributions documents glossary..."); + await updateContributionsDocumentsGlossary(); + logger.info("=== Ending glossary update ==="); }; diff --git a/targets/frontend/src/modules/glossary/updateContributionsDocumentsGlossary.ts b/targets/frontend/src/modules/glossary/updateContributionsDocumentsGlossary.ts index 2639d163c..f0b914f53 100644 --- a/targets/frontend/src/modules/glossary/updateContributionsDocumentsGlossary.ts +++ b/targets/frontend/src/modules/glossary/updateContributionsDocumentsGlossary.ts @@ -2,21 +2,26 @@ import { SOURCES } from "@socialgouv/cdtn-sources"; import { fetchDocumentBySource } from "./fetchDocumentBySource"; import { addGlossaryContent, fetchGlossary } from "@shared/utils"; import { updateDocument } from "./updateDocument"; +import { logger } from "@shared/utils"; export const updateContributionsDocumentsGlossary = async () => { const glossary = await fetchGlossary(); const contributions = await fetchDocumentBySource(SOURCES.CONTRIBUTIONS); - for (const contribution of contributions) { - const document = contribution.document; - if (document.content_type === "ANSWER") { + logger.info(`Found ${contributions.length} contributions`); + for (let i = 0; i < contributions.length; i++) { + const document = contributions[i].document; + if (document.contentType === "ANSWER") { const contentWithGlossary = addGlossaryContent( glossary, document.content ?? "" ); - await updateDocument(contribution.cdtn_id, { + await updateDocument(contributions[i].cdtn_id, { ...document, contentWithGlossary, }); + logger.info( + `Updated contribution ${i + 1}/${contributions.length} with glossary` + ); } } }; diff --git a/targets/frontend/src/modules/glossary/updateDocument.ts b/targets/frontend/src/modules/glossary/updateDocument.ts index 556a71e2a..c6ae3209e 100644 --- a/targets/frontend/src/modules/glossary/updateDocument.ts +++ b/targets/frontend/src/modules/glossary/updateDocument.ts @@ -2,7 +2,7 @@ import { gqlClient } from "@shared/utils"; const updateDocumentWithCdtnId = ` mutation editDocument($cdtnId: String!, $document: jsonb) { - update_documents(where: {cdtn_id: {_eq: $cdtnId}, _set: {document: $document}) { + update_documents(where: {cdtn_id: {_eq: $cdtnId}}, _set: {document: $document}) { affected_rows } } diff --git a/targets/frontend/src/modules/glossary/updateEditorialContentDocumentsGlossary.ts b/targets/frontend/src/modules/glossary/updateEditorialContentDocumentsGlossary.ts index b4dd918a1..413c3d36b 100644 --- a/targets/frontend/src/modules/glossary/updateEditorialContentDocumentsGlossary.ts +++ b/targets/frontend/src/modules/glossary/updateEditorialContentDocumentsGlossary.ts @@ -1,6 +1,10 @@ import { SOURCES } from "@socialgouv/cdtn-sources"; import { fetchDocumentBySource } from "./fetchDocumentBySource"; -import { addGlossaryContentToMarkdown, fetchGlossary } from "@shared/utils"; +import { + addGlossaryContentToMarkdown, + fetchGlossary, + logger, +} from "@shared/utils"; import { updateDocument } from "./updateDocument"; export const updateEditorialContentDocumentsGlossary = async () => { @@ -8,28 +12,37 @@ export const updateEditorialContentDocumentsGlossary = async () => { const editorialContents = await fetchDocumentBySource( SOURCES.EDITORIAL_CONTENT ); - for (const editorialContent of editorialContents) { - const document = editorialContent.document; - await updateDocument(editorialContent.cdtn_id, { + logger.info(`Found ${editorialContents.length} editorial contents`); + for (let i = 0; i < editorialContents.length; i++) { + const document = editorialContents[i].document; + await updateDocument(editorialContents[i].cdtn_id, { ...document, introWithGlossary: addGlossaryContentToMarkdown( glossary, document.intro ?? "" ), - blocks: document.blocks.map((block: any) => { - if ("markdown" in block) { - return { - ...block, - htmlWithGlossary: addGlossaryContentToMarkdown( - glossary, - block.markdown - ), - }; - } + contents: document.contents.map((content: any) => { return { - ...block, + ...content, + blocks: content.blocks.map((block: any) => { + if ("markdown" in block) { + return { + ...block, + htmlWithGlossary: addGlossaryContentToMarkdown( + glossary, + block.markdown + ), + }; + } + return block; + }), }; }), }); + logger.info( + `Updated editorial content ${i + 1}/${ + editorialContents.length + } with glossary` + ); } }; diff --git a/targets/frontend/src/pages/api/glossary.ts b/targets/frontend/src/pages/api/glossary.ts new file mode 100644 index 000000000..ddec2408c --- /dev/null +++ b/targets/frontend/src/pages/api/glossary.ts @@ -0,0 +1,27 @@ +import { getServerSession } from "next-auth/next"; +import { authOptions } from "./auth/[...nextauth]"; +import { NextApiRequest, NextApiResponse } from "next"; +import { updateGlossary } from "src/modules/glossary"; + +export default async function handler( + req: NextApiRequest, + res: NextApiResponse +) { + if (req.method !== "POST") { + res.status(405).json({ message: "Method not allowed" }); + return; + } + + const session = await getServerSession(req, res, authOptions); + + if (!session) { + res.status(401).json({ message: "You must be logged in." }); + return; + } + + updateGlossary(); + + return res.json({ + message: "Success", + }); +} diff --git a/targets/frontend/src/pages/glossary/index.js b/targets/frontend/src/pages/glossary/index.tsx similarity index 78% rename from targets/frontend/src/pages/glossary/index.js rename to targets/frontend/src/pages/glossary/index.tsx index 822f85323..6e7e24727 100644 --- a/targets/frontend/src/pages/glossary/index.js +++ b/targets/frontend/src/pages/glossary/index.tsx @@ -1,6 +1,6 @@ import Link from "next/link"; import { useEffect, useMemo, useState } from "react"; -import { IoMdAdd, IoMdCloseCircleOutline } from "react-icons/io"; +import { IoMdAdd, IoMdSync, IoMdCloseCircleOutline } from "react-icons/io"; import { Button, IconButton } from "src/components/button"; import { TermList } from "src/components/glossary/TermList"; import { Layout } from "src/components/layout/auth.layout"; @@ -35,7 +35,7 @@ function getTermsByLetter(glossary = []) { return alphabet.map((letter) => ({ letter, terms: glossary.filter( - ({ slug }) => slug.substring(0, 1).toUpperCase() === letter + ({ slug }: any) => slug.substring(0, 1).toUpperCase() === letter ), })); } @@ -55,7 +55,7 @@ export function GlossaryPage() { if (!displayedTerms && !search) return setDisplayedTerms(glossary); setSearching(true); setDebouncedDisplayedTerms( - glossary.filter((entry) => + glossary.filter((entry: any) => entry.term.toLowerCase().includes(search.toLowerCase().trim()) ) ); @@ -80,16 +80,16 @@ export function GlossaryPage() { <> + - {termsByLetters.map(({ letter, terms }) => ( @@ -106,10 +106,9 @@ export function GlossaryPage() { )} ))} - - +