Skip to content

Commit

Permalink
fix(data): use slug to find contrib (#3265)
Browse files Browse the repository at this point in the history 
when generating ccn page we need to use slug to find contribution theme
rather than index, since index lead to multiple contribution (all-in-one
document and each ccn contrib document)

also add a memoizee to addGlossary to avoid pass to much time on it
since there is a lot a contrib document

remove console.log wich broke ingest
  • Loading branch information
Lionel authored Jan 19, 2021
1 parent 3d01ded commit 4e5daa4
Show file tree
Hide file tree
Showing 4 changed files with 89 additions and 12 deletions.
6 changes: 2 additions & 4 deletions packages/code-du-travail-data/indexing/cdtnDocuments.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ async function* cdtnDocumentsGen() {
return Promise.reject(data);
});

console.log("themes fetched");
console.error("themes fetched");

const themes = themesQueryResult.data.themes;

Expand Down Expand Up @@ -154,9 +154,7 @@ async function* cdtnDocumentsGen() {
return {
...content,
answers: content.answers.map((data) => {
const contrib = contributions.find(
({ index }) => data.index === index
);
const contrib = contributions.find(({ slug }) => data.slug === slug);
if (!contrib) {
throw "unknown contribution";
}
Expand Down
8 changes: 6 additions & 2 deletions packages/code-du-travail-data/indexing/glossary.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
import memoizee from "memoizee";

const conventionMatchers = [
"convention collective",
"conventions collectives",
Expand All @@ -22,7 +24,7 @@ const endTag = `(?=[^<]*</)`;
*/
const DISABLE_GLOSSARY = process.env.DISABLE_GLOSSARY || false;

export const createGlossaryTransform = (glossaryTerms) =>
export const createGlossaryTransform = (glossaryTerms) => {
function addGlossary(htmlContent) {
if (DISABLE_GLOSSARY) {
return htmlContent;
Expand Down Expand Up @@ -105,4 +107,6 @@ export const createGlossaryTransform = (glossaryTerms) =>
});

return finalContent;
};
}
return memoizee(addGlossary);
};
1 change: 1 addition & 0 deletions packages/code-du-travail-data/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
"@socialgouv/kali-data": "^2.32.0",
"esm": "^3.2.25",
"lodash.uniqby": "^4.7.0",
"memoizee": "^0.4.15",
"node-fetch": "^2.6.1",
"p-queue": "^6.6.2",
"p-retry": "^4.2.0",
Expand Down
86 changes: 80 additions & 6 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@
dependencies:
cross-fetch "3.0.6"

"@babel/[email protected]", "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.5.5":
"@babel/[email protected]", "@babel/code-frame@^7.0.0", "@babel/code-frame@^7.10.4", "@babel/code-frame@^7.12.11", "@babel/code-frame@^7.5.5":
version "7.12.11"
resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.12.11.tgz#f4ad435aa263db935b8f10f2c552d23fb716a63f"
integrity sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==
Expand All @@ -72,7 +72,29 @@
resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.12.7.tgz#9329b4782a7d6bbd7eef57e11addf91ee3ef1e41"
integrity sha512-YaxPMGs/XIWtYqrdEOZOCPsVWfEoriXopnsz3/i7apYPXQ3698UFhS6dVT1KN5qOsWmVgw/FOrmQgpRaZayGsw==

"@babel/[email protected]", "@babel/core@>=7.2.2", "@babel/core@>=7.9.0", "@babel/core@^7.1.0", "@babel/core@^7.12.1", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.7.5":
"@babel/[email protected]":
version "7.11.6"
resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.11.6.tgz#3a9455dc7387ff1bac45770650bc13ba04a15651"
integrity sha512-Wpcv03AGnmkgm6uS6k8iwhIwTrcP0m17TL1n1sy7qD0qelDu4XNeW0dN0mHfa+Gei211yDaLoEe/VlbXQzM4Bg==
dependencies:
"@babel/code-frame" "^7.10.4"
"@babel/generator" "^7.11.6"
"@babel/helper-module-transforms" "^7.11.0"
"@babel/helpers" "^7.10.4"
"@babel/parser" "^7.11.5"
"@babel/template" "^7.10.4"
"@babel/traverse" "^7.11.5"
"@babel/types" "^7.11.5"
convert-source-map "^1.7.0"
debug "^4.1.0"
gensync "^1.0.0-beta.1"
json5 "^2.1.2"
lodash "^4.17.19"
resolve "^1.3.2"
semver "^5.4.1"
source-map "^0.5.0"

"@babel/core@>=7.2.2", "@babel/core@>=7.9.0", "@babel/core@^7.1.0", "@babel/core@^7.12.1", "@babel/core@^7.12.10", "@babel/core@^7.12.3", "@babel/core@^7.7.5":
version "7.12.10"
resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.12.10.tgz#b79a2e1b9f70ed3d84bbfb6d8c4ef825f606bccd"
integrity sha512-eTAlQKq65zHfkHZV0sIVODCPGVgoo1HdBlbSLi9CqOzuZanMv2ihzY+4paiKr1mH+XmYESMAmJ/dpZ68eN6d8w==
Expand All @@ -93,6 +115,15 @@
semver "^5.4.1"
source-map "^0.5.0"

"@babel/generator@^7.11.6", "@babel/generator@^7.12.11":
version "7.12.11"
resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.12.11.tgz#98a7df7b8c358c9a37ab07a24056853016aba3af"
integrity sha512-Ggg6WPOJtSi8yYQvLVjG8F/TlpWDlKx0OpS4Kt+xMQPs5OaGYWy+v1A+1TvxI6sAMGZpKWWoAQ1DaeQbImlItA==
dependencies:
"@babel/types" "^7.12.11"
jsesc "^2.5.1"
source-map "^0.5.0"

"@babel/generator@^7.12.1", "@babel/generator@^7.12.10":
version "7.12.10"
resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.12.10.tgz#2b188fc329fb8e4f762181703beffc0fe6df3460"
Expand Down Expand Up @@ -190,13 +221,29 @@
"@babel/template" "^7.10.4"
"@babel/types" "^7.10.4"

"@babel/helper-function-name@^7.12.11":
version "7.12.11"
resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.12.11.tgz#1fd7738aee5dcf53c3ecff24f1da9c511ec47b42"
integrity sha512-AtQKjtYNolKNi6nNNVLQ27CP6D9oFR6bq/HPYSizlzbp7uC1M59XJe8L+0uXjbIaZaUJF99ruHqVGiKXU/7ybA==
dependencies:
"@babel/helper-get-function-arity" "^7.12.10"
"@babel/template" "^7.12.7"
"@babel/types" "^7.12.11"

"@babel/helper-get-function-arity@^7.10.4":
version "7.10.4"
resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.10.4.tgz#98c1cbea0e2332f33f9a4661b8ce1505b2c19ba2"
integrity sha512-EkN3YDB+SRDgiIUnNgcmiD361ti+AVbL3f3Henf6dqqUyr5dMsorno0lJWJuLhDhkI5sYEpgj6y9kB8AOU1I2A==
dependencies:
"@babel/types" "^7.10.4"

"@babel/helper-get-function-arity@^7.12.10":
version "7.12.10"
resolved "https://registry.yarnpkg.com/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.10.tgz#b158817a3165b5faa2047825dfa61970ddcc16cf"
integrity sha512-mm0n5BPjR06wh9mPQaDdXWDoll/j5UpCAPl1x8fS71GHm7HA6Ua2V4ylG1Ju8lvcTOietbPNNPaSilKj+pj+Ag==
dependencies:
"@babel/types" "^7.12.10"

"@babel/helper-hoist-variables@^7.10.4":
version "7.10.4"
resolved "https://registry.yarnpkg.com/@babel/helper-hoist-variables/-/helper-hoist-variables-7.10.4.tgz#d49b001d1d5a68ca5e6604dda01a6297f7c9381e"
Expand All @@ -218,7 +265,7 @@
dependencies:
"@babel/types" "^7.12.5"

"@babel/helper-module-transforms@^7.12.1":
"@babel/helper-module-transforms@^7.11.0", "@babel/helper-module-transforms@^7.12.1":
version "7.12.1"
resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.12.1.tgz#7954fec71f5b32c48e4b303b437c34453fd7247c"
integrity sha512-QQzehgFAZ2bbISiCpmVGfiGux8YVFXQ0abBic2Envhej22DVXV9nCFaS5hIQbkyo1AdGb+gNME2TSh3hYJVV/w==
Expand Down Expand Up @@ -292,6 +339,13 @@
dependencies:
"@babel/types" "^7.11.0"

"@babel/helper-split-export-declaration@^7.12.11":
version "7.12.11"
resolved "https://registry.yarnpkg.com/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.11.tgz#1b4cc424458643c47d37022223da33d76ea4603a"
integrity sha512-LsIVN8j48gHgwzfocYUSkO/hjYAOJqlpJEc7tGXcIm4cubjVUf8LGW6eWRyxEu7gA25q02p0rQUWoCI33HNS5g==
dependencies:
"@babel/types" "^7.12.11"

"@babel/helper-validator-identifier@^7.10.4", "@babel/helper-validator-identifier@^7.12.11":
version "7.12.11"
resolved "https://registry.yarnpkg.com/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz#c9a1f021917dcb5ccf0d4e453e399022981fc9ed"
Expand All @@ -312,7 +366,7 @@
"@babel/traverse" "^7.10.4"
"@babel/types" "^7.10.4"

"@babel/helpers@^7.12.5":
"@babel/helpers@^7.10.4", "@babel/helpers@^7.12.5":
version "7.12.5"
resolved "https://registry.yarnpkg.com/@babel/helpers/-/helpers-7.12.5.tgz#1a1ba4a768d9b58310eda516c449913fe647116e"
integrity sha512-lgKGMQlKqA8meJqKsW6rUnc4MdUk35Ln0ATDqdM1a/UpARODdI4j5Y5lVfUScnSNkJcdCRAaWkspykNoFg9sJA==
Expand All @@ -335,6 +389,11 @@
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.10.tgz#824600d59e96aea26a5a2af5a9d812af05c3ae81"
integrity sha512-PJdRPwyoOqFAWfLytxrWwGrAxghCgh/yTNCYciOz8QgjflA7aZhECPZAa2VUedKg2+QMWkI0L9lynh2SNmNEgA==

"@babel/parser@^7.11.5", "@babel/parser@^7.12.11":
version "7.12.11"
resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.12.11.tgz#9ce3595bcd74bc5c466905e86c535b8b25011e79"
integrity sha512-N3UxG+uuF4CMYoNj8AhnbAcJF0PiuJ9KHuy1lQmkYsxTer/MAH9UBNHsBoAX/4s6NvlDD047No8mYVGGzLL4hg==

"@babel/plugin-proposal-async-generator-functions@^7.12.1":
version "7.12.1"
resolved "https://registry.yarnpkg.com/@babel/plugin-proposal-async-generator-functions/-/plugin-proposal-async-generator-functions-7.12.1.tgz#dc6c1170e27d8aca99ff65f4925bd06b1c90550e"
Expand Down Expand Up @@ -1069,6 +1128,21 @@
globals "^11.1.0"
lodash "^4.17.19"

"@babel/traverse@^7.11.5":
version "7.12.12"
resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.12.12.tgz#d0cd87892704edd8da002d674bc811ce64743376"
integrity sha512-s88i0X0lPy45RrLM8b9mz8RPH5FqO9G9p7ti59cToE44xFm1Q+Pjh5Gq4SXBbtb88X7Uy7pexeqRIQDDMNkL0w==
dependencies:
"@babel/code-frame" "^7.12.11"
"@babel/generator" "^7.12.11"
"@babel/helper-function-name" "^7.12.11"
"@babel/helper-split-export-declaration" "^7.12.11"
"@babel/parser" "^7.12.11"
"@babel/types" "^7.12.12"
debug "^4.1.0"
globals "^11.1.0"
lodash "^4.17.19"

"@babel/[email protected]":
version "7.8.3"
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.8.3.tgz#5a383dffa5416db1b73dedffd311ffd0788fb31c"
Expand All @@ -1078,7 +1152,7 @@
lodash "^4.17.13"
to-fast-properties "^2.0.0"

"@babel/types@^7.0.0", "@babel/types@^7.10.4", "@babel/types@^7.11.0", "@babel/types@^7.12.1", "@babel/types@^7.12.10", "@babel/types@^7.12.11", "@babel/types@^7.12.5", "@babel/types@^7.12.6", "@babel/types@^7.12.7", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0":
"@babel/types@^7.0.0", "@babel/types@^7.10.4", "@babel/types@^7.11.0", "@babel/types@^7.11.5", "@babel/types@^7.12.1", "@babel/types@^7.12.10", "@babel/types@^7.12.11", "@babel/types@^7.12.12", "@babel/types@^7.12.5", "@babel/types@^7.12.6", "@babel/types@^7.12.7", "@babel/types@^7.3.0", "@babel/types@^7.3.3", "@babel/types@^7.4.4", "@babel/types@^7.7.0":
version "7.12.12"
resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.12.12.tgz#4608a6ec313abbd87afa55004d373ad04a96c299"
integrity sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==
Expand Down Expand Up @@ -16383,7 +16457,7 @@ resolve-url@^0.2.1:
resolved "https://registry.yarnpkg.com/resolve-url/-/resolve-url-0.2.1.tgz#2c637fe77c893afd2a663fe21aa9080068e2052a"
integrity sha1-LGN/53yJOv0qZj/iGqkIAGjiBSo=

resolve@^1.1.6, resolve@^1.10.0, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.18.1, resolve@^1.19.0:
resolve@^1.1.6, resolve@^1.10.0, resolve@^1.12.0, resolve@^1.13.1, resolve@^1.17.0, resolve@^1.18.1, resolve@^1.19.0, resolve@^1.3.2:
version "1.19.0"
resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.19.0.tgz#1af5bf630409734a067cae29318aac7fa29a267c"
integrity sha512-rArEXAgsBG4UgRGcynxWIWKFvh/XZCcS8UJdHhwy91zwAvCZIbcs+vAbflgBnNjYMs/i/i+/Ux6IZhML1yPvxg==
Expand Down

0 comments on commit 4e5daa4

Please sign in to comment.