feat(data): add datafiller results to api (#906)

* fix(es): make slug field a text field * feat(api): include datafiller results * doc * cleanup + docs * fix review * wth * fix: snapshot * lintfix * snapshots * data: update prequalified.json * update snapshots * update prequalified.json * update snapshots * allow 70% ES relevance fix #887
SocialGouv · Jul 1, 2019 · c4f5810 · c4f5810
1 parent cc0bcc2
commit c4f5810
Show file tree

Hide file tree

Showing 14 changed files with 24,878 additions and 105 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -23,6 +23,7 @@ COPY ./packages/code-du-travail-data/dataset/kali/package.json /app/packages/cod
 COPY ./packages/code-du-travail-data/dataset/stop_words/package.json /app/packages/code-du-travail-data/dataset/stop_words/package.json
 COPY ./packages/code-du-travail-data/dataset/synonyms/package.json /app/packages/code-du-travail-data/dataset/synonyms/package.json
 COPY ./packages/code-du-travail-data/dataset/themes/package.json /app/packages/code-du-travail-data/dataset/themes/package.json
+COPY ./packages/code-du-travail-data/dataset/datafiller/package.json /app/packages/code-du-travail-data/dataset/datafiller/package.json
 COPY ./packages/code-du-travail-data/package.json /app/packages/code-du-travail-data/package.json
 COPY ./packages/react-fiche-service-public/package.json /app/packages/react-fiche-service-public/package.json
 

diff --git a/packages/code-du-travail-api/src/server/routes/__tests__/__snapshots__/items.spec.js.snap b/packages/code-du-travail-api/src/server/routes/__tests__/__snapshots__/items.spec.js.snap
@@ -4,7 +4,7 @@ exports[`return a faq item with additionnal content from its source and slug 1`]
 Object {
   "_id": "6",
   "_index": "cdtn_document_test",
-  "_score": 1.9169226,
+  "_score": 4.344671,
   "_source": Object {
     "author": "DIRECCTE",
     "date": "01/12/2017",
@@ -82,7 +82,7 @@ exports[`return item from source and slug  1`] = `
 Object {
   "_id": "2",
   "_index": "cdtn_document_test",
-  "_score": 1.5804504,
+  "_score": 4.0081987,
   "_source": Object {
     "breadcrumbs": Array [
       Object {

diff --git a/packages/code-du-travail-api/src/server/routes/__tests__/__snapshots__/search.spec.js.snap b/packages/code-du-travail-api/src/server/routes/__tests__/__snapshots__/search.spec.js.snap
@@ -48,32 +48,7 @@ Object {
 
 exports[`return faq search results for demission  1`] = `
 Object {
-  "hits": Array [
-    Object {
-      "_id": "6",
-      "_index": "cdtn_document_test",
-      "_score": 4.38116,
-      "_source": Object {
-        "slug": "dans-les-hcr-quel-est-le-preavis-de-demission",
-        "source": "faq",
-        "title": "Dans les HCR, quel est le préavis de démission ?",
-      },
-      "_type": "cdtn_document_test",
-    },
-    Object {
-      "_id": "7",
-      "_index": "cdtn_document_test",
-      "_score": 4.1424417,
-      "_source": Object {
-        "slug": "joccupe-un-poste-de-vendeuse-en-boulangerie-patisserie-et-je-viens-de-poser-ma-demission-quelles-sont-les-consequences-dun-arret-maladie-pendant-la-periode-de-preavis-est-ce-que-je-peux-etre-sanctionnee-en-cas-darret-de-travail",
-        "source": "faq",
-        "title": "J'occupe un poste de vendeuse en boulangerie-pâtisserie et je viens de poser ma démission. Quelles sont les conséquences d’un arrêt maladie pendant la période de préavis ? Est-ce que je peux être sanctionnée en cas d’arrêt de travail ?",
-      },
-      "_type": "cdtn_document_test",
-    },
-  ],
-  "max_score": 4.38116,
-  "total": 2,
+  "hits": Array [],
 }
 `;
 
@@ -142,125 +117,218 @@ exports[`return search results for demission 1`] = `
 Object {
   "hits": Array [
     Object {
-      "_id": "2",
-      "_index": "cdtn_document_test",
-      "_score": 4.844017,
+      "_id": "VNQWlGsBPUX8CkMHsmI2",
+      "_index": "code_du_travail_numerique",
+      "_score": 16.53278,
+      "_source": Object {
+        "slug": "demission-dun-salarie",
+        "source": "fiches_service_public",
+        "title": "Démission d'un salarié",
+        "url": "https://www.service-public.fr/particuliers/vosdroits/F2883",
+      },
+      "_type": "code_du_travail_numerique",
+      "relevance": 5,
+    },
+    Object {
+      "_id": "ONQWlGsBPUX8CkMHuGRZ",
+      "_index": "code_du_travail_numerique",
+      "_score": 31.28897,
       "_source": Object {
         "anchor": "#Comment-presenter-une-demission",
         "slug": "demission-comment-presenter-une-demission",
         "source": "fiches_ministere_travail",
-        "title": "Démission : comment présenter une démission ?",
+        "title": "Démission : comment présenter une démission ?",
         "url": "https://travail-emploi.gouv.fr/droit-du-travail/la-rupture-du-contrat-de-travail/article/la-demission#Comment-presenter-une-demission",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 5,
     },
     Object {
-      "_id": "4",
-      "_index": "cdtn_document_test",
-      "_score": 4.7127514,
+      "_id": "69QWlGsBPUX8CkMHkTf1",
+      "_index": "code_du_travail_numerique",
+      "_score": 9.976059,
       "_source": Object {
-        "slug": "demission-dun-salarie",
+        "slug": "l1237-1",
+        "source": "code_du_travail",
+        "title": "Article L1237-1",
+        "url": "https://www.legifrance.gouv.fr/affichCodeArticle.do?idArticle=LEGIARTI000006901174&cidTexte=LEGITEXT000006072050",
+      },
+      "_type": "code_du_travail_numerique",
+      "relevance": 5,
+    },
+    Object {
+      "_id": "QdQWlGsBPUX8CkMHkTfx",
+      "_index": "code_du_travail_numerique",
+      "_score": 12.340258,
+      "_source": Object {
+        "slug": "l1231-1",
+        "source": "code_du_travail",
+        "title": "Article L1231-1",
+        "url": "https://www.legifrance.gouv.fr/affichCodeArticle.do?idArticle=LEGIARTI000019071194&cidTexte=LEGITEXT000006072050",
+      },
+      "_type": "code_du_travail_numerique",
+      "relevance": 5,
+    },
+    Object {
+      "_id": "T9QWlGsBPUX8CkMHrGHT",
+      "_index": "code_du_travail_numerique",
+      "_score": 29.38531,
+      "_source": Object {
+        "slug": "peut-on-demissionner-pendant-ses-conges-payes",
         "source": "fiches_service_public",
-        "title": "Démission d'un salarié",
-        "url": "https://www.service-public.fr/particuliers/vosdroits/F2883",
+        "title": "Peut-on démissionner pendant ses congés payés ?",
+        "url": "https://www.service-public.fr/particuliers/vosdroits/F12598",
+      },
+      "_type": "code_du_travail_numerique",
+      "relevance": 4,
+    },
+    Object {
+      "_id": "sdQWlGsBPUX8CkMHrGHV",
+      "_index": "code_du_travail_numerique",
+      "_score": 27.494377,
+      "_source": Object {
+        "slug": "peut-on-demissionner-pendant-un-conge-maternite",
+        "source": "fiches_service_public",
+        "title": "Peut-on démissionner pendant un congé maternité ?",
+        "url": "https://www.service-public.fr/particuliers/vosdroits/F1730",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 4,
     },
     Object {
-      "_id": "6",
-      "_index": "cdtn_document_test",
-      "_score": 4.38116,
+      "_id": "ddQWlGsBPUX8CkMHsmI3",
+      "_index": "code_du_travail_numerique",
+      "_score": 27.990566,
       "_source": Object {
-        "slug": "dans-les-hcr-quel-est-le-preavis-de-demission",
-        "source": "faq",
-        "title": "Dans les HCR, quel est le préavis de démission ?",
+        "slug": "peut-on-demissionner-pendant-un-arret-maladie",
+        "source": "fiches_service_public",
+        "title": "Peut-on démissionner pendant un arrêt maladie ?",
+        "url": "https://www.service-public.fr/particuliers/vosdroits/F31679",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 4,
     },
     Object {
-      "_id": "0",
-      "_index": "cdtn_document_test",
-      "_score": 4.2591395,
+      "_id": "4NQWlGsBPUX8CkMHsmI5",
+      "_index": "code_du_travail_numerique",
+      "_score": 27.392416,
       "_source": Object {
-        "slug": "themes/rupture-de-contrat/la-demission",
-        "source": "themes",
-        "title": "La démission",
+        "slug": "peut-on-demissionner-pendant-un-conge-parental",
+        "source": "fiches_service_public",
+        "title": "Peut-on démissionner pendant un congé parental ?",
+        "url": "https://www.service-public.fr/particuliers/vosdroits/F540",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 4,
     },
     Object {
-      "_id": "5",
-      "_index": "cdtn_document_test",
-      "_score": 4.233489,
+      "_id": "O9QWlGsBPUX8CkMHuGRZ",
+      "_index": "code_du_travail_numerique",
+      "_score": 28.318968,
+      "_source": Object {
+        "anchor": "#Faut-il-respecter-un-preavis",
+        "slug": "demission-faut-il-respecter-un-preavis",
+        "source": "fiches_ministere_travail",
+        "title": "Démission : faut-il respecter un préavis ?",
+        "url": "https://travail-emploi.gouv.fr/droit-du-travail/la-rupture-du-contrat-de-travail/article/la-demission#Faut-il-respecter-un-preavis",
+      },
+      "_type": "code_du_travail_numerique",
+      "relevance": 4,
+    },
+    Object {
+      "_id": "PtQWlGsBPUX8CkMHuGRZ",
+      "_index": "code_du_travail_numerique",
+      "_score": 33.23731,
+      "_source": Object {
+        "anchor": "#Le-salarie-peut-il-lt-lt-demissionner-dans-le-cadre-d-un-contrat-a-nbsp",
+        "slug": "demission-le-salarie-peut-il-demissionner-dans-le-cadre-dun-contrat-a-duree-determinee",
+        "source": "fiches_ministere_travail",
+        "title": "Démission : le salarié peut-il « démissionner » dans le cadre d’un contrat à durée déterminée ?",
+        "url": "https://travail-emploi.gouv.fr/droit-du-travail/la-rupture-du-contrat-de-travail/article/la-demission#Le-salarie-peut-il-lt-lt-demissionner-dans-le-cadre-d-un-contrat-a-nbsp",
+      },
+      "_type": "code_du_travail_numerique",
+      "relevance": 4,
+    },
+    Object {
+      "_id": "pdQWlGsBPUX8CkMHsmI4",
+      "_index": "code_du_travail_numerique",
+      "_score": 25.200104,
       "_source": Object {
         "slug": "demission-dune-assistante-maternelle",
         "source": "fiches_service_public",
         "title": "Démission d'une assistante maternelle",
         "url": "https://www.service-public.fr/particuliers/vosdroits/F33164",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 3,
     },
     Object {
-      "_id": "7",
-      "_index": "cdtn_document_test",
-      "_score": 4.1424417,
+      "_id": "l9QWlGsBPUX8CkMHsmI3",
+      "_index": "code_du_travail_numerique",
+      "_score": 29.233229,
       "_source": Object {
-        "slug": "joccupe-un-poste-de-vendeuse-en-boulangerie-patisserie-et-je-viens-de-poser-ma-demission-quelles-sont-les-consequences-dun-arret-maladie-pendant-la-periode-de-preavis-est-ce-que-je-peux-etre-sanctionnee-en-cas-darret-de-travail",
-        "source": "faq",
-        "title": "J'occupe un poste de vendeuse en boulangerie-pâtisserie et je viens de poser ma démission. Quelles sont les conséquences d’un arrêt maladie pendant la période de préavis ? Est-ce que je peux être sanctionnée en cas d’arrêt de travail ?",
+        "slug": "demission-du-salarie-a-domicile-employe-par-un-particulier",
+        "source": "fiches_service_public",
+        "title": "Démission du salarié à domicile employé par un particulier",
+        "url": "https://www.service-public.fr/particuliers/vosdroits/F32431",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 3,
     },
     Object {
-      "_id": "8",
-      "_index": "cdtn_document_test",
-      "_score": 3.5526776,
+      "_id": "OtQWlGsBPUX8CkMHuGRZ",
+      "_index": "code_du_travail_numerique",
+      "_score": 39.01729,
       "_source": Object {
-        "slug": "r1225-18",
-        "source": "code_du_travail",
-        "title": "Article R1225-18",
-        "url": "https://www.legifrance.gouv.fr/affichCodeArticle.do?idArticle=LEGIARTI000018537778&cidTexte=LEGITEXT000006072050",
+        "anchor": "#Le-refus-par-le-salarie-d-accepter-un-changement-des-conditions-de-nbsp",
+        "slug": "demission-le-refus-par-le-salarie-daccepter-un-changement-des-conditions-de-travail-est-il-une-demission",
+        "source": "fiches_ministere_travail",
+        "title": "Démission : le refus, par le salarié, d’accepter un changement des conditions de travail est-il une démission ?",
+        "url": "https://travail-emploi.gouv.fr/droit-du-travail/la-rupture-du-contrat-de-travail/article/la-demission#Le-refus-par-le-salarie-d-accepter-un-changement-des-conditions-de-nbsp",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 3,
     },
     Object {
-      "_id": "9",
-      "_index": "cdtn_document_test",
-      "_score": 3.4196491,
+      "_id": "7NQWlGsBPUX8CkMHkTf1",
+      "_index": "code_du_travail_numerique",
+      "_score": 10.407834,
       "_source": Object {
-        "slug": "r1442-22-4",
+        "slug": "l1237-2",
         "source": "code_du_travail",
-        "title": "Article R1442-22-4",
-        "url": "https://www.legifrance.gouv.fr/affichCodeArticle.do?idArticle=LEGIARTI000033748111&cidTexte=LEGITEXT000006072050",
+        "title": "Article L1237-2",
+        "url": "https://www.legifrance.gouv.fr/affichCodeArticle.do?idArticle=LEGIARTI000006901175&cidTexte=LEGITEXT000006072050",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 3,
     },
     Object {
-      "_id": "3",
-      "_index": "cdtn_document_test",
-      "_score": 1.3544811,
+      "_id": "99QWlGsBPUX8CkMHsmI5",
+      "_index": "code_du_travail_numerique",
+      "_score": 29.657969,
       "_source": Object {
-        "anchor": "#L-absence-prolongee-du-salarie-est-elle-une-demission",
-        "slug": "demission-labsence-prolongee-du-salarie-est-elle-une-demission",
-        "source": "fiches_ministere_travail",
-        "title": "Démission : l’absence prolongée du salarié est-elle une démission ?",
-        "url": "https://travail-emploi.gouv.fr/droit-du-travail/la-rupture-du-contrat-de-travail/article/la-demission#L-absence-prolongee-du-salarie-est-elle-une-demission",
+        "slug": "peut-on-percevoir-lallocation-chomage-en-cas-de-demission",
+        "source": "fiches_service_public",
+        "title": "Peut-on percevoir l'allocation chômage en cas de démission ?",
+        "url": "https://www.service-public.fr/particuliers/vosdroits/F89",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 2,
     },
     Object {
-      "_id": "1",
-      "_index": "cdtn_document_test",
-      "_score": 1.2437692,
+      "_id": "PNQWlGsBPUX8CkMHuGRZ",
+      "_index": "code_du_travail_numerique",
+      "_score": 26.313335,
       "_source": Object {
-        "slug": "themes/labandon-de-poste-est-il-une-demission",
-        "source": "themes",
-        "title": "L’abandon de poste est-il une démission ?",
+        "anchor": "#Et-les-heures-pour-recherche-d-emploi",
+        "slug": "demission-et-les-heures-pour-recherche-demploi",
+        "source": "fiches_ministere_travail",
+        "title": "Démission : et les heures pour recherche d’emploi ?",
+        "url": "https://travail-emploi.gouv.fr/droit-du-travail/la-rupture-du-contrat-de-travail/article/la-demission#Et-les-heures-pour-recherche-d-emploi",
       },
-      "_type": "cdtn_document_test",
+      "_type": "code_du_travail_numerique",
+      "relevance": 2,
     },
   ],
-  "max_score": 4.844017,
-  "total": 10,
 }
 `;
diff --git a/packages/code-du-travail-api/src/server/routes/items/searchBySourceSlug.elastic.js b/packages/code-du-travail-api/src/server/routes/items/searchBySourceSlug.elastic.js
@@ -3,10 +3,14 @@ function getSearchBody({ source, slug }) {
     size: 1,
     query: {
       bool: {
-        must: {
-          match: { source }
-        },
-        filter: { term: { slug } }
+        must: [
+          {
+            match: { source }
+          },
+          {
+            match: { slug }
+          }
+        ]
       }
     },
     _source: [

diff --git a/packages/code-du-travail-api/src/server/routes/search/index.js b/packages/code-du-travail-api/src/server/routes/search/index.js
@@ -4,6 +4,7 @@ const API_BASE_URL = require("../v1.prefix");
 const elasticsearchClient = require("../../conf/elasticsearch.js");
 const getSearchBody = require("./search.elastic");
 const getFacetsBody = require("./facets.elastic");
+const getKnownQuery = require("./search.prequalified");
 
 const index =
   process.env.ELASTICSEARCH_DOCUMENT_INDEX || "code_du_travail_numerique";
@@ -24,12 +25,20 @@ const router = new Router({ prefix: API_BASE_URL });
  */
 router.get("/search", async ctx => {
   const query = ctx.request.query.q;
+  const excludeSources = (ctx.request.query.excludeSources || "").split(",");
+
+  // shortcut ES if we find a known query
+  const knownQueryResult = getKnownQuery(query, excludeSources);
+
+  if (knownQueryResult) {
+    ctx.body = knownQueryResult;
+    return;
+  }
+
   // we add 1 to maxResults in case we have a snippet document in the results
   // we filter results to remove snippet document from main results
   const size = Math.min(ctx.request.query.size || MAX_RESULTS + 1, 100);
 
-  const excludeSources = (ctx.request.query.excludeSources || "").split(",");
-
   const body = getSearchBody({ query, size, excludeSources });
   const facetBody = getFacetsBody({ query });