fix the 10k max results problem (#705)

* fix the 10k max results problem * remove debug code
wellcomecollection · Aug 23, 2023 · a053cdf · a053cdf
1 parent b1ce8c8
commit a053cdf
Show file tree

Hide file tree

Showing 2 changed files with 45 additions and 3 deletions.
diff --git a/search/src/main/scala/weco/api/search/services/TemplateSearchBuilder.scala b/search/src/main/scala/weco/api/search/services/TemplateSearchBuilder.scala
@@ -11,14 +11,13 @@ trait TemplateSearchBuilder extends Encoders {
   // called "query".
   // This preserves the existing behaviour of /search-templates.json
   val queryTemplate: String
-
   // Importantly, this is *not* JSON, due to the `{{#` sequences, so must be created and sent as a string.
   lazy protected val source: String =
     s"""
        |{ {{#query}}
        |  "query": $queryTemplate,
        |  {{/query}}
-       |
+       |  "track_total_hits": true,
        |  "from": "{{from}}",
        |  "size": "{{size}}",
        |  "_source": {

diff --git a/search/src/test/scala/weco/api/search/elasticsearch/WorksQueryTest.scala b/search/src/test/scala/weco/api/search/elasticsearch/WorksQueryTest.scala
@@ -1,13 +1,15 @@
 package weco.api.search.elasticsearch
 
 import com.sksamuel.elastic4s.Index
+import io.circe.Json
+import io.circe.syntax.EncoderOps
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.EitherValues
 import weco.api.search.fixtures.{IndexFixtures, TestDocumentFixtures}
 import weco.api.search.generators.SearchOptionsGenerators
 import weco.api.search.models.index.IndexedWork
-import weco.api.search.models.{SearchQuery}
+import weco.api.search.models.SearchQuery
 import weco.api.search.services.WorksService
 import weco.fixtures.TestWith
 
@@ -242,6 +244,47 @@ class WorksQueryTest
         }
       }
     }
+
+    it("accurately reports total hits") {
+      info(
+        """By default, Elasticsearch only returns total hits up to 10000.
+          | Ensure that the number we return to the API is an accurate reflection
+          | of the number of matches, and has not been cut down to this 10k max.
+          |""".stripMargin
+      )
+      withLocalWorksIndex { index =>
+        val docs: Seq[TestDocument] = (0 to 10010) map { i =>
+          val docId = s"id$i"
+          val doc = Json.obj(
+            "display" -> Json
+              .obj("id" -> docId.asJson, "title" -> s"work number $i".asJson),
+            "query" -> Json.obj(
+              "title" -> Seq(s"work number $i").asJson
+            ),
+            "type" -> "Visible".asJson
+          )
+
+          TestDocument(
+            docId,
+            doc
+          )
+        }
+        indexLoadedTestDocuments(index, docs)
+
+        val future = worksService.listOrSearch(
+          index,
+          searchOptions = createWorksSearchOptionsWith(
+            searchQuery = Some(SearchQuery("work"))
+          )
+        )
+
+        whenReady(future) { r =>
+          r.right.value.totalResults should be > 10000
+        }
+      }
+
+    }
+
   }
 
   private def assertForQueryResults[R](index: Index, query: String)(