diff --git a/.gitignore b/.gitignore
index 48a309237..5a9b95519 100644
--- a/.gitignore
+++ b/.gitignore
@@ -146,7 +146,6 @@ assets/upload/*
!assets/upload/index.html
*_dev
-snippets
# profiler
*.profile
diff --git a/apps/cms/templatetags/readux_templatetags.py b/apps/cms/templatetags/readux_templatetags.py
index e9b708e0b..66d0e3d2b 100644
--- a/apps/cms/templatetags/readux_templatetags.py
+++ b/apps/cms/templatetags/readux_templatetags.py
@@ -2,7 +2,23 @@
register = Library()
+
@register.filter_function
def order_by(queryset, args):
- args = [x.strip() for x in args.split(',')]
+ args = [x.strip() for x in args.split(",")]
return queryset.order_by(*args)
+
+
+@register.filter
+def dict_item(dictionary, key):
+ """'Template filter to allow accessing dictionary value by variable key.
+ Example use::
+
+ {{ mydict|dict_item:keyvar }}
+ """
+ # adapted from Princeton-CDH/geniza project https://github.com/Princeton-CDH/geniza/
+ try:
+ return dictionary[key]
+ except AttributeError:
+ # fail silently if something other than a dict is passed
+ return None
diff --git a/apps/iiif/canvases/management/commands/rebuild_ocr.py b/apps/iiif/canvases/management/commands/rebuild_ocr.py
index b345470f4..955b3595e 100644
--- a/apps/iiif/canvases/management/commands/rebuild_ocr.py
+++ b/apps/iiif/canvases/management/commands/rebuild_ocr.py
@@ -123,4 +123,5 @@ def __rebuild(self, canvas, testing=False):
anno.content = word['content']
anno.save()
prog_bar.next()
+ canvas.save()
prog_bar.finish()
diff --git a/apps/iiif/canvases/models.py b/apps/iiif/canvases/models.py
index e6f59e0f9..6032daed3 100644
--- a/apps/iiif/canvases/models.py
+++ b/apps/iiif/canvases/models.py
@@ -1,5 +1,6 @@
"""Django models representing IIIF canvases and IIIF image server info."""
import os
+from functools import cached_property
from urllib.parse import quote
from boto3 import resource
from bs4 import BeautifulSoup
@@ -144,23 +145,21 @@ def thumbnail_crop_volume(self):
# landscape
return f'{self.resource_id}/pct:25,15,50,85/,600/0/default.jpg'
- @property
+ @cached_property
def result(self):
- """Empty attribute to hold the result of requests to get OCR data."""
- words = Annotation.objects.filter(
- owner=USER.objects.get(username='ocr'),
- canvas=self.id).order_by('order')
+ """Cached property containing OCR text content from associated annotations."""
+ words = self.annotation_set.filter(owner__username="ocr").order_by("order")
clean_words = []
for word in words:
- clean_word = BeautifulSoup(word.content, 'html.parser').text
+ clean_word = BeautifulSoup(word.content, "html.parser").text
clean_words.append(clean_word)
- return ' '.join(clean_words)
+ return " ".join(clean_words)
def save(self, *args, **kwargs): # pylint: disable = signature-differs
"""
Override save function to set `resource_id` add OCR,
set as manifest's `start_canvas` if manifest does not have one,
- and set
+ and set position
"""
self.__check_image_server()
diff --git a/apps/iiif/canvases/tasks.py b/apps/iiif/canvases/tasks.py
index af06a2315..78acfdbab 100644
--- a/apps/iiif/canvases/tasks.py
+++ b/apps/iiif/canvases/tasks.py
@@ -16,6 +16,7 @@ def add_ocr_task(canvas_id, *args, **kwargs):
if ocr is not None:
add_ocr_annotations(canvas, ocr)
+ canvas.save() # trigger reindex
@app.task(name='adding_oa_ocr_to_canvas', retry_backoff=5)
def add_oa_ocr_task(annotation_list_url):
diff --git a/apps/iiif/manifests/documents.py b/apps/iiif/manifests/documents.py
index f6f94e7cd..4c5100fb3 100644
--- a/apps/iiif/manifests/documents.py
+++ b/apps/iiif/manifests/documents.py
@@ -4,11 +4,14 @@
from django_elasticsearch_dsl import Document, fields
from django_elasticsearch_dsl.registries import registry
from elasticsearch_dsl import analyzer
+from django.db.models.query import Prefetch
from django.utils.html import strip_tags
from unidecode import unidecode
+from apps.iiif.annotations.models import Annotation
+from apps.iiif.canvases.models import Canvas
from apps.iiif.kollections.models import Collection
-from .models import Manifest
+from apps.iiif.manifests.models import Manifest
# TODO: Better English stemming (e.g. Rome to match Roman), multilingual stemming.
stemmer = analyzer(
@@ -25,9 +28,14 @@ class ManifestDocument(Document):
# fields to map explicitly in Elasticsearch
authors = fields.KeywordField(multi=True) # only used for faceting/filtering
author = fields.TextField() # only used for searching
- collections = fields.NestedField(properties={
- "label": fields.KeywordField(),
- })
+ canvas_set = fields.NestedField(
+ properties={
+ "result": fields.TextField(analyzer=stemmer),
+ "position": fields.IntegerField(),
+ "pid": fields.KeywordField(),
+ }
+ ) # canvas_set.result = OCR annotation text on each canvas
+ collections = fields.NestedField(properties={"label": fields.KeywordField()})
date_earliest = fields.DateField()
date_latest = fields.DateField()
has_pdf = fields.BooleanField()
@@ -38,10 +46,12 @@ class ManifestDocument(Document):
class Index:
"""Settings for Elasticsearch"""
+
name = "manifests"
class Django:
"""Settings for automatically pulling data from Django"""
+
model = Manifest
# fields to map dynamically in Elasticsearch
@@ -57,7 +67,7 @@ class Django:
"publisher",
"viewingdirection",
]
- related_models = [Collection]
+ related_models = [Collection, Canvas]
def prepare_authors(self, instance):
"""convert authors string into list"""
@@ -88,12 +98,30 @@ def prepare_summary(self, instance):
def get_queryset(self):
"""prefetch related to improve performance"""
- return super().get_queryset().prefetch_related(
- "collections"
+ return (
+ super()
+ .get_queryset()
+ .prefetch_related(
+ "collections",
+ "image_server",
+ "languages",
+ Prefetch(
+ "canvas_set",
+ queryset=Canvas.objects.prefetch_related(
+ Prefetch(
+ "annotation_set",
+ queryset=Annotation.objects.select_related("owner"),
+ ),
+ ),
+ ),
+ )
)
def get_instances_from_related(self, related_instance):
- """Retrieving item to index from related collections"""
+ """Retrieving item to index from related objects"""
if isinstance(related_instance, Collection):
# many to many relationship
return related_instance.manifests.all()
+ elif isinstance(related_instance, Canvas):
+ # many to many relationship
+ return related_instance.manifest
diff --git a/apps/readux/forms.py b/apps/readux/forms.py
index b093a186c..3c2e810e0 100644
--- a/apps/readux/forms.py
+++ b/apps/readux/forms.py
@@ -57,6 +57,21 @@ class ManifestSearchForm(forms.Form):
},
),
)
+ scope = forms.ChoiceField(
+ label="Limit search to",
+ required=False,
+ initial="all",
+ choices=(
+ ("all", "All"),
+ ("metadata", "Metadata only"),
+ ("text", "Textual contents only"),
+ ),
+ widget=forms.Select(
+ attrs={
+ "class": "uk-select",
+ },
+ ),
+ )
language = FacetedMultipleChoiceField(
label="Language",
required=False,
diff --git a/apps/readux/views.py b/apps/readux/views.py
index 0d3f24a7f..add5499ae 100644
--- a/apps/readux/views.py
+++ b/apps/readux/views.py
@@ -407,42 +407,74 @@ def get_queryset(self):
form_data = form.cleaned_data
# default to empty string if no query in form data
- search_query = form_data.get("q", "")
+ search_query = form_data.get("q") or ""
+ scope = form_data.get("scope") or "all"
if search_query:
- multimatch_query = MultiMatch(query=search_query, fields=self.query_search_fields)
- volumes = volumes.query(multimatch_query)
+ queries = []
+ if scope in ["all", "metadata"]:
+ # query for root level fields
+ multimatch_query = Q(
+ "multi_match", query=search_query, fields=self.query_search_fields
+ )
+ queries.append(multimatch_query)
+
+ if scope in ["all", "text"]:
+ # query for nested fields (i.e. canvas position and text)
+ nested_query = Q(
+ "nested",
+ path="canvas_set",
+ query=Q(
+ "multi_match",
+ query=search_query,
+ fields=["canvas_set.result"],
+ ),
+ inner_hits={
+ "name": "canvases",
+ "size": 3, # max number of pages shown in full-text results
+ "highlight": {"fields": {"canvas_set.result": {}}},
+ },
+ # sum scores if in full text only search, so vols with most hits show up first.
+ # if also searching metadata, use avg (default) instead, to not over-inflate.
+ score_mode="sum" if scope == "text" else "avg",
+ )
+ queries.append(nested_query)
+
+ # combine them with bool: { should }
+ q = Q("bool", should=queries)
+ volumes = volumes.query(q)
# highlight
volumes = volumes.highlight_options(
require_field_match=False,
fragment_size=200,
number_of_fragments=10,
+ max_analyzed_offset=999999,
).highlight(
"label", "author", "summary"
)
# filter on authors
- author_filter = form_data.get("author", "")
+ author_filter = form_data.get("author") or ""
if author_filter:
volumes = volumes.filter("terms", authors=author_filter)
# filter on languages
- language_filter = form_data.get("language", "")
+ language_filter = form_data.get("language") or ""
if language_filter:
volumes = volumes.filter("terms", languages=language_filter)
# filter on collections
- collection_filter = form_data.get("collection", "")
+ collection_filter = form_data.get("collection") or ""
if collection_filter:
volumes = volumes.filter("nested", path="collections", query=Q(
"terms", **{"collections.label": collection_filter}
))
# filter on date published
- min_date_filter = form_data.get("start_date", "")
+ min_date_filter = form_data.get("start_date") or ""
if min_date_filter:
volumes = volumes.filter("range", date_earliest={"gte": min_date_filter})
- max_date_filter = form_data.get("end_date", "")
+ max_date_filter = form_data.get("end_date") or ""
if max_date_filter:
volumes = volumes.filter("range", date_latest={"lte": max_date_filter})
diff --git a/apps/static/css/project.css b/apps/static/css/project.css
index cfef32f3b..bda4bf286 100644
--- a/apps/static/css/project.css
+++ b/apps/static/css/project.css
@@ -41,6 +41,20 @@ em {
border: 1px solid transparent;
}
+#search-form .scope-and-keyword {
+ display: flex;
+ flex-flow: row nowrap;
+ margin: 1rem 0 0;
+ gap: 0.5rem;
+}
+
+#search-form .scope-and-keyword .scope {
+ flex: 0 1 auto;
+}
+#search-form .scope-and-keyword .keyword {
+ flex: 1 0 auto;
+}
+
.uk-tab > .uk-active > a {
border-color: var(--link-color) !important;
color: var(--link-color) !important; }
@@ -1829,12 +1843,38 @@ ol#search-results dl {
margin-left: 2rem; }
/* Clamp summary to 3 lines */
-ol#search-results dd.result-volume-summary {
+ol#search-results .result-volume-summary {
display: -webkit-box;
-webkit-box-orient: vertical;
-webkit-line-clamp: 3;
overflow: hidden; }
+/* search result highlighting */
+ol#search-results em {
+ color: #510029 !important;
+ font-weight: bold;
+ font-style: normal; }
+
+/* results within full text */
+ol#search-results .result-page a {
+ display: flex;
+ flex-flow: row nowrap;
+ justify-content: flex-start;
+ align-items: flex-start;
+ font-size: 0.8rem;
+ gap: 1rem;
+ margin: 0.5rem 0;
+}
+
+ol#search-results .result-page .page-number {
+ min-width: 4rem;
+ max-width: 4rem;
+}
+
+ol#search-results .result-page ul li {
+ list-style: disc;
+}
+
.sr-only {
position: absolute;
width: 1px;
diff --git a/apps/templates/search_results.html b/apps/templates/search_results.html
index 091a3f2e7..a08889a76 100644
--- a/apps/templates/search_results.html
+++ b/apps/templates/search_results.html
@@ -48,16 +48,21 @@ Search
accept-charset="utf-8"
>