config: add GND to concepts and places

* Adds GND to concepts and places. * Dependencies updates. * Adds safety ignore for sentry-sdk version 1.45.1. Co-Authored-by: Peter Weber <[email protected]>
rero · Sep 9, 2024 · e8647c4 · e8647c4
1 parent 04968b3
commit e8647c4
Show file tree

Hide file tree

Showing 10 changed files with 253 additions and 243 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/rero_ils/config.py b/rero_ils/config.py
@@ -31,68 +31,55 @@
 
 from celery.schedules import crontab
 from flask import request
-from invenio_circulation.pidstore.pids import (
-    CIRCULATION_LOAN_FETCHER,
-    CIRCULATION_LOAN_MINTER,
-    CIRCULATION_LOAN_PID_TYPE,
-)
-from invenio_circulation.transitions.transitions import (
-    CreatedToPending,
-    ItemAtDeskToItemOnLoan,
-    ItemInTransitHouseToItemReturned,
-    ItemOnLoanToItemInTransitHouse,
-    ItemOnLoanToItemOnLoan,
-    ItemOnLoanToItemReturned,
-    PendingToItemAtDesk,
-    PendingToItemInTransitPickup,
-    ToCancelled,
-    ToItemOnLoan,
-)
+from invenio_circulation.pidstore.pids import CIRCULATION_LOAN_FETCHER, \
+    CIRCULATION_LOAN_MINTER, CIRCULATION_LOAN_PID_TYPE
+from invenio_circulation.transitions.transitions import CreatedToPending, \
+    ItemAtDeskToItemOnLoan, ItemInTransitHouseToItemReturned, \
+    ItemOnLoanToItemInTransitHouse, ItemOnLoanToItemOnLoan, \
+    ItemOnLoanToItemReturned, PendingToItemAtDesk, \
+    PendingToItemInTransitPickup, ToCancelled, ToItemOnLoan
 from invenio_records_rest.facets import range_filter, terms_filter
 from invenio_records_rest.utils import allow_all, deny_all
 
 from rero_ils.modules.acquisition.acq_accounts.api import AcqAccount
-from rero_ils.modules.acquisition.acq_accounts.permissions import (
-    AcqAccountPermissionPolicy,
-)
+from rero_ils.modules.acquisition.acq_accounts.permissions import \
+    AcqAccountPermissionPolicy
 from rero_ils.modules.acquisition.acq_invoices.api import AcquisitionInvoice
-from rero_ils.modules.acquisition.acq_invoices.permissions import (
-    AcqInvoicePermissionPolicy,
-)
+from rero_ils.modules.acquisition.acq_invoices.permissions import \
+    AcqInvoicePermissionPolicy
 from rero_ils.modules.acquisition.acq_order_lines.api import AcqOrderLine
-from rero_ils.modules.acquisition.acq_order_lines.permissions import (
-    AcqOrderLinePermissionPolicy,
-)
+from rero_ils.modules.acquisition.acq_order_lines.permissions import \
+    AcqOrderLinePermissionPolicy
 from rero_ils.modules.acquisition.acq_orders.api import AcqOrder
-from rero_ils.modules.acquisition.acq_orders.permissions import AcqOrderPermissionPolicy
+from rero_ils.modules.acquisition.acq_orders.permissions import \
+    AcqOrderPermissionPolicy
 from rero_ils.modules.acquisition.acq_receipt_lines.api import AcqReceiptLine
-from rero_ils.modules.acquisition.acq_receipt_lines.permissions import (
-    AcqReceiptLinePermissionPolicy,
-)
+from rero_ils.modules.acquisition.acq_receipt_lines.permissions import \
+    AcqReceiptLinePermissionPolicy
 from rero_ils.modules.acquisition.acq_receipts.api import AcqReceipt
-from rero_ils.modules.acquisition.acq_receipts.permissions import (
-    AcqReceiptPermissionPolicy,
-)
+from rero_ils.modules.acquisition.acq_receipts.permissions import \
+    AcqReceiptPermissionPolicy
 from rero_ils.modules.acquisition.budgets.api import Budget
-from rero_ils.modules.acquisition.budgets.permissions import BudgetPermissionPolicy
+from rero_ils.modules.acquisition.budgets.permissions import \
+    BudgetPermissionPolicy
 from rero_ils.modules.entities.local_entities.api import LocalEntity
-from rero_ils.modules.entities.local_entities.permissions import (
-    LocalEntityPermissionPolicy,
-)
+from rero_ils.modules.entities.local_entities.permissions import \
+    LocalEntityPermissionPolicy
 from rero_ils.modules.entities.models import EntityFieldWithRef, EntityType
 from rero_ils.modules.entities.remote_entities.api import RemoteEntity
-from rero_ils.modules.entities.remote_entities.permissions import (
-    RemoteEntityPermissionPolicy,
-)
+from rero_ils.modules.entities.remote_entities.permissions import \
+    RemoteEntityPermissionPolicy
 from rero_ils.modules.loans.api import LoansSearch
 
 from .modules.circ_policies.api import CircPolicy
-from .modules.circ_policies.permissions import CirculationPolicyPermissionPolicy
+from .modules.circ_policies.permissions import \
+    CirculationPolicyPermissionPolicy
 from .modules.collections.api import Collection
 from .modules.collections.permissions import CollectionPermissionPolicy
 from .modules.documents.api import Document
 from .modules.documents.permissions import DocumentPermissionPolicy
-from .modules.documents.query import acquisition_filter, nested_identified_filter
+from .modules.documents.query import acquisition_filter, \
+    nested_identified_filter
 from .modules.holdings.api import Holding
 from .modules.holdings.models import HoldingCirculationAction
 from .modules.holdings.permissions import HoldingsPermissionPolicy
@@ -103,43 +90,38 @@
 from .modules.items.api import Item
 from .modules.items.models import ItemCirculationAction
 from .modules.items.permissions import ItemPermissionPolicy
-from .modules.items.utils import item_location_retriever, same_location_validator
+from .modules.items.utils import item_location_retriever, \
+    same_location_validator
 from .modules.libraries.api import Library
 from .modules.libraries.permissions import LibraryPermissionPolicy
 from .modules.loans.api import Loan
 from .modules.loans.models import LoanState
 from .modules.loans.permissions import LoanPermissionPolicy
 from .modules.loans.query import misc_status_filter
-from .modules.loans.utils import (
-    can_be_requested,
-    get_default_loan_duration,
-    get_extension_params,
-    is_item_available_for_checkout,
-    loan_build_document_ref,
-    loan_build_item_ref,
-    loan_build_patron_ref,
-    validate_item_pickup_transaction_locations,
-    validate_loan_duration,
-)
+from .modules.loans.utils import can_be_requested, get_default_loan_duration, \
+    get_extension_params, is_item_available_for_checkout, \
+    loan_build_document_ref, loan_build_item_ref, loan_build_patron_ref, \
+    validate_item_pickup_transaction_locations, validate_loan_duration
 from .modules.local_fields.api import LocalField
 from .modules.local_fields.permissions import LocalFieldPermissionPolicy
 from .modules.locations.api import Location
 from .modules.locations.permissions import LocationPermissionPolicy
 from .modules.notifications.api import Notification
-from .modules.notifications.dispatcher import Dispatcher as NotificationDispatcher
+from .modules.notifications.dispatcher import \
+    Dispatcher as NotificationDispatcher
 from .modules.notifications.models import NotificationType
 from .modules.notifications.permissions import NotificationPermissionPolicy
 from .modules.operation_logs.api import OperationLog
 from .modules.operation_logs.permissions import OperationLogPermissionPolicy
 from .modules.organisations.api import Organisation
 from .modules.organisations.permissions import OrganisationPermissionPolicy
 from .modules.patron_transaction_events.api import PatronTransactionEvent
-from .modules.patron_transaction_events.permissions import (
-    PatronTransactionEventPermissionPolicy,
-)
+from .modules.patron_transaction_events.permissions import \
+    PatronTransactionEventPermissionPolicy
 from .modules.patron_transaction_events.utils import total_facet_filter_builder
 from .modules.patron_transactions.api import PatronTransaction
-from .modules.patron_transactions.permissions import PatronTransactionPermissionPolicy
+from .modules.patron_transactions.permissions import \
+    PatronTransactionPermissionPolicy
 from .modules.patron_types.api import PatronType
 from .modules.patron_types.permissions import PatronTypePermissionPolicy
 from .modules.patrons.api import Patron
@@ -150,26 +132,19 @@
 from .modules.stats.api.api import Stat
 from .modules.stats.permissions import StatisticsPermissionPolicy
 from .modules.stats_cfg.api import StatConfiguration
-from .modules.stats_cfg.permissions import StatisticsConfigurationPermissionPolicy
+from .modules.stats_cfg.permissions import \
+    StatisticsConfigurationPermissionPolicy
 from .modules.templates.permissions import TemplatePermissionPolicy
-from .modules.users.api import get_profile_countries, get_readonly_profile_fields
+from .modules.users.api import get_profile_countries, \
+    get_readonly_profile_fields
 from .modules.users.models import UserRole
 from .modules.vendors.api import Vendor
 from .modules.vendors.permissions import VendorPermissionPolicy
-from .permissions import (
-    librarian_delete_permission_factory,
-    librarian_permission_factory,
-    librarian_update_permission_factory,
-    wiki_edit_ui_permission,
-    wiki_edit_view_permission,
-)
-from .query import (
-    and_i18n_term_filter,
-    and_term_filter,
-    exclude_terms_filter,
-    i18n_terms_filter,
-    or_terms_filter_by_criteria,
-)
+from .permissions import librarian_delete_permission_factory, \
+    librarian_permission_factory, librarian_update_permission_factory, \
+    wiki_edit_ui_permission, wiki_edit_view_permission
+from .query import and_i18n_term_filter, and_term_filter, \
+    exclude_terms_filter, i18n_terms_filter, or_terms_filter_by_criteria
 from .utils import TranslatedList, get_current_language
 
 APP_THEME = ["bootstrap3"]
@@ -3557,32 +3532,34 @@ def _(x):
 #: page is displayed on RERO-ILS frontpage.
 RERO_ILS_UI_GIT_HASH = None
 
+#: RERO_ILS MEF base url.
+RERO_ILS_MEF_REF_BASE_URL = "mef.rero.ch"
 #: RERO_ILS MEF specific configurations.
 RERO_ILS_MEF_CONFIG = {
     "agents": {
         "base_url": os.environ.get(
-            "RERO_ILS_MEF_AGENTS_URL", "https://mef.rero.ch/api/agents"
+            "RERO_ILS_MEF_AGENTS_URL", f"https:/{RERO_ILS_MEF_REF_BASE_URL}/api/agents"
         ),
         "sources": ["idref", "gnd"],
     },
     "concepts": {
         "base_url": os.environ.get(
-            "RERO_ILS_MEF_CONCEPTS_URL", "https://mef.rero.ch/api/concepts"
+            "RERO_ILS_MEF_CONCEPTS_URL", f"https:/{RERO_ILS_MEF_REF_BASE_URL}/api/concepts"
         ),
-        "sources": ["idref"],
+        "sources": ["idref", "gnd"],
     },
-    "concepts-genreForm": {
+    "concepts_genreForm": {
         "base_url": os.environ.get(
-            "RERO_ILS_MEF_CONCEPTS_URL", "https://mef.rero.ch/api/concepts"
+            "RERO_ILS_MEF_CONCEPTS_URL", f"https:/{RERO_ILS_MEF_REF_BASE_URL}/api/concepts"
         ),
-        "sources": ["idref"],
+        "sources": ["idref", "gnd"],
         "filters": [{"idref.bnf_type": "genre/forme Rameau"}],
     },
     "places": {
         "base_url": os.environ.get(
-            "RERO_ILS_MEF_PLACES_URL", "https://mef.rero.ch/api/places"
+            "RERO_ILS_MEF_PLACES_URL", f"https:/{RERO_ILS_MEF_REF_BASE_URL}/api/places"
         ),
-        "sources": ["idref"],
+        "sources": ["idref", "gnd"],
     },
 }
 RERO_ILS_ENTITY_TYPES = {

diff --git a/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_link-v0.0.1.json b/rero_ils/modules/documents/jsonschemas/documents/document_genre_form_link-v0.0.1.json
@@ -25,7 +25,7 @@
               "options": [
                 {
                   "label": "Genre, form",
-                  "value": "concepts-genreForm"
+                  "value": "concepts_genreForm"
                 }
               ]
             },

diff --git a/rero_ils/modules/entities/local_entities/proxy.py b/rero_ils/modules/entities/local_entities/proxy.py
@@ -27,7 +27,7 @@
     "person": Q("term", type=EntityType.PERSON),
     "organisation": Q("term", type=EntityType.ORGANISATION),
     "concepts": Q("term", type=EntityType.TOPIC),
-    "concepts-genreForm": Q("term", type=EntityType.TOPIC) & Q("term", genreForm=True),
+    "concepts_genreForm": Q("term", type=EntityType.TOPIC) & Q("term", genreForm=True),
 }
 
 

diff --git a/rero_ils/modules/entities/remote_entities/proxy.py b/rero_ils/modules/entities/remote_entities/proxy.py
@@ -74,7 +74,7 @@ def create_proxy(category):
                 "class": MefConceptsProxy,
                 "entities": (EntityType.TEMPORAL,),
             },
-            "concepts-genreForm": {
+            "concepts_genreForm": {
                 "class": MefConceptsGenreFormProxy,
                 "entities": (EntityType.TOPIC,),
             },
@@ -110,10 +110,10 @@ class MEFProxyMixin:
 
     # Headers that should be excluded from remote MEF system response.
     excluded_headers = [
-        "Content-Encoding",
-        "Content-Length",
-        "Transfer-Encoding",
-        "Connection",
+        "CONTENT-Encoding".upper(),
+        "Content-Length".upper(),
+        "Transfer-Encoding".upper(),
+        "Connection".upper(),
     ]
     mef_entrypoint = None  # Must be overridden by subclasses
 
@@ -163,13 +163,12 @@ def search(self, term):
         content = json.loads(response.content)
         for hit in content.get("hits", {}).get("hits", []):
             self._post_process_result_hit(hit)
-
         # Finally, return a flask `Response` from a `request.Response`. All
         # remote server response headers were cloned in the new response except
         # some inconsistent headers.
         flask_response = jsonify(content)
         for header_name, header_value in response.headers.items():
-            if header_name not in self.excluded_headers:
+            if header_name.upper() not in self.excluded_headers:
                 flask_response.headers[header_name] = header_value
         return flask_response
 
@@ -215,10 +214,14 @@ def _post_process_result_hit(self, hit):
         # We would like to add the direct MEF URI for each source of the hit.
         # This URI is the direct access for the source metadata on the remote
         # MEF authority server.
-        # TODO :: this URI should be returned by MEF API
         if not (metadata := hit.get("metadata")):
             return
+        # for the local link we need https://mef.rero.ch
+        ref_base_url = current_app.config.get("RERO_ILS_MEF_REF_BASE_URL", "mef.rero.ch")
         base_url = get_mef_url(self.mef_entrypoint)
+        base_url_split = base_url.split('/')
+        base_url_split[2] = ref_base_url
+        base_url = '/'.join(base_url_split)
         for source_name in self.sources:
             if not (src_data := metadata.get(source_name)):
                 continue
@@ -317,7 +320,7 @@ def _post_process_result_hit(self, hit):
 class MefConceptsGenreFormProxy(MefConceptsProxy):
     """Proxy on RERO-MEF authority system for specific `genreForm` concepts."""
 
-    mef_entrypoint = "concepts-genreForm"
+    mef_entrypoint = "concepts_genreForm"
 
 
 class MefPlacesProxy(MEFProxyMixin):

diff --git a/scripts/setup b/scripts/setup
@@ -357,6 +357,8 @@ eval ${PREFIX} invenio reroils index run --raise-on-error
 #: don't forget to recreat the documents json schema files:
 # SIZE=big # SIZE=small
 # export RERO_ILS_MEF_AGENTS_URL=https://mef.rero.ch/api/agents
+# export RERO_ILS_MEF_CONCEPTS_URL=https://mef.rero.ch/api/concepts
+# export RERO_ILS_MEF_PLACES_URL=https://mef.rero.ch/api/places
 # invenio reroils utils marc21tojson -t rero ${DATA_PATH}/documents_${SIZE}.xml ${DATA_PATH}/documents_${SIZE}.json ${DATA_PATH}/documents_${SIZE}_errors.xml -v -r
 # Save the entities after setup for later use.
 # invenio reroils utils export -t rement -o ${DATA_PATH}/remote_entities_${SIZE}.json -v

diff --git a/scripts/test b/scripts/test
@@ -80,11 +80,17 @@ function pretests () {
   # -> Vulnerability found in werkzeug version 2.2.3
   #    Vulnerability ID: 62019
   # -> Vulnerability found in werkzeug version 2.2.3
-  #   Vulnerability ID: 71595
+  #    Vulnerability ID: 71595
+  # -> Vulnerability found in werkzeug version 2.2.3
+  #    Vulnerability ID: 71594
+  # -> Vulnerability found in jinja2 version 3.1.4
+  #    Vulnerability ID: 70612
+  # -> Vulnerability found in py version 1.11.0
+  #    Vulnerability ID: 51457
   # -> Vulnerability found in sentry-sdk version 1.45.1
-  #   Vulnerability ID: 72260
+  #    Vulnerability ID: 72260
   info_msg "Check vulnerabilities:"
-  safety_exceptions="-i 40459 -i 70624 -i 51668 -i 42194 -i 42852 -i 71594 -i 62019 -i 71595 -i 70612 -i 51457 -i 72260"
+  safety_exceptions="-i 40459 -i 70624 -i 51668 -i 42194 -i 42852 -i 62019 -i 71595 -i 71594 -i 70612 -i 51457 -i 72260"
   msg=$(safety check -o text ${safety_exceptions}) || {
     echo "Safety vulnerabilites found for packages:" $(safety check -o bare ${safety_exceptions})
     echo "Run: \"safety check -o screen ${safety_exceptions} | grep -i vulnerability\" for more details"

diff --git a/tests/api/entities/local_entities/test_local_entities_rest.py b/tests/api/entities/local_entities/test_local_entities_rest.py
@@ -147,7 +147,7 @@ def test_local_search_by_proxy(client, local_entity_genre_form, local_entity_org
     response = client.get(
         url_for(
             "api_local_entities.local_search_proxy",
-            entity_type="concepts-genreForm",
+            entity_type="concepts_genreForm",
             term="personal",
             size="dummy_qs_arg",
         )
@@ -159,7 +159,7 @@ def test_local_search_by_proxy(client, local_entity_genre_form, local_entity_org
     response = client.get(
         url_for(
             "api_local_entities.local_search_proxy",
-            entity_type="concepts-genreForm",
+            entity_type="concepts_genreForm",
             term="personal",
             size="0",
         )
@@ -170,7 +170,7 @@ def test_local_search_by_proxy(client, local_entity_genre_form, local_entity_org
     response = client.get(
         url_for(
             "api_local_entities.local_search_proxy",
-            entity_type="concepts-genreForm",
+            entity_type="concepts_genreForm",
             term="dummy_key",
         )
     )

diff --git a/tests/api/entities/remote_entities/test_remote_entities_rest.py b/tests/api/entities/remote_entities/test_remote_entities_rest.py
@@ -95,7 +95,7 @@ def test_remote_search_proxy(
     response = client.get(
         url_for(
             "api_remote_entities.remote_search_proxy",
-            entity_type="concepts-genreForm",
+            entity_type="concepts_genreForm",
             term="side-car",
         )
     )