config: add GND to concepts and places

* Adds GND to concepts and places.
* Dependencies updates.
* Adds safety ignore for sentry-sdk version 1.45.1.

Co-Authored-by: Peter Weber <[email protected]>

rero_ils/config.py

@@ -3557,34 +3557,29 @@ def _(x):
 #: page is displayed on RERO-ILS frontpage.
 RERO_ILS_UI_GIT_HASH = None
 
+#: RERO_ILS MEF base url could be changed.
+RERO_ILS_MEF_REF_BASE_URL = os.environ.get("RERO_ILS_MEF_REF_BASE_URL", "mef.rero.ch")
 #: RERO_ILS MEF specific configurations.
 RERO_ILS_MEF_CONFIG = {
     "agents": {
-        "base_url": os.environ.get(
-            "RERO_ILS_MEF_AGENTS_URL", "https://mef.rero.ch/api/agents"
-        ),
+        "base_url": f"https://{RERO_ILS_MEF_REF_BASE_URL}/api/agents",
         "sources": ["idref", "gnd"],
     },
     "concepts": {
-        "base_url": os.environ.get(
-            "RERO_ILS_MEF_CONCEPTS_URL", "https://mef.rero.ch/api/concepts"
-        ),
-        "sources": ["idref"],
+        "base_url": f"https://{RERO_ILS_MEF_REF_BASE_URL}/api/concepts",
+        "sources": ["idref", "gnd"],
     },
-    "concepts-genreForm": {
-        "base_url": os.environ.get(
-            "RERO_ILS_MEF_CONCEPTS_URL", "https://mef.rero.ch/api/concepts"
-        ),
-        "sources": ["idref"],
+    "concepts_genreForm": {
+        "base_url": f"https://{RERO_ILS_MEF_REF_BASE_URL}/api/concepts",
+        "sources": ["idref", "gnd"],
         "filters": [{"idref.bnf_type": "genre/forme Rameau"}],
     },
     "places": {
-        "base_url": os.environ.get(
-            "RERO_ILS_MEF_PLACES_URL", "https://mef.rero.ch/api/places"
-        ),
-        "sources": ["idref"],
+        "base_url": f"https://{RERO_ILS_MEF_REF_BASE_URL}/api/places",
+        "sources": ["idref", "gnd"],
     },
 }
+
 RERO_ILS_ENTITY_TYPES = {
     "bf:Person": "agents",
     "bf:Organisation": "agents",

rero_ils/dojson/utils.py

@@ -468,16 +468,16 @@ def get_mef_link(bibid, reroid, entity_type, ids, key):
     from rero_ils.modules.utils import requests_retry_session
 
     # In dojson we dont have app. mef_url should be the same as
-    # RERO_ILS_MEF_AGENTS_URL in config.py
-    # https://mef.test.rero.ch/api/agents/mef/?q=rero.rero_pid:A012327677
+    # RERO_ILS_MEF_APP_BASE_URL in config.py
+    # https://mef.rero.ch/api/agents/mef/?q=rero.rero_pid:A012327677
     if not ids:
         return
     try:
+        # Try to get RERO_ILS_ENTITY_TYPES and RERO_ILS_MEF_CONFIG from current app
+        # In the dojson cli is no current app and we have to get the value directly
+        # from config.py
         entity_types = current_app.config.get("RERO_ILS_ENTITY_TYPES", {})
         mef_config = current_app.config.get("RERO_ILS_MEF_CONFIG")
-    # Try to get RERO_ILS_ENTITY_TYPES and RERO_ILS_MEF_CONFIG from current app
-    # In the dojson cli is no current app and we have to get the value directly
-    # from config.py
     except Exception:
         from rero_ils.config import RERO_ILS_ENTITY_TYPES as entity_types
         from rero_ils.config import RERO_ILS_MEF_CONFIG as mef_config

rero_ils/modules/documents/jsonschemas/documents/document_genre_form_link-v0.0.1.json

@@ -25,7 +25,7 @@
               "options": [
                 {
                   "label": "Genre, form",
-                  "value": "concepts-genreForm"
+                  "value": "concepts_genreForm"
                 }
               ]
             },

rero_ils/modules/entities/local_entities/proxy.py

@@ -27,7 +27,7 @@
     "person": Q("term", type=EntityType.PERSON),
     "organisation": Q("term", type=EntityType.ORGANISATION),
     "concepts": Q("term", type=EntityType.TOPIC),
-    "concepts-genreForm": Q("term", type=EntityType.TOPIC) & Q("term", genreForm=True),
+    "concepts_genreForm": Q("term", type=EntityType.TOPIC) & Q("term", genreForm=True),
 }
 
 

rero_ils/modules/entities/remote_entities/proxy.py

@@ -74,7 +74,7 @@ def create_proxy(category):
                 "class": MefConceptsProxy,
                 "entities": (EntityType.TEMPORAL,),
             },
-            "concepts-genreForm": {
+            "concepts_genreForm": {
                 "class": MefConceptsGenreFormProxy,
                 "entities": (EntityType.TOPIC,),
             },
@@ -110,26 +110,19 @@ class MEFProxyMixin:
 
     # Headers that should be excluded from remote MEF system response.
     excluded_headers = [
-        "Content-Encoding",
-        "Content-Length",
-        "Transfer-Encoding",
-        "Connection",
+        "CONTENT-Encoding".upper(),
+        "Content-Length".upper(),
+        "Transfer-Encoding".upper(),
+        "Connection".upper(),
     ]
     mef_entrypoint = None  # Must be overridden by subclasses
 
     def __init__(self, *args):
         """Magic initialization method."""
         self.entity_types = args
-        self.sources = (
-            current_app.config.get("RERO_ILS_MEF_CONFIG", {})
-            .get(self.mef_entrypoint, {})
-            .get("sources", [])
-        )
-        self.filters = (
-            current_app.config.get("RERO_ILS_MEF_CONFIG", {})
-            .get(self.mef_entrypoint, {})
-            .get("filters", [])
-        )
+        mef_config = current_app.config.get("RERO_ILS_MEF_CONFIG", {})
+        self.sources = mef_config.get(self.mef_entrypoint, {}).get("sources", [])
+        self.filters = mef_config.get(self.mef_entrypoint, {}).get("filters", [])
 
     def search(self, term):
         """Search specific term on MEF authority system.
@@ -163,13 +156,12 @@ def search(self, term):
         content = json.loads(response.content)
         for hit in content.get("hits", {}).get("hits", []):
             self._post_process_result_hit(hit)
-
         # Finally, return a flask `Response` from a `request.Response`. All
         # remote server response headers were cloned in the new response except
         # some inconsistent headers.
         flask_response = jsonify(content)
         for header_name, header_value in response.headers.items():
-            if header_name not in self.excluded_headers:
+            if header_name.upper() not in self.excluded_headers:
                 flask_response.headers[header_name] = header_value
         return flask_response
 
@@ -215,10 +207,12 @@ def _post_process_result_hit(self, hit):
         # We would like to add the direct MEF URI for each source of the hit.
         # This URI is the direct access for the source metadata on the remote
         # MEF authority server.
-        # TODO :: this URI should be returned by MEF API
         if not (metadata := hit.get("metadata")):
             return
-        base_url = get_mef_url(self.mef_entrypoint)
+        mef_ref_base_url = current_app.config.get("RERO_ILS_MEF_REF_BASE_URL")
+        base_url = get_mef_url(self.mef_entrypoint).replace(
+            mef_ref_base_url, "mef.rero.ch"
+        )
         for source_name in self.sources:
             if not (src_data := metadata.get(source_name)):
                 continue
@@ -317,7 +311,7 @@ def _post_process_result_hit(self, hit):
 class MefConceptsGenreFormProxy(MefConceptsProxy):
     """Proxy on RERO-MEF authority system for specific `genreForm` concepts."""
 
-    mef_entrypoint = "concepts-genreForm"
+    mef_entrypoint = "concepts_genreForm"
 
 
 class MefPlacesProxy(MEFProxyMixin):

rero_ils/modules/ext.py

@@ -304,6 +304,13 @@ def init_config(self, app):
             from werkzeug.serving import WSGIRequestHandler
 
             WSGIRequestHandler.protocol_version = "HTTP/1.1"
+        # Correct MEF url in RERO_ILS_MEF_CONFIG from config.py
+        mef_ref_base_url = app.config.get("RERO_ILS_MEF_REF_BASE_URL")
+        for mef_entity_type in app.config.get("RERO_ILS_MEF_CONFIG", {}):
+            base_url = app.config["RERO_ILS_MEF_CONFIG"][mef_entity_type][
+                "base_url"
+            ].replace("mef.rero.ch", mef_ref_base_url)
+            app.config["RERO_ILS_MEF_CONFIG"][mef_entity_type]["base_url"] = base_url
 
     def register_signals(self, app):
         """Register signals."""

rero_ils/modules/views.py

@@ -142,6 +142,7 @@ def proxy():
     """Proxy to get record metadata from MEF server."""
     if not (url := request.args.get("url")):
         abort(400, "Missing `url` parameter")
+    url = url.replace("mef.rero.ch", current_app.config["RERO_ILS_MEF_REF_BASE_URL"])
     response = requests.get(url)
     return make_response(response.content, response.status_code)
 

scripts/setup

@@ -356,7 +356,6 @@ eval ${PREFIX} invenio reroils index run --raise-on-error
 #: - xml to json transformation for rero marcxml --------------------------------
 #: don't forget to recreat the documents json schema files:
 # SIZE=big # SIZE=small
-# export RERO_ILS_MEF_AGENTS_URL=https://mef.rero.ch/api/agents
 # invenio reroils utils marc21tojson -t rero ${DATA_PATH}/documents_${SIZE}.xml ${DATA_PATH}/documents_${SIZE}.json ${DATA_PATH}/documents_${SIZE}_errors.xml -v -r
 # Save the entities after setup for later use.
 # invenio reroils utils export -t rement -o ${DATA_PATH}/remote_entities_${SIZE}.json -v

scripts/test

@@ -80,11 +80,17 @@ function pretests () {
   # -> Vulnerability found in werkzeug version 2.2.3
   #    Vulnerability ID: 62019
   # -> Vulnerability found in werkzeug version 2.2.3
-  #   Vulnerability ID: 71595
+  #    Vulnerability ID: 71595
+  # -> Vulnerability found in werkzeug version 2.2.3
+  #    Vulnerability ID: 71594
+  # -> Vulnerability found in jinja2 version 3.1.4
+  #    Vulnerability ID: 70612
+  # -> Vulnerability found in py version 1.11.0
+  #    Vulnerability ID: 51457
   # -> Vulnerability found in sentry-sdk version 1.45.1
-  #   Vulnerability ID: 72260
+  #    Vulnerability ID: 72260
   info_msg "Check vulnerabilities:"
-  safety_exceptions="-i 40459 -i 70624 -i 51668 -i 42194 -i 42852 -i 71594 -i 62019 -i 71595 -i 70612 -i 51457 -i 72260"
+  safety_exceptions="-i 40459 -i 70624 -i 51668 -i 42194 -i 42852 -i 62019 -i 71595 -i 71594 -i 70612 -i 51457 -i 72260"
   msg=$(safety check -o text ${safety_exceptions}) || {
     echo "Safety vulnerabilites found for packages:" $(safety check -o bare ${safety_exceptions})
     echo "Run: \"safety check -o screen ${safety_exceptions} | grep -i vulnerability\" for more details"

tests/api/entities/local_entities/test_local_entities_rest.py

@@ -147,7 +147,7 @@ def test_local_search_by_proxy(client, local_entity_genre_form, local_entity_org
     response = client.get(
         url_for(
             "api_local_entities.local_search_proxy",
-            entity_type="concepts-genreForm",
+            entity_type="concepts_genreForm",
             term="personal",
             size="dummy_qs_arg",
         )
@@ -159,7 +159,7 @@ def test_local_search_by_proxy(client, local_entity_genre_form, local_entity_org
     response = client.get(
         url_for(
             "api_local_entities.local_search_proxy",
-            entity_type="concepts-genreForm",
+            entity_type="concepts_genreForm",
             term="personal",
             size="0",
         )
@@ -170,7 +170,7 @@ def test_local_search_by_proxy(client, local_entity_genre_form, local_entity_org
     response = client.get(
         url_for(
             "api_local_entities.local_search_proxy",
-            entity_type="concepts-genreForm",
+            entity_type="concepts_genreForm",
             term="dummy_key",
         )
     )

tests/api/entities/remote_entities/test_remote_entities_rest.py

@@ -95,7 +95,7 @@ def test_remote_search_proxy(
     response = client.get(
         url_for(
             "api_remote_entities.remote_search_proxy",
-            entity_type="concepts-genreForm",
+            entity_type="concepts_genreForm",
             term="side-car",
         )
     )

tests/conftest.py

@@ -204,14 +204,17 @@ def app_config(app_config):
         },
         "concepts": {
             "base_url": "https://mef.rero.ch/api/concepts",
-            "sources": ["idref"],
+            "sources": ["idref", "gnd"],
         },
-        "concepts-genreForm": {
+        "concepts_genreForm": {
             "base_url": "https://mef.rero.ch/api/concepts",
-            "sources": ["idref"],
+            "sources": ["idref", "gnd"],
             "filters": [{"idref.bnf_type": "sujet Rameau"}],
         },
-        "places": {"base_url": "https://mef.rero.ch/api/places", "sources": ["idref"]},
+        "places": {
+            "base_url": "https://mef.rero.ch/api/places",
+            "sources": ["idref", "gnd"],
+        },
     }
     return app_config
 
@@ -267,6 +270,12 @@ def mef_concepts_url(app):
     return app.config.get("RERO_ILS_MEF_CONFIG", {}).get("concepts", {}).get("base_url")
 
 
+@pytest.fixture(scope="module")
+def mef_places_url(app):
+    """Get MEF agent URL from config."""
+    return app.config.get("RERO_ILS_MEF_CONFIG", {}).get("places", {}).get("base_url")
+
+
 @pytest.fixture(scope="module")
 def bnf_ean_any_123():
     """Load bnf ean any 123 xml file."""

tests/unit/test_cql_parser.py

@@ -173,7 +173,6 @@ def test_get_query_clause_with_sorting(app):
     assert query.term.value == '"cat"'
     with pytest.raises(Diagnostic) as err:
         query.to_es()
-    print("--------", err.value)
     assert str(err.value) == "info:srw/diagnostic/1/80 [Sort not supported]: "