VIAF: add sources

* Adds WorldCat, Wikipedia and sourc URL's Co-Authored-by: Peter Weber <[email protected]>
rero · Feb 25, 2023 · 79711d2 · 79711d2
1 parent aa0dbdf
commit 79711d2
Show file tree

Hide file tree

Showing 23 changed files with 38,401 additions and 3,535 deletions.
diff --git a/data/aggnd.json b/data/aggnd.json
diff --git a/data/aggnd_metadata.csv b/data/aggnd_metadata.csv
diff --git a/data/aggnd_pidstore.csv b/data/aggnd_pidstore.csv
diff --git a/data/agrero.json b/data/agrero.json
diff --git a/data/aidref.json b/data/aidref.json
diff --git a/data/aidref_metadata.csv b/data/aidref_metadata.csv
diff --git a/data/aidref_pidstore.csv b/data/aidref_pidstore.csv
diff --git a/data/mef.json b/data/mef.json
diff --git a/data/mef_id.csv b/data/mef_id.csv
diff --git a/data/mef_metadata.csv b/data/mef_metadata.csv
diff --git a/data/mef_pidstore.csv b/data/mef_pidstore.csv
diff --git a/data/viaf.json b/data/viaf.json
diff --git a/data/viaf_metadata.csv b/data/viaf_metadata.csv
diff --git a/data/viaf_pidstore.csv b/data/viaf_pidstore.csv
diff --git a/rero_mef/agents/cli.py b/rero_mef/agents/cli.py
@@ -40,8 +40,7 @@ def agents():
 @agents.command()
 @click.option('-k', '--enqueue', 'enqueue', is_flag=True, default=False,
               help="Enqueue record creation.")
-@click.option('-o', '--online', 'online', multiple=True,
-              default=['aidref', 'aggnd', 'agrero'])
+@click.option('-o', '--online', 'online', multiple=True, default=[])
 @click.option('-v', '--verbose', 'verbose', is_flag=True, default=False)
 @click.option('-V', '--online_verbose', 'online_verbose', is_flag=True,
               default=False)
@@ -155,7 +154,7 @@ def create_csv_viaf(viaf_file, output_directory, verbose):
         err=True)
 
     count = create_viaf_files(
-        viaf_input_file=viaf_file,
+        viaf_input_file_name=viaf_file,
         viaf_pidstore_file_name=pidstore,
         viaf_metadata_file_name=metadata,
         verbose=verbose

diff --git a/rero_mef/agents/utils.py b/rero_mef/agents/utils.py
@@ -203,7 +203,7 @@ def create_viaf_files(
             ) as viaf_pidstore,
             open(
                 viaf_metadata_file_name, 'w', encoding='utf-8'
-                ) as viaf_metadata,\
+                ) as viaf_metadata,
             open(
                 viaf_input_file_name, 'r', encoding='utf-8'
             ) as viaf_in_file):
@@ -233,7 +233,19 @@ def create_viaf_files(
                 previous_viaf_pid = viaf_pid
             corresponding = fields[1].split('|')
             if len(corresponding) == 2:
-                corresponding_data[corresponding[0]] = corresponding[1]
+                corresponding_data.setdefault(
+                    corresponding[0], {'pid': corresponding[1]})
+            corresponding = fields[1].split('@')
+            if len(corresponding) == 2:
+                if corresponding[0] == 'Wikipedia':
+                    # multiple wikipedia
+                    corresponding_data.setdefault(corresponding[0], {})
+                    corresponding_data[corresponding[0]].setdefault('url', [])
+                    corresponding_data[corresponding[0]]['url'].append(
+                        corresponding[1])
+                else:
+                    corresponding_data.setdefault(
+                        corresponding[0], {'url': corresponding[1]})
         # save the last record
         agent_pid += 1
         if write_link_json(

diff --git a/rero_mef/agents/viaf/api.py b/rero_mef/agents/viaf/api.py
@@ -30,6 +30,7 @@
 from ..api import Action, ReroIndexer, ReroMefRecord
 from ..mef.api import AgentMefRecord
 from ..utils import get_entity_class
+from ...filter import exists_filter
 from ...utils import get_entity_class, progressbar
 
 
@@ -56,6 +57,83 @@ class AgentViafRecord(ReroMefRecord):
     name = 'viaf'
     model_cls = ViafMetadata
     search = AgentViafSearch
+    # https://viaf.org/
+    sources = {
+        'DNB': 'gnd',  # German National Library
+        'SUDOC': 'idref',  # Sudoc [ABES], France
+        'RERO': 'rero',  # RERO - Library Network of Western Switzerland
+        'SZ': 'sz',  # Swiss National Library
+        'BNE': 'bne',  # National Library of Spain
+        'BNF': 'bnf',  # National Library of France
+        'ICCU': 'iccu',  # Central Institute for the Union Catalogue of the Italian libraries  # noqa
+        'ISNI': 'isni',  # ISNI
+        'WKP': 'wiki'  # Wikidata
+        # 'LC': 'loc',  # Library of Congress
+        # 'SELIBR': 'selibr',  # National Library of Sweden
+        # 'NLA': 'nla',  # National Library of Australia
+        # 'PTBNP': 'ptbnp',  # National Library of Portugal
+        # 'BLBNB': 'BLBNB',  # National Library of Brazil
+        # 'NKC': 'nkc',  # National Library of the Czech Republic
+        # 'J9U': 'j9u',  # National Library of Israel
+        # 'EGAXA': 'egaxa',  # Library of Alexandria, Egypt
+        # 'BAV': 'bav',  # Vatican Library
+        # 'CAOONL': 'caoonl',  # Library and Archives Canada/PFAN
+        # 'JPG': 'jpg',  # Union List of Artist Names [Getty Research Institute]  # noqa
+        # 'NUKAT': 'nukat',  # NUKAT Center of Warsaw University Library
+        # 'NSZL': 'NSZL',  # National Széchényi Library, Hungary
+        # 'VLACC': 'vlacc',  # Flemish Public Libraries National Library of Russia  # noqa
+        # 'NTA': 'nta',  # National Library of Netherlands
+        # 'BIBSYS': 'bibsys',  # BIBSYS
+        # 'GRATEVE': 'grateve',  # National Library of Greece
+        # 'ARBABN': 'arbabn',  # National Library of Argentina
+        # 'W2Z': 'w2z',  # National Library of Norway
+        # 'DBC': 'dbc',  # DBC (Danish Bibliographic Center)
+        # 'NDL': 'ndl',  # National Diet Library, Japan
+        # 'NII': 'nii',  # NII (Japan)
+        # 'NLB': 'nlb',  # National Library Board, Singapore
+        # 'LNB': 'lnb',  # National Library of Latvia
+        # 'PLWABN': 'plwabn',  # National Library of Poland
+        # 'BNC': 'BNC',  # National Library of Catalonia
+        # 'LNL': 'lnl',  # Lebanese National Library
+        # 'PERSEUS': 'perseus',  # Perseus Digital Library
+        # 'SRP': 'srp',  # Syriac Reference Portal
+        # 'N6I': 'n6i',  # National Library of Ireland
+        # 'NSK': 'nsk',  # National and University Library in Zagreb
+        # 'CYT': 'cyt',  # National Central Library, Taiwan
+        # 'B2Q': 'b2q',  # National Library and Archives of Québec
+        # 'KRLNK': 'krlnk',  # National Library of Korea
+        # 'BNL': 'BNL',  # National Library of Luxembourg
+        # 'BNCHL': 'bnchl',  # National Library of Chile
+        # 'MRBNR': 'mrbnr',  # National Library of Morocco
+        # 'XA': 'xa',  # xA Extended Authorities
+        # 'XR': 'xr',  # xR Extended Relationships
+        # 'FAST': 'fast',  # FAST Subjects
+        # 'ERRR': 'errr',  # National Library of Estonia
+        # 'UIY': 'uiy',  # National and University Library of Iceland (NULI)
+        # 'NYNYRILM': 'nynyrilm',  # Repertoire International de Litterature Musicale, Inc. (RILM)  # noqa
+        # 'DE663': 'de663',  # International Inventory of Musical Sources (RISM)  # noqa
+        # 'SIMACOB': 'simacob',  # NUK/COBISS.SI, Slovenia
+        # 'LIH': 'lih',  # National Library of Lithuania
+        # 'SKMASNL': 'skmasnl',  # Slovak National Library
+        # 'UAE': 'uae',  # United Arab Emirates University
+    }
+    sources_used = ('DNB', 'SUDOC', 'RERO')
+
+    @classmethod
+    def filters(cls):
+        """Filters for sources."""
+        return {
+            source: exists_filter(f'{source}_pid')
+            for source in cls.sources.values()
+        }
+
+    @classmethod
+    def aggregations(cls):
+        """Aggregations for sources."""
+        return {
+            source: dict(filter=dict(exists=dict(field=f'{source}_pid')))
+            for source in cls.sources.values()
+        }
 
     def create_mef_and_agents(self, dbcommit=False, reindex=False,
                               online=None, verbose=False,
@@ -195,11 +273,6 @@ def get_online_record(cls, viaf_source_code, pid, format=None):
                        link = get the VIAF link record
         :returns: VIAF record as json
         """
-        source_code = {
-            'DNB': 'gnd_pid',
-            'SUDOC': 'idref_pid',
-            'RERO': 'rero_pid'
-        }
         viaf_format = '/viaf.json'
         if format == 'link':
             viaf_format = '/justlinks.json'
@@ -217,19 +290,34 @@ def get_online_record(cls, viaf_source_code, pid, format=None):
             if isinstance(sources, dict):
                 sources = [sources]
             for source in sources:
-                text = source.get('#text', '|').split('|')
-                if text[0] in source_code:
-                    result[source_code[text[0]]] = text[1]
+                # get pid
+                text = source.get('#text', '|')
+                text = text.split('|')
+                if bib_source := cls.sources.get(text[0]):
+                    result[f'{bib_source}_pid'] = text[1]
+                    # get URL
+                    if nsid := source.get('@nsid'):
+                        if nsid.startswith('http'):
+                            result[bib_source] = nsid
+            # get Wikipedia and WorldCat URLs
+            if result.get('wiki_pid'):
+                x_links = data_json.get('xLinks', {}).get('xLink', [])
+                for x_link in x_links:
+                    if 'worldcat' in x_link:
+                        result['worldcat'] = x_link
+                    elif isinstance(x_link, dict):
+                        text = x_link.get('#text')
+                        if text and 'wikipedia' in text:
+                            result.setdefault('wiki', []).append(text)
         # make sure we got a VIAF with the same pid for source
-        if result.get(source_code.get(viaf_source_code)) == pid:
+        if result.get(f'{cls.sources.get(viaf_source_code)}_pid') == pid:
             return result
 
     @classmethod
     def get_viaf(cls, agent):
         """Get VIAF record by agent.
 
         :param agent: Agency do get corresponding VIAF record.
-        :param online: Try to get VIAF record online if not exist.
         """
         if isinstance(agent, AgentMefRecord):
             return [cls.get_record_by_pid(agent.get('viaf_pid'))]
@@ -349,14 +437,13 @@ def get_missing_agent_pids(cls, agent, verbose=False):
     def get_pids_with_multiple_viaf(cls, verbose=False):
         """Get agent pids with multiple MEF records.
 
-        :params record_types: Record types (pid_types).
         :param verbose: Verbose.
-        :returns: pids, multiple pids, missing pids.
+        :returns: pids.
         """
         multiple_pids = {
-            'gnd_pid': {},
-            'idref_pid': {},
-            'rero_pid': {}
+            'DNB': 'gnd_pid',
+            'SUDOC': 'idref_pid',
+            'RERO': 'rero_pid'
         }
         progress = progressbar(
             items=AgentViafSearch()

diff --git a/rero_mef/agents/viaf/jsonschemas/viaf/viaf-v0.0.1.json b/rero_mef/agents/viaf/jsonschemas/viaf/viaf-v0.0.1.json
@@ -1,6 +1,6 @@
 {
   "$schema": "http://json-schema.org/draft-04/schema#",
-  "title": "Schema for VIAF authority",
+  "title": "Schema for VIAF contributor",
   "type": "object",
   "required": [
     "$schema",
@@ -10,7 +10,7 @@
   "properties": {
     "$schema": {
       "title": "Schema",
-      "description": "Schema to VIAF authority record.",
+      "description": "Schema to VIAF contributor record.",
       "type": "string",
       "minLength": 7
     },
@@ -20,20 +20,125 @@
       "minLength": 1
     },
     "gnd_pid": {
-      "title": "GND authority ID",
+      "title": "GND contributor ID",
       "type": "string",
       "minLength": 1
     },
+    "gnd": {
+      "title": "GND contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
+    "idref_pid": {
+      "title": "IDREF contributor ID",
+      "type": "string",
+      "minLength": 1
+    },
+    "IDREF": {
+      "title": "IDREF contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
     "rero_pid": {
-      "title": "RERO authority ID",
+      "title": "RERO contributor ID",
       "type": "string",
       "minLength": 1
     },
-    "idref_pid": {
-      "title": "IDREF authority ID",
+    "rero": {
+      "title": "RERO contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
+    "sz_pid": {
+      "title": "SZ contributor ID",
+      "type": "string",
+      "minLength": 1
+    },
+    "sz": {
+      "title": "SZ contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
+    "bne_pid": {
+      "title": "BNE contributor ID",
+      "type": "string",
+      "minLength": 1
+    },
+    "bne": {
+      "title": "BNE contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
+    "bnf_pid": {
+      "title": "BNF contributor ID",
+      "type": "string",
+      "minLength": 1
+    },
+    "bnf": {
+      "title": "BNF contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
+    "iccu_pid": {
+      "title": "ICCU contributor ID",
+      "type": "string",
+      "minLength": 1
+    },
+    "iccu": {
+      "title": "ICCU contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
+    "isni_pid": {
+      "title": "ISNI contributor ID",
+      "type": "string",
+      "minLength": 1
+    },
+    "isni": {
+      "title": "ISNI contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
+    "wiki_pid": {
+      "title": "Wikipedia contributor ID",
       "type": "string",
       "minLength": 1
     },
+    "wiki": {
+      "title": "Wikipedia links",
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "title": "Wikipedia link",
+        "type": "string",
+        "format": "uri",
+        "pattern": "^(https?)://.*$",
+        "minLength": 7
+      }
+    },
+    "worldcat": {
+      "title": "WorldCat contributor URL",
+      "type": "string",
+      "format": "uri",
+      "pattern": "^(https?)://.*$",
+      "minLength": 7
+    },
     "md5": {
       "title": "MD5",
       "type": "string",