agent: remove bf:Agent

Co-Authored-by: Peter Weber <[email protected]>
rero · Feb 12, 2024 · 2650e5c · 2650e5c
1 parent 42c43c2
commit 2650e5c
Show file tree

Hide file tree

Showing 27 changed files with 8,179 additions and 16,085 deletions.
diff --git a/data/aggnd.json b/data/aggnd.json
diff --git a/data/aggnd_metadata.csv b/data/aggnd_metadata.csv
diff --git a/data/agrero.json b/data/agrero.json
diff --git a/data/agrero_metadata.csv b/data/agrero_metadata.csv
diff --git a/data/aidref.json b/data/aidref.json
diff --git a/data/aidref_metadata.csv b/data/aidref_metadata.csv
diff --git a/poetry.lock b/poetry.lock
diff --git a/rero_mef/agents/gnd/mappings/v7/agents_gnd/gnd-agent-v0.0.1.json b/rero_mef/agents/gnd/mappings/v7/agents_gnd/gnd-agent-v0.0.1.json
@@ -14,9 +14,6 @@
       "pid": {
         "type": "keyword"
       },
-      "bf:Agent": {
-        "type": "keyword"
-      },
       "type": {
         "type": "keyword"
       },

diff --git a/rero_mef/agents/idref/mappings/v7/agents_idref/idref-agent-v0.0.1.json b/rero_mef/agents/idref/mappings/v7/agents_idref/idref-agent-v0.0.1.json
@@ -14,9 +14,6 @@
       "pid": {
         "type": "keyword"
       },
-      "bf:Agent": {
-        "type": "keyword"
-      },
       "type": {
         "type": "keyword"
       },

diff --git a/rero_mef/agents/mef/api.py b/rero_mef/agents/mef/api.py
@@ -192,12 +192,6 @@ def get_latest(cls, pid_type, pid):
                 if search.count() > 0:
                     new_data = next(search.scan()).to_dict()
                     new_pid = new_data.get('idref', {}).get('pid')
-            for agent in cls.entities:
-                if agent_data := data.get(agent):
-                    if not agent_data.get('type'):
-                        data[agent]['type'] = agent_data['bf:Agent']
-                        if not data.get('type'):
-                            data['type'] = agent_data['bf:Agent']
             return cls.get_latest(pid_type=pid_type, pid=new_pid) \
                 if new_pid else data
         return {}

diff --git a/rero_mef/agents/mef/mappings/v7/mef/mef-v0.0.1.json b/rero_mef/agents/mef/mappings/v7/mef/mef-v0.0.1.json
@@ -56,9 +56,6 @@
           "pid": {
             "type": "keyword"
           },
-          "bf:Agent": {
-            "type": "keyword"
-          },
           "type": {
             "type": "keyword"
           },
@@ -164,9 +161,6 @@
           "pid": {
             "type": "keyword"
           },
-          "bf:Agent": {
-            "type": "keyword"
-          },
           "type": {
             "type": "keyword"
           },
@@ -272,9 +266,6 @@
           "pid": {
             "type": "keyword"
           },
-          "bf:Agent": {
-            "type": "keyword"
-          },
           "type": {
             "type": "keyword"
           },

diff --git a/rero_mef/agents/rero/mappings/v7/agents_rero/rero-agent-v0.0.1.json b/rero_mef/agents/rero/mappings/v7/agents_rero/rero-agent-v0.0.1.json
@@ -14,9 +14,6 @@
       "pid": {
         "type": "keyword"
       },
-      "bf:Agent": {
-        "type": "keyword"
-      },
       "type": {
         "type": "keyword"
       },

diff --git a/rero_mef/agents/serializers.py b/rero_mef/agents/serializers.py
@@ -65,10 +65,6 @@ def serialize(self, pid, record, links_factory=None, **kwargs):
         :param record: Record instance.
         :param links_factory: Factory function for record links.
         """
-        # TODO: could be deleted in the future
-        if not record.get('type'):
-            record['type'] = record['bf:Agent']
-
         return super(ReroMefSerializer, self).serialize(
             pid=pid, record=record, links_factory=add_links, **kwargs)
 

diff --git a/rero_mef/alembic/28982ef7ce63_delete_bf_agent.py b/rero_mef/alembic/28982ef7ce63_delete_bf_agent.py
@@ -0,0 +1,115 @@
+#
+# This file is part of Invenio.
+# Copyright (C) 2016-2018 CERN.
+#
+# Invenio is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+"""delete bf:Agent"""
+
+import click
+from invenio_db import db
+
+from rero_mef.agents import AgentMefIndexer, AgentMefRecord
+from rero_mef.utils import get_entity_class, get_entity_indexer_class, \
+    progressbar
+
+# revision identifiers, used by Alembic.
+revision = '28982ef7ce63'
+down_revision = '0d0b3009dbe0'
+branch_labels = ()
+depends_on = None
+
+COMMIT_COUNT = 1000
+
+agents = {
+    'aggnd': 'GND',
+    'aidref': 'IDREF',
+    'agrero': 'RERO'
+}
+
+
+def commit_changes(indexer, ids):
+    db.session.commit()
+    indexer.bulk_index(ids)
+    indexer.process_bulk_queue()
+
+
+def upgrade():
+    """Upgrade database."""
+    for agent, name in agents.items():
+        agent_class = get_entity_class(agent)
+        indexer = get_entity_indexer_class(agent)()
+        click.echo(
+            f'Delete bf:Agent from {agent}: {agent_class.count()}')
+        progress_bar = progressbar(
+            items=agent_class.get_all_records(),
+            length=agent_class.count(),
+            verbose=True
+        )
+        ids = []
+        for idx, rec in enumerate(progress_bar, 1):
+            rec.pop('bf:Agent', None)
+            ids.append(rec.id)
+            rec.update(data=rec, dbcommit=False, reindex=False)
+            if idx % COMMIT_COUNT == 0:
+                commit_changes(indexer, ids)
+                ids = []
+        commit_changes(indexer, ids)
+
+    click.echo(f'Delete bf:Agent from mef: {AgentMefRecord.count()}')
+    progress_bar = progressbar(
+        items=AgentMefRecord.get_all_records(),
+        length=AgentMefRecord.count(),
+        verbose=True
+    )
+    ids = []
+    indexer = AgentMefIndexer()
+    for idx, rec in enumerate(progress_bar, 1):
+        rec.pop('bf:Agent', None)
+        ids.append(rec.id)
+        rec.update(data=rec, dbcommit=False, reindex=False)
+        if idx % COMMIT_COUNT == 0:
+            commit_changes(indexer, ids)
+            ids = []
+    commit_changes(indexer, ids)
+
+
+def downgrade():
+    """Downgrade database."""
+    for agent, name in agents.items():
+        agent_class = get_entity_class(agent)
+        indexer = get_entity_indexer_class(agent)()
+        click.echo(
+            f'Add bf:Agent to {agent}: {agent_class.count()}')
+        progress_bar = progressbar(
+            items=agent_class.get_all_records(),
+            length=agent_class.count(),
+            verbose=True
+        )
+        ids = []
+        for idx, rec in enumerate(progress_bar, 1):
+            rec['bf:Agent'] = rec['type']
+            ids.append(rec.id)
+            rec.update(data=rec, dbcommit=False, reindex=False)
+            if idx % COMMIT_COUNT == 0:
+                commit_changes(indexer, ids)
+                ids = []
+        commit_changes(indexer, ids)
+
+    click.echo(f'Add bf:Agent to mef: {AgentMefRecord.count()}')
+    progress_bar = progressbar(
+        items=AgentMefRecord.get_all_records(),
+        length=AgentMefRecord.count(),
+        verbose=True
+    )
+    ids = []
+    indexer = AgentMefIndexer()
+    for idx, rec in enumerate(progress_bar, 1):
+        rec['bf:Agent'] = rec['type']
+        ids.append(rec.id)
+        rec.update(data=rec, dbcommit=False, reindex=False)
+        if idx % COMMIT_COUNT == 0:
+            commit_changes(indexer, ids)
+            ids = []
+    commit_changes(indexer, ids)
diff --git a/rero_mef/api.py b/rero_mef/api.py
@@ -137,7 +137,7 @@ def create_or_update(cls, data, id_=None, delete_pid=True, dbcommit=False,
             data = add_schema(data, agent_record.provider.pid_type)
             # save the records old data if the new one is empty
             copy_fields = ['pid', '$schema', 'identifier', 'identifiedBy',
-                           'authorized_access_point', 'type', 'bf:Agent',
+                           'authorized_access_point', 'type',
                            'relation_pid', 'deleted']
             original_data = {
                 k: v for k, v in agent_record.items() if k in copy_fields}

diff --git a/rero_mef/jsonschemas/common/agent-v0.0.1.json b/rero_mef/jsonschemas/common/agent-v0.0.1.json
@@ -22,14 +22,6 @@
       "type": "string",
       "minLength": 1
     },
-    "bf:Agent": {
-      "title": "Type",
-      "type": "string",
-      "enum": [
-        "bf:Person",
-        "bf:Organisation"
-      ]
-    },
     "type": {
       "title": "Type",
       "type": "string",

diff --git a/rero_mef/marctojson/do_gnd_agent.py b/rero_mef/marctojson/do_gnd_agent.py
@@ -368,7 +368,6 @@ def trans_gnd_authorized_access_point(self):
                 subfields=subfields
             )
             for authorized_access_point in authorized_access_points:
-                self.json_dict['bf:Agent'] = agent
                 self.json_dict['type'] = agent
                 if self.json_dict.get('authorized_access_point'):
                     variant_access_points.append(authorized_access_point)

diff --git a/rero_mef/marctojson/do_idref_agent.py b/rero_mef/marctojson/do_idref_agent.py
@@ -388,7 +388,6 @@ def trans_idref_authorized_access_point(self):
             tag_grouping=tag_grouping
         )
         for authorized_access_point in authorized_access_points:
-            self.json_dict['bf:Agent'] = agent
             self.json_dict['type'] = agent
             if self.json_dict.get('authorized_access_point'):
                 variant_access_points.append(authorized_access_point)

diff --git a/rero_mef/marctojson/do_rero_agent.py b/rero_mef/marctojson/do_rero_agent.py
@@ -192,7 +192,6 @@ def trans_rero_authorized_access_point(self):
         if self.logger and self.verbose:
             self.logger.info(
                 'Call Function', 'trans_rero_authorized_access_point')
-        self.json_dict['bf:Agent'] = agent
         self.json_dict['type'] = agent
         authorized_access_points = build_string_list_from_fields(
             record=self.marc,