concepts: add MEF

* Adds MEF concepts class. * Splits serialiazers. * Adds coveralls to github actions. * Adds tests to increase test coverage. Co-Authored-by: Peter Weber <[email protected]>
rero · Jan 31, 2022 · 12a3d70 · 12a3d70
1 parent c534f0e
commit 12a3d70
Show file tree

Hide file tree

Showing 108 changed files with 28,023 additions and 1,011 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -0,0 +1,10 @@
+[run]
+relative_files = True
+omit =
+    rero_mef/cli.py
+    rero_mef/agents/cli.py
+    rero_mef/concepts/cli.py
+    rero_mef/marctojson/logger.py
+    rero_mef/marctojson/helper.py
+    rero_mef/marctojson/records.py
+    rero_mef/marctojson/do_skeleton.py
diff --git a/.github/workflows/continuous-integration-test.yml b/.github/workflows/continuous-integration-test.yml
@@ -9,7 +9,7 @@ jobs:
         dependencies: ['development', 'deploy']
     steps:
     - name: Cancel Previous Runs
-      uses: styfle/cancel-workflow-action@0.5.0
+      uses: styfle/cancel-workflow-action@0.9.1
       with:
         access_token: ${{ github.token }}
 
@@ -52,31 +52,20 @@ jobs:
       if: ${{ matrix.dependencies == 'development' }}
       run: |
         poetry run bootstrap --ci
+        pip install --upgrade coveralls
 
     - name: Bootstrap deploy
       if: ${{ matrix.dependencies == 'deploy' }}
       run: |
         poetry run bootstrap --ci --deploy E2E=yes
 
-    - name: Update coveralls
-      run: |
-        poetry run pip install --upgrade coveralls
-
     - name: Run Test
       run: poetry run run-tests
 
-    # - name: Upload Coverage ${{ matrix.tests }}
-    #   if: ${{ matrix.dependencies == 'locked' }}
-    #   env:
-    #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    #     COVERALLS_FLAG_NAME: test
-    #     COVERALLS_PARALLEL: true
-    #   run: poetry run coveralls
-    #
-    # - name: Finished Coverage ${{ matrix.tests }}
-    #   if: ${{ matrix.dependencies == 'locked' }}
-    #   env:
-    #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    #     COVERALLS_PARALLEL_FINISHED: true
-    #   run: |
-    #     poetry run coveralls --finish
+    - name: Coveralls
+      if: ${{ matrix.dependencies == 'development' }}
+      env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COVERALLS_SERVICE_NAME: github
+      run: |
+          coveralls
diff --git a/README.rst b/README.rst
@@ -18,14 +18,21 @@
  RERO MEF
 ==========
 
-.. image:: https://img.shields.io/travis/rero/rero-mef.svg
-        :target: https://travis-ci.org/rero/rero-mef
+.. image:: https://github.com/rero/rero-mef/workflows/build/badge.svg
+        :alt: Github actions status
+        :target: https://github.com/rero/rero-mef/actions?query=workflow%3Abuild
 
 .. image:: https://img.shields.io/coveralls/rero/rero-mef.svg
-        :target: https://coveralls.io/r/rero/rero-mef
+        :alt: Coveralls
+        :target: https://coveralls.io/rero/rero-mef
 
-.. image:: https://img.shields.io/github/license/rero/rero-mef.svg
-        :target: https://github.com/rero/rero-mef/blob/master/LICENSE
+.. image:: https://img.shields.io/github/tag/rero/rero-mef.svg
+        :alt: Release Number
+        :target: https://github.com/rero/rero-mef/releases/latest
+
+.. image:: https://img.shields.io/badge/License-AGPL%20v3-blue.svg
+        :alt: License
+        :target: http://www.gnu.org/licenses/agpl-3.0.html
 
 MEF (Multilingual Entity File) server with records for persons, works, etc. for reuse in integrated library systems (ILS).
 

diff --git a/pytest.ini b/pytest.ini
@@ -17,8 +17,7 @@
 
 [pytest]
 live_server_scope = module
-
-addopts = --pydocstyle --doctest-glob="*.rst" --doctest-modules --cov=rero_mef --cov-report=term-missing --ignore=setup.py
+addopts = --pycodestyle --pydocstyle --doctest-glob="*.rst" --doctest-modules --cov=rero_mef --cov-report=term-missing --ignore=setup.py --ignore=docs/conf.py
 testpaths = docs tests rero_mef
 
 # not displaying all the PendingDeprecationWarnings from invenio

diff --git a/rero_mef/agents/api.py b/rero_mef/agents/api.py
@@ -16,15 +16,12 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 """API for manipulating records."""
 
-from copy import deepcopy
-
 import click
 from flask import current_app
 from invenio_pidstore.models import PersistentIdentifier, PIDStatus
 from invenio_search import current_search
 
-from ..api import Action, ReroIndexer, ReroMefRecord, ReroMefRecordError
-from ..utils import add_md5, add_schema
+from ..api import Action, ReroIndexer, ReroMefRecord
 
 
 class AgentRecord(ReroMefRecord):
@@ -58,40 +55,20 @@ def create(cls, data, id_=None, delete_pid=False, dbcommit=False,
     def update_indexes(cls):
         """Update indexes."""
         try:
-            index = f'agents_{cls.agent}'
+            index = f'agents_{cls.name}'
             current_search.flush_and_refresh(index=index)
         except Exception as err:
             current_app.logger.error(f'ERROR flush and refresh: {err}')
 
-    def update_test_md5(self, data, dbcommit=False, reindex=False):
-        """Update existing record."""
-        return_record = self
-        if not data.get('md5'):
-            data = add_md5(data)
-        if data.get('md5', 'data') == self.get('md5', 'agent'):
-            # record has no changes
-            return return_record, Action.UPTODATE
-        data = add_schema(data, self.agent)
-        return_record = self.update(
-            data=data, dbcommit=dbcommit, reindex=reindex)
-        return return_record, Action.UPDATE
-
-    def replace_test_md5(self, data, dbcommit=False, reindex=False):
-        """Replace data in record."""
-        new_data = deepcopy(data)
-        pid = new_data.get('pid')
-        if not pid:
-            raise ReroMefRecordError.PidMissing(f'missing pid={self.pid}')
-        self.clear()
-        self, action = self.update_test_md5(
-            data=new_data, dbcommit=dbcommit, reindex=reindex)
-        if action == Action.UPTODATE:
-            self = new_data
-        return self
-
     def create_or_update_mef_viaf_record(self, dbcommit=False, reindex=False,
                                          online=False):
-        """Create or update MEF and VIAF record."""
+        """Create or update MEF and VIAF record.
+
+        :param dbcommit: Commit changes to DB.
+        :param reindex: Reindex record.
+        :param online: Try to get VIAF record online.
+        :returns: MEF record, MEF action, VIAF record, VIAF
+        """
         from .viaf.api import AgentViafRecord
         AgentViafRecord.update_indexes()
         viaf_record, got_online = AgentViafRecord.get_viaf_by_agent(
@@ -104,10 +81,7 @@ def create_or_update_mef_viaf_record(self, dbcommit=False, reindex=False,
             agent_pid=self.pid
         )
         mef_data = {self.agent: {'$ref': ref_string}}
-        mef_record = AgentMefRecord.get_mef_by_agent_pid(
-            agent_pid=self.pid,
-            agent_name=self.name
-        )
+        mef_record = AgentMefRecord.get_mef_by_entity_pid(self.pid, self.name)
         if viaf_record:
             mef_data['viaf_pid'] = viaf_record.pid
             if not mef_record:
@@ -142,10 +116,7 @@ def delete_from_mef(self, dbcommit=False, reindex=False, verbose=False):
         from .mef.api import AgentMefRecord
         mef_action = Action.DISCARD
         old_mef_pid = 'None'
-        mef_record = AgentMefRecord.get_mef_by_agent_pid(
-            agent_pid=self.pid,
-            agent_name=self.name
-        )
+        mef_record = AgentMefRecord.get_mef_by_entity_pid(self.pid, self.name)
         if mef_record:
             old_mef_pid = mef_record.pid
             if not mef_record.deleted:
@@ -157,7 +128,7 @@ def delete_from_mef(self, dbcommit=False, reindex=False, verbose=False):
                     reindex=reindex
                 )
                 mef_record = AgentMefRecord.create_deleted(
-                    agent=self,
+                    record=self,
                     dbcommit=dbcommit,
                     reindex=reindex
                 )
@@ -166,7 +137,7 @@ def delete_from_mef(self, dbcommit=False, reindex=False, verbose=False):
         else:
             # MEF record is missing create one
             mef_record = AgentMefRecord.create_deleted(
-                agent=self,
+                record=self,
                 dbcommit=dbcommit,
                 reindex=reindex
             )
@@ -209,43 +180,46 @@ def create_or_update_agent_mef_viaf(cls, data, id_=None, delete_pid=True,
             reindex=reindex,
             test_md5=test_md5
         )
-        if record.deleted:
-            mef_record, mef_action = record.delete_from_mef(
-                dbcommit=dbcommit,
-                reindex=reindex,
-                verbose=verbose
-            )
-            # record.delete(
-            #     dbcommit=dbcommit,
-            #     delindex=True,
-            # )
-            # record = None
-            action = Action.DELETE
-            viaf_record = None
-            online = False
+        if action == Action.ERROR:
+            return None, action, None, Action.ERROR, None, False
         else:
-            if action == Action.UPTODATE:
-                mef_record = AgentMefRecord.get_mef_by_agent_pid(
-                    agent_pid=record.pid,
-                    agent_name=record.name
+            if record.deleted:
+                mef_record, mef_action = record.delete_from_mef(
+                    dbcommit=dbcommit,
+                    reindex=reindex,
+                    verbose=verbose
                 )
-                mef_action = Action.UPTODATE
-                viaf_record, online = AgentViafRecord.get_viaf_by_agent(record)
+                # record.delete(
+                #     dbcommit=dbcommit,
+                #     delindex=True,
+                # )
+                # record = None
+                action = Action.DELETE
+                viaf_record = None
+                online = False
             else:
-                mef_record, mef_action, viaf_record, online = \
-                    record.create_or_update_mef_viaf_record(
-                        dbcommit=dbcommit,
-                        reindex=reindex,
-                        online=online
-                    )
-        return record, action, mef_record, mef_action, viaf_record, online
+                if action == Action.UPTODATE:
+                    mef_record = AgentMefRecord.get_mef_by_entity_pid(
+                        record.pid, record.name)
+                    mef_action = Action.UPTODATE
+                    viaf_record, online = AgentViafRecord.get_viaf_by_agent(
+                        record)
+                else:
+                    mef_record, mef_action, viaf_record, online = \
+                        record.create_or_update_mef_viaf_record(
+                            dbcommit=dbcommit,
+                            reindex=reindex,
+                            online=online
+                        )
+            return record, action, mef_record, mef_action, viaf_record, online
 
     @classmethod
     def get_online_record(cls, id, verbose=False):
         """Get online Record.
 
         Has to be overloaded in agent class.
         """
+        raise NotImplementedError()
 
     @property
     def deleted(self):

diff --git a/rero_mef/agents/cli.py b/rero_mef/agents/cli.py
@@ -22,11 +22,11 @@
 import os
 
 import click
+from flask import current_app
 from flask.cli import with_appcontext
 
 from .mef.api import AgentMefRecord
-from .tasks import create_from_viaf as task_mef_and_agents_from_viaf
-from .tasks import create_mef as task_mef_from_agent
+from .tasks import task_create_mef_for_agent, task_create_mef_from_viaf_agent
 from .utils import create_mef_files, create_viaf_files
 from .viaf.api import AgentViafRecord
 from ..utils import get_entity_class, get_entity_classes, progressbar
@@ -77,7 +77,7 @@ def create_from_viaf(test_md5, enqueue, online, verbose, progress, wait,
         )
     for pid in progress_bar:
         if enqueue:
-            task = task_mef_and_agents_from_viaf.delay(
+            task = task_create_mef_from_viaf_agent.delay(
                 pid=pid,
                 dbcommit=True,
                 reindex=True,
@@ -86,7 +86,7 @@ def create_from_viaf(test_md5, enqueue, online, verbose, progress, wait,
             )
             click.echo(f'viaf pid: {pid} task:{task}')
         else:
-            task_mef_and_agents_from_viaf(
+            task_create_mef_from_viaf_agent(
                 pid=pid,
                 dbcommit=True,
                 reindex=True,
@@ -127,14 +127,12 @@ def create_from_viaf(test_md5, enqueue, online, verbose, progress, wait,
 @with_appcontext
 def create_mef(pid_type, enqueue, online, verbose, progress, wait, missing):
     """Create MEF from agents."""
+    AGENTS = current_app.config.get('AGENTS', [])
     if missing:
         missing_pids, to_much_pids = \
-            AgentMefRecord.get_all_missing_agents_pids(
-                agents=pid_type,
-                verbose=progress
-            )
+            AgentMefRecord.get_all_missing_pids(pid_type, verbose=progress)
     for agent in pid_type:
-        if agent not in ['aidref', 'aggnd', 'agrero']:
+        if agent not in AGENTS:
             click.secho(
                 f'Error create MEF from {agent}. Wrong agent!',
                 fg='red'
@@ -162,7 +160,7 @@ def create_mef(pid_type, enqueue, online, verbose, progress, wait, missing):
                 )
             for pid in progress_bar:
                 if enqueue:
-                    task = task_mef_from_agent.delay(
+                    task = task_create_mef_for_agent.delay(
                         pid=pid,
                         agent=agent,
                         dbcommit=True,
@@ -172,7 +170,7 @@ def create_mef(pid_type, enqueue, online, verbose, progress, wait, missing):
                     if verbose:
                         click.echo(f'{agent} pid: {pid} task:{task}')
                 else:
-                    msg = task_mef_from_agent(
+                    msg = task_create_mef_for_agent(
                         pid=pid,
                         agent=agent,
                         dbcommit=True,
@@ -235,7 +233,7 @@ def create_csv_mef(viaf_metadata_file, output_directory, verbose):
     :param output_directory: Output directory.
     :param verbose: Verbose.
     """
-    click.secho(f'  Create MEF CSV files from JSON.', err=True)
+    click.secho(f'  Create MEF CSV files from VIAF metadata.', err=True)
     pidstore = os.path.join(output_directory, 'mef_pidstore.csv')
     metadata = os.path.join(output_directory, 'mef_metadata.csv')
     ids = os.path.join(output_directory, 'mef_id.csv')
@@ -247,7 +245,7 @@ def create_csv_mef(viaf_metadata_file, output_directory, verbose):
     message = f'  CSV output files: {pidstore}, {metadata}'
 
     count = create_mef_files(
-            viaf_pidstore_file=viaf_metadata_file,
+            viaf_metadata_file_name=viaf_metadata_file,
             input_directory=output_directory,
             mef_pidstore_file_name=pidstore,
             mef_metadata_file_name=metadata,

diff --git a/rero_mef/agents/gnd/jsonresolvers/gnd_resolver.py b/rero_mef/agents/gnd/jsonresolvers/gnd_resolver.py
@@ -27,7 +27,7 @@
 from ..api import AgentGndRecord
 
 
-@jsonresolver.route('/api/gnd/<path:path>', host=get_host())
+@jsonresolver.route('/api/agents/gnd/<path:path>', host=get_host())
 def resolve_gnd(path):
     """Resolve GND records."""
     return resolve_record(path, AgentGndRecord)