Implement a PluginVersionProvider for processes (aiidateam#3131)

This new utility class is used by the `Runner` class to keep a mapping
of certain process classes, either `Process` sub classes or process
functions onto a dictionary of "version" information. This dictionary
now includes the version of `aiida-core` that is running as well as the
version of the top level module of the plugin, if defined. If the latter
cannot be determined for whatever reason, only the version of
`aiida-core` is returned.

This information is then retrieved by the `Process` class during the
creation process. It will store this information in the attributes of
the process node. Currently, no logic in `aiida-core` will act on this
information. Its sole purpose is to give the user slightly more info on
with what versions of core and the plugin a certain process node was
generated. The attributes can also be used in querying to filter nodes
for a sub set generated with a specific version of the plugin.

Note that "plugin" here refers to the entire package, e.g the entire
`aiida-quantumespresso` plugin. Each plugin can contain multiple entry
points for the various entry point categories, that are sometimes
individually also referred to as plugins. In the future, the version
dictionary returned by the `PluginVersionProvider` may be enriched with
a specific `entry_point` version, once that level of version granulariy
will become supported. For the time being it is not included.

.pre-commit-config.yaml

@@ -60,7 +60,6 @@
         aiida/backends/sqlalchemy/models/node.py|
         aiida/backends/sqlalchemy/models/settings.py|
         aiida/backends/sqlalchemy/models/user.py|
-        aiida/backends/sqlalchemy/models/utils.py|
         aiida/backends/sqlalchemy/queries.py|
         aiida/backends/sqlalchemy/tests/test_generic.py|
         aiida/backends/sqlalchemy/tests/__init__.py|
@@ -128,7 +127,6 @@
         aiida/plugins/entry.py|
         aiida/plugins/info.py|
         aiida/plugins/registry.py|
-        aiida/plugins/utils.py|
         aiida/schedulers/datastructures.py|
         aiida/schedulers/plugins/direct.py|
         aiida/schedulers/plugins/lsf.py|

aiida/backends/tests/__init__.py

@@ -141,6 +141,7 @@
         'orm.utils.repository': ['aiida.backends.tests.orm.utils.test_repository'],
         'parsers.parser': ['aiida.backends.tests.parsers.test_parser'],
         'plugin_loader': ['aiida.backends.tests.test_plugin_loader'],
+        'plugins.utils': ['aiida.backends.tests.plugins.test_utils'],
         'query': ['aiida.backends.tests.test_query'],
         'restapi': ['aiida.backends.tests.test_restapi'],
         'tools.data.orbital': ['aiida.backends.tests.tools.data.orbital.test_orbitals'],

aiida/backends/tests/engine/test_process_function.py

@@ -120,6 +120,18 @@ def tearDown(self):
         super(TestProcessFunction, self).tearDown()
         self.assertIsNone(Process.current())
 
+    def test_plugin_version(self):
+        """Test the version attributes of a process function."""
+        from aiida import __version__ as version_core
+
+        _, node = self.function_args_with_default.run_get_node()
+
+        # Since the "plugin" i.e. the process function is defined in `aiida-core` the `version.plugin` is the same as
+        # the version of `aiida-core` itself
+        version_info = node.get_attribute('version')
+        self.assertEqual(version_info['core'], version_core)
+        self.assertEqual(version_info['plugin'], version_core)
+
     def test_process_state(self):
         """Test the process state for a process function."""
         _, node = self.function_args_with_default.run_get_node()

aiida/backends/tests/plugins/test_utils.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+###########################################################################
+# Copyright (c), The AiiDA team. All rights reserved.                     #
+# This file is part of the AiiDA code.                                    #
+#                                                                         #
+# The code is hosted on GitHub at https://github.com/aiidateam/aiida_core #
+# For further information on the license, see the LICENSE.txt file        #
+# For further information please visit http://www.aiida.net               #
+###########################################################################
+"""Tests for utilities dealing with plugins and entry points."""
+from __future__ import division
+from __future__ import absolute_import
+from __future__ import print_function
+
+from aiida import __version__ as version_core
+from aiida.backends.testbase import AiidaTestCase
+from aiida.engine import calcfunction, WorkChain
+from aiida.plugins import CalculationFactory
+from aiida.plugins.utils import PluginVersionProvider
+
+
+class TestPluginVersionProvider(AiidaTestCase):
+    """Tests for the :py:class:`~aiida.plugins.utils.PluginVersionProvider` utility class."""
+
+    # pylint: disable=no-init,old-style-class,too-few-public-methods,no-member
+
+    def setUp(self):
+        super(TestPluginVersionProvider, self).setUp()
+        self.provider = PluginVersionProvider()
+
+    @staticmethod
+    def create_dynamic_plugin_module(plugin, plugin_version, add_module_to_sys=True, add_version=True):
+        """Create a fake dynamic module with a certain `plugin` entity, a class or function and the given version."""
+        import sys
+        import types
+        import uuid
+
+        # Create a new module with a unique name and add the `plugin` and `plugin_version` as attributes
+        module_name = 'TestModule{}'.format(str(uuid.uuid4())[:5])
+        dynamic_module = types.ModuleType(module_name, 'Dynamically created module for testing purposes')
+        setattr(plugin, '__module__', dynamic_module.__name__)
+        setattr(dynamic_module, plugin.__name__, plugin)
+
+        # For tests that need to fail, this flag can be set to `False`
+        if add_version:
+            setattr(dynamic_module, '__version__', plugin_version)
+
+        # Get the `DummyClass` plugin from the dynamically created test module
+        dynamic_plugin = getattr(dynamic_module, plugin.__name__)
+
+        # Make the dynamic module importable unless the test requests not to, to test an unimportable module
+        if add_module_to_sys:
+            sys.modules[dynamic_module.__name__] = dynamic_module
+
+        return dynamic_plugin
+
+    def test_external_module_import_fail(self):
+        """Test that mapper does not except even if external module cannot be imported."""
+
+        class DummyCalcJob():
+            pass
+
+        version_plugin = '0.1.01'
+        dynamic_plugin = self.create_dynamic_plugin_module(DummyCalcJob, version_plugin, add_module_to_sys=False)
+
+        expected_version = {'version': {'core': version_core}}
+        self.assertEqual(self.provider.get_version_info(dynamic_plugin), expected_version)
+
+    def test_external_module_no_version_attribute(self):
+        """Test that mapper does not except even if external module does not define `__version__` attribute."""
+
+        class DummyCalcJob():
+            pass
+
+        version_plugin = '0.1.02'
+        dynamic_plugin = self.create_dynamic_plugin_module(DummyCalcJob, version_plugin, add_version=False)
+
+        expected_version = {'version': {'core': version_core}}
+        self.assertEqual(self.provider.get_version_info(dynamic_plugin), expected_version)
+
+    def test_external_module_class(self):
+        """Test the mapper works for a class from an external module."""
+
+        class DummyCalcJob():
+            pass
+
+        version_plugin = '0.1.17'
+        dynamic_plugin = self.create_dynamic_plugin_module(DummyCalcJob, version_plugin)
+
+        expected_version = {'version': {'core': version_core, 'plugin': version_plugin}}
+        self.assertEqual(self.provider.get_version_info(dynamic_plugin), expected_version)
+
+    def test_external_module_function(self):
+        """Test the mapper works for a function from an external module."""
+
+        @calcfunction
+        def test_calcfunction():
+            return
+
+        version_plugin = '0.2.19'
+        dynamic_plugin = self.create_dynamic_plugin_module(test_calcfunction, version_plugin)
+
+        expected_version = {'version': {'core': version_core, 'plugin': version_plugin}}
+        self.assertEqual(self.provider.get_version_info(dynamic_plugin), expected_version)
+
+    def test_calcfunction(self):
+        """Test the mapper for a `calcfunction`."""
+
+        @calcfunction
+        def test_calcfunction():
+            return
+
+        expected_version = {'version': {'core': version_core, 'plugin': version_core}}
+        self.assertEqual(self.provider.get_version_info(test_calcfunction), expected_version)
+
+    def test_calc_job(self):
+        """Test the mapper for a `CalcJob`."""
+        AddArithmeticCalculation = CalculationFactory('arithmetic.add')  # pylint: disable=invalid-name
+
+        expected_version = {'version': {'core': version_core, 'plugin': version_core}}
+        self.assertEqual(self.provider.get_version_info(AddArithmeticCalculation), expected_version)
+
+    def test_work_chain(self):
+        """Test the mapper for a `WorkChain`."""
+
+        class SomeWorkChain(WorkChain):
+            """Need to create a dummy class since there is no built-in work chain with entry point in `aiida-core`."""
+
+        expected_version = {'version': {'core': version_core, 'plugin': version_core}}
+        self.assertEqual(self.provider.get_version_info(SomeWorkChain), expected_version)

aiida/engine/processes/process.py

@@ -633,6 +633,9 @@ def _setup_db_record(self):
 
     def _setup_metadata(self):
         """Store the metadata on the ProcessNode."""
+        version_info = self.runner.plugin_version_provider.get_version_info(self)
+        self.node.set_attribute_many(version_info)
+
         for name, metadata in self.metadata.items():
             if name in ['store_provenance', 'dry_run', 'call_link_label']:
                 continue

aiida/engine/runners.py

@@ -7,7 +7,7 @@
 # For further information on the license, see the LICENSE.txt file        #
 # For further information please visit http://www.aiida.net               #
 ###########################################################################
-# pylint: disable=cyclic-import,global-statement
+# pylint: disable=global-statement
 """Runners that can run and submit processes."""
 from __future__ import division
 from __future__ import print_function
@@ -22,9 +22,10 @@
 
 from aiida.common import exceptions
 from aiida.orm import load_node
+from aiida.plugins.utils import PluginVersionProvider
+
 from .processes import futures
 from .processes.calcjobs import manager
-from .utils import instantiate_process
 from . import transports
 from . import utils
 
@@ -36,7 +37,7 @@
 ResultAndPk = collections.namedtuple('ResultAndPk', ['result', 'pk'])
 
 
-class Runner(object):  # pylint: disable=useless-object-inheritance
+class Runner(object):  # pylint: disable=useless-object-inheritance,too-many-public-methods
     """Class that can launch processes by running in the current interpreter or by submitting them to the daemon."""
 
     _persister = None
@@ -66,6 +67,7 @@ def __init__(self, poll_interval=0, loop=None, communicator=None, rmq_submit=Fal
         self._transport = transports.TransportQueue(self._loop)
         self._job_manager = manager.JobManager(self._transport)
         self._persister = persister
+        self._plugin_version_provider = PluginVersionProvider()
 
         if communicator is not None:
             self._communicator = communicator
@@ -108,6 +110,10 @@ def communicator(self):
         """
         return self._communicator
 
+    @property
+    def plugin_version_provider(self):
+        return self._plugin_version_provider
+
     @property
     def job_manager(self):
         return self._job_manager
@@ -147,6 +153,10 @@ def close(self):
         self.stop()
         self._closed = True
 
+    def instantiate_process(self, process, *args, **inputs):
+        from .utils import instantiate_process
+        return instantiate_process(self, process, *args, **inputs)
+
     def submit(self, process, *args, **inputs):
         """
         Submit the process with the supplied inputs to this runner immediately returning control to
@@ -159,7 +169,7 @@ def submit(self, process, *args, **inputs):
         assert not utils.is_process_function(process), 'Cannot submit a process function'
         assert not self._closed
 
-        process = instantiate_process(self, process, *args, **inputs)
+        process = self.instantiate_process(process, *args, **inputs)
 
         if not process.metadata.store_provenance:
             raise exceptions.InvalidOperation('cannot submit a process with `store_provenance=False`')
@@ -187,7 +197,7 @@ def schedule(self, process, *args, **inputs):
         assert not utils.is_process_function(process), 'Cannot submit a process function'
         assert not self._closed
 
-        process = instantiate_process(self, process, *args, **inputs)
+        process = self.instantiate_process(process, *args, **inputs)
         self.loop.add_callback(process.step_until_terminated)
         return process.node
 
@@ -207,7 +217,7 @@ def _run(self, process, *args, **inputs):
             return result, node
 
         with utils.loop_scope(self.loop):
-            process = instantiate_process(self, process, *args, **inputs)
+            process = self.instantiate_process(process, *args, **inputs)
 
             def kill_process(_num, _frame):
                 """Send the kill signal to the process in the current scope."""