Process functions: Allow nested output namespaces (#5954)

Up till now, the following was not possible:

    @calcfunction
    def add(x, y):
        return {'nested.output': x + y}

An exception would be raised by `plumpy` because the `nested` output
namespace, defined by the `ProcessSpec` that is automatically generated
from the function signature, does not contain the port `output`. The
automatically generated process spec _does_ mark the outputs namespace
as dynamic, but this was not being applied recursively.

Not only is this functionality for users, the `Parser.parse_from_node`
functonality is currently broken if the `Parser` returns output nodes in
nested namespaces. The reason is that the `parse_from_node` creates a
`calcfunction` on-the-fly which raises when it gets the outputs with the
nested labels. Even though the original `Process` class may have
specified these nested output namespaces, the on-the-fly calcfunction to
capture the manual re-parsing does not support this.

The addition of this functionality requires a change in `plumpy` which
was released with `plumpy==0.21.6` which is therefore made the minimum
required version.

This version also includes another fix where the recently introduced
check on the type of the return value of a workchain conditional
predicate, was changed from raising an exception, to logging a
deprecation warning. The correspondig test is updated to check that the
user warning is emitted instead of catching the exception.

docs/source/topics/processes/functions.rst

@@ -220,6 +220,13 @@ As always, all the values returned by a calculation function have to be storable
     Because of the calculation/workflow duality in AiiDA, a ``calcfunction``, which is a calculation-like process, can only *create* and not *return* data nodes.
     This means that if a node is returned from a ``calcfunction`` that *is already stored*, the engine will throw an exception.
 
+.. versionadded:: 2.3
+
+    Outputs can be attached with nested namespaces in the output labels:
+
+    .. include:: include/snippets/functions/calcfunction_nested_outputs.py
+        :code: python
+
 .. _topics:processes:functions:exit_codes:
 
 Exit codes

docs/source/topics/processes/include/snippets/functions/calcfunction_nested_outputs.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+from aiida.engine import calcfunction
+from aiida.orm import Int
+
+
+@calcfunction
+def add(alpha, beta):
+    return {'nested.sum': alpha + beta}
+
+result = add(Int(1), Int(2))
+assert result['nested']['sum'] == 3

environment.yml

@@ -24,7 +24,7 @@ dependencies:
 - importlib-resources~=5.0
 - numpy~=1.19
 - paramiko>=2.7.2,~=2.7
-- plumpy~=0.21.4
+- plumpy~=0.21.6
 - pgsu~=0.2.1
 - psutil~=5.6
 - psycopg2-binary~=2.8

pyproject.toml

@@ -43,7 +43,7 @@ dependencies = [
     "importlib-resources~=5.0;python_version<'3.9'",
     "numpy~=1.19",
     "paramiko~=2.7,>=2.7.2",
-    "plumpy~=0.21.4",
+    "plumpy~=0.21.6",
     "pgsu~=0.2.1",
     "psutil~=5.6",
     "psycopg2-binary~=2.8",

requirements/requirements-py-3.10.txt

@@ -90,7 +90,7 @@ pickleshare==0.7.5
 Pillow==9.3.0
 plotly==5.4.0
 pluggy==1.0.0
-plumpy==0.21.4
+plumpy==0.21.6
 prometheus-client==0.12.0
 prompt-toolkit==3.0.37
 psutil==5.8.0

requirements/requirements-py-3.11.txt

@@ -107,7 +107,7 @@ Pillow==9.3.0
 platformdirs==2.5.4
 plotly==5.11.0
 pluggy==1.0.0
-plumpy==0.21.4
+plumpy==0.21.6
 prometheus-client==0.15.0
 prompt-toolkit==3.0.37
 psutil==5.9.4

requirements/requirements-py-3.8.txt

@@ -92,7 +92,7 @@ pickleshare==0.7.5
 Pillow==9.3.0
 plotly==5.4.0
 pluggy==1.0.0
-plumpy==0.21.4
+plumpy==0.21.6
 prometheus-client==0.12.0
 prompt-toolkit==3.0.37
 psutil==5.8.0

requirements/requirements-py-3.9.txt

@@ -91,7 +91,7 @@ pickleshare==0.7.5
 Pillow==9.3.0
 plotly==5.4.0
 pluggy==1.0.0
-plumpy==0.21.4
+plumpy==0.21.6
 prometheus-client==0.12.0
 prompt-toolkit==3.0.37
 psutil==5.8.0

tests/engine/test_process_function.py

@@ -123,6 +123,11 @@ def function_out_unstored():
     return orm.Int(DEFAULT_INT)
 
 
+@workfunction
+def function_return_nested():
+    return {'nested.output': orm.Int(DEFAULT_INT).store()}
+
+
 def test_properties():
     """Test that the `is_process_function` and `node_class` attributes are set."""
     assert function_return_input.is_process_function
@@ -444,6 +449,13 @@ def test_function_out_unstored():
         function_out_unstored()
 
 
+def test_function_return_nested():
+    """Test that a process function can returned outputs in nested namespaces."""
+    results, node = function_return_nested.run_get_node()
+    assert results['nested']['output'] == DEFAULT_INT
+    assert node.outputs.nested.output == DEFAULT_INT
+
+
 def test_simple_workflow():
     """Test construction of simple workflow by chaining process functions."""
 

tests/engine/test_work_chain.py

@@ -514,39 +514,46 @@ def read_context(self):
     def test_str(self):
         assert isinstance(str(Wf.spec()), str)
 
-    def test_invalid_if_predicate(self):
+    def test_invalid_if_predicate(self, recwarn):
         """Test that workchain raises if the predicate of an ``if_`` condition does not return a boolean."""
 
         class TestWorkChain(WorkChain):
 
             @classmethod
             def define(cls, spec):
                 super().define(spec)
-                spec.outline(if_(cls.predicate))
+                spec.outline(if_(cls.predicate)(cls.run_step))
 
             def predicate(self):
                 """Invalid predicate whose return value is not a boolean."""
                 return 'true'
 
-        with pytest.raises(TypeError, match=r'The conditional predicate `predicate` did not return a boolean'):
-            launch.run(TestWorkChain)
+            def run_step(self):
+                pass
+
+        launch.run(TestWorkChain)
+        assert len(recwarn) == 1
 
-    def test_invalid_while_predicate(self):
+    def test_invalid_while_predicate(self, recwarn):
         """Test that workchain raises if the predicate of an ``while_`` condition does not return a boolean."""
 
         class TestWorkChain(WorkChain):
 
             @classmethod
             def define(cls, spec):
                 super().define(spec)
-                spec.outline(while_(cls.predicate))
+                spec.outline(while_(cls.predicate)(cls.run_step))
 
             def predicate(self):
                 """Invalid predicate whose return value is not a boolean."""
                 return 'true'
 
-        with pytest.raises(TypeError, match=r'The conditional predicate `predicate` did not return a boolean'):
-            launch.run(TestWorkChain)
+            def run_step(self):
+                # Need to return an exit code to abort the workchain, otherwise we would be stuck in an infinite loop
+                return ExitCode(1)
+
+        launch.run(TestWorkChain)
+        assert len(recwarn) == 1
 
     def test_malformed_outline(self):
         """