From 1ed957e34a2686bf7ca5e600932ad329cd8fec05 Mon Sep 17 00:00:00 2001 From: Sebastiaan Huber Date: Tue, 4 Apr 2023 11:14:00 +0200 Subject: [PATCH] Process functions: Allow nested output namespaces (#5954) Up till now, the following was not possible: @calcfunction def add(x, y): return {'nested.output': x + y} An exception would be raised by `plumpy` because the `nested` output namespace, defined by the `ProcessSpec` that is automatically generated from the function signature, does not contain the port `output`. The automatically generated process spec _does_ mark the outputs namespace as dynamic, but this was not being applied recursively. Not only is this functionality for users, the `Parser.parse_from_node` functonality is currently broken if the `Parser` returns output nodes in nested namespaces. The reason is that the `parse_from_node` creates a `calcfunction` on-the-fly which raises when it gets the outputs with the nested labels. Even though the original `Process` class may have specified these nested output namespaces, the on-the-fly calcfunction to capture the manual re-parsing does not support this. The addition of this functionality requires a change in `plumpy` which was released with `plumpy==0.21.6` which is therefore made the minimum required version. This version also includes another fix where the recently introduced check on the type of the return value of a workchain conditional predicate, was changed from raising an exception, to logging a deprecation warning. The correspondig test is updated to check that the user warning is emitted instead of catching the exception. --- docs/source/topics/processes/functions.rst | 7 ++++++ .../functions/calcfunction_nested_outputs.py | 11 +++++++++ environment.yml | 2 +- pyproject.toml | 2 +- requirements/requirements-py-3.10.txt | 2 +- requirements/requirements-py-3.11.txt | 2 +- requirements/requirements-py-3.8.txt | 2 +- requirements/requirements-py-3.9.txt | 2 +- tests/engine/test_process_function.py | 12 ++++++++++ tests/engine/test_work_chain.py | 23 ++++++++++++------- 10 files changed, 51 insertions(+), 14 deletions(-) create mode 100644 docs/source/topics/processes/include/snippets/functions/calcfunction_nested_outputs.py diff --git a/docs/source/topics/processes/functions.rst b/docs/source/topics/processes/functions.rst index 513da6554c..3b53c46c1e 100644 --- a/docs/source/topics/processes/functions.rst +++ b/docs/source/topics/processes/functions.rst @@ -220,6 +220,13 @@ As always, all the values returned by a calculation function have to be storable Because of the calculation/workflow duality in AiiDA, a ``calcfunction``, which is a calculation-like process, can only *create* and not *return* data nodes. This means that if a node is returned from a ``calcfunction`` that *is already stored*, the engine will throw an exception. +.. versionadded:: 2.3 + + Outputs can be attached with nested namespaces in the output labels: + + .. include:: include/snippets/functions/calcfunction_nested_outputs.py + :code: python + .. _topics:processes:functions:exit_codes: Exit codes diff --git a/docs/source/topics/processes/include/snippets/functions/calcfunction_nested_outputs.py b/docs/source/topics/processes/include/snippets/functions/calcfunction_nested_outputs.py new file mode 100644 index 0000000000..22b11d81f8 --- /dev/null +++ b/docs/source/topics/processes/include/snippets/functions/calcfunction_nested_outputs.py @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +from aiida.engine import calcfunction +from aiida.orm import Int + + +@calcfunction +def add(alpha, beta): + return {'nested.sum': alpha + beta} + +result = add(Int(1), Int(2)) +assert result['nested']['sum'] == 3 diff --git a/environment.yml b/environment.yml index a480331aa8..33f4810589 100644 --- a/environment.yml +++ b/environment.yml @@ -24,7 +24,7 @@ dependencies: - importlib-resources~=5.0 - numpy~=1.19 - paramiko>=2.7.2,~=2.7 -- plumpy~=0.21.4 +- plumpy~=0.21.6 - pgsu~=0.2.1 - psutil~=5.6 - psycopg2-binary~=2.8 diff --git a/pyproject.toml b/pyproject.toml index 756122e108..634d0ff848 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,7 +43,7 @@ dependencies = [ "importlib-resources~=5.0;python_version<'3.9'", "numpy~=1.19", "paramiko~=2.7,>=2.7.2", - "plumpy~=0.21.4", + "plumpy~=0.21.6", "pgsu~=0.2.1", "psutil~=5.6", "psycopg2-binary~=2.8", diff --git a/requirements/requirements-py-3.10.txt b/requirements/requirements-py-3.10.txt index 1cb25a4e3d..269f4f6f42 100644 --- a/requirements/requirements-py-3.10.txt +++ b/requirements/requirements-py-3.10.txt @@ -90,7 +90,7 @@ pickleshare==0.7.5 Pillow==9.3.0 plotly==5.4.0 pluggy==1.0.0 -plumpy==0.21.4 +plumpy==0.21.6 prometheus-client==0.12.0 prompt-toolkit==3.0.37 psutil==5.8.0 diff --git a/requirements/requirements-py-3.11.txt b/requirements/requirements-py-3.11.txt index 57ff117a21..aec787177c 100644 --- a/requirements/requirements-py-3.11.txt +++ b/requirements/requirements-py-3.11.txt @@ -107,7 +107,7 @@ Pillow==9.3.0 platformdirs==2.5.4 plotly==5.11.0 pluggy==1.0.0 -plumpy==0.21.4 +plumpy==0.21.6 prometheus-client==0.15.0 prompt-toolkit==3.0.37 psutil==5.9.4 diff --git a/requirements/requirements-py-3.8.txt b/requirements/requirements-py-3.8.txt index 1d8f39da88..3e0a2b3713 100644 --- a/requirements/requirements-py-3.8.txt +++ b/requirements/requirements-py-3.8.txt @@ -92,7 +92,7 @@ pickleshare==0.7.5 Pillow==9.3.0 plotly==5.4.0 pluggy==1.0.0 -plumpy==0.21.4 +plumpy==0.21.6 prometheus-client==0.12.0 prompt-toolkit==3.0.37 psutil==5.8.0 diff --git a/requirements/requirements-py-3.9.txt b/requirements/requirements-py-3.9.txt index ec3b9c464f..09180f7494 100644 --- a/requirements/requirements-py-3.9.txt +++ b/requirements/requirements-py-3.9.txt @@ -91,7 +91,7 @@ pickleshare==0.7.5 Pillow==9.3.0 plotly==5.4.0 pluggy==1.0.0 -plumpy==0.21.4 +plumpy==0.21.6 prometheus-client==0.12.0 prompt-toolkit==3.0.37 psutil==5.8.0 diff --git a/tests/engine/test_process_function.py b/tests/engine/test_process_function.py index ee6514adf0..3810fbae59 100644 --- a/tests/engine/test_process_function.py +++ b/tests/engine/test_process_function.py @@ -123,6 +123,11 @@ def function_out_unstored(): return orm.Int(DEFAULT_INT) +@workfunction +def function_return_nested(): + return {'nested.output': orm.Int(DEFAULT_INT).store()} + + def test_properties(): """Test that the `is_process_function` and `node_class` attributes are set.""" assert function_return_input.is_process_function @@ -444,6 +449,13 @@ def test_function_out_unstored(): function_out_unstored() +def test_function_return_nested(): + """Test that a process function can returned outputs in nested namespaces.""" + results, node = function_return_nested.run_get_node() + assert results['nested']['output'] == DEFAULT_INT + assert node.outputs.nested.output == DEFAULT_INT + + def test_simple_workflow(): """Test construction of simple workflow by chaining process functions.""" diff --git a/tests/engine/test_work_chain.py b/tests/engine/test_work_chain.py index 7fd010cbb4..bf8c363d9f 100644 --- a/tests/engine/test_work_chain.py +++ b/tests/engine/test_work_chain.py @@ -514,7 +514,7 @@ def read_context(self): def test_str(self): assert isinstance(str(Wf.spec()), str) - def test_invalid_if_predicate(self): + def test_invalid_if_predicate(self, recwarn): """Test that workchain raises if the predicate of an ``if_`` condition does not return a boolean.""" class TestWorkChain(WorkChain): @@ -522,16 +522,19 @@ class TestWorkChain(WorkChain): @classmethod def define(cls, spec): super().define(spec) - spec.outline(if_(cls.predicate)) + spec.outline(if_(cls.predicate)(cls.run_step)) def predicate(self): """Invalid predicate whose return value is not a boolean.""" return 'true' - with pytest.raises(TypeError, match=r'The conditional predicate `predicate` did not return a boolean'): - launch.run(TestWorkChain) + def run_step(self): + pass + + launch.run(TestWorkChain) + assert len(recwarn) == 1 - def test_invalid_while_predicate(self): + def test_invalid_while_predicate(self, recwarn): """Test that workchain raises if the predicate of an ``while_`` condition does not return a boolean.""" class TestWorkChain(WorkChain): @@ -539,14 +542,18 @@ class TestWorkChain(WorkChain): @classmethod def define(cls, spec): super().define(spec) - spec.outline(while_(cls.predicate)) + spec.outline(while_(cls.predicate)(cls.run_step)) def predicate(self): """Invalid predicate whose return value is not a boolean.""" return 'true' - with pytest.raises(TypeError, match=r'The conditional predicate `predicate` did not return a boolean'): - launch.run(TestWorkChain) + def run_step(self): + # Need to return an exit code to abort the workchain, otherwise we would be stuck in an infinite loop + return ExitCode(1) + + launch.run(TestWorkChain) + assert len(recwarn) == 1 def test_malformed_outline(self): """