Temporary directory autoremoval (#148)

This PR adds the option `keep_tmpdirs` to the `workflow` block. If set to `False`, then the workflow, once finished, will delete all the `outdirs` from the calculations it performed. `keep_tmpdirs` is `True` by default.

This PR also changes the default value of `from_scratch` from `False` to `True`, because I think this is better choice for the default.

Unrelatedly, this PR also moves some old test routines from `koopmans/` into `tests/`, which means they are now run as part of the test suite.

codecov.yml

@@ -11,3 +11,6 @@ coverage:
     project: off
     patch: off
     changes: off
+
+ignore:
+  - "koopmans/testing"

koopmans/settings/_workflow.py

@@ -85,7 +85,10 @@ def __init__(self, **kwargs) -> None:
             Setting('from_scratch',
                     'if True, will delete any preexisting workflow and start again; '
                     'if False, will resume a workflow from where it was last up to',
-                    bool, False, (True, False)),
+                    bool, True, (True, False)),
+            Setting('keep_tmpdirs',
+                    'If False, delete all of the temporary directories at the end of the calculation',
+                    bool, True, (True, False)),
             Setting('orbital_groups',
                     'a list of integers the same length as the total number of bands, '
                     'denoting which bands to assign the same screening parameter to',

koopmans/utils/_os.py

@@ -72,7 +72,7 @@ def chdir(path: Union[Path, str]):
     # context will be executed in the directory "path"
 
     # Ensure path is a Path object
-    if isinstance(path, str):
+    if not isinstance(path, Path):
         path = Path(path)
 
     this_dir = Path.cwd()

koopmans/workflows/_workflow.py

@@ -9,6 +9,7 @@
 
 from abc import ABC, abstractmethod
 import os
+import shutil
 import copy
 import operator
 from functools import reduce
@@ -112,7 +113,8 @@ def __init__(self, atoms: Atoms,
         else:
             if self.parameters.pseudo_library is None:
                 utils.warn(
-                    'Neither a pseudopotential library nor a list of pseudopotentials was provided; defaulting to sg15_v1.2')
+                    'Neither a pseudopotential library nor a list of pseudopotentials was provided; defaulting to '
+                    'sg15_v1.2')
                 self.parameters.pseudo_library = 'sg15_v1.2'
             self.pseudopotentials = {}
             for symbol, tag in set([(a.symbol, a.tag) for a in self.atoms]):
@@ -258,6 +260,8 @@ def run(self) -> None:
             self._run_sanity_checks()
         self._run()
         self.print_conclusion()
+        if not self._is_a_subworkflow:
+            self._teardown()
 
     @abstractmethod
     def _run(self) -> None:
@@ -1041,8 +1045,10 @@ def toinputjson(self) -> Dict[str, Dict]:
                 projections = self.projections.get_subset(filling, spin)
                 if len(projections) > 1:
                     proj_kwarg = {'projections_blocks': [p.projections for p in projections]}
-                else:
+                elif len(projections) == 1:
                     proj_kwarg = {'projections': projections[0].projections}
+                else:
+                    proj_kwarg = {}
                 reduce(operator.getitem, nested_keys[:-1], bigdct['w90'])[k].update(**proj_kwarg)
             else:
                 raise NotImplementedError(
@@ -1161,6 +1167,18 @@ def plot_bandstructure(self,
         legends = [ax.get_legend() for ax in axes if ax.get_legend() is not None]
         utils.savefig(fname=filename + '.png', bbox_extra_artists=legends, bbox_inches='tight')
 
+    def _teardown(self):
+        '''
+        Performs final tasks before the workflow completes
+        '''
+
+        # Removing tmpdirs
+        if not self.parameters.keep_tmpdirs:
+            all_outdirs = [calc.parameters.get('outdir', None) for calc in self.calculations]
+            outdirs = set([o.resolve() for o in all_outdirs if o is not None and o.resolve().exists()])
+            for outdir in outdirs:
+                shutil.rmtree(outdir)
+
 
 def get_version(module):
     if isinstance(module, ModuleType):

koopmans/qei_to_json/example.cpi → tests/data/qei_to_json/example.cpi

@@ -6,7 +6,7 @@
    outdir           = 'TMP-CP/'
    prefix           = 'kc'
    disk_io          = 'high'
-   pseudo_dir       = '../../pseudos/sg15_v1.0/pbe/'
+   pseudo_dir       = '../../../pseudos/sg15_v1.0/pbe/'
    ndr              = 99
    ndw              = 50
    write_hr         = .false.

koopmans/qei_to_json/example.pwi → tests/data/qei_to_json/example.pwi

@@ -2,7 +2,7 @@
    calculation      = 'scf'
    outdir           = 'TMP/'
    prefix           = 'kc'
-   pseudo_dir       = '../../pseudos/sg15_v1.2/pbe/'
+   pseudo_dir       = '../../../pseudos/sg15_v1.2/pbe/'
 /
 &SYSTEM
    ibrav            = 2

tests/io/test_json.py

@@ -0,0 +1,19 @@
+'''
+
+Tests for koopmans.io
+
+'''
+
+from ase.build import molecule
+from koopmans import utils
+from koopmans.workflows import SinglepointWorkflow
+from koopmans.io import write, read
+
+
+def test_write_then_read_json(tmpdir):
+    with utils.chdir(tmpdir):
+        atoms = molecule('H2O', vacuum=10, tags=['0', '1', '1'])
+        workflow_out = SinglepointWorkflow(atoms, parameters={'pseudo_library': 'sg15'}, name='test')
+        write(workflow_out, 'test.json')
+        workflow_in = read('test.json')
+        assert workflow_out == workflow_in

tests/test_qei_to_json.py

@@ -0,0 +1,13 @@
+from pathlib import Path
+from koopmans import utils
+from koopmans.qei_to_json import qei_to_json
+
+
+def test_pwi_to_json(tmpdir, datadir):
+    with utils.chdir(tmpdir):
+        qei_to_json(datadir / 'qei_to_json' / 'example.pwi', 'example.json')
+
+
+def test_cpi_to_json(tmpdir, datadir):
+    with utils.chdir(tmpdir):
+        qei_to_json(datadir / 'qei_to_json' / 'example.cpi', 'example.json')

tests/workflows/test_convergence.py

@@ -7,7 +7,7 @@ def test_convergence_h2o(water, workflow_patch, tmp_path, sys2file):
         parameters = {
             "functional": "dft",
             "task": "convergence",
-            "from_scratch": True,
+            "keep_tmpdirs": False,
             "convergence_observable": "homo energy",
             "convergence_threshold": "0.1 eV",
             "convergence_parameters": ["ecutwfc", "cell_size"]}

tests/workflows/test_dft.py

@@ -5,6 +5,6 @@
 def test_pwbandstructure_si(silicon, workflow_patch, tmp_path, sys2file):
     with utils.chdir(tmp_path):
         wf = workflows.PWBandStructureWorkflow(
-            parameters={'pseudo_library': 'pseudo_dojo_standard', 'base_functional': 'pbesol', 'from_scratch': True},
+            parameters={'pseudo_library': 'pseudo_dojo_standard', 'base_functional': 'pbesol', 'keep_tmpdirs': False},
             name='si', **silicon)
         wf.run()

tests/workflows/test_singlepoint.py

@@ -13,7 +13,7 @@ def test_singlepoint_h2o_ki_dscf_explicit(water, espresso_patch, tmp_path, sys2f
     with chdir(tmp_path):
         parameters = {'functional': 'ki',
                       'n_max_sc_steps': 1,
-                      'from_scratch': True,
+                      'keep_tmpdirs': False,
                       'orbital_groups_self_hartree_tol': 100.0
                       }
         wf = workflows.SinglepointWorkflow(parameters=parameters, **water)
@@ -24,7 +24,7 @@ def test_singlepoint_h2o_all_dscf(water, workflow_patch, tmp_path, sys2file):
     with chdir(tmp_path):
         parameters = {'functional': 'all',
                       'n_max_sc_steps': 2,
-                      'from_scratch': True,
+                      'keep_tmpdirs': False,
                       'orbital_groups_self_hartree_tol': 100.0
                       }
         wf = workflows.SinglepointWorkflow(parameters=parameters, **water)
@@ -35,7 +35,7 @@ def test_singlepoint_si_ki_dscf(silicon, workflow_patch, tmp_path, sys2file):
     with chdir(tmp_path):
         parameters = {'functional': 'ki',
                       'method': 'dscf',
-                      'from_scratch': True,
+                      'keep_tmpdirs': False,
                       'mp_correction': True,
                       'eps_inf': 13.02,
                       'init_orbitals': 'mlwfs',
@@ -53,7 +53,7 @@ def test_singlepoint_si_ki_dfpt_explicit(silicon, espresso_patch, tmp_path, sys2
 
         parameters = {'functional': 'ki',
                       'method': 'dfpt',
-                      'from_scratch': True,
+                      'keep_tmpdirs': False,
                       'eps_inf': 13.02,
                       'init_orbitals': 'mlwfs',
                       'alpha_guess': 0.077,
@@ -68,7 +68,7 @@ def test_singlepoint_si_ki_dfpt(silicon, workflow_patch, tmp_path, sys2file):
 
         parameters = {'functional': 'ki',
                       'method': 'dfpt',
-                      'from_scratch': True,
+                      'keep_tmpdirs': False,
                       'eps_inf': 13.02,
                       'init_orbitals': 'mlwfs',
                       'alpha_guess': 0.077,
@@ -82,7 +82,7 @@ def test_singlepoint_ozone_ki_dfpt(ozone, workflow_patch, tmp_path, sys2file):
     with chdir(tmp_path):
         parameters = {'functional': 'ki',
                       'method': 'dfpt',
-                      'from_scratch': True,
+                      'keep_tmpdirs': False,
                       'init_orbitals': 'kohn-sham',
                       'npool': 1,
                       'orbital_groups_self_hartree_tol': 100.0,
@@ -97,7 +97,7 @@ def test_singlepoint_gaas_wan2odd(gaas, espresso_patch, tmp_path, sys2file):
     with chdir(tmp_path):
         parameters = {'functional': 'ki',
                       'method': 'dscf',
-                      'from_scratch': True,
+                      'keep_tmpdirs': False,
                       'calculate_alpha': False,
                       'init_orbitals': 'mlwfs',
                       'npool': 1,

tests/workflows/test_wannierize.py

@@ -11,6 +11,7 @@ def test_wannierize_tio2(tio2, tmp_path, sys2file, workflow_patch):
         parameters = {
             "init_orbitals": "mlwfs",
             "init_empty_orbitals": "projwfs",
+            "keep_tmpdirs": False,
             "pseudo_library": "pseudo_dojo_standard"}
         wf = workflows.WannierizeWorkflow(parameters=parameters, kgrid=[2, 2, 2], kpath='GXG', **tio2)
         wf.run()