From b14fd615bd2c8e5e3ab97ef9075bbbd72e6751ec Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Wed, 2 Feb 2022 19:04:56 -0700 Subject: [PATCH 01/14] Add missing script header --- examples/scripts/deconv_tv_admm_tune.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/examples/scripts/deconv_tv_admm_tune.py b/examples/scripts/deconv_tv_admm_tune.py index 69ba01b10..0c36c19e7 100644 --- a/examples/scripts/deconv_tv_admm_tune.py +++ b/examples/scripts/deconv_tv_admm_tune.py @@ -1,3 +1,19 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# This file is part of the SCICO package. Details of the copyright +# and user license can be found in the 'LICENSE.txt' file distributed +# with the package. + +r""" +Image Deconvolution Parameter Tuning +==================================== + +This example demonstrates the use of +[scico.ray.tune](../_autosummary/scico.ray.tune.rst) to tune parameters +for the companion example script `deconv_tv_admm`. +""" + + import numpy as np import jax From d56893e84f0ef5fe8b7d7a4b4d866a49f93c1154 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Wed, 2 Feb 2022 19:09:15 -0700 Subject: [PATCH 02/14] Header docstring improvement --- examples/scripts/deconv_tv_admm_tune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/scripts/deconv_tv_admm_tune.py b/examples/scripts/deconv_tv_admm_tune.py index 0c36c19e7..aff321f14 100644 --- a/examples/scripts/deconv_tv_admm_tune.py +++ b/examples/scripts/deconv_tv_admm_tune.py @@ -10,7 +10,7 @@ This example demonstrates the use of [scico.ray.tune](../_autosummary/scico.ray.tune.rst) to tune parameters -for the companion example script `deconv_tv_admm`. +for the companion [example script](deconv_tv_admm.rst). """ From 50780d152d4351bce257926391ee024b088fba6f Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Wed, 2 Feb 2022 19:18:44 -0700 Subject: [PATCH 03/14] Make plotting compatible with automatic notebook generation --- examples/scripts/deconv_tv_admm_tune.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/scripts/deconv_tv_admm_tune.py b/examples/scripts/deconv_tv_admm_tune.py index aff321f14..3942c4c89 100644 --- a/examples/scripts/deconv_tv_admm_tune.py +++ b/examples/scripts/deconv_tv_admm_tune.py @@ -135,7 +135,7 @@ def eval_params(config): mec="blue", fig=fig, ) -_, ax = plot.plot( +plot.plot( best_config["lambda"], best_config["rho"], ptyp="loglog", @@ -149,6 +149,7 @@ def eval_params(config): mec="red", fig=fig, ) +ax = fig.axes[0] ax.set_xlim([config["rho"].lower, config["rho"].upper]) ax.set_ylim([config["lambda"].lower, config["lambda"].upper]) fig.show() From a9a4fbe348e4608aa76142970c3e4b76a20a3734 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Wed, 2 Feb 2022 21:07:02 -0700 Subject: [PATCH 04/14] Improve result record name --- scico/ray/tune.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scico/ray/tune.py b/scico/ray/tune.py index 7238e0f8f..6d450d4b9 100644 --- a/scico/ray/tune.py +++ b/scico/ray/tune.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright (C) 2021 by SCICO Developers +# Copyright (C) 2021-2022 by SCICO Developers # All rights reserved. BSD 3-clause License. # This file is part of the SCICO package. Details of the copyright and # user license can be found in the 'LICENSE' file distributed with the @@ -117,10 +117,17 @@ def run( def _run(config, checkpoint_dir=None): run_or_experiment(config) + if isinstance(run_or_experiment, str): + name = run_or_experiment + else: + name = run_or_experiment.__name__ + name += "_" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + return ray.tune.run( _run, metric=metric, mode=mode, + name=name, time_budget_s=time_budget_s, num_samples=num_samples, resources_per_trial=resources_per_trial, From 140f5b0e22c47931af7cc3907592fc1a1538b1f0 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Wed, 2 Feb 2022 21:39:03 -0700 Subject: [PATCH 05/14] Change default result directory --- scico/ray/tune.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scico/ray/tune.py b/scico/ray/tune.py index 6d450d4b9..f1c351c02 100644 --- a/scico/ray/tune.py +++ b/scico/ray/tune.py @@ -8,6 +8,7 @@ """Parameter tuning using :doc:`ray.tune `.""" import datetime +import os from typing import Any, Callable, Dict, List, Mapping, Optional, Type, Union import ray @@ -71,6 +72,7 @@ def run( config: Optional[Dict[str, Any]] = None, hyperopt: bool = True, verbose: bool = True, + local_dir: Optional[str] = None, ) -> ray.tune.ExperimentAnalysis: """Simplified wrapper for :func:`ray.tune.run`. @@ -97,6 +99,8 @@ def run( running, and terminated trials are indicated by "P:", "R:", and "T:" respectively, followed by the current best metric value and the parameters at which it was reported. + local_dir: Directory in which to save tuning results. Defaults to + "./ray_results". Returns: Result of parameter search. @@ -123,6 +127,9 @@ def _run(config, checkpoint_dir=None): name = run_or_experiment.__name__ name += "_" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + if local_dir is None: + local_dir = os.path.join(".", "ray_results") + return ray.tune.run( _run, metric=metric, @@ -130,6 +137,7 @@ def _run(config, checkpoint_dir=None): name=name, time_budget_s=time_budget_s, num_samples=num_samples, + local_dir=local_dir, resources_per_trial=resources_per_trial, max_concurrent_trials=max_concurrent_trials, reuse_actors=True, From 93255f27a8130d65c904bdf4adf3554df499e976 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 07:23:21 -0700 Subject: [PATCH 06/14] Change default result directory --- scico/ray/tune.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scico/ray/tune.py b/scico/ray/tune.py index f1c351c02..97b03cbfc 100644 --- a/scico/ray/tune.py +++ b/scico/ray/tune.py @@ -9,6 +9,7 @@ import datetime import os +import tempfile from typing import Any, Callable, Dict, List, Mapping, Optional, Type, Union import ray @@ -100,7 +101,9 @@ def run( and "T:" respectively, followed by the current best metric value and the parameters at which it was reported. local_dir: Directory in which to save tuning results. Defaults to - "./ray_results". + a subdirectory "ray_results" within the path returned by + `tempfile.gettempdir()`, corresponding e.g. to + "/tmp/ray_results" under Linux. Returns: Result of parameter search. @@ -128,7 +131,7 @@ def _run(config, checkpoint_dir=None): name += "_" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S") if local_dir is None: - local_dir = os.path.join(".", "ray_results") + local_dir = os.path.join(tempfile.gettempdir(), "ray_results") return ray.tune.run( _run, From 0845f647223861d3ce9b09742d6f09773aafd8d1 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 07:33:22 -0700 Subject: [PATCH 07/14] Add deconv_tv_admm_tune.py to examples index --- docs/source/examples.rst | 19 +++---------------- examples/scripts/README.rst | 6 ++++++ examples/scripts/index.rst | 3 +++ 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/docs/source/examples.rst b/docs/source/examples.rst index bee7a025a..82deea618 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -6,22 +6,6 @@ Usage Examples .. toctree:: :maxdepth: 1 -.. _example_dependencies: - -Example Dependencies --------------------- - -Some examples use additional dependencies, which are listed in `examples_requirements.txt `_. -The additional requirements should be installed via pip, with the exception of ``astra-toolbox``, -which should be installed via conda: - - :: - - conda install -c astra-toolbox astra-toolbox - pip install -r examples/examples_requirements.txt # Installs other example requirements - -The dependencies can also be installed individually as required. - Organized by Application ------------------------ @@ -57,6 +41,7 @@ Deconvolution examples/deconv_ppp_bm3d_pgm examples/deconv_ppp_dncnn_admm examples/deconv_tv_admm + examples/deconv_tv_admm_tune Sparse Coding @@ -117,6 +102,7 @@ Total Variation examples/deconv_microscopy_tv_admm examples/deconv_microscopy_allchn_tv_admm examples/deconv_tv_admm + examples/deconv_tv_admm_tune examples/denoise_tv_iso_admm examples/denoise_tv_iso_pgm examples/denoise_tv_iso_multi @@ -158,6 +144,7 @@ ADMM examples/deconv_ppp_bm3d_admm examples/deconv_ppp_dncnn_admm examples/deconv_tv_admm + examples/deconv_tv_admm_tune examples/demosaic_ppp_bm3d_admm examples/denoise_tv_iso_admm examples/denoise_tv_iso_multi diff --git a/examples/scripts/README.rst b/examples/scripts/README.rst index a7d95c288..9fc969338 100644 --- a/examples/scripts/README.rst +++ b/examples/scripts/README.rst @@ -40,6 +40,8 @@ Deconvolution Image Deconvolution (ADMM Plug-and-Play Priors w/ DnCNN) `deconv_tv_admm.py `_ Image Deconvolution (ADMM w/ Total Variation) + `deconv_tv_admm_tune.py `_ + Image Deconvolution Parameter Tuning Sparse Coding @@ -106,6 +108,8 @@ Total Variation Deconvolution Microscopy (All Channels) `deconv_tv_admm.py `_ Image Deconvolution (ADMM w/ Total Variation) + `deconv_tv_admm_tune.py `_ + Image Deconvolution Parameter Tuning `denoise_tv_iso_admm.py `_ Isotropic Total Variation (ADMM) `denoise_tv_iso_pgm.py `_ @@ -156,6 +160,8 @@ ADMM Image Deconvolution (ADMM Plug-and-Play Priors w/ DnCNN) `deconv_tv_admm.py `_ Image Deconvolution (ADMM w/ Total Variation) + `deconv_tv_admm_tune.py `_ + Image Deconvolution Parameter Tuning `demosaic_ppp_bm3d_admm.py `_ Image Demosaicing (ADMM Plug-and-Play Priors w/ BM3D) `denoise_tv_iso_admm.py `_ diff --git a/examples/scripts/index.rst b/examples/scripts/index.rst index 09ac78cce..f096f4800 100644 --- a/examples/scripts/index.rst +++ b/examples/scripts/index.rst @@ -27,6 +27,7 @@ Deconvolution - deconv_ppp_bm3d_pgm.py - deconv_ppp_dncnn_admm.py - deconv_tv_admm.py + - deconv_tv_admm_tune.py Sparse Coding @@ -72,6 +73,7 @@ Total Variation - deconv_microscopy_tv_admm.py - deconv_microscopy_allchn_tv_admm.py - deconv_tv_admm.py + - deconv_tv_admm_tune.py - denoise_tv_iso_admm.py - denoise_tv_iso_pgm.py - denoise_tv_iso_multi.py @@ -104,6 +106,7 @@ ADMM - deconv_ppp_bm3d_admm.py - deconv_ppp_dncnn_admm.py - deconv_tv_admm.py + - deconv_tv_admm_tune.py - demosaic_ppp_bm3d_admm.py - denoise_tv_iso_admm.py - denoise_tv_iso_multi.py From d64895b1c5ac7d986ad24c942f4bb449f31fd176 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 08:20:57 -0700 Subject: [PATCH 08/14] Clean up --- examples/scripts/deconv_tv_admm_tune.py | 4 ++-- scico/ray/tune.py | 5 +---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/examples/scripts/deconv_tv_admm_tune.py b/examples/scripts/deconv_tv_admm_tune.py index 3942c4c89..b1d3b55bb 100644 --- a/examples/scripts/deconv_tv_admm_tune.py +++ b/examples/scripts/deconv_tv_admm_tune.py @@ -59,7 +59,7 @@ """ -def eval_params(config): +def eval_params(config, reporter): # Extract solver parameters from config dict. λ, ρ = config["lambda"], config["rho"] # Get main arrays from ray object store. @@ -84,7 +84,7 @@ def eval_params(config): # Perform 50 iterations, reporting performance to ray.tune every 5 iterations. for step in range(10): x_admm = solver.solve() - tune.report(psnr=float(metric.psnr(x_gt, x_admm))) + reporter(psnr=float(metric.psnr(x_gt, x_admm))) """ diff --git a/scico/ray/tune.py b/scico/ray/tune.py index 97b03cbfc..a701bf7fd 100644 --- a/scico/ray/tune.py +++ b/scico/ray/tune.py @@ -121,9 +121,6 @@ def run( else: kwargs.update({"verbose": 0}) - def _run(config, checkpoint_dir=None): - run_or_experiment(config) - if isinstance(run_or_experiment, str): name = run_or_experiment else: @@ -134,7 +131,7 @@ def _run(config, checkpoint_dir=None): local_dir = os.path.join(tempfile.gettempdir(), "ray_results") return ray.tune.run( - _run, + run_or_experiment, metric=metric, mode=mode, name=name, From 3d72df87e516eba92228f69bebcfbaea9d68e547 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 08:21:29 -0700 Subject: [PATCH 09/14] Improve tests --- scico/test/test_ray_tune.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scico/test/test_ray_tune.py b/scico/test/test_ray_tune.py index 000a76648..9bfdf4969 100644 --- a/scico/test/test_ray_tune.py +++ b/scico/test/test_ray_tune.py @@ -1,3 +1,6 @@ +import os +import tempfile + import numpy as np import pytest @@ -11,11 +14,13 @@ pytest.skip("ray.tune not installed", allow_module_level=True) -def eval_params(config): +def eval_params(config, reporter): x, y = config["x"], config["y"] cost = x ** 2 + (y - 0.5) ** 2 - tune.report(cost=cost) + reporter(cost=cost) + +tune.ray.tune.register_trainable("eval_func", eval_params) config = {"x": tune.uniform(-1, 1), "y": tune.uniform(-1, 1)} resources = {"gpu": 0, "cpu": 1} @@ -24,7 +29,7 @@ def eval_params(config): @pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning") def test_random(): analysis = tune.run( - eval_params, + "eval_func", metric="cost", mode="min", num_samples=100, @@ -32,6 +37,7 @@ def test_random(): resources_per_trial=resources, hyperopt=False, verbose=False, + local_dir=os.path.join(tempfile.gettempdir(), "ray_test"), ) best_config = analysis.get_best_config(metric="cost", mode="min") assert np.abs(best_config["x"]) < 0.25 From 56761bb7b29b7a50eabbc8eeb1a39a995120f9b8 Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 16:33:53 -0700 Subject: [PATCH 10/14] Change tune configuration --- examples/scripts/deconv_tv_admm_tune.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/scripts/deconv_tv_admm_tune.py b/examples/scripts/deconv_tv_admm_tune.py index b1d3b55bb..18061c084 100644 --- a/examples/scripts/deconv_tv_admm_tune.py +++ b/examples/scripts/deconv_tv_admm_tune.py @@ -78,11 +78,11 @@ def eval_params(config, reporter): C_list=[C], rho_list=[ρ], x0=A.adj(y), - maxiter=5, + maxiter=10, subproblem_solver=LinearSubproblemSolver(), ) # Perform 50 iterations, reporting performance to ray.tune every 5 iterations. - for step in range(10): + for step in range(5): x_admm = solver.solve() reporter(psnr=float(metric.psnr(x_gt, x_admm))) @@ -91,7 +91,7 @@ def eval_params(config, reporter): Define parameter search space and resources per trial. """ config = {"lambda": tune.loguniform(1e-2, 1e0), "rho": tune.loguniform(1e-1, 1e1)} -resources = {"gpu": 0, "cpu": 1} # gpus per trial, cpus per trial +resources = {"cpu": 4, "gpu": 0} # cpus per trial, gpus per trial """ From 9eed66bf75e0ac75253d66b3679890a82aba76ce Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 16:34:20 -0700 Subject: [PATCH 11/14] Update submodule --- data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data b/data index 43df469cc..01165b7e7 160000 --- a/data +++ b/data @@ -1 +1 @@ -Subproject commit 43df469cccf54bc615438381a6c066d72e020fd6 +Subproject commit 01165b7e7d24c952586acd6f01a6e23d558ae5ee From b2aa9dc960b8d8cb524c37c15a6669e502215dfc Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 19:37:11 -0700 Subject: [PATCH 12/14] Include example depencies instructions, resolve #195 --- docs/source/exampledepend.rst | 15 +++++++++++++++ docs/source/examples.rst | 2 ++ examples/makeindex.py | 8 +++++++- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 docs/source/exampledepend.rst diff --git a/docs/source/exampledepend.rst b/docs/source/exampledepend.rst new file mode 100644 index 000000000..2528e0330 --- /dev/null +++ b/docs/source/exampledepend.rst @@ -0,0 +1,15 @@ +.. _example_dependencies: + +Example Dependencies +-------------------- + +Some examples use additional dependencies, which are listed in `examples_requirements.txt `_. +The additional requirements should be installed via pip, with the exception of ``astra-toolbox``, +which should be installed via conda: + + :: + + conda install -c astra-toolbox astra-toolbox + pip install -r examples/examples_requirements.txt # Installs other example requirements + +The dependencies can also be installed individually as required. diff --git a/docs/source/examples.rst b/docs/source/examples.rst index 82deea618..fae2d5bcd 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -6,6 +6,8 @@ Usage Examples .. toctree:: :maxdepth: 1 +.. include:: exampledepend.rst + Organized by Application ------------------------ diff --git a/examples/makeindex.py b/examples/makeindex.py index 944c23686..6e0894a7e 100755 --- a/examples/makeindex.py +++ b/examples/makeindex.py @@ -76,12 +76,18 @@ print(".. _example_notebooks:\n", file=dstfile) with open(src, "r") as srcfile: for line in srcfile: + # Add toctree and include statements after main heading + if line[0:3] == "===": + print(line, end="", file=dstfile) + print("\n.. toctree::\n :maxdepth: 1", file=dstfile) + print("\n.. include:: exampledepend.rst", file=dstfile) + continue # Detect lines containing script filenames m = re.match(r"(\s+)- ([^\s]+).py", line) if m: print(" " + prfx + m.group(2), file=dstfile) else: print(line, end="", file=dstfile) - # Add toctree statements after section headings + # Add toctree statement after section headings if line[0:3] == line[0] * 3 and line[0] in ["=", "-", "^"]: print("\n.. toctree::\n :maxdepth: 1", file=dstfile) From 0e9bf5b748228e1fdb64b430ffb4df841c9f1bfe Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 19:38:46 -0700 Subject: [PATCH 13/14] Correct comment --- examples/scripts/deconv_tv_admm_tune.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/scripts/deconv_tv_admm_tune.py b/examples/scripts/deconv_tv_admm_tune.py index 18061c084..99235f606 100644 --- a/examples/scripts/deconv_tv_admm_tune.py +++ b/examples/scripts/deconv_tv_admm_tune.py @@ -81,7 +81,7 @@ def eval_params(config, reporter): maxiter=10, subproblem_solver=LinearSubproblemSolver(), ) - # Perform 50 iterations, reporting performance to ray.tune every 5 iterations. + # Perform 50 iterations, reporting performance to ray.tune every 10 iterations. for step in range(5): x_admm = solver.solve() reporter(psnr=float(metric.psnr(x_gt, x_admm))) From 32dd02be4834c783e821b9e65a4239b5fbec1f8e Mon Sep 17 00:00:00 2001 From: Brendt Wohlberg Date: Thu, 3 Feb 2022 19:41:29 -0700 Subject: [PATCH 14/14] Update submodule --- data | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data b/data index 01165b7e7..a5550f049 160000 --- a/data +++ b/data @@ -1 +1 @@ -Subproject commit 01165b7e7d24c952586acd6f01a6e23d558ae5ee +Subproject commit a5550f04949e05486f9b4ab54114ae3a7f50edb7