From 525e7ade2725f9ed2ee6b6cc92c21d75528388bd Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Wed, 1 Mar 2023 11:50:46 -0500 Subject: [PATCH 1/9] io: replace 'pickle' with 'dill' --- frank/io.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frank/io.py b/frank/io.py index ab65b837..8fe7357e 100644 --- a/frank/io.py +++ b/frank/io.py @@ -22,7 +22,7 @@ import os import numpy as np -import pickle +import dill import logging def load_uvtable(data_file): @@ -131,7 +131,8 @@ def load_sol(sol_file): (see frank.radial_fitters.FrankFitter) """ - sol = np.load(sol_file, allow_pickle=True) + with open(sol_file, 'rb') as f: + sol = dill.load(f) return sol @@ -185,11 +186,11 @@ def save_fit(u, v, vis, weights, sol, prefix, save_solution=True, if save_solution: with open(prefix + '_frank_sol.obj', 'wb') as f: - pickle.dump(sol, f) + dill.dump(sol, f) if save_iteration_diag: with open(prefix + '_frank_iteration_diagnostics.obj', 'wb') as f: - pickle.dump(iteration_diag, f) + dill.dump(iteration_diag, f) if save_profile_fit: np.savetxt(prefix + '_frank_profile_fit.txt', From 89b3dfe82e1a3d492bad062dbda5a81a3de938c4 Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Wed, 1 Mar 2023 12:33:07 -0500 Subject: [PATCH 2/9] add sol save/load test for standard and debris FF --- frank/tests.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/frank/tests.py b/frank/tests.py index 0a98a94d..b075e862 100644 --- a/frank/tests.py +++ b/frank/tests.py @@ -25,12 +25,13 @@ from frank.constants import rad_to_arcsec from frank.hankel import DiscreteHankelTransform from frank.radial_fitters import FourierBesselFitter, FrankFitter +from frank.debris_fitters import FrankDebrisFitter from frank.geometry import ( FixedGeometry, FitGeometryGaussian, FitGeometryFourierBessel ) from frank.constants import deg_to_rad from frank.utilities import UVDataBinner, generic_dht -from frank.io import load_uvtable, save_uvtable +from frank.io import load_uvtable, save_uvtable, load_sol, save_fit from frank.statistical_models import VisibilityMapping from frank import fit @@ -157,7 +158,6 @@ def load_AS209(uv_cut=None): return uv_AS209_DSHARP, geometry - def test_fit_geometry(): """Check the geometry fit on a subset of the AS209 data""" AS209, _ = load_AS209() @@ -426,6 +426,35 @@ def test_uvbin(): np.testing.assert_allclose(w, uvbin.weights[i]) np.testing.assert_allclose(len(widx), uvbin.bin_counts[i]) +def test_save_load_sol(): + """Check saving/loading a frank 'sol' object""" + AS209, AS209_geometry = load_AS209(uv_cut=1e6) + u, v, vis, weights = [AS209[k][::100] for k in ['u', 'v', 'V', 'weights']] + Rmax, N = 1.6, 20 + + # generate a sol from a standard frank fit + FF = FrankFitter(Rmax, N, AS209_geometry, alpha=1.05, weights_smooth=1e-2) + sol = FF.fit(u, v, vis, weights) + + # and from a frank debris fit (has additional keys over a standard fit sol) + FF_deb = FrankDebrisFitter(Rmax, N, AS209_geometry, lambda x : 0.05 * x, + alpha=1.05, weights_smooth=1e-2) + sol_deb = FF_deb.fit(u, v, vis, weights) + + tmp_dir = '/tmp/frank/tests' + os.makedirs(tmp_dir, exist_ok=True) + + save_prefix = [os.path.join(tmp_dir, 'standard'), os.path.join(tmp_dir, 'debris')] + sols = [sol, sol_deb] + + for ii, jj in enumerate(save_prefix): + # save the 'sol' object + save_fit(u, v, vis, weights, sols[ii], prefix=jj, + save_profile_fit=False, save_vis_fit=False, save_uvtables=False + ) + # load it + load_sol(jj + '_frank_sol.obj') + def _run_pipeline(geometry='gaussian', fit_phase_offset=True, fit_inc_pa=True, make_figs=False, From 0738f1cba07ecd6c814df06bde6bb4cbf5eea250 Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Wed, 1 Mar 2023 12:44:44 -0500 Subject: [PATCH 3/9] add dill to dependencies --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 75402faf..1c353efb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,7 +40,8 @@ install_requires = numpy>=1.12 matplotlib>=3.1.0 scipy>=1.2.0 - + dill>=0.3.4 + # additional files to include in install [options.package_data] * = default_parameters.json, parameter_descriptions.json, frank.mplstyle From e7fbc2bb524b6ace9a8b4c5c3c9e06a44387709b Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Wed, 1 Mar 2023 12:55:34 -0500 Subject: [PATCH 4/9] change self._scale_height in VisMap to array --- frank/statistical_models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frank/statistical_models.py b/frank/statistical_models.py index e2b98628..981625c7 100644 --- a/frank/statistical_models.py +++ b/frank/statistical_models.py @@ -102,8 +102,8 @@ def __init__(self, DHT, geometry, if scale_height is None: raise ValueError('You requested a model with a non-zero scale height' ' but did not specify H(R) (scale_height=None)') - self._scale_height = scale_height - self._H2 = 0.5*(2*np.pi*scale_height(self.r) / rad_to_arcsec)**2 + self._scale_height = scale_height(self.r) + self._H2 = 0.5*(2*np.pi*self._scale_height / rad_to_arcsec)**2 if self._verbose: logging.info(' Assuming an optically thin model but geometrically: ' @@ -278,7 +278,7 @@ def check_hash(self, hash, multi_freq=False, geometry=None): if hash[4] is None: return False else: - return np.alltrue(self._scale_height(self.r) == hash[4](self.r)) + return np.alltrue(self._scale_height == hash[4]) def predict_visibilities(self, I, q, k=None, geometry=None): @@ -500,7 +500,7 @@ def size(self): def scale_height(self): "Vertial thickness of the disc, unit = arcsec" if self._scale_height is not None: - return self._scale_height(self.r) + return self._scale_height else: return None From d39a6b40714bb90acb2babeae75baed8ad051e91 Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Wed, 1 Mar 2023 12:57:55 -0500 Subject: [PATCH 5/9] revert dill to pickle in io.py --- frank/io.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frank/io.py b/frank/io.py index 8fe7357e..320d7572 100644 --- a/frank/io.py +++ b/frank/io.py @@ -22,7 +22,7 @@ import os import numpy as np -import dill +import pickle import logging def load_uvtable(data_file): @@ -132,7 +132,7 @@ def load_sol(sol_file): """ with open(sol_file, 'rb') as f: - sol = dill.load(f) + sol = pickle.load(f) return sol @@ -186,11 +186,11 @@ def save_fit(u, v, vis, weights, sol, prefix, save_solution=True, if save_solution: with open(prefix + '_frank_sol.obj', 'wb') as f: - dill.dump(sol, f) + pickle.dump(sol, f) if save_iteration_diag: with open(prefix + '_frank_iteration_diagnostics.obj', 'wb') as f: - dill.dump(iteration_diag, f) + pickle.dump(iteration_diag, f) if save_profile_fit: np.savetxt(prefix + '_frank_profile_fit.txt', From e897fec0d5c77238936b7a15a9a9d4a92ea7a073 Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Wed, 1 Mar 2023 12:58:02 -0500 Subject: [PATCH 6/9] remove dill dep --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 1c353efb..75402faf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -40,8 +40,7 @@ install_requires = numpy>=1.12 matplotlib>=3.1.0 scipy>=1.2.0 - dill>=0.3.4 - + # additional files to include in install [options.package_data] * = default_parameters.json, parameter_descriptions.json, frank.mplstyle From 896be9a62354e6b356c410ea261f7ca66a43baee Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Wed, 1 Mar 2023 23:08:38 -0500 Subject: [PATCH 7/9] remove sol save/load test --- frank/tests.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/frank/tests.py b/frank/tests.py index b075e862..309904a1 100644 --- a/frank/tests.py +++ b/frank/tests.py @@ -426,35 +426,6 @@ def test_uvbin(): np.testing.assert_allclose(w, uvbin.weights[i]) np.testing.assert_allclose(len(widx), uvbin.bin_counts[i]) -def test_save_load_sol(): - """Check saving/loading a frank 'sol' object""" - AS209, AS209_geometry = load_AS209(uv_cut=1e6) - u, v, vis, weights = [AS209[k][::100] for k in ['u', 'v', 'V', 'weights']] - Rmax, N = 1.6, 20 - - # generate a sol from a standard frank fit - FF = FrankFitter(Rmax, N, AS209_geometry, alpha=1.05, weights_smooth=1e-2) - sol = FF.fit(u, v, vis, weights) - - # and from a frank debris fit (has additional keys over a standard fit sol) - FF_deb = FrankDebrisFitter(Rmax, N, AS209_geometry, lambda x : 0.05 * x, - alpha=1.05, weights_smooth=1e-2) - sol_deb = FF_deb.fit(u, v, vis, weights) - - tmp_dir = '/tmp/frank/tests' - os.makedirs(tmp_dir, exist_ok=True) - - save_prefix = [os.path.join(tmp_dir, 'standard'), os.path.join(tmp_dir, 'debris')] - sols = [sol, sol_deb] - - for ii, jj in enumerate(save_prefix): - # save the 'sol' object - save_fit(u, v, vis, weights, sols[ii], prefix=jj, - save_profile_fit=False, save_vis_fit=False, save_uvtables=False - ) - # load it - load_sol(jj + '_frank_sol.obj') - def _run_pipeline(geometry='gaussian', fit_phase_offset=True, fit_inc_pa=True, make_figs=False, From 4ceb2a863b077bf96b6bd9d0c5dff0528b094d98 Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Wed, 1 Mar 2023 23:10:01 -0500 Subject: [PATCH 8/9] remove unused test imports --- frank/tests.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/frank/tests.py b/frank/tests.py index 309904a1..8b00c068 100644 --- a/frank/tests.py +++ b/frank/tests.py @@ -25,13 +25,11 @@ from frank.constants import rad_to_arcsec from frank.hankel import DiscreteHankelTransform from frank.radial_fitters import FourierBesselFitter, FrankFitter -from frank.debris_fitters import FrankDebrisFitter from frank.geometry import ( FixedGeometry, FitGeometryGaussian, FitGeometryFourierBessel ) -from frank.constants import deg_to_rad from frank.utilities import UVDataBinner, generic_dht -from frank.io import load_uvtable, save_uvtable, load_sol, save_fit +from frank.io import load_uvtable, save_uvtable from frank.statistical_models import VisibilityMapping from frank import fit From 3e79f22506787f7ec04cf5b16e0ae884ada3a37c Mon Sep 17 00:00:00 2001 From: Jeff Jennings Date: Thu, 2 Mar 2023 10:20:40 -0500 Subject: [PATCH 9/9] restore io test --- frank/tests.py | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/frank/tests.py b/frank/tests.py index 8b00c068..a9091cc7 100644 --- a/frank/tests.py +++ b/frank/tests.py @@ -25,11 +25,12 @@ from frank.constants import rad_to_arcsec from frank.hankel import DiscreteHankelTransform from frank.radial_fitters import FourierBesselFitter, FrankFitter +from frank.debris_fitters import FrankDebrisFitter from frank.geometry import ( FixedGeometry, FitGeometryGaussian, FitGeometryFourierBessel ) from frank.utilities import UVDataBinner, generic_dht -from frank.io import load_uvtable, save_uvtable +from frank.io import load_uvtable, save_uvtable, load_sol, save_fit from frank.statistical_models import VisibilityMapping from frank import fit @@ -424,6 +425,35 @@ def test_uvbin(): np.testing.assert_allclose(w, uvbin.weights[i]) np.testing.assert_allclose(len(widx), uvbin.bin_counts[i]) +def test_save_load_sol(): + """Check saving/loading a frank 'sol' object""" + AS209, AS209_geometry = load_AS209(uv_cut=1e6) + u, v, vis, weights = [AS209[k][::100] for k in ['u', 'v', 'V', 'weights']] + Rmax, N = 1.6, 20 + + # generate a sol from a standard frank fit + FF = FrankFitter(Rmax, N, AS209_geometry, alpha=1.05, weights_smooth=1e-2) + sol = FF.fit(u, v, vis, weights) + + # and from a frank debris fit (has additional keys over a standard fit sol) + FF_deb = FrankDebrisFitter(Rmax, N, AS209_geometry, lambda x : 0.05 * x, + alpha=1.05, weights_smooth=1e-2) + sol_deb = FF_deb.fit(u, v, vis, weights) + + tmp_dir = '/tmp/frank/tests' + os.makedirs(tmp_dir, exist_ok=True) + + save_prefix = [os.path.join(tmp_dir, 'standard'), os.path.join(tmp_dir, 'debris')] + sols = [sol, sol_deb] + + for ii, jj in enumerate(save_prefix): + # save the 'sol' object + save_fit(u, v, vis, weights, sols[ii], prefix=jj, + save_profile_fit=False, save_vis_fit=False, save_uvtables=False + ) + # load it + load_sol(jj + '_frank_sol.obj') + def _run_pipeline(geometry='gaussian', fit_phase_offset=True, fit_inc_pa=True, make_figs=False,