From 766cb516da26cc0f4e30b66138d0db934bf64966 Mon Sep 17 00:00:00 2001 From: Juan Cabanela Date: Tue, 22 Aug 2023 14:43:03 -0500 Subject: [PATCH 1/4] Added a pydantic-validated Camera class and test. --- CHANGES.rst | 6 +- stellarphot/core.py | 128 ++++++++++++++++++++++++--------- stellarphot/tests/test_core.py | 24 +++++-- 3 files changed, 117 insertions(+), 41 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index daa8b79f..b14f0621 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -4,10 +4,10 @@ New Features ^^^^^^^^^^^^ + Development of new data classes for handling source list, photometry, and catalog data which include data format validation. [#125] -+ Aperture photometry streamlined into ``single_image_photometry`` ``multi_image_photometry`` functions that use the new data classes. [#141] -+ Photometry related notebooks updated to use new data classes. [#151] ++ Aperture photometry streamlined into ``single_image_photometry`` and ``multi_image_photometry`` functions that use the new data classes. [#141] + ``multi_image_photometry`` now is a wrapper for single_image_photometry instead of a completely separate function. -+ Logging has been implemented for photmetry, so all the output can now be logged to a file. [#150] ++ Photometry related notebooks updated to use new data classes and new functions. [#151] ++ Logging has been implemented for photometry, so all the output can now be logged to a file. [#150] Other Changes and Additions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/stellarphot/core.py b/stellarphot/core.py index 3f7e48ec..1796e4e9 100644 --- a/stellarphot/core.py +++ b/stellarphot/core.py @@ -2,6 +2,9 @@ from astropy.coordinates import EarthLocation, SkyCoord from astropy.table import Column, QTable, Table from astropy.time import Time +from astropy.units import Quantity + +from pydantic import BaseModel, root_validator import numpy as np @@ -9,7 +12,45 @@ 'SourceListData'] -class Camera: +# Approach to validation of units was inspired by the GammaPy project +# which did it before we did: +# https://docs.gammapy.org/dev/_modules/gammapy/analysis/config.html + +class QuantityType(Quantity): + # Validator for Quantity type + @classmethod + def __get_validators__(cls): + yield cls.validate + + @classmethod + def validate(cls, v): + try: + v = Quantity(v) + except TypeError: + raise ValueError(f"Invalid value for Quantity: {v}") + + return v + + +class PixelScaleType(Quantity): + # Validator for pixel scale type + @classmethod + def __get_validators__(cls): + yield cls.validate + + @classmethod + def validate(cls, v): + try: + v = Quantity(v) + except TypeError: + raise ValueError(f"Invalid value for Quantity: {v}") + if (len(v.unit.bases) != 2 or + v.unit.bases[0].physical_type != "angle" or + v.unit.bases[1].name != "pix" ): + raise ValueError(f"Invalid unit for pixel scale: {v.unit!r}") + return v + +class Camera(BaseModel): """ A class to represent a CCD-based camera. @@ -17,15 +58,15 @@ class Camera: ---------- gain : `astropy.quantity.Quantity` - The gain of the camera in units such that the unit of the product `gain` - times the image data matches the unit of the `read_noise`. + The gain of the camera in units such the product of `gain` + times the image data has units equal to that of the `read_noise`. read_noise : `astropy.quantity.Quantity` The read noise of the camera with units. dark_current : `astropy.quantity.Quantity` The dark current of the camera in units such that, when multiplied by - exposure time, the unit matches the unit of the `read_noise`. + exposure time, the unit matches the units of the `read_noise`. pixel_scale : `astropy.quantity.Quantity` The pixel scale of the camera in units of arcseconds per pixel. @@ -34,15 +75,15 @@ class Camera: ---------- gain : `astropy.quantity.Quantity` - The gain of the camera in units such that the unit of the product - `gain` times the image data matches the unit of the `read_noise`. + The gain of the camera in units such the product of `gain` + times the image data has units equal to that of the `read_noise`. read_noise : `astropy.quantity.Quantity` - The read noise of the camera in units of electrons. + The read noise of the camera with units. dark_current : `astropy.quantity.Quantity` - The dark current of the camera in units such that, when multiplied - by exposure time, the unit matches the unit of the `read_noise`. + The dark current of the camera in units such that, when multiplied by + exposure time, the unit matches the units of the `read_noise`. pixel_scale : `astropy.quantity.Quantity` The pixel scale of the camera in units of arcseconds per pixel. @@ -70,31 +111,50 @@ class Camera: """ - def __init__(self, gain=1.0 * u.electron / u.adu, - read_noise=1.0 * u.electron, - dark_current=0.01 * u.electron / u.second, - pixel_scale=1.0 * u.arcsec / u.pixel): - super().__init__() - - # Check that input has units and if not crap out. - if isinstance(gain, u.Quantity): - self.gain = gain - else: - raise TypeError("gain must have units.") - - if isinstance(read_noise, u.Quantity): - self.read_noise = read_noise - else: - raise TypeError("read_noise must have units.") - - if isinstance(dark_current, u.Quantity): - self.dark_current = dark_current - else: - raise TypeError("dark_current must have units.") - if isinstance(pixel_scale, u.Quantity): - self.pixel_scale = pixel_scale - else: - raise TypeError("pixel_scale must have units.") + gain: QuantityType + read_noise: QuantityType + dark_current: QuantityType + pixel_scale: PixelScaleType + + class Config: + validate_all = True + validate_assignment = True + extra = "forbid" + json_encoders = { + Quantity: lambda v: f"{v.value} {v.unit}", + QuantityType: lambda v: f"{v.value} {v.unit}", + PixelScaleType: lambda v: f"{v.value} {v.unit}" + } + + # When the switch to pydantic v2 happens, this root_validator will need + # to be replaced by a model_validator decorator. + @root_validator() + def validate_gain(cls, values): + # Get read noise units + try: + rn_unit = Quantity(values['read_noise']).unit + except KeyError: + raise ValueError("Valid keys for values are: ", values.keys()) + + # Check that gain and read noise have compatible units, that is that + # gain is read noise per adu. + gain = Quantity(values['gain']) + if (len(gain.unit.bases) != 2 or gain.unit.bases[0] != rn_unit or + gain.unit.bases[1] != u.adu): + raise ValueError(f"Gain units {gain.unit} are not compatible with " + f"read noise units {rn_unit}.") + + # Check that dark current and read noise have compatible units, that is + # that dark current is read noise per second. + dark_current = Quantity(values['dark_current']) + if (len(dark_current.unit.bases) != 2 or + dark_current.unit.bases[0] != rn_unit or + dark_current.unit.bases[1] != u.s): + raise ValueError(f"Dark current units {dark_current.unit} are not " + f"compatible with read noise units {rn_unit}.") + + # Dark current validates against read noise + return values def copy(self): return Camera(gain=self.gain, diff --git a/stellarphot/tests/test_core.py b/stellarphot/tests/test_core.py index 7637fd67..987d060d 100644 --- a/stellarphot/tests/test_core.py +++ b/stellarphot/tests/test_core.py @@ -32,28 +32,44 @@ def test_camera_unitscheck(): dark_current = 0.01 * u.electron / u.second pixel_scale = 0.563 * u.arcsec / u.pix - with pytest.raises(TypeError): + with pytest.raises(ValueError): c = Camera(gain=gain.value, read_noise=read_noise, dark_current=dark_current, pixel_scale=pixel_scale) - with pytest.raises(TypeError): + with pytest.raises(ValueError): c = Camera(gain=gain, read_noise=read_noise.value, dark_current=dark_current, pixel_scale=pixel_scale) - with pytest.raises(TypeError): + with pytest.raises(ValueError): c = Camera(gain=gain, read_noise=read_noise, dark_current=dark_current.value, pixel_scale=pixel_scale) - with pytest.raises(TypeError): + with pytest.raises(ValueError): c = Camera(gain=gain, read_noise=read_noise, dark_current=dark_current, pixel_scale=pixel_scale.value) +def test_camera_altunitscheck(): + # Check to see that 'count' is allowed instead of 'electron' + gain = 2.0 * u.count / u.adu + read_noise = 10 * u.count + dark_current = 0.01 * u.count / u.second + pixel_scale = 0.563 * u.arcsec / u.pix + c = Camera(gain=gain, + read_noise=read_noise, + dark_current=dark_current, + pixel_scale=pixel_scale) + assert c.gain == gain + assert c.dark_current == dark_current + assert c.read_noise == read_noise + assert c.pixel_scale == pixel_scale + + # Create several test descriptions for use in base_enhanced_table tests. test_descript = {'id': None, 'ra': u.deg, From fdc603780c3753a9e9ecd544dfd95af64b53d4f4 Mon Sep 17 00:00:00 2001 From: Juan Cabanela Date: Thu, 24 Aug 2023 21:55:01 -0500 Subject: [PATCH 2/4] Fixed error message to be clearer (was originally debugging statement). --- stellarphot/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stellarphot/core.py b/stellarphot/core.py index 1796e4e9..11fc23cf 100644 --- a/stellarphot/core.py +++ b/stellarphot/core.py @@ -134,7 +134,7 @@ def validate_gain(cls, values): try: rn_unit = Quantity(values['read_noise']).unit except KeyError: - raise ValueError("Valid keys for values are: ", values.keys()) + raise ValueError("read_noise must be specified including units.") # Check that gain and read noise have compatible units, that is that # gain is read noise per adu. From 45fe3d83747aff7f8939dedd5c42d60943e2ec1d Mon Sep 17 00:00:00 2001 From: Juan Cabanela Date: Mon, 28 Aug 2023 14:44:22 -0500 Subject: [PATCH 3/4] Responding to Matt's Comments on the PR. --- stellarphot/core.py | 17 ++++++++--------- stellarphot/tests/test_core.py | 10 +++++----- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/stellarphot/core.py b/stellarphot/core.py index 11fc23cf..6d9a7908 100644 --- a/stellarphot/core.py +++ b/stellarphot/core.py @@ -28,7 +28,9 @@ def validate(cls, v): v = Quantity(v) except TypeError: raise ValueError(f"Invalid value for Quantity: {v}") - + else: + if not v.unit.bases: + raise ValueError("Must provided a unit") return v @@ -128,17 +130,14 @@ class Config: # When the switch to pydantic v2 happens, this root_validator will need # to be replaced by a model_validator decorator. - @root_validator() + @root_validator(skip_on_failure=True) def validate_gain(cls, values): # Get read noise units - try: - rn_unit = Quantity(values['read_noise']).unit - except KeyError: - raise ValueError("read_noise must be specified including units.") + rn_unit = Quantity(values['read_noise']).unit # Check that gain and read noise have compatible units, that is that # gain is read noise per adu. - gain = Quantity(values['gain']) + gain = values['gain'] if (len(gain.unit.bases) != 2 or gain.unit.bases[0] != rn_unit or gain.unit.bases[1] != u.adu): raise ValueError(f"Gain units {gain.unit} are not compatible with " @@ -146,7 +145,7 @@ def validate_gain(cls, values): # Check that dark current and read noise have compatible units, that is # that dark current is read noise per second. - dark_current = Quantity(values['dark_current']) + dark_current = values['dark_current'] if (len(dark_current.unit.bases) != 2 or dark_current.unit.bases[0] != rn_unit or dark_current.unit.bases[1] != u.s): @@ -867,5 +866,5 @@ def drop_x_y(self): self.meta['has_x_y'] = False self['xcenter'] = Column(data=np.full(len(self), np.nan), name='ra', unit=u.deg) - self['ycenter'] = Column(data=np.full(len(self), np.nan), name='dec', + self['ycenter'] = Column(data=np.full(len(self), np.nan), name='dec', unit=u.deg) diff --git a/stellarphot/tests/test_core.py b/stellarphot/tests/test_core.py index 987d060d..2b8a7f39 100644 --- a/stellarphot/tests/test_core.py +++ b/stellarphot/tests/test_core.py @@ -6,7 +6,7 @@ from astropy.io import ascii from astropy.coordinates import EarthLocation from astropy.utils.data import get_pkg_data_filename - +from pydantic import ValidationError from stellarphot.core import (Camera, BaseEnhancedTable, PhotometryData, CatalogData, SourceListData) @@ -32,22 +32,22 @@ def test_camera_unitscheck(): dark_current = 0.01 * u.electron / u.second pixel_scale = 0.563 * u.arcsec / u.pix - with pytest.raises(ValueError): + with pytest.raises(ValidationError, match="gain"): c = Camera(gain=gain.value, read_noise=read_noise, dark_current=dark_current, pixel_scale=pixel_scale) - with pytest.raises(ValueError): + with pytest.raises(ValidationError, match="read_noise"): c = Camera(gain=gain, read_noise=read_noise.value, dark_current=dark_current, pixel_scale=pixel_scale) - with pytest.raises(ValueError): + with pytest.raises(ValidationError, match="dark_current"): c = Camera(gain=gain, read_noise=read_noise, dark_current=dark_current.value, pixel_scale=pixel_scale) - with pytest.raises(ValueError): + with pytest.raises(ValidationError, match="pixel_scale"): c = Camera(gain=gain, read_noise=read_noise, dark_current=dark_current, From 74f3439b1f968aeff5de4e2f3ddb086700371943 Mon Sep 17 00:00:00 2001 From: Juan Cabanela Date: Mon, 28 Aug 2023 14:46:09 -0500 Subject: [PATCH 4/4] Added class method decorator to @root_validator in Camera --- stellarphot/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stellarphot/core.py b/stellarphot/core.py index 6d9a7908..7c568b9d 100644 --- a/stellarphot/core.py +++ b/stellarphot/core.py @@ -131,6 +131,7 @@ class Config: # When the switch to pydantic v2 happens, this root_validator will need # to be replaced by a model_validator decorator. @root_validator(skip_on_failure=True) + @classmethod def validate_gain(cls, values): # Get read noise units rn_unit = Quantity(values['read_noise']).unit