From 9956922f6c3c028d8f87bcfb7d920242766209fa Mon Sep 17 00:00:00 2001 From: Wei Ji Date: Sat, 30 May 2020 12:04:56 +1200 Subject: [PATCH] :white_check_mark: Properly tested deltatime_to_utctime function Turn the ICESat-2 delta_time to utc_time conversion code in our jupyter notebook into a well tested function! The cool bit is that we can pass in either a dask or numpy backed xarray.DataArray, and get the equivalent output, with dimensions and coordinates preserved! Gotta love [NEP18](https://numpy.org/neps/nep-0018-array-function-protocol.html). Added a chunks statement to test_catalog.yaml, and ensure the file is cached in a relative path. Had to make sure to read the test atl11_dataset using dask and close it properly after each test (?) or subsequent tests will fail, seeing a numpy.array instead of a dask.array (?). Should do proper setup/teardown next time. Also bumping up cftime from 1.1.1.2 to 1.1.3 and certifi from 2019.11.28 to 2020.4.5.1 to bust the CI cache, just in case. --- atl11_play.ipynb | 18 ++----- atl11_play.py | 8 +--- deepicedrain/__init__.py | 2 +- deepicedrain/spatiotemporal.py | 17 +++++++ deepicedrain/tests/test_calculate_delta.py | 14 ++++-- deepicedrain/tests/test_catalog.yaml | 6 ++- .../tests/test_spatiotemporal_conversions.py | 47 +++++++++++++++++++ poetry.lock | 40 ++++++++-------- 8 files changed, 103 insertions(+), 49 deletions(-) create mode 100644 deepicedrain/tests/test_spatiotemporal_conversions.py diff --git a/atl11_play.ipynb b/atl11_play.ipynb index 329e62d..8769a21 100644 --- a/atl11_play.ipynb +++ b/atl11_play.ipynb @@ -20,7 +20,6 @@ "import glob\n", "\n", "import deepicedrain\n", - "import pointCollection.is2_calendar\n", "\n", "import dask\n", "import dask.array\n", @@ -214,18 +213,7 @@ "metadata": {}, "outputs": [], "source": [ - "ICESAT2_EPOCH = np.datetime64(pointCollection.is2_calendar.t_0())\n", - "# ICESAT2_EPOCH = np.datetime64(datetime.datetime(2018, 1, 1, 0, 0, 0))" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "utc_time = dask.array.asarray(ICESAT2_EPOCH) + ds.delta_time.data\n", - "ds[\"utc_time\"] = xr.DataArray(data=utc_time, coords=ds.delta_time.coords)" + "ds[\"utc_time\"] = deepicedrain.deltatime_to_utctime(dataarray=ds.delta_time)" ] }, { @@ -284,7 +272,9 @@ " ymin=-699564.516934089,\n", " ymax=-365489.6822096751,\n", " ),\n", - " \"antarctica\": deepicedrain.Region(\"Antarctica\", -2700000, 2800000, -2200000, 2300000),\n", + " \"antarctica\": deepicedrain.Region(\n", + " \"Antarctica\", -2700000, 2800000, -2200000, 2300000\n", + " ),\n", " \"siple_coast\": deepicedrain.Region(\n", " \"Siple Coast\", -1000000, 250000, -1000000, -100000\n", " ),\n", diff --git a/atl11_play.py b/atl11_play.py index 13717c5..ebe7035 100644 --- a/atl11_play.py +++ b/atl11_play.py @@ -24,7 +24,6 @@ import glob import deepicedrain -import pointCollection.is2_calendar import dask import dask.array @@ -118,12 +117,7 @@ # in the future. # %% -ICESAT2_EPOCH = np.datetime64(pointCollection.is2_calendar.t_0()) -# ICESAT2_EPOCH = np.datetime64(datetime.datetime(2018, 1, 1, 0, 0, 0)) - -# %% -utc_time = dask.array.asarray(ICESAT2_EPOCH) + ds.delta_time.data -ds["utc_time"] = xr.DataArray(data=utc_time, coords=ds.delta_time.coords) +ds["utc_time"] = deepicedrain.deltatime_to_utctime(dataarray=ds.delta_time) # %% [markdown] # ## Mask out low quality height data diff --git a/deepicedrain/__init__.py b/deepicedrain/__init__.py index 8711e68..c23e436 100644 --- a/deepicedrain/__init__.py +++ b/deepicedrain/__init__.py @@ -5,7 +5,7 @@ import deepicedrain from deepicedrain.deltamath import calculate_delta -from deepicedrain.spatiotemporal import Region +from deepicedrain.spatiotemporal import Region, deltatime_to_utctime __version__: str = "0.1.0" diff --git a/deepicedrain/spatiotemporal.py b/deepicedrain/spatiotemporal.py index 2aa0fe9..3bafaf2 100644 --- a/deepicedrain/spatiotemporal.py +++ b/deepicedrain/spatiotemporal.py @@ -3,6 +3,7 @@ Does bounding box region subsets, coordinate/time conversions, and more! """ import dataclasses +import datetime import numpy as np import xarray as xr @@ -55,3 +56,19 @@ def subset( ) return ds.where(cond=cond, drop=drop) + + +def deltatime_to_utctime( + dataarray: xr.DataArray, + start_epoch: np.datetime64 = np.datetime64("2018-01-01T00:00:00.000000"), +) -> xr.DataArray: + """ + Converts GPS time in nanoseconds from an epoch (default is 2018 Jan 1st) + to Coordinated Universal Time (UTC). + + Note, does not account for leap seconds! There are none declared since the + last one announced on 31/12/2016, so it should be fine for now as of 2020. + """ + utc_time: xr.DataArray = dataarray.__class__(start_epoch) + dataarray + + return utc_time diff --git a/deepicedrain/tests/test_calculate_delta.py b/deepicedrain/tests/test_calculate_delta.py index 1425f25..0fbf4d9 100644 --- a/deepicedrain/tests/test_calculate_delta.py +++ b/deepicedrain/tests/test_calculate_delta.py @@ -14,7 +14,7 @@ def test_calculate_delta_height(): """ Check that calculating change in elevation works. """ - atl11_dataset: xr.Dataset = catalog.test_data.atl11_test_case.read() + atl11_dataset: xr.Dataset = catalog.test_data.atl11_test_case.to_dask() delta_height = calculate_delta( dataset=atl11_dataset, oldcyclenum=3, newcyclenum=4, variable="h_corr" ) @@ -25,12 +25,14 @@ def test_calculate_delta_height(): npt.assert_allclose(actual=delta_height.mean().data, desired=-0.90124122) npt.assert_allclose(actual=delta_height.max().data, desired=9.49908442) + atl11_dataset.close() + def test_calculate_delta_time(): """ Check that calculating change in time works. """ - atl11_dataset: xr.Dataset = catalog.test_data.atl11_test_case.read() + atl11_dataset: xr.Dataset = catalog.test_data.atl11_test_case.to_dask() delta_time = calculate_delta( dataset=atl11_dataset, oldcyclenum=3, newcyclenum=4, variable="delta_time" ) @@ -38,11 +40,13 @@ def test_calculate_delta_time(): assert isinstance(delta_time, xr.DataArray) assert delta_time.shape == (1404,) npt.assert_equal( - actual=delta_time.min().data, desired=np.timedelta64(7846786703322903) + actual=np.asarray(delta_time.min()), desired=np.timedelta64(7846786703322903) ) npt.assert_equal( - actual=delta_time.mean().data, desired=np.timedelta64(7846786865357197) + actual=np.asarray(delta_time.mean()), desired=np.timedelta64(7846786865357197), ), npt.assert_equal( - actual=delta_time.max().data, desired=np.timedelta64(7846787022726588) + actual=np.asarray(delta_time.max()), desired=np.timedelta64(7846787022726588) ) + + atl11_dataset.close() diff --git a/deepicedrain/tests/test_catalog.yaml b/deepicedrain/tests/test_catalog.yaml index 21e0d7d..14c2238 100644 --- a/deepicedrain/tests/test_catalog.yaml +++ b/deepicedrain/tests/test_catalog.yaml @@ -2,16 +2,18 @@ metadata: version: 1 sources: atl11_test_case: + description: 'An example ATL11 hdf5 file for testing various calculations' args: + chunks: + cycle_number: 2 urlpath: simplecache::https://github.com/suzanne64/ATL11/raw/125ee1a653d78e6b86864b35c9d0fcfd72d64a85/ATL11_test_case/ATL11_078805_0304_02_v002.h5 xarray_kwargs: engine: h5netcdf group: /pt2/corrected_h storage_options: simplecache: - cache_storage: tests/test_data + cache_storage: '{{ CATALOG_DIR }}/test_data' same_names: True - description: 'An example ATL11 hdf5 file for testing various calculations' driver: intake_xarray.netcdf.NetCDFSource metadata: coords: diff --git a/deepicedrain/tests/test_spatiotemporal_conversions.py b/deepicedrain/tests/test_spatiotemporal_conversions.py new file mode 100644 index 0000000..b863a37 --- /dev/null +++ b/deepicedrain/tests/test_spatiotemporal_conversions.py @@ -0,0 +1,47 @@ +""" +Tests various conversions between geospatial and temporal units +""" +import datetime + +import dask +import numpy as np +import numpy.testing as npt +import pandas as pd +import xarray as xr + +from deepicedrain import catalog, deltatime_to_utctime + + +def test_deltatime_to_utctime(): + """ + Test that converting from ICESat-2 delta_time to utc_time works, + and that the xarray dimensions are preserved in the process. + """ + atl11_dataset: xr.Dataset = catalog.test_data.atl11_test_case.to_dask() + + utc_time: xr.DataArray = deltatime_to_utctime(dataarray=atl11_dataset.delta_time) + + assert utc_time.shape == (1404, 2) + assert utc_time.dims == ("ref_pt", "cycle_number") + assert dask.is_dask_collection(utc_time) + + utc_time = utc_time.compute() + + npt.assert_equal( + actual=utc_time.data.min(), + desired=np.datetime64("2019-05-19T20:53:51.039891534"), + ) + npt.assert_equal( + actual=np.datetime64(pd.DataFrame(utc_time.data)[0].mean()), + desired=np.datetime64("2019-05-19 20:54:00.925868"), + ) + npt.assert_equal( + actual=np.datetime64(pd.DataFrame(utc_time.data)[1].mean()), + desired=np.datetime64("2019-08-18 16:33:47.791226"), + ) + npt.assert_equal( + actual=utc_time.data.max(), + desired=np.datetime64("2019-08-18T16:33:57.834610209"), + ) + + atl11_dataset.close() diff --git a/poetry.lock b/poetry.lock index e568586..df849c0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -172,7 +172,7 @@ description = "Python package for providing Mozilla's CA Bundle." name = "certifi" optional = false python-versions = "*" -version = "2019.11.28" +version = "2020.4.5.1" [[package]] category = "main" @@ -180,7 +180,7 @@ description = "Time-handling functionality from netcdf4-python" name = "cftime" optional = false python-versions = "*" -version = "1.1.1.2" +version = "1.1.3" [package.dependencies] numpy = "*" @@ -1901,26 +1901,26 @@ cartopy = [ {file = "Cartopy-0.18.0.tar.gz", hash = "sha256:7ffa317e8f8011e0d965a3ef1179e57a049f77019867ed677d49dcc5c0744434"}, ] certifi = [ - {file = "certifi-2019.11.28-py2.py3-none-any.whl", hash = "sha256:017c25db2a153ce562900032d5bc68e9f191e44e9a0f762f373977de9df1fbb3"}, - {file = "certifi-2019.11.28.tar.gz", hash = "sha256:25b64c7da4cd7479594d035c08c2d809eb4aab3a26e5a990ea98cc450c320f1f"}, + {file = "certifi-2020.4.5.1-py2.py3-none-any.whl", hash = "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304"}, + {file = "certifi-2020.4.5.1.tar.gz", hash = "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519"}, ] cftime = [ - {file = "cftime-1.1.1.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:33a78faacac37cdfb68f998400c81ece5acf368cbf803a9fde7cf01e527d0860"}, - {file = "cftime-1.1.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:ba44e4ae980db94e999ae91e160d734c7865ab437e48591a96fe98ad46b541cb"}, - {file = "cftime-1.1.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:9d49569ed1c2e8fc3a4c0f95bdbe14a2e146a794ed7a6970dcb54fea455acb88"}, - {file = "cftime-1.1.1.2-cp36-none-win32.whl", hash = "sha256:a47357917b1f28af3a8b53e3fa0004fa1c7b4e454d1744a26fda46571991def5"}, - {file = "cftime-1.1.1.2-cp36-none-win_amd64.whl", hash = "sha256:43644b85c8a9351f5b208dda7442132c78c782ed2ed86e6cdc8580306f2d9afb"}, - {file = "cftime-1.1.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:056a3843f3b74789f73770180d4a80416c0558841c55a22c05a99a04a44d0f1c"}, - {file = "cftime-1.1.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:a915c640da4baff6221b87a2ab09f1aef1132611072e300cfd6ccc6960693bc9"}, - {file = "cftime-1.1.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1a3e9025d315ef7d023a2865f377f105d75907f0c66913fca56158fce8aa6f2e"}, - {file = "cftime-1.1.1.2-cp37-none-win32.whl", hash = "sha256:10974085d22c8345fbaf7ae5dbf8560ddca5477fc899126a2bcd57f65f1f52b0"}, - {file = "cftime-1.1.1.2-cp37-none-win_amd64.whl", hash = "sha256:9c3698285c77e24a3250d0366d872526470e5333d66874d0c0733ce99e242114"}, - {file = "cftime-1.1.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d09d0469e617c78115d860c2483fcc6e5e55154738c7ea7146a3591254aa366f"}, - {file = "cftime-1.1.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:21a28362757b3a7db4db0e8f174843798116b07c93976f6194d0afe3eb6fe1c0"}, - {file = "cftime-1.1.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:298c97970031c2073064249184395cc040b0ec5f6439720bab2ac49354285715"}, - {file = "cftime-1.1.1.2-cp38-none-win32.whl", hash = "sha256:89ee6f94dbb81c22576f81c939a1b8d985c3bd31f63063babeda7e969339ff3b"}, - {file = "cftime-1.1.1.2-cp38-none-win_amd64.whl", hash = "sha256:2405ca220a9f90edbf4836f175fd5535455d8cdd1863213a09f41613c03530fb"}, - {file = "cftime-1.1.1.2.tar.gz", hash = "sha256:35711b5ec3928b9e724817bfa1b7325da205788ee04eae9166cbcd96ea7976a6"}, + {file = "cftime-1.1.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:106d8bd1c144c83de1288c51225fd6846539074b60bb9e05b5e357d5e1eed6b9"}, + {file = "cftime-1.1.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:ead81301dbfcfef2b452ce6997f4f82cc0c2d968d27a2795251b5cdfc4b17295"}, + {file = "cftime-1.1.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:623255e0264de5cde085f0027ab03365965f3f416de407b604b00abe676f20f2"}, + {file = "cftime-1.1.3-cp36-none-win32.whl", hash = "sha256:bdd3a0b85fda45585529825e0954d6140d6df40b7bc489770e72fa6ce79ab9ea"}, + {file = "cftime-1.1.3-cp36-none-win_amd64.whl", hash = "sha256:3fb1f637aed7391c9a5d718175014f4cf705970e1d596bbc80bcac078eefefb0"}, + {file = "cftime-1.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:9ad8521598be2b354d159538e9572afb4c0d4ac2b0b6240eef7197e1d709a89e"}, + {file = "cftime-1.1.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:fb9a5ed38dee8ac43235ba161bf5fb61274e965b9b396a650ba4d515b9d70a7c"}, + {file = "cftime-1.1.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:9e1c863e38d410de0b59c5be7799ef05cd6f241cc254c6d656dfdae7546c1025"}, + {file = "cftime-1.1.3-cp37-none-win32.whl", hash = "sha256:555c101a0d03f6f6231253d0d84620a5ffb69757680be532086a70b033b9969b"}, + {file = "cftime-1.1.3-cp37-none-win_amd64.whl", hash = "sha256:ad0b9db793eae28ed1f53157e87044119c8153bf211420414f2c6b6e27fc86db"}, + {file = "cftime-1.1.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dd44afef96467f2cb3ec9324e9f471653e5daa55b05198f8da389ccb85d38157"}, + {file = "cftime-1.1.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:7c4d0de4ab846b76f73a9a45d9085e5c6321610e74548bdeaaf6c588fe841587"}, + {file = "cftime-1.1.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c9b27142cbd41049346128c8a5b78d292b675a3c185e16ed58645c273156e16"}, + {file = "cftime-1.1.3-cp38-none-win32.whl", hash = "sha256:d65d0fcc7120db05ddd34625310bc56038063fddcf63f3129379949d2ee762e9"}, + {file = "cftime-1.1.3-cp38-none-win_amd64.whl", hash = "sha256:eab0fb0268fc4743f65eb6efac5998af2b0805d914c2e2f83226531add57fd98"}, + {file = "cftime-1.1.3.tar.gz", hash = "sha256:fd84b8631dca1db9b40a75e18671b9edafd3515580d8ab33ce1ebafee75451f0"}, ] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},