From 798532041fa7dd3f1b2d63ac971f34b5389b8a4c Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Thu, 2 Sep 2021 20:47:43 -0400 Subject: [PATCH 1/7] add helper method for creating empty canonical dataframe --- .../nwis_client/src/hydrotools/nwis_client/iv.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/nwis_client/src/hydrotools/nwis_client/iv.py b/python/nwis_client/src/hydrotools/nwis_client/iv.py index 4794488a..b45f30bc 100644 --- a/python/nwis_client/src/hydrotools/nwis_client/iv.py +++ b/python/nwis_client/src/hydrotools/nwis_client/iv.py @@ -809,3 +809,17 @@ def flatten_and_stringify(v): value_groups = np.array_split(values, n_groups) return list(map(lambda i: ",".join(i), value_groups)) + + +def _create_empty_canonical_df() -> pd.DataFrame: + """Returns an empty hydrotools canonical dataframe with correct field datatypes.""" + cols = { + "value_time": pd.Series(dtype="datetime64[ns]"), + "variable_name": pd.Series(dtype="category"), + "usgs_site_code": pd.Series(dtype="category"), + "measurement_unit": pd.Series(dtype="category"), + "value": pd.Series(dtype="float32"), + "qualifiers": pd.Series(dtype="category"), + "series": pd.Series(dtype="category"), + } + return pd.DataFrame(cols, index=[]) From de351009faf4fa683d681f720d45711635c2f12e Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Thu, 2 Sep 2021 20:55:39 -0400 Subject: [PATCH 2/7] remove redundant code that returned an inproperly formatted dataframe --- python/nwis_client/src/hydrotools/nwis_client/iv.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/nwis_client/src/hydrotools/nwis_client/iv.py b/python/nwis_client/src/hydrotools/nwis_client/iv.py index b45f30bc..671b5c53 100644 --- a/python/nwis_client/src/hydrotools/nwis_client/iv.py +++ b/python/nwis_client/src/hydrotools/nwis_client/iv.py @@ -249,12 +249,6 @@ def list_to_df_helper(item: dict): list_of_frames = list(map(list_to_df_helper, raw_data)) - # Empty list. No data was returned in the request - if not list_of_frames: - warning_message = "No data was returned by the request." - warnings.warn(warning_message) - return pd.DataFrame(None) - # Concatenate list in single pd.DataFrame dfs = pd.concat(list_of_frames, ignore_index=True) From 58ee56a635ea5a7d2b194e17f80fe9641aef16c6 Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Thu, 2 Sep 2021 20:57:00 -0400 Subject: [PATCH 3/7] return empty canonical dataframe if no data is retrieved from nwis. fixes #133 --- python/nwis_client/src/hydrotools/nwis_client/iv.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/nwis_client/src/hydrotools/nwis_client/iv.py b/python/nwis_client/src/hydrotools/nwis_client/iv.py index 671b5c53..04a4e2c5 100644 --- a/python/nwis_client/src/hydrotools/nwis_client/iv.py +++ b/python/nwis_client/src/hydrotools/nwis_client/iv.py @@ -252,6 +252,10 @@ def list_to_df_helper(item: dict): # Concatenate list in single pd.DataFrame dfs = pd.concat(list_of_frames, ignore_index=True) + # skip data processing steps if no data was retrieved and return empty canonical df + if dfs.empty: + return _create_empty_canonical_df() + # Convert values to numbers dfs.loc[:, "value"] = pd.to_numeric(dfs["value"], downcast="float") From 85d3e618035a665f8838afa9b189f02eb46d4a9c Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Thu, 2 Sep 2021 21:06:50 -0400 Subject: [PATCH 4/7] add empty dataframe test --- python/nwis_client/tests/test_nwis.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/python/nwis_client/tests/test_nwis.py b/python/nwis_client/tests/test_nwis.py index 9da45eb4..fef48d8f 100644 --- a/python/nwis_client/tests/test_nwis.py +++ b/python/nwis_client/tests/test_nwis.py @@ -423,3 +423,17 @@ def test_get_raw_countyCd(setup_iv): def test_splitting_bbox(test, validation): v = iv._bbox_split(test) assert v == validation + + +def test_get_returns_empty_canonical_dataframe(setup_iv, monkeypatch): + """Verify that `get` can returns an empty canonical dataframe.""" + + def get_raw_mock(*args, **kwargs): + return [{"values": []}] + + monkeypatch.setattr(iv.IVDataService, "get_raw", get_raw_mock) + df = setup_iv.get( + sites=["01189000"], startDt="2015-12-01T00:00", endDt="2015-12-31T23:45" + ) + canonical_df = iv._create_empty_canonical_df() + assert df.equals(canonical_df) From 67d76875d006e03f0bfa4d05bc3db0b7862e5528 Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Thu, 2 Sep 2021 21:22:18 -0400 Subject: [PATCH 5/7] re-introduce code from de35100. this covers the case when (literally) no data is returned from nwis. However, now returns a canonical df --- python/nwis_client/src/hydrotools/nwis_client/iv.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/python/nwis_client/src/hydrotools/nwis_client/iv.py b/python/nwis_client/src/hydrotools/nwis_client/iv.py index 04a4e2c5..1c9f97d9 100644 --- a/python/nwis_client/src/hydrotools/nwis_client/iv.py +++ b/python/nwis_client/src/hydrotools/nwis_client/iv.py @@ -249,6 +249,12 @@ def list_to_df_helper(item: dict): list_of_frames = list(map(list_to_df_helper, raw_data)) + # Empty list. No data was returned in the request + if not list_of_frames: + warning_message = "No data was returned by the request." + warnings.warn(warning_message) + return _create_empty_canonical_df() + # Concatenate list in single pd.DataFrame dfs = pd.concat(list_of_frames, ignore_index=True) From cd559b35e61b35834da21f5c8572a39e0fb6f096 Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Thu, 2 Sep 2021 21:24:27 -0400 Subject: [PATCH 6/7] add empty df warning message helper function. call it where empty dataframes can be returned by IVDataService.get --- python/nwis_client/src/hydrotools/nwis_client/iv.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/nwis_client/src/hydrotools/nwis_client/iv.py b/python/nwis_client/src/hydrotools/nwis_client/iv.py index 1c9f97d9..cf5c69d0 100644 --- a/python/nwis_client/src/hydrotools/nwis_client/iv.py +++ b/python/nwis_client/src/hydrotools/nwis_client/iv.py @@ -247,12 +247,15 @@ def list_to_df_helper(item: dict): return df + def empty_df_warning_helper(): + warning_message = "No data was returned by the request." + warnings.warn(warning_message) + list_of_frames = list(map(list_to_df_helper, raw_data)) # Empty list. No data was returned in the request if not list_of_frames: - warning_message = "No data was returned by the request." - warnings.warn(warning_message) + empty_df_warning_helper() return _create_empty_canonical_df() # Concatenate list in single pd.DataFrame @@ -260,6 +263,7 @@ def list_to_df_helper(item: dict): # skip data processing steps if no data was retrieved and return empty canonical df if dfs.empty: + empty_df_warning_helper() return _create_empty_canonical_df() # Convert values to numbers From b12a305dd6b31a014c0cb5254c907120d24c8f1b Mon Sep 17 00:00:00 2001 From: Austin Raney Date: Thu, 2 Sep 2021 21:25:05 -0400 Subject: [PATCH 7/7] bump patch version 3.0.4 -> 3.0.5 --- python/nwis_client/setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/nwis_client/setup.cfg b/python/nwis_client/setup.cfg index 82af7682..2ee804c5 100644 --- a/python/nwis_client/setup.cfg +++ b/python/nwis_client/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = hydrotools.nwis_client -version = 3.0.4 +version = 3.0.5 author = Austin Raney author_email = aaraney@protonmail.com description = A convenient interface to the USGS NWIS Instantaneous Values (IV) REST Service API.