Merge pull request #80 from tethys-ts/dev

this time I really fixed the mod date query

conda/meta.yaml

@@ -1,5 +1,5 @@
 {% set name = "tethysts" %}
-{% set version = "4.5.8" %}
+{% set version = "4.5.9" %}
 # {% set sha256 = "ae2cc83fb5a75e8dc3e1b2c2137deea412c8a4c7c9acca52bf4ec59de52a80c9" %}
 
 # sha256 is the prefered checksum -- you can get it for a file with:
@@ -32,10 +32,10 @@ build:
 
 requirements:
   build:
-    - python
+    - python >=3.8
     - setuptools
   run:
-    - python
+    - python >=3.8
     - zstandard
     - pandas
     - xarray

setup.py

@@ -10,7 +10,7 @@
 name = 'tethysts'
 main_package = 'tethysts'
 datasets = 'datasets/time_series'
-version = '4.5.8'
+version = '4.5.9'
 descrip = 'tethys time series S3 extraction'
 
 # The below code is for readthedocs. To have sphinx/readthedocs interact with

tethysts/utils.py

@@ -856,6 +856,29 @@ def xr_concat(datasets: List[xr.Dataset]):
     return xr3
 
 
+def filter_mod_dates(results, from_mod_date=None, to_mod_date=None):
+    """
+    Need to do this because xarray "where" is useless...
+    """
+    if ((from_mod_date is not None) or (to_mod_date is not None)) and ('modified_date' in results):
+        mod_dates = results['modified_date'].copy().load()
+
+        if (from_mod_date is not None) and (to_mod_date is not None):
+            mod_bool = (mod_dates >= pd.Timestamp(from_mod_date)) & (mod_dates <= pd.Timestamp(to_mod_date))
+        elif (from_mod_date is not None):
+            mod_bool = (mod_dates >= pd.Timestamp(from_mod_date))
+        elif (to_mod_date is not None):
+            mod_bool = (mod_dates <= pd.Timestamp(to_mod_date))
+
+        data_vars1 = [var for var in results.data_vars if 'time' in results[var].dims]
+
+        results[data_vars1] = results[data_vars1].where(mod_bool)
+
+        return results.dropna('time', how='all')
+    else:
+        return results
+
+
 def results_concat(results_list, output_path=None, from_date=None, to_date=None, from_mod_date=None, to_mod_date=None, compression='lzf'):
     """
 
@@ -872,16 +895,7 @@ def results_concat(results_list, output_path=None, from_date=None, to_date=None,
 
     ## Deal with mod dates filters
     if ((from_mod_date is not None) or (to_mod_date is not None)) and ('modified_date' in xr3):
-        mod_dates = xr3['modified_date'].copy().load()
-
-        if (from_mod_date is not None) and (to_mod_date is not None):
-            mod_bool = (mod_dates >= pd.Timestamp(from_mod_date)) & (mod_dates <= pd.Timestamp(to_mod_date))
-        elif (from_mod_date is not None):
-            mod_bool = (mod_dates >= pd.Timestamp(from_mod_date))
-        elif (to_mod_date is not None):
-            mod_bool = (mod_dates <= pd.Timestamp(to_mod_date))
-
-        xr3 = xr3.where(mod_bool, drop=True)
+        xr3 = filter_mod_dates(xr3, from_mod_date, to_mod_date)
 
     return xr3