Merge pull request #68 from fluves/SVH-pandas-warning

Fix handling of time series with mixed timezone offsets

.gitignore

@@ -12,6 +12,7 @@ __pycache__/*
 .cache/*
 .*.swp
 */.ipynb_checkpoints/*
+.ipynb_checkpoints/*
 .DS_Store
 
 # Project files

.pre-commit-config.yaml

@@ -2,13 +2,12 @@ exclude: '^docs/conf.py'
 
 repos:
 - repo: https://github.com/psf/black
-  rev: 23.1.0
+  rev: 24.8.0
   hooks:
     - id: black
-      language_version: python3.9
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.6.0
   hooks:
   - id: trailing-whitespace
   - id: check-added-large-files
@@ -24,7 +23,7 @@ repos:
     args: ['--fix=no']
 
 - repo: https://github.com/pycqa/flake8
-  rev: 6.0.0
+  rev: 7.1.1
   hooks:
   - id: flake8
     args: ['--max-line-length=88']  # default of Black

setup.py

@@ -6,6 +6,7 @@
     PyScaffold helps you to put up the scaffold of your new Python project.
     Learn more under: https://pyscaffold.org/
 """
+
 from setuptools import setup
 
 if __name__ == "__main__":

src/pywaterinfo/waterinfo.py

@@ -100,7 +100,7 @@ def __init__(
                 self._request = requests_cache.CachedSession(
                     cache_name="pywaterinfo_cache.sqlite",
                     use_memory=False,
-                    cache_control=True,
+                    cache_control=False,
                     expire_after=CACHE_RETENTION,
                     stale_if_error=False,
                     use_cache_dir=True,
@@ -631,7 +631,10 @@ def get_timeseries_values(
                     df[key_name] = section[key_name]
             # convert datetime objects to Pandas timestamp
             if "Timestamp" in df.columns:
-                df["Timestamp"] = pd.to_datetime(df["Timestamp"])
+                # round trip via UTC to handle mixed time series
+                df["Timestamp"] = pd.to_datetime(
+                    df["Timestamp"], utc=True
+                ).dt.tz_convert(timezone)
             time_series.append(df)
 
         return pd.concat(time_series)

tests/conftest.py

@@ -20,7 +20,7 @@
 
 @pytest.fixture
 def patch_retention(monkeypatch):
-    retention = datetime.timedelta(seconds=0.0001)
+    retention = datetime.timedelta(seconds=2)
     monkeypatch.setattr("pywaterinfo.waterinfo.CACHE_RETENTION", retention)
 
 

tests/test_cache.py

@@ -27,9 +27,6 @@ def test_cache_hic(hic_cached_connection):
     assert res.from_cache
 
 
-{"request": "getTimeseriesValueLayer", "timeseriesgroup_id": "156207"}
-
-
 def test_clear_cache(vmm_cached_connection):
     """Cache is cleared."""
     data, res = vmm_cached_connection.request_kiwis({"request": "getRequestInfo"})
@@ -66,15 +63,18 @@ def test_cache_retention(patch_retention):
     expired cache and check for `from_cache` in a new request.
     """
     vmm = Waterinfo("vmm", cache=True)
-    # patch/exclude cache control to use retention time only
-    vmm._request.settings.cache_control = False
     vmm.clear_cache()
     _, res = vmm.request_kiwis({"request": "getRequestInfo"})
     assert not res.from_cache
 
     time.sleep(1)
-    vmm._request.cache
     _, res = vmm.request_kiwis({"request": "getRequestInfo"})
+    assert vmm._request.cache.contains(  # noqa
+        url="https://download.waterinfo.be/tsmdownload/KiWIS/KiWIS?"
+        "request=getRequestInfo&service=kisters&type=QueryServices&"
+        "format=json&datasource=1&timezone=UTC"
+    )
+    time.sleep(1)
     assert res.is_expired
 
     vmm._request.cache.delete(expired=True)

tests/test_waterinfo.py

@@ -353,10 +353,24 @@ def test_overwrite_timezone(self, connection, request):
             end="20190501 14:10",
             timezone="CET",
         )
-        assert is_datetime64tz_dtype(df["Timestamp"])
-        assert df["Timestamp"].dt.tz == datetime.timezone(
-            datetime.timedelta(seconds=7200)
+        assert isinstance(df["Timestamp"].dtype, pd.DatetimeTZDtype)
+        assert df["Timestamp"].dt.tz == pytz.timezone("CET")
+
+    def test_timezone_mixed_timezone(self, connection, request):
+        """Queries resulting in mixed timezone offsets are handled without warning
+
+        See also https://github.com/fluves/pywaterinfo/issues/67
+        """
+        connection = request.getfixturevalue(connection)
+        df = connection.get_timeseries_values(
+            "69928042",
+            start="2013-03-21 00:00:00",
+            end="2013-04-01 23:00:00",
+            timezone="CET",
+            returnfields="Timestamp,Value",
         )
+        assert isinstance(df["Timestamp"].dtype, pd.DatetimeTZDtype)
+        assert df["Timestamp"].dt.tz == pytz.timezone("CET")
 
     def test_start_end_timezone(self, connection, request):
         """pywaterinfo can handle start/end dates already containing tz info"""