adding metadata to timeseries object originating from ddlpy (#360)

* adding metadata to timeseries object originating from ddlpy

* updated tests

* updated whatsnew

docs/whats-new.md

@@ -4,6 +4,7 @@
 
 ### Feat
 - `datetime.timezone` support in `hatyan.write_components()` to support components from ddlpy timeseries in [#359](https://github.com/Deltares/hatyan/pull/359)
+- adding metdata to timeseries from ddlpy in `hatyan.ddlpy_to_hatyan()` in [#360](https://github.com/Deltares/hatyan/pull/360)
 
 
 ## 2.9.0 (2024-09-11)

hatyan/ddlpy_helpers.py

@@ -5,6 +5,7 @@
 """
 
 import pandas as pd
+from hatyan.metadata import metadata_from_ddlpy
 
 __all__ = ["ddlpy_to_hatyan"]
 
@@ -34,6 +35,8 @@ def ddlpy_to_hatyan(ddlpy_meas, ddlpy_meas_exttyp=None):
 
     ts_pd = ddlpy_to_hatyan_plain(ddlpy_meas, isnumeric=True)
     ts_pd['values'] /= 100 #convert from cm to m
+    metadata = metadata_from_ddlpy(ddlpy_meas)
+    ts_pd.attrs = metadata
     if ddlpy_meas_exttyp is None:
         return ts_pd
 
@@ -70,6 +73,7 @@ def ddlpy_to_hatyan_plain(ddlpy_meas, isnumeric=True):
                           'qualitycode':pd.to_numeric(ddlpy_meas['WaarnemingMetadata.KwaliteitswaardecodeLijst'],downcast='integer'),
                           'status':ddlpy_meas['WaarnemingMetadata.StatuswaardeLijst'].str[0],
                           })
+
     return ts_pd
 
 

hatyan/metadata.py

@@ -33,6 +33,23 @@ def metadata_from_diablocks(diablocks_pd, block_id):
     return metadata
 
 
+def metadata_from_ddlpy(ddlpy_meas):
+    dict_translate = {'grootheid':'Grootheid.Code',
+                      'groepering':'Groepering.Code',
+                      'eenheid':'Eenheid.Code',
+                      'vertref':'Hoedanigheid.Code',
+                      'station':'Code',
+                      }
+    metadata = {}
+    for k,v in dict_translate.items():
+        meta_values = ddlpy_meas[v]
+        meta_val = meta_values.iloc[0]
+        assert (meta_values == meta_val).all()
+        metadata[k] = meta_val
+    metadata['origin'] = 'ddlpy'
+    return metadata
+
+
 def metadata_from_obj(obj):
     metadata = obj.attrs.copy()
     return metadata

tests/test_ddlpy_helpers.py

@@ -45,10 +45,21 @@ def test_ddlpy_to_hatyan(locations):
     assert ptypes.is_integer_dtype(ts_measwl["qualitycode"])
     assert ptypes.is_object_dtype(ts_measwl["status"])
 
+    # check if metadata is complete and correct
+    meta_fromts = ts_measwl.attrs
+    meta_expected = {
+        'grootheid': 'WATHTE',
+        'groepering': 'NVT',
+        'eenheid': 'cm',
+        'vertref': 'NAP',
+        'station': 'HOEKVHLD',
+        'origin': 'ddlpy',
+        }
+    assert meta_fromts == meta_expected
+
 
 @pytest.mark.unittest
 def test_convert_hwlwstr2num(locations):
-
     tstart_dt = "2022-12-19"
     tstop_dt = "2022-12-31"
 
@@ -74,10 +85,28 @@ def test_convert_hwlwstr2num(locations):
 
     assert "HWLWcode" in ts_measwlHWLW.columns
     assert np.array_equal(ts_measwlHWLW["HWLWcode"].values, hwlwcode_expected)
+
+    # check if metadata is complete and correct
+    meta_fromts = ts_measwlHWLW.attrs
+    meta_expected = {
+        'grootheid': 'WATHTE',
+        'groepering': 'GETETM2',
+        'eenheid': 'cm',
+        'vertref': 'NAP',
+        'station': 'HOEKVHLD',
+        'origin': 'ddlpy',
+        }
+    assert meta_fromts == meta_expected
 
 
 @pytest.mark.systemtest
-def test_ddlpy_to_components(locations):
+def test_ddlpy_to_components(tmp_path, locations):
+    """
+    this test shows the metadata of the hatyan timeseries is properly created and passed
+    such that the component file can be written without altering or adding metadata.
+    This was achieved by fixing issue https://github.com/Deltares/hatyan/issues/357 
+    and issue https://github.com/Deltares/hatyan/issues/358.
+    """
     start_dt  = "2022-01-01 00:00:00 +01:00"
     end_dt  = "2022-03-31 23:50:00 +01:00"
 
@@ -90,13 +119,19 @@ def test_ddlpy_to_components(locations):
     locs_ddl_one = locs_ddl.loc[donar_loccode]
     ddl_df = ddlpy.measurements(locs_ddl_one, start_date=start_dt, end_date=end_dt)
     df_meas = hatyan.ddlpy_to_hatyan(ddl_df)
-
-    ts_comp_nfac1_fualltimes0_xfac1_peryear0 = hatyan.analysis(ts=df_meas, const_list='month', nodalfactors=True, fu_alltimes=False, xfac=True, analysis_perperiod=False)
 
-    # TODO: manually setting attrs should not be necessary after improving hatyan.ddlpy_to_hatyan()
-    # https://github.com/Deltares/hatyan/issues/358
-    ts_comp_nfac1_fualltimes0_xfac1_peryear0.attrs['grootheid'] = "WATHTE"
-    ts_comp_nfac1_fualltimes0_xfac1_peryear0.attrs['eenheid'] = "cm"
-    ts_comp_nfac1_fualltimes0_xfac1_peryear0.attrs['vertref'] = "NAP"
-    ts_comp_nfac1_fualltimes0_xfac1_peryear0.attrs['station'] = donar_loccode
-    hatyan.write_components(ts_comp_nfac1_fualltimes0_xfac1_peryear0, filename='components_%.ana'%(donar_loccode))
+    # check if metadata is complete and correct
+    meta_fromts = df_meas.attrs
+    meta_expected = {
+        'grootheid': 'WATHTE',
+        'groepering': 'NVT',
+        'eenheid': 'cm',
+        'vertref': 'NAP',
+        'station': 'VLISSGN',
+        'origin': 'ddlpy'
+        }
+    assert meta_fromts == meta_expected
+
+    comp = hatyan.analysis(ts=df_meas, const_list='month', nodalfactors=True, fu_alltimes=False, xfac=True, analysis_perperiod=False)
+    file_comp = tmp_path / f'components_{donar_loccode}.ana'
+    hatyan.write_components(comp, filename=file_comp)