Updated dependencies, refactoring and bugfixes

.gitignore

@@ -1,4 +1,5 @@
 # Custom
+.idea/
 dist/
 _archive/
 _raw_photoshop/

CHANGELOG.md

@@ -0,0 +1,18 @@
+# CHANGELOG
+
+## v1.2.0 | 6 Mar 2024
+
+- Refactored code to work with newest package versions
+- Several small bugfixes
+- Now using `poetry` for dependency management
+- Now using Python `3.9.18`
+- All dependencies were updated to their newest possible versions
+- Added example for using the class `DynamicLagCompensation` to run `dyco` directly from
+  code (`example.example_kwargs.example`)
+
+## v1.1.2 | 16 Jun 2021
+
+### Release version for publication in JOSS
+
+- JOSS: https://joss.theoj.org/
+- DYCO open review: openjournals/joss-reviews#2575

LICENSE

@@ -632,7 +632,7 @@ state the exclusion of warranty; and each file should have at least
 the "copyright" line and a pointer to where the full notice is found.
 
     DYCO Dynamic Lag Compensation
-    Copyright (C) 2020  holukas
+    Copyright (C) 2020-2024 Lukas Hörtnagl
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -645,14 +645,14 @@ the "copyright" line and a pointer to where the full notice is found.
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 Also add information on how to contact you by electronic and paper mail.
 
   If the program does terminal interaction, make it output a short
 notice like this when it starts in an interactive mode:
 
-    DYCO Dynamic Lag Compensation  Copyright (C) 2020  holukas
+    DYCO Dynamic Lag Compensation  Copyright (C) 2020-2024 Lukas Hörtnagl
     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
     This is free software, and you are welcome to redistribute it
     under certain conditions; type `show c' for details.
@@ -664,7 +664,7 @@ might be different; for a GUI interface, you would use an "about box".
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU GPL, see
-<http://www.gnu.org/licenses/>.
+<https://www.gnu.org/licenses/>.
 
   The GNU General Public License does not permit incorporating your program
 into proprietary programs.  If your program is a subroutine library, you

dyco/analyze.py

@@ -1,6 +1,6 @@
 """
     DYCO Dynamic Lag Compensation
-    Copyright (C) 2020  holukas
+    Copyright (C) 2020-2024 Lukas Hörtnagl
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
 
@@ -99,18 +99,18 @@ def plot_final_instantaneous_lagtimes(outdir, phase, df):
 
         # Accepted reference lags
         ax.plot_date(pd.to_datetime(df.index), df['INSTANTANEOUS_LAG'],
-                     alpha=1, marker='o', ms=6, color='black', lw=0, ls='-',
+                     alpha=1, fmt='o', ms=6, color='black', lw=0, ls='-',
                      label=f'final reference time lag (absolute limit {abs_limit})', markeredgecolor='None', zorder=100)
 
         # Found lags in Phase 3
         ax.plot_date(pd.to_datetime(df.index), df['PEAK-COVABSMAX_SHIFT'],
-                     alpha=1, marker='o', ms=12, color='#FFC107', lw=0, ls='-', markeredgecolor='None', zorder=99,
+                     alpha=1, fmt='o', ms=12, color='#FFC107', lw=0, ls='-', markeredgecolor='None', zorder=99,
                      label=f'found Phase 3 time lag (search between {lagsearch_start} and {lagsearch_end})')
 
         # Marks lags that were outside limit and therefore set to the default lag
         set_to_default_lags = df.loc[df['SET_TO_DEFAULT'] == True, ['INSTANTANEOUS_LAG']]
         ax.plot_date(pd.to_datetime(set_to_default_lags.index), set_to_default_lags['INSTANTANEOUS_LAG'],
-                     alpha=1, marker='o', ms=12, color='#8BC34A', lw=0, ls='-', markeredgecolor='None', zorder=99,
+                     alpha=1, fmt='o', ms=12, color='#8BC34A', lw=0, ls='-', markeredgecolor='None', zorder=99,
                      label=f'found Phase 3 time lag was set to default')
 
         plot.default_format(ax=ax, label_color='black', fontsize=12,
@@ -337,6 +337,8 @@ def make_lut_agg(self, target_lag: int, segment_lagtimes_df: pd.DataFrame):
             subset = peaks_hq_S[filter_around_this_day]
             num_vals = len(subset)
 
+            # lut_df.loc[this_date, 'median'] = 9
+
             if num_vals >= 5:
                 # print(f"{this_date}    {num_vals}    {subset.median()}")
                 lut_df.loc[this_date, 'median'] = subset.median()
@@ -360,7 +362,9 @@ def make_lut_agg(self, target_lag: int, segment_lagtimes_df: pd.DataFrame):
                                         col='correction')
         self.logger.warning(f"No correction could be generated from data for dates: {missing_df.index.to_list()}")
         self.logger.warning(f"Filling missing corrections for dates: {missing_df.index.to_list()}")
-        lut_df['correction'].fillna(method='ffill', inplace=True, limit=1)
+        lut_df['correction'] = lut_df['correction'].ffill(limit=1)
+        lut_df['correction'] = lut_df['correction'].bfill(limit=1)
+        # lut_df['correction'] = lut_df['correction'].fillna(method='ffill', inplace=False, limit=1)  # deprecated
 
         try:
             lut_df['recommended_default_winsize'] = \
@@ -407,7 +411,7 @@ def get_hq_peaks(self, df):
 
         """
         df.set_index('start', inplace=True)
-        df.index = pd.to_datetime(df.index)
+        df.index = pd.to_datetime(df.index, format="mixed")
         peaks_hq_S = df.loc[df['PEAK-COVABSMAX_SHIFT'] == df['PEAK-AUTO_SHIFT'],
                             'PEAK-COVABSMAX_SHIFT']
         peaks_hq_S.index = peaks_hq_S.index.to_pydatetime()  # Convert to DatetimeIndex

dyco/cli.py

@@ -1,6 +1,6 @@
 """
     DYCO Dynamic Lag Compensation
-    Copyright (C) 2020  holukas
+    Copyright (C) 2020-2024 Lukas Hörtnagl
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
 
@@ -43,7 +43,7 @@ def validate_args(args):
 
 def get_args():
     """Get args from CLI input"""
-    parser = argparse.ArgumentParser(description="BICO - Conversion of ETH binary files to ASCII",
+    parser = argparse.ArgumentParser(description="dyco - dynamic lag compensation",
                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter)
 
     # Positional args
@@ -62,7 +62,7 @@ def get_args():
 
     # Optional args
     parser.add_argument('-i', '--indir', type=Path,
-                        help="Path to the source folder that contains the data files, e.g. 'C:/bico/input'")
+                        help="Path to the source folder that contains the data files, e.g. 'C:/dyco/input'")
     parser.add_argument('-o', '--outdir', type=Path,
                         help="Path to output folder, e.g. C:/bico/output")
     parser.add_argument('-fnd', '--filenamedateformat', type=str, default='%Y%m%d%H%M%S',

dyco/correction.py

@@ -1,6 +1,6 @@
 """
     DYCO Dynamic Lag Compensation
-    Copyright (C) 2020  holukas
+    Copyright (C) 2020-2024 Lukas Hörtnagl
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
 
@@ -86,6 +86,7 @@ def run(self):
 
             if not self.phase == 3:
                 this_date = file_info_row['start'].date()
+                this_date = pd.to_datetime(this_date)
                 shift_correction = self.lut_df.loc[this_date][self.lut_col]
             else:
                 this_date = file_info_row['start']
@@ -184,18 +185,19 @@ def read_lut_time_lags(self):
         """Read csv file that contains the look-up table for lag correction"""
         filepath = self.outdirs[
                        f'{self.phase}-6_{self.phase_files}_normalization_lookup_table'] / f'LUT_default_agg_time_lags.csv'
-        parse = lambda x: dt.datetime.strptime(x, '%Y-%m-%d')
+        # parse = lambda x: dt.datetime.strptime(x, '%Y-%m-%d')  # now deprecated
         df = pd.read_csv(filepath,
                          skiprows=None,
                          header=0,
                          # names=header_cols_list,
                          # na_values=-9999,
                          encoding='utf-8',
                          delimiter=',',
-                         mangle_dupe_cols=True,
+                         # mangle_dupe_cols=True,
                          # keep_date_col=False,
                          parse_dates=True,
-                         date_parser=parse,
+                         # date_parser=parse,  # now deprecated
+                         date_format='%Y-%m-%d',
                          index_col=0,
                          dtype=None,
                          engine='c')

dyco/dyco.py

@@ -1,6 +1,6 @@
 """
     DYCO Dynamic Lag Compensation
-    Copyright (C) 2020  holukas
+    Copyright (C) 2020-2024 Lukas Hörtnagl
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
 import os
@@ -109,7 +109,7 @@ def _update_args(self, args, prev_phase, prev_phase_files, prev_outdir_files, pr
                 args['lgs_winsize'] = 100  # Small window for instantaneous search in Phase 3
                 args['lgs_num_iter'] = 1
 
-            args['indir'] = prev_outdir_files / f"{prev_phase}-7_{prev_phase_files}_normalized"
+            args['indir'] = Path(prev_outdir_files) / f"{prev_phase}-7_{prev_phase_files}_normalized"
             args['var_lagged'] = f"{args['var_lagged']}_DYCO"  # Use normalized signal
             filename, file_extension = os.path.splitext(args['fnm_pattern'])
             args['fnm_pattern'] = f"{filename}_DYCO{file_extension}"  # Search normalized files
@@ -165,13 +165,13 @@ def __init__(self,
                  fnm_date_format: str = '%Y%m%d%H%M%S',
                  fnm_pattern: str = '*.csv',
                  files_how_many: None or int = None,
-                 file_generation_res: str = '30T',
-                 file_duration: str = '30T',
+                 file_generation_res: str = '30min',
+                 file_duration: str = '30min',
                  dat_recs_timestamp_format: None or str = None,
                  dat_recs_nominal_timeres: float = 0.05,
-                 lgs_segment_dur: str = '30T',
+                 lgs_segment_dur: str = '30min',
                  lgs_winsize: int = 1000,
-                 lgs_num_iter: int = 3,
+                 lgs_num_iter: int = 1,
                  lgs_hist_remove_fringe_bins: bool = True,
                  lgs_hist_perc_thres: float = 0.9,
                  target_lag: int = 0,
@@ -226,16 +226,16 @@ def __init__(self,
             Frequency at which new files were generated. This does not
             relate to the data records but to the file creation time.
             Examples:
-                * '30T' means a new file was generated every 30 minutes.
-                * '1H' means a new file was generated every hour.
-                * '6H' means a new file was generated every six hours.
+                * '30min' means a new file was generated every 30 minutes.
+                * '1h' means a new file was generated every hour.
+                * '6h' means a new file was generated every six hours.
             For pandas DateOffset options see:
                 https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases
 
         file_duration: str (pandas DateOffset)
             Duration of one data file.
             Example:
-                * '30T': data file contains data from 30 minutes.
+                * '30min': data file contains data from 30 minutes.
             For pandas DateOffset options see:
                 https://pandas.pydata.org/pandas-docs/stable/user_guide/timeseries.html#offset-aliases
 
@@ -246,10 +246,10 @@ def __init__(self,
             *file_duration*, then the file data is split into segments
             and the lag time is calculated for each segment separately.
             Examples:
-                * '10T': calculates lag times for 10-minute segments.
+                * '10m': calculates lag times for 10-minute segments.
                 * With the settings
-                    file_duration = '30T' and
-                    lgs_segments_dur = '10T'
+                    file_duration = '30min' and
+                    lgs_segments_dur = '10m'
                     the 30-minute data file is split into three 10-minute
                     segments and the lag time is determined in each of the
                     segments, yielding three lag times.
@@ -435,10 +435,8 @@ def setup(self):
         return logfile_path, files_overview_df
 
     def calculate_lags(self):
-        """
-        Calculate covariances and detect covariance peaks to determine lags
-        for each file segment
-        """
+        """Calculate covariances and detect covariance peaks to determine lags
+        for each file segment."""
         for iteration in range(1, 1 + self.lgs_num_iter):
             loop_iter = loop.Loop(dyco_instance=self,
                                   iteration=iteration)

dyco/files.py

@@ -1,6 +1,6 @@
 """
     DYCO Dynamic Lag Compensation
-    Copyright (C) 2020  holukas
+    Copyright (C) 2020-2024 Lukas Hörtnagl
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -13,7 +13,7 @@
     GNU General Public License for more details.
 
     You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 """
 
@@ -47,7 +47,7 @@ def read_segment_lagtimes_file(filepath):
                                 # na_values=-9999,
                                 encoding='utf-8',
                                 delimiter=',',
-                                mangle_dupe_cols=True,
+                                # mangle_dupe_cols=True,
                                 # keep_date_col=False,
                                 parse_dates=False,
                                 # date_parser=parse,
@@ -96,12 +96,12 @@ def read_raw_data(filepath, data_timestamp_format):
                               num_missing_header_cols=num_missing_header_cols)
 
     if data_timestamp_format:
-        parse = lambda x: dt.datetime.strptime(x, data_timestamp_format)
-        date_parser = parse
+        # parse = lambda x: dt.datetime.strptime(x, data_timestamp_format)  # now deprecated
+        # date_parser = parse  # now deprecated
         parse_dates = True
         index_col = 0
     else:
-        date_parser = None
+        # date_parser = None  # now deprecated
         parse_dates = False
         index_col = None
 
@@ -112,10 +112,11 @@ def read_raw_data(filepath, data_timestamp_format):
                           na_values=-9999,
                           encoding='utf-8',
                           delimiter=',',
-                          mangle_dupe_cols=True,
-                          keep_date_col=False,
+                          # mangle_dupe_cols=True,  # now deprecated
+                          # keep_date_col=False,  # now deprecated
                           parse_dates=parse_dates,
-                          date_parser=date_parser,
+                          # date_parser=date_parser,  # now deprecated
+                          date_format=data_timestamp_format,
                           index_col=index_col,
                           dtype=None,
                           engine='c',