time range cofiguration

src/main/java/edu/ie3/tools/Converter.java

@@ -35,6 +35,8 @@
  * @version 4.0
  */
 public class Converter implements Runnable {
+  private ZonedDateTime convertFrom;
+  private ZonedDateTime convertUntil;
   public static final Logger logger = LogManager.getLogger(Converter.class);
   public static final Logger fileStatusLogger = LogManager.getLogger("FileStatus");
 
@@ -50,23 +52,11 @@ public class Converter implements Runnable {
   private final ExecutorService fileEraserExecutor =
       Executors.newFixedThreadPool((int) Math.ceil(noOfProcessors / 3d));
 
-  /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 - TS 01 | ") */
-  public static String getFormattedTimestep(@NotNull ZonedDateTime modelrun, int timestep) {
-    return "MR "
-        + MODEL_RUN_FORMATTER.format(modelrun)
-        + " - TS "
-        + String.format("%02d", timestep)
-        + " |    ";
-  }
-
-  /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 | ") */
-  public static String getFormattedModelrun(@NotNull ZonedDateTime modelrun) {
-    return "MR " + MODEL_RUN_FORMATTER.format(modelrun) + "         |    ";
-  }
+  public Converter(){}
 
-  /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 - TS 01 | ") */
-  public static String getFormattedTimestep(@NotNull FileModel file) {
-    return getFormattedTimestep(file.getModelrun(), file.getTimestep());
+  public Converter(ZonedDateTime convertFrom, ZonedDateTime convertUntil) {
+    this.convertFrom = convertFrom;
+    this.convertUntil = convertUntil;
   }
 
   @Override
@@ -76,10 +66,22 @@ public void run() {
       logger.setLevel(Main.debug ? Level.ALL : Level.INFO);
       printInit();
       validateConnectionProperties();
-      convert();
+      if (this.convertFrom != null && this.convertUntil != null){
+        convert(this.convertFrom, this.convertUntil);
+      }
+      else {
+        convertUntilNewest();
+      }
     } else logger.info("Converter is already running.");
   }
 
+  public void printInit() {
+    logger.info("________________________________________________________________________________");
+    logger.info("Converter started");
+    logger.trace("Program arguments:");
+    Main.printProgramArguments().forEach(s -> logger.trace("   " + s));
+  }
+
   /** Validates Connection Properties from user input */
   private void validateConnectionProperties() {
     Properties receivedProperties = new Properties();
@@ -103,51 +105,49 @@ private void validateConnectionProperties() {
     dbController = new DatabaseController(PERSISTENCE_UNIT_NAME, receivedProperties);
   }
 
-  public void printInit() {
-    logger.info("________________________________________________________________________________");
-    logger.info("Converter started");
-    logger.trace("Program arguments:");
-    Main.printProgramArguments().forEach(s -> logger.trace("   " + s));
+  private void convertUntilNewest() {
+    // retrieves the starting modelrun ( = oldest modelrun where persisted==false or no converter
+    // run info available)
+    ZonedDateTime currentModelrun =
+            (ZonedDateTime)
+                    dbController.execSingleResultNamedQuery(
+                            FileModel.OldestModelrunWithUnprocessedFiles, Collections.emptyList());
+
+    // retrieves the newest possible modelrun ( = newest downloaded modelrun)
+    ZonedDateTime newestPossibleModelrun =
+            (ZonedDateTime)
+                    dbController.execSingleResultNamedQuery(
+                            FileModel.NewestDownloadedModelrun, Collections.emptyList());
+
+    convert(currentModelrun, newestPossibleModelrun);
   }
 
-  private void convert() {
+
+  private void convert(ZonedDateTime start, ZonedDateTime end) {
     String formattedModelrun = "";
     try {
       fileEraser = new FileEraser(edu.ie3.tools.Main.directory, dbController);
 
-      // retrieves the newest possible modelrun ( = newest downloaded modelrun)
-      ZonedDateTime newestPossibleModelrun =
-          (ZonedDateTime)
-              dbController.execSingleResultNamedQuery(
-                  FileModel.NewestDownloadedModelrun, Collections.emptyList());
-
-      // retrieves the starting modelrun ( = oldest modelrun where persisted==false or no converter
-      // run info available)
-      ZonedDateTime currentModelrun =
-          (ZonedDateTime)
-              dbController.execSingleResultNamedQuery(
-                  FileModel.OldestModelrunWithUnprocessedFiles, Collections.emptyList());
-
-      if (currentModelrun != null) {
+      if (start != null) {
         coordinates = getCoordinates();
-        while (currentModelrun.isBefore(newestPossibleModelrun)
-            || currentModelrun.isEqual(newestPossibleModelrun)) {
+        while (start.isBefore(end)
+            || start.isEqual(end)) {
           logger.info(
               "############################### "
-                  + MODEL_RUN_FORMATTER.format(currentModelrun)
+                  + MODEL_RUN_FORMATTER.format(start)
                   + " ###############################");
-          formattedModelrun = getFormattedModelrun(currentModelrun);
+          formattedModelrun = getFormattedModelrun(start);
           long tic;
           long toc;
           tic = System.currentTimeMillis();
           for (int timestep = 0; timestep < edu.ie3.tools.Main.timesteps; timestep++) {
-            handleTimestep(currentModelrun, timestep);
+            handleTimestep(start, timestep);
             dbController.flush();
           }
 
           toc = System.currentTimeMillis();
           logger.debug(formattedModelrun + "This modelrun took " + (toc - tic) / 60000 + "m \n");
-          currentModelrun = currentModelrun.plusHours(3); // increment modelrun
+          start = start.plusHours(3); // increment modelrun
         }
       }
     } catch (Exception e) {
@@ -157,6 +157,11 @@ private void convert() {
     }
   }
 
+  /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 | ") */
+  public static String getFormattedModelrun(@NotNull ZonedDateTime modelrun) {
+    return "MR " + MODEL_RUN_FORMATTER.format(modelrun) + "         |    ";
+  }
+
   /** opens archive files and converts the data for one timestep */
   private void handleTimestep(ZonedDateTime currentModelrun, int timestep) {
     String formattedTimestep = getFormattedTimestep(currentModelrun, timestep);
@@ -175,6 +180,20 @@ private void handleTimestep(ZonedDateTime currentModelrun, int timestep) {
     logger.info(formattedTimestep + "Timestep finished");
   }
 
+  /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 - TS 01 | ") */
+  public static String getFormattedTimestep(@NotNull ZonedDateTime modelrun, int timestep) {
+    return "MR "
+            + MODEL_RUN_FORMATTER.format(modelrun)
+            + " - TS "
+            + String.format("%02d", timestep)
+            + " |    ";
+  }
+
+  /** @return timestamp for logging output (e.g "MR 09.10.2018 18:00 - TS 01 | ") */
+  public static String getFormattedTimestep(@NotNull FileModel file) {
+    return getFormattedTimestep(file.getModelrun(), file.getTimestep());
+  }
+
   /** Decompresses all grib files that are intact for every parameter.
    *
    * @param currentModelrun the model run for which to decompress files
@@ -186,38 +205,26 @@ private void decompressGribFiles(ZonedDateTime currentModelrun, int timestep) {
             + File.separator
             + FILENAME_DATE_FORMATTER.format(currentModelrun)
             + File.separator;
-    List<Decompressor> tasks = new ArrayList<>();
+    List<Decompressor> fileDecompressors = new ArrayList<>();
     List<FileModel> files = new ArrayList<>();
     for (Parameter param : Parameter.values()) {
       FileModel file =
           dbController.find(
               FileModel.class, FileModel.createFileName(currentModelrun, timestep, param));
       if (file != null) {
         files.add(file);
-        if (file.isSufficient_size() && (file.isValid_file() == null || file.isValid_file())) {
+        // add file to the files to decompress if it's valid
+        if (isFileIntact(file)) {
           if (!file.isPersisted() && !file.isArchivefile_deleted() && !file.isDecompressed()) {
-            tasks.add(new Decompressor(file, folderpath));
-          }
-        } else if (file.getDownload_fails() > 3
-            || file.getModelrun().isBefore(ZonedDateTime.now().minusDays(1))) {
-          if (edu.ie3.tools.Main.deleteDownloadedFiles) {
-            logger.trace(
-                "Delete file "
-                    + file.getName()
-                    + " because it did not have a valid size or content.");
-            fileEraserExecutor.submit(fileEraser.eraseCallable(file));
-          } else {
-            logger.trace(
-                "File "
-                    + file.getName()
-                    + " did not have a valid size or content. If you want to delete it pass -del as argument!");
+            fileDecompressors.add(new Decompressor(file, folderpath));
           }
-        }
+        } else handleBrokenFile(file);
       }
     }
 
     try {
-      decompressionExecutor.invokeAll(tasks);
+      // actual decompression of files
+      decompressionExecutor.invokeAll(fileDecompressors);
     } catch (InterruptedException e) {
       e.printStackTrace();
     }
@@ -233,6 +240,37 @@ private void decompressGribFiles(ZonedDateTime currentModelrun, int timestep) {
         });
   }
 
+  /** Checks if file is sufficiently large and is valid
+   *
+   * @param file the file to check
+   * @return whether it is intact or not
+   */
+  private Boolean isFileIntact(FileModel file){
+    return file.isSufficient_size() && (file.isValid_file() == null || file.isValid_file());
+  }
+
+  /** Either delete broken file or log it depending on configuration.
+   *
+   * @param file the broken file to handle
+   */
+  private void handleBrokenFile(FileModel file){
+    if (file.getDownloadFails() > 3
+          || file.getModelrun().isBefore(ZonedDateTime.now().minusDays(1))) {
+      if (edu.ie3.tools.Main.deleteDownloadedFiles) {
+        logger.trace(
+                "Delete file "
+                        + file.getName()
+                        + " because it did not have a valid size or content.");
+        fileEraserExecutor.submit(fileEraser.eraseCallable(file));
+      } else {
+        logger.trace(
+                "File "
+                        + file.getName()
+                        + " did not have a valid size or content. If you want to delete it pass -del as argument!");
+      }
+    }
+  }
+
   /**
    * Calls {@link Converter#convertTimeStep(ZonedDateTime, int, String)} with default folderpath
    * <br>
@@ -491,10 +529,10 @@ private void shutdownAllExecutors() {
 
   private Collection<CoordinateModel> getCoordinates() {
     HashMap<String, Object> namedCoordinateParams = new HashMap<>();
-    namedCoordinateParams.put("minLatitude", Main.minLatitude);
-    namedCoordinateParams.put("maxLatitude", Main.maxLatitude);
-    namedCoordinateParams.put("minLongitude", Main.minLongitude);
-    namedCoordinateParams.put("maxLongitude", Main.maxLongitude);
+    namedCoordinateParams.put("minLatitude", Main.MIN_LATITUDE);
+    namedCoordinateParams.put("maxLatitude", Main.MAX_LATITUDE);
+    namedCoordinateParams.put("minLongitude", Main.MIN_LONGITUDE);
+    namedCoordinateParams.put("maxLongitude", Main.MAX_LONGITUDE);
     return dbController.execNamedQuery(
         CoordinateModel.CoordinatesInRectangle, namedCoordinateParams);
   }

src/main/java/edu/ie3/tools/Downloader.java

@@ -181,7 +181,7 @@ public boolean download(ZonedDateTime modelrun, Parameter param) {
    */
   public boolean downloadFile(String folder, FileModel filemodel) {
     boolean success = false;
-    if (!filemodel.isSufficient_size() && filemodel.getDownload_fails() < 3) {
+    if (!filemodel.isSufficient_size() && filemodel.getDownloadFails() < 3) {
       String url = filemodel.getURL();
       try {
         if (isUrlReachable(url)) {

src/main/java/edu/ie3/tools/Main.java

@@ -7,9 +7,14 @@
 package edu.ie3.tools;
 
 import java.io.File;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
 import java.util.HashSet;
+import java.util.Locale;
 import java.util.Set;
 import java.util.TimeZone;
+
 import picocli.CommandLine;
 
 @CommandLine.Command(
@@ -57,6 +62,16 @@ public class Main {
       description = "Converts grib2 files")
   public static boolean doConvert;
 
+  @CommandLine.Option(
+          names = {"-convert_from"},
+          description = "Start datetime of conversion.")
+  public static String convertFrom;
+
+  @CommandLine.Option(
+          names = {"-convert_until"},
+          description = "End datetime of conversion.")
+  public static String convertUntil;
+
   @CommandLine.Option(
       names = {"download"},
       description = "Downloads grib2 files")
@@ -114,10 +129,10 @@ public class Main {
   public static boolean debug = false;
 
   // Coordinate-Rectangle to convert
-  public static final double minLongitude = 4.29694;
-  public static final double maxLongitude = 18.98635;
-  public static final double minLatitude = 45.71457;
-  public static final double maxLatitude = 57.65129;
+  public static final double MIN_LONGITUDE = 4.29694;
+  public static final double MAX_LONGITUDE = 18.98635;
+  public static final double MIN_LATITUDE = 45.71457;
+  public static final double MAX_LATITUDE = 57.65129;
 
   public static void main(String[] args) {
     // ICON-EU data timezone is UTC time,
@@ -143,8 +158,21 @@ public static void main(String[] args) {
       if (!eccodes.isEmpty()) eccodes += File.separator;
       eccodes += "grib_get_data";
     }
-    if (doDownload) new Downloader().run();
-    if (doConvert) new Converter().run();
+
+    if (doConvert) {
+      if (convertFrom != null && convertUntil != null) {
+        DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss").withZone(ZoneId.of("UTC")).withLocale(Locale.GERMANY);
+        ZonedDateTime from = ZonedDateTime.parse(convertFrom, dateTimeFormatter);
+        ZonedDateTime until = ZonedDateTime.parse(convertUntil, dateTimeFormatter);
+        new Converter(from, until).run();
+      }
+      else if (convertFrom != null || convertUntil != null){
+        throw new IllegalArgumentException("Either convertFrom or convertTo is missing. We need both to convert data for a specific interval");
+      }
+      else {
+        new Downloader().run();
+      }
+    }
   }
 
   public static Set<String> printProgramArguments() {

src/main/java/edu/ie3/tools/models/persistence/FileModel.java

@@ -106,6 +106,13 @@ public void setName(String name) {
     this.name = name;
   }
 
+  /** Creates the correct file name within the icon model for a parameter within a given time step for a given model run
+   *
+   * @param modelrun the model run to consider
+   * @param timestep the time step to check
+   * @param parameter the considered parameter
+   * @return a String of the expected file name
+   */
   public static String createFileName(ZonedDateTime modelrun, int timestep, Parameter parameter) {
     String name = parameter.getPrefix(); // ie icon-eu_europe_regular-lat-lon_single-level_
     name += FILENAME_DATE_FORMATTER.format(modelrun) + "_"; // ie 2018090512_
@@ -134,11 +141,11 @@ public void setParameter(Parameter parameter) {
     this.parameter = parameter;
   }
 
-  public int getDownload_fails() {
+  public int getDownloadFails() {
     return download_fails;
   }
 
-  public void setDownload_fails(int download_fails) {
+  public void setDownloadFails(int download_fails) {
     this.download_fails = download_fails;
   }
 
@@ -198,6 +205,10 @@ public void setPersisted(boolean persisted) {
     this.persisted = persisted;
   }
 
+  /** Checks if the raw file downloaded from the model run is deleted.
+   *
+   * @return whether it is deleted or not
+   */
   public boolean isArchivefile_deleted() {
     return archivefile_deleted;
   }