init importfilmlists

mediathekview · Oct 13, 2023 · dc43ded · dc43ded
1 parent 7fe4fa4
commit dc43ded
Show file tree

Hide file tree

Showing 11 changed files with 351 additions and 49 deletions.
diff --git a/MServer-Config.yaml b/MServer-Config.yaml
@@ -83,8 +83,25 @@ writeFilmlistIdFileEnabled: true
 #The fimlist id file path
 filmlistIdFilePath: target/filmlists/filmlist.id
 
+
+# import additional filmlist sources
+importFilmlistConfigurations :
+    - active: false
+      path: "someCrawlerlist.json"
+      format: OLD_JSON
+      createDiff: false
+    - active: false
+      path: "filme-org.xz"
+      format: OLD_JSON_COMPRESSED_XZ
+      createDiff: true
+
+#     filmlistImporEnabled = true;
+#    filmlistImportFormat = FilmlistFormats.OLD_JSON_COMPRESSED_XZ;
+#    filmlistImportLocation = "https://verteiler1.mediathekview.de/Filmliste-akt.xz";
+
+
 # Sets if a filmlist should be imported
-filmlistImporEnabled: false
+#filmlistImporEnabled: false
 
 # The format of the film list to import.
 # Possible are: JSON, OLD_JSON, JSON_COMPRESSED_XZ, OLD_JSON_COMPRESSED_XZ, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP

diff --git a/src/main/java/de/mediathekview/mserver/base/config/ImportFilmlistConfiguration.java b/src/main/java/de/mediathekview/mserver/base/config/ImportFilmlistConfiguration.java
@@ -0,0 +1,67 @@
+package de.mediathekview.mserver.base.config;
+
+import java.util.Objects;
+
+import de.mediathekview.mlib.filmlisten.FilmlistFormats;
+
+public class ImportFilmlistConfiguration {
+  private final Boolean active;
+  private final String path;
+  private final FilmlistFormats format;
+  private final Boolean createDiff;
+
+
+  public ImportFilmlistConfiguration(Boolean active, String path, FilmlistFormats format, Boolean createDiff) {
+    this.active = active;
+    this.path = path;
+    this.format = format;
+    this.createDiff = createDiff;
+  }
+
+  public ImportFilmlistConfiguration() {
+    this.active = null;
+    this.path = null;
+    this.format = null;
+    this.createDiff = null;
+  }
+
+  public Boolean isActive() {
+    return active;
+  }
+  public String getPath() {
+    return path;
+  }
+  public FilmlistFormats getFormat() {
+    return format;
+  }
+  public Boolean isCreateDiff() {
+    return createDiff;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!(obj instanceof final ImportFilmlistConfiguration that)) {
+      return false;
+    }
+    if (!super.equals(obj)) {
+      return false;
+    }
+    return Objects.equals(isActive(), that.isActive())
+        && Objects.equals(getPath(), that.getPath())
+        && Objects.equals(getFormat(), that.getFormat())
+        && Objects.equals(isCreateDiff(), that.isCreateDiff());
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(
+        super.hashCode(),
+        isActive(),
+        getPath(),
+        getFormat(),
+        isCreateDiff());
+  }
+}
diff --git a/src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java b/src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java
@@ -18,6 +18,8 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO
   private final String ignoreFilmlistPath;
   /** add livestreams from external list **/
   private final ImportLivestreamConfiguration importLivestreamConfiguration;
+  /** add additional filmlist from external **/
+  private final List<ImportFilmlistConfiguration> ImportFilmlistConfigurations;
   /** The maximum amount of cpu threads to be used. */
   private Integer maximumCpuThreads;
   /**
@@ -32,12 +34,10 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO
   private Set<FilmlistFormats> filmlistSaveFormats;
   private Map<FilmlistFormats, String> filmlistSavePaths;
   private Map<FilmlistFormats, String> filmlistDiffSavePaths;
-  private FilmlistFormats filmlistImportFormat;
   private String filmlistImportLocation;
   private MServerLogSettingsDTO logSettings;
   private Map<CrawlerUrlType, URL> crawlerURLs;
   private Map<CrawlerApiParam, String> crawlerApiParams;
-  private Boolean filmlistImporEnabled;
 
   public MServerConfigDTO() {
     super();
@@ -79,16 +79,14 @@ public MServerConfigDTO() {
     filmlistSavePaths.put(FilmlistFormats.JSON_COMPRESSED_BZIP, "filmliste.json.bz");
     filmlistSavePaths.put(FilmlistFormats.OLD_JSON_COMPRESSED_BZIP, "filmliste_old.json.bz");
 
-    filmlistImporEnabled = true;
-    filmlistImportFormat = FilmlistFormats.OLD_JSON_COMPRESSED_XZ;
-    filmlistImportLocation = "https://verteiler1.mediathekview.de/Filmliste-akt.xz";
-
     writeFilmlistHashFileEnabled = false;
     filmlistHashFilePath = "filmlist.hash";
     writeFilmlistIdFileEnabled = true;
     filmlistIdFilePath = "filmlist.id";
     ignoreFilmlistPath = "ignoreFilmlist.txt";
     importLivestreamConfiguration = new ImportLivestreamConfiguration(false, "live-streams.json", FilmlistFormats.OLD_JSON);
+    ImportFilmlistConfigurations = new ArrayList<ImportFilmlistConfiguration>();
+    //ImportFilmlistConfigurations.add(new ImportFilmlistConfiguration(false, "https://verteiler1.mediathekview.de/Filmliste-akt.xz", FilmlistFormats.OLD_JSON_COMPRESSED_XZ, true));
 
     Arrays.stream(Sender.values())
         .forEach(sender -> senderConfigurations.put(sender, new MServerBasicConfigDTO(this)));
@@ -122,14 +120,6 @@ public void setFilmlistDiffSavePaths(final Map<FilmlistFormats, String> filmlist
     this.filmlistDiffSavePaths = filmlistDiffSavePaths;
   }
 
-  public FilmlistFormats getFilmlistImportFormat() {
-    return filmlistImportFormat;
-  }
-
-  public void setFilmlistImportFormat(final FilmlistFormats filmlistImportFormat) {
-    this.filmlistImportFormat = filmlistImportFormat;
-  }
-
   public String getFilmlistImportLocation() {
     return filmlistImportLocation;
   }
@@ -242,6 +232,9 @@ public ImportLivestreamConfiguration getImportLivestreamConfiguration() {
     return importLivestreamConfiguration;
   }
 
+  public List<ImportFilmlistConfiguration> getImportFilmlistConfigurations() {
+    return ImportFilmlistConfigurations;
+  }
 
   /**
    * Loads the {@link Sender} specific configuration and if it not exist creates one.
@@ -255,14 +248,6 @@ public MServerBasicConfigDTO getSenderConfig(final Sender aSender) {
     return senderConfigurations.get(aSender);
   }
 
-  public Boolean getFilmlistImporEnabled() {
-    return filmlistImporEnabled;
-  }
-
-  public void setFilmlistImporEnabled(final Boolean filmlistImporEnabled) {
-    this.filmlistImporEnabled = filmlistImporEnabled;
-  }
-
   @Override
   public boolean equals(final Object o) {
     if (this == o) {
@@ -284,17 +269,16 @@ public boolean equals(final Object o) {
         && Objects.equals(getFilmlistSaveFormats(), that.getFilmlistSaveFormats())
         && Objects.equals(getFilmlistSavePaths(), that.getFilmlistSavePaths())
         && Objects.equals(getFilmlistDiffSavePaths(), that.getFilmlistDiffSavePaths())
-        && getFilmlistImportFormat() == that.getFilmlistImportFormat()
         && Objects.equals(getFilmlistImportLocation(), that.getFilmlistImportLocation())
         && Objects.equals(getLogSettings(), that.getLogSettings())
         && Objects.equals(getCrawlerURLs(), that.getCrawlerURLs())
-        && Objects.equals(getFilmlistImporEnabled(), that.getFilmlistImporEnabled())
         && Objects.equals(getWriteFilmlistHashFileEnabled(), that.getWriteFilmlistHashFileEnabled())
         && Objects.equals(getFilmlistHashFilePath(), that.getFilmlistHashFilePath())
         && Objects.equals(getWriteFilmlistIdFileEnabled(), that.getWriteFilmlistIdFileEnabled())
         && Objects.equals(getFilmlistIdFilePath(), that.getFilmlistIdFilePath())
         && Objects.equals(getIgnoreFilmslistPath(), that.getIgnoreFilmslistPath())
-        && Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration());
+        && Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration())
+        && Objects.equals(getImportFilmlistConfigurations(), that.getImportFilmlistConfigurations());
   }
 
   @Override
@@ -310,17 +294,16 @@ public int hashCode() {
         getFilmlistSaveFormats(),
         getFilmlistSavePaths(),
         getFilmlistDiffSavePaths(),
-        getFilmlistImportFormat(),
         getFilmlistImportLocation(),
         getLogSettings(),
         getCrawlerURLs(),
-        getFilmlistImporEnabled(),
         getWriteFilmlistHashFileEnabled(),
         getFilmlistHashFilePath(),
         getWriteFilmlistIdFileEnabled(),
         getFilmlistIdFilePath(),
         getIgnoreFilmslistPath(),
-        getImportLivestreamConfiguration());
+        getImportLivestreamConfiguration(),
+        getImportFilmlistConfigurations());
   }
 
   public void initializeSenderConfigurations() {

diff --git a/src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java b/src/main/java/de/mediathekview/mserver/crawler/CrawlerManager.java
@@ -2,10 +2,12 @@
 
 import de.mediathekview.mlib.daten.Film;
 import de.mediathekview.mlib.daten.Filmlist;
+import de.mediathekview.mlib.daten.Podcast;
 import de.mediathekview.mlib.daten.Sender;
 import de.mediathekview.mlib.filmlisten.FilmlistFormats;
 import de.mediathekview.mlib.filmlisten.FilmlistManager;
 import de.mediathekview.mlib.progress.ProgressListener;
+import de.mediathekview.mserver.base.config.ImportFilmlistConfiguration;
 import de.mediathekview.mserver.base.config.MServerConfigDTO;
 import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.base.config.MServerCopySettings;
@@ -37,6 +39,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.time.LocalDateTime;
 import java.util.*;
 import java.util.Map.Entry;
 import java.util.concurrent.*;
@@ -65,7 +68,7 @@ public class CrawlerManager extends AbstractManager {
 
   private final FilmlistManager filmlistManager;
   private final Collection<ProgressListener> copyProgressListeners;
-  private Filmlist differenceList;
+  private final Filmlist differenceList;
 
   private CrawlerManager() {
     super();
@@ -77,6 +80,7 @@ private CrawlerManager() {
 
     crawlerMap = new EnumMap<>(Sender.class);
     filmlist = new Filmlist();
+    differenceList = new Filmlist();
     filmlistManager = FilmlistManager.getInstance();
     copyProgressListeners = new ArrayList<>();
     initializeCrawler(rootConfig);
@@ -151,9 +155,12 @@ public void filterFilmlist() {
    * {@link MServerConfigDTO#getFilmlistImportLocation()}.
    */
   public void importFilmlist() {
-    if (checkConfigForFilmlistImport()) {
-      importFilmlist(config.getFilmlistImportFormat(), config.getFilmlistImportLocation());
+    for (ImportFilmlistConfiguration importFilmlistConfiguration : config.getImportFilmlistConfigurations()) {
+      if (checkImportFilmlistConfig(importFilmlistConfiguration) && importFilmlistConfiguration.isActive()) {
+        importFilmlist(importFilmlistConfiguration);
+      }  
     }
+
   }
 
   public void importLivestreamFilmlist() {
@@ -188,18 +195,22 @@ public void importLivestreamFilmlist(final FilmlistFormats aFormat, final String
    *     </code> or <code>https</code> it tries to import from URL. Otherwise it tries to import
    *     from the given Location as a file path.
    */
-  public void importFilmlist(final FilmlistFormats aFormat, final String aFilmlistLocation) {
+  public void importFilmlist(final ImportFilmlistConfiguration importFilmlistConfiguration) {
     try {
       final Optional<Filmlist> importedFilmlist;
-      if (aFilmlistLocation.startsWith(HTTP)) {
-        importedFilmlist = importFilmListFromURl(aFormat, aFilmlistLocation);
+      if (importFilmlistConfiguration.getPath().startsWith(HTTP)) {
+        importedFilmlist = importFilmListFromURl(importFilmlistConfiguration.getFormat(), importFilmlistConfiguration.getPath());
       } else {
-        importedFilmlist = importFilmlistFromFile(aFormat, aFilmlistLocation);
+        importedFilmlist = importFilmlistFromFile(importFilmlistConfiguration.getFormat(), importFilmlistConfiguration.getPath());
+      }
+      //
+      final Filmlist difflist = new Filmlist(UUID.randomUUID(), LocalDateTime.now());
+      importedFilmlist.ifPresent(value -> addAllToFilmlist(mergeTwoFilmlists(filmlist,value),difflist));
+      if (importFilmlistConfiguration.isCreateDiff()) {
+        addAllToFilmlist(difflist, differenceList);
       }
-
-      importedFilmlist.ifPresent(value -> differenceList = filmlist.merge(value));
     } catch (final IOException ioException) {
-      LOG.fatal(String.format(FILMLIST_IMPORT_ERROR_TEMPLATE, aFilmlistLocation), ioException);
+      LOG.fatal(String.format(FILMLIST_IMPORT_ERROR_TEMPLATE, importFilmlistConfiguration.getPath()), ioException);
     }
   }
 
@@ -348,14 +359,23 @@ private boolean checkAllUsedFormatsHaveSavePaths() {
     return missingSavePathFormats.isEmpty();
   }
 
-  private boolean checkConfigForFilmlistImport() {
-    if (config.getFilmlistImportFormat() == null) {
-      printMessage(ServerMessages.NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG);
+  private boolean checkImportFilmlistConfig(ImportFilmlistConfiguration config) {
+    if (config.getPath() == null || config.getPath().isBlank() ) {
+      printMessage(ServerMessages.NO_FILMLIST_IMPORT_LOCATION_IN_CONFIG);
       return false;
     }
 
-    if (config.getFilmlistImportLocation() == null) {
-      printMessage(ServerMessages.NO_FILMLIST_IMPORT_LOCATION_IN_CONFIG);
+    if (config.getFormat() == null) {
+      printMessage(ServerMessages.NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG);
+      return false;
+    }
+    // @TODO define message
+    if (config.isActive() == null) {
+      printMessage(ServerMessages.NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG);
+      return false;
+    }
+    if (config.isCreateDiff() == null) {
+      printMessage(ServerMessages.NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG);
       return false;
     }
 
@@ -532,4 +552,70 @@ private void runCrawlers(final AbstractCrawler... aCrawlers) {
   public Filmlist getFilmlist() {
     return filmlist;
   }
+  // added to allow JUNIT tests
+  public Filmlist getDifferenceList() {
+    return differenceList;
+  }
+
+  // move this to MLIB after PR merge
+  public static void addAllToFilmlist(final Filmlist source,final Filmlist target) {
+    target.addAllFilms(source.getFilms().values());
+    target.addAllLivestreams(source.getLivestreams().values());
+    target.addAllPodcasts(source.getPodcasts().values());
+  }
+
+  // move this to MLIB and replace merge function after PR merge
+  public static Filmlist mergeTwoFilmlists(final Filmlist aThis, final Filmlist aFilmlist) {
+    final Filmlist toBeAdded = new Filmlist(UUID.randomUUID(), LocalDateTime.now());
+    final Filmlist diff = new Filmlist(UUID.randomUUID(), LocalDateTime.now());
+    // add all from old list not in the new list
+    aFilmlist.getFilms().entrySet().stream()
+        .filter(e -> !containsFilm(aThis, e.getValue()))
+        .forEachOrdered(e -> toBeAdded.getFilms().put(e.getKey(), e.getValue()));
+    // the diff list contains all new entries (fresh list) which are not already in the old list
+    aThis.getFilms().entrySet().stream()
+    .filter(e -> !containsFilm(aFilmlist,e.getValue()))
+    .forEachOrdered(e -> diff.getFilms().put(e.getKey(), e.getValue()));
+    // add the history to the current list
+    aThis.getFilms().putAll(toBeAdded.getFilms());
+    //
+    // the same for podcast
+    aFilmlist.getPodcasts().entrySet().stream()
+        .filter(e -> !containsPodcast(aThis,e.getValue()))
+        .forEachOrdered(e -> toBeAdded.getPodcasts().put(e.getKey(), e.getValue()));
+    aThis.getFilms().entrySet().stream()
+    .filter(e -> !containsPodcast(aFilmlist,e.getValue()))
+    .forEachOrdered(e -> diff.getPodcasts().put(e.getKey(), e.getValue()));
+    aThis.getPodcasts().putAll(toBeAdded.getPodcasts());
+    //
+    return diff;
+  }
+
+  public static boolean containsFilm(Filmlist athis, Film film) {
+    Optional<Film> check = athis.getFilms().entrySet().stream()
+        .filter(entry -> 
+            film.getTitel().equalsIgnoreCase(entry.getValue().getTitel()) &&
+            film.getThema().equalsIgnoreCase(entry.getValue().getThema()) &&
+            film.getSender().equals(entry.getValue().getSender()) &&
+            film.getDuration().equals(entry.getValue().getDuration()))
+        .map(Map.Entry::getValue)
+        .findFirst();
+
+    return check.isPresent();
+  }
+
+  public static boolean containsPodcast(Filmlist athis, Podcast prodcast) {
+    Optional<Podcast> check = athis.getPodcasts().entrySet().stream()
+        .filter(entry -> 
+        prodcast.getTitel().equalsIgnoreCase(entry.getValue().getTitel()) &&
+        prodcast.getThema().equalsIgnoreCase(entry.getValue().getThema()) &&
+        prodcast.getSender().equals(entry.getValue().getSender()) &&
+        prodcast.getDuration().equals(entry.getValue().getDuration()))
+        .map(Map.Entry::getValue)
+        .findFirst();
+
+    return check.isPresent();
+  } 
+
+
 }