ZDF Film to Partner logic

MServer-Config.yaml

@@ -1,7 +1,7 @@
 #### Server configurations ####
 
 # The maximum amount of cpu threads to be used.
-maximumCpuThreads: 1
+maximumCpuThreads: 10
 
 # The maximum duration in minutes the server should run.<br>
 # If set to 0 the server runs without a time limit.
@@ -28,11 +28,11 @@ senderIncluded:
   #- FUNK
   #- KIKA
   # - DW
-   - ORF
+  # - ORF
   #- PHOENIX
   #- SRF
   #- SR
-  #- ZDF
+  - ZDF
 
 #SRF,SR,PHONIX,ORF,KIKA,DW,3SAT<
 
@@ -130,7 +130,7 @@ maximumCrawlDurationInMinutes: 120
 
 # Enables the topics search
 # maximumSubpages limits the depth of the topics search
-topicsSearchEnabled: true
+topicsSearchEnabled: false
 
 # The maximum amount of sub pages to be crawled.<br>
 # Example: If a Sendung overview side has 10 pages with videos for this Sendung and
@@ -176,7 +176,8 @@ senderConfigurations:
     maximumSubpages: 2
     maximumRequestsPerSecond: 8.0
   ZDF:
-    maximumRequestsPerSecond: 10.0
+    maximumDaysForSendungVerpasstSection: 21
+    maximumRequestsPerSecond: 20
   FUNK:
     maximumUrlsPerTask: 99
   DREISAT:

src/main/java/de/mediathekview/mserver/base/utils/FilmlistDebugHelper.java

@@ -0,0 +1,111 @@
+package de.mediathekview.mserver.base.utils;
+
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import de.mediathekview.mlib.daten.Film;
+import de.mediathekview.mlib.daten.Filmlist;
+import de.mediathekview.mlib.daten.Resolution;
+import de.mediathekview.mserver.crawler.kika.KikaApiCrawler;
+
+public class FilmlistDebugHelper {
+  private static final Logger LOG = LogManager.getLogger(FilmlistDebugHelper.class);
+
+
+
+  public static Film getFilmFromSet(Set<Film> base, Film searchFilm) {
+    for (Film e : base) {
+        if (e.equals(searchFilm)) {
+            return e;
+        }
+    }
+    return null;
+  }
+
+  public static void printFilmlistForSender(Filmlist list, String sender, boolean printFullDetails) {
+    list.getFilms().values().stream().forEach( e -> {
+      if (e.getSenderName().equalsIgnoreCase(sender)) {
+        if (printFullDetails) {
+          LOG.debug("{} {} {}", e.getTitel(), e.getThema(), e);
+        } else {
+          LOG.debug("{} {}", e.getTitel(), e.getThema());
+        }
+      }
+    });
+  }
+
+  public static void printFilmlistForThema(Filmlist list, String thema, boolean printFullDetails) {
+    list.getFilms().values().stream().forEach( e -> {
+      if (e.getThema().equalsIgnoreCase(thema)) {
+        if (printFullDetails) {
+          LOG.debug("{} {} {} {}", e.getSenderName(), e.getTitel(), e.getThema(), e);
+        } else {
+          LOG.debug("{} {} {}", e.getSenderName(), e.getTitel(), e.getThema());
+        }
+      }
+    });
+  }
+
+
+
+
+  public static void compareFilmlist(Filmlist aFilmlist, Filmlist bFilmlist) {
+    ArrayList<Film> bFilms = new ArrayList<>(bFilmlist.getFilms().values());
+    aFilmlist.getFilms().values().forEach( f -> {
+      if (bFilms.indexOf(f) == -1) {
+        LOG.info("Missing Film in source list");
+        LOG.info(f.toString());
+      } else {
+        Film expectedFilm = bFilms.get(bFilms.indexOf(f));
+        compare(f, expectedFilm);
+      }
+    });
+    ArrayList<Film> aFilms = new ArrayList<>(aFilmlist.getFilms().values());
+    bFilms.forEach( f -> {
+      if (aFilms.indexOf(f) == -1) {
+        LOG.info("Missing Film in target list");
+        LOG.info(f.toString());
+      }
+    });
+  }
+
+  private static void compare(Film aFilm, Film bFilm) {
+    String error = "";
+    if (!aFilm.getSenderName().equalsIgnoreCase(bFilm.getSenderName())) {
+      error = "Incorrect Sender";
+    } else if (!aFilm.getTitel().equalsIgnoreCase(bFilm.getTitel())){
+      error = "Incorrect Title";
+    } else if (!aFilm.getThema().equalsIgnoreCase(bFilm.getThema())){
+      error = "Incorrect Topic";
+    } else if (!aFilm.getDuration().equals(bFilm.getDuration())){
+      error = "Incorrect Duration";
+    } else if (!aFilm.getBeschreibung().equalsIgnoreCase(bFilm.getBeschreibung())){
+      error = "Incorrect Description";
+    } else if (!aFilm.getWebsite().toString().equalsIgnoreCase(bFilm.getWebsite().toString())){
+      error = "Incorrect website";
+    } else if (!aFilm.getTime().equals(bFilm.getTime())){
+      error = "Incorrect Time";
+    } else if (!aFilm.getSubtitles().equals(bFilm.getSubtitles())){
+      error = "Incorrect subtitle";
+    } else if (bFilm.getUrl(Resolution.SMALL) != null && 
+              !aFilm.getUrl(Resolution.SMALL).toString().equalsIgnoreCase(bFilm.getUrl(Resolution.SMALL).toString())) {
+      error = "URL SMALL";
+    } else if (bFilm.getUrl(Resolution.NORMAL) != null &&
+              !aFilm.getUrl(Resolution.NORMAL).toString().equalsIgnoreCase(bFilm.getUrl(Resolution.NORMAL).toString())) {
+      error = "URL NORMAL";
+    } else if (bFilm.getUrl(Resolution.HD) != null &&
+                !aFilm.getUrl(Resolution.HD).toString().equalsIgnoreCase(bFilm.getUrl(Resolution.HD).toString())) {
+      error = "URL HD";
+    }
+    //
+    if (error != "") {
+      LOG.info(error);
+      LOG.info(aFilm.toString());
+      LOG.info(bFilm.toString());
+    }
+  }
+
+}

src/main/java/de/mediathekview/mserver/crawler/dreisat/DreiSatCrawler.java

@@ -26,7 +26,7 @@ public DreiSatCrawler(
       final Collection<MessageListener> aMessageListeners,
       final Collection<SenderProgressListener> aProgressListeners,
       final MServerConfigManager rootConfig) {
-    super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+    super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig, DreisatConstants.PARTNER_TO_SENDER);
   }
 
   /**

src/main/java/de/mediathekview/mserver/crawler/dreisat/DreisatConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.dreisat;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class DreisatConstants {
 
   /** Base url of the 3Sat website. */
@@ -13,5 +18,11 @@ public final class DreisatConstants {
 
   public static final String URL_HTML_DAY = URL_BASE + "/programm?airtimeDate=%s";
 
+  public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+  static {
+    PARTNER_TO_SENDER.put("3sat", Sender.DREISAT);
+  }
+
   private DreisatConstants() {}
 }

src/main/java/de/mediathekview/mserver/crawler/phoenix/PhoenixConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.phoenix;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class PhoenixConstants {
   private PhoenixConstants() {}
 
@@ -10,4 +15,11 @@ private PhoenixConstants() {}
   public static final String URL_FILM_DETAIL_JSON = "/response/id/";
 
   public  static final String URL_VIDEO_DETAILS = "%s/php/mediaplayer/data/beitrags_details.php?id=%s";
+
+  public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+  static {
+    PARTNER_TO_SENDER.put("Phoenix", Sender.PHOENIX);
+  }
+
 }

src/main/java/de/mediathekview/mserver/crawler/phoenix/tasks/PhoenixFilmDetailTask.java

@@ -72,7 +72,7 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
                 this.filmDetailHost,
                 filmDetailDto.getBaseName())));
     final ZdfFilmDetailTask zdfFilmDetailTask =
-        new ZdfFilmDetailTask(this.crawler, "", shows, null);
+        new ZdfFilmDetailTask(this.crawler, "", shows, null, PhoenixConstants.PARTNER_TO_SENDER);
     final Set<Film> films = zdfFilmDetailTask.invoke();
     films.forEach(
         film -> {

src/main/java/de/mediathekview/mserver/crawler/zdf/AbstractZdfCrawler.java

@@ -1,6 +1,7 @@
 package de.mediathekview.mserver.crawler.zdf;
 
 import de.mediathekview.mlib.daten.Film;
+import de.mediathekview.mlib.daten.Sender;
 import de.mediathekview.mlib.messages.listener.MessageListener;
 import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.base.messages.ServerMessages;
@@ -19,6 +20,7 @@
 import java.time.temporal.ChronoUnit;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -29,13 +31,16 @@
 public abstract class AbstractZdfCrawler extends AbstractCrawler {
 
   private static final Logger LOG = LogManager.getLogger(AbstractZdfCrawler.class);
+  private final Map<String, Sender> partner2Sender;
 
   protected AbstractZdfCrawler(
       final ForkJoinPool aForkJoinPool,
       final Collection<MessageListener> aMessageListeners,
       final Collection<SenderProgressListener> aProgressListeners,
-      final MServerConfigManager rootConfig) {
+      final MServerConfigManager rootConfig,
+      final Map<String, Sender> partner2Sender) {
     super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+    this.partner2Sender = partner2Sender;
   }
 
   @Override
@@ -60,7 +65,8 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
           this,
           getApiUrlBase(),
           new ConcurrentLinkedQueue<>(shows),
-          configuration.getVideoAuthKey().orElse(null));
+          configuration.getVideoAuthKey().orElse(null),
+          partner2Sender);
     } catch (final InterruptedException ex) {
       LOG.debug("{} crawler interrupted.", getSender().getName(), ex);
       Thread.currentThread().interrupt();

src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.zdf;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class ZdfConstants {
 
   /** Name of the header required for authentification. */
@@ -27,6 +32,16 @@ public final class ZdfConstants {
   /** The language key of german audio description. */
   public static final String LANGUAGE_GERMAN_AD = LANGUAGE_GERMAN + LANGUAGE_SUFFIX_AD;
   public static final String LANGUAGE_GERMAN_DGS = LANGUAGE_GERMAN + LANGUAGE_SUFFIX_DGS;
+
+  public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+  static {
+    PARTNER_TO_SENDER.put("ZDFinfo", Sender.ZDF);
+    PARTNER_TO_SENDER.put("ZDFneo", Sender.ZDF);
+    PARTNER_TO_SENDER.put("ZDF", Sender.ZDF); 
+    PARTNER_TO_SENDER.put("EMPTY", Sender.ZDF);
+    // IGNORED Sender [KI.KA, WDR, PHOENIX, one, HR, 3sat, SWR, arte, BR, RBB, ARD, daserste, alpha, MDR, radiobremen, funk, ZDF, NDR, SR]
+  }
 
   private ZdfConstants() {}
 }

src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfCrawler.java

@@ -30,7 +30,7 @@ public ZdfCrawler(
       final Collection<MessageListener> aMessageListeners,
       final Collection<SenderProgressListener> aProgressListeners,
       final MServerConfigManager rootConfig) {
-    super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+    super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig, ZdfConstants.PARTNER_TO_SENDER);
   }
 
   @Override

src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java

@@ -44,6 +44,7 @@ public class ZdfFilmDetailDeserializer implements JsonDeserializer<Optional<ZdfF
   private static final String JSON_ELEMENT_TITLE = "title";
   private static final String JSON_ELEMENT_TEASER_TEXT = "teasertext";
   private static final String JSON_ATTRIBUTE_TEMPLATE = "http://zdf.de/rels/streams/ptmd-template";
+  private static final String JSON_ELEMENT_TVSERVICE = "tvService";
 
   private static final String PLACEHOLDER_PLAYER_ID = "{playerId}";
   private static final String PLAYER_ID = "android_native_5";
@@ -63,11 +64,11 @@ public class ZdfFilmDetailDeserializer implements JsonDeserializer<Optional<ZdfF
   private static final String[] SEASONNUMBER = {"http://zdf.de/rels/cmdm/season", "seasonNumber"};
 
   private final String apiUrlBase;
-  private final Sender sender;
+  private final Map<String, Sender> partner2Sender;
 
-  public ZdfFilmDetailDeserializer(final String apiUrlBase, final Sender sender) {
+  public ZdfFilmDetailDeserializer(final String apiUrlBase, Map<String, Sender> partner2Sender) {
     this.apiUrlBase = apiUrlBase;
-    this.sender = sender;
+    this.partner2Sender= partner2Sender; 
   }
 
   @Override
@@ -93,7 +94,12 @@ public Optional<ZdfFilmDto> deserialize(
         mainVideoTarget = mainVideo.get(JSON_ELEMENT_TARGET).getAsJsonObject();
       }
     }
-
+    final Optional<String> tvService = JsonUtils.getElementValueAsString(aJsonObject, JSON_ELEMENT_TVSERVICE);
+    //System.out.println(tvService + " " + partner2Sender.get(tvService.orElse("EMPTY")));
+    if (!partner2Sender.containsKey(tvService.orElse("EMPTY"))) {
+      return Optional.empty();
+    }
+
     final Optional<String> title = parseTitle(rootNode, programItemTarget);
     final Optional<String> topic = parseTopic(rootNode);
     final Optional<String> description = parseDescription(rootNode);
@@ -106,7 +112,7 @@ public Optional<ZdfFilmDto> deserialize(
 
     if (title.isPresent()) {
       final Optional<Film> film =
-          createFilm(topic, title.get(), description, website, time, duration);
+          createFilm(partner2Sender.get(tvService.orElse("EMPTY")), topic, title.get(), description, website, time, duration);
 
       if (film.isPresent() && downloadUrl.containsKey(DOWNLOAD_URL_DEFAULT)) {
         return Optional.of(new ZdfFilmDto(film.get(), downloadUrl.get(DOWNLOAD_URL_DEFAULT), downloadUrl.get(DOWNLOAD_URL_DGS)));
@@ -157,6 +163,7 @@ private String finalizeDownloadUrl(final String url) {
   }
 
   private Optional<Film> createFilm(
+      final Sender sender,
       final Optional<String> aTopic,
       final String aTitle,
       final Optional<String> aDescription,