Merge pull request #997 from mediathekview/bugfix/zdfFilm2Partner

Bugfix/zdf film2 partner

MServer-Config.yaml

@@ -1,7 +1,7 @@
 #### Server configurations ####
 
 # The maximum amount of cpu threads to be used.
-maximumCpuThreads: 1
+maximumCpuThreads: 10
 
 # The maximum duration in minutes the server should run.<br>
 # If set to 0 the server runs without a time limit.
@@ -28,11 +28,11 @@ senderIncluded:
   #- FUNK
   #- KIKA
   # - DW
-   - ORF
+  # - ORF
   #- PHOENIX
   #- SRF
   #- SR
-  #- ZDF
+  - ZDF
 
 #SRF,SR,PHONIX,ORF,KIKA,DW,3SAT<
 
@@ -130,7 +130,7 @@ maximumCrawlDurationInMinutes: 120
 
 # Enables the topics search
 # maximumSubpages limits the depth of the topics search
-topicsSearchEnabled: true
+topicsSearchEnabled: false
 
 # The maximum amount of sub pages to be crawled.<br>
 # Example: If a Sendung overview side has 10 pages with videos for this Sendung and
@@ -176,7 +176,8 @@ senderConfigurations:
     maximumSubpages: 2
     maximumRequestsPerSecond: 8.0
   ZDF:
-    maximumRequestsPerSecond: 10.0
+    maximumDaysForSendungVerpasstSection: 21
+    maximumRequestsPerSecond: 20
   FUNK:
     maximumUrlsPerTask: 99
   DREISAT:

src/main/java/de/mediathekview/mserver/base/utils/CheckUrlAvailability.java

@@ -2,6 +2,8 @@
 
 
 import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -19,9 +21,9 @@ public class CheckUrlAvailability {
   private final FileSizeDeterminer fsd;
   private int numberOfThreads = 10;
   private Long minFileSize = 2048L;
-  private int removedCounter = 0;
+  private AtomicInteger removedCounter = new AtomicInteger(0);
   private long timeoutInMS = 1*60*1000L;
-  private boolean timeout = false;
+  private AtomicBoolean timeout = new AtomicBoolean(false);
   private long start = 0;
 
   public CheckUrlAvailability(final long minFileSize, final long timeoutInSec, final int numberOfThreads) {
@@ -45,13 +47,13 @@ public Filmlist getAvaiableFilmlist(final Filmlist importList) {
             .join();
     customThreadPool.shutdown();
     //
-    LOG.debug("checked {} urls and removed {} in {} sec and timeout was reached: {}", importList.getFilms().size(), removedCounter, ((System.currentTimeMillis()-start)/1000), timeout);
+    LOG.debug("checked {} urls and removed {} in {} sec and timeout was reached: {}", importList.getFilms().size(), removedCounter.get(), ((System.currentTimeMillis()-start)/1000), timeout.get());
     return filteredFilmlist;
   }
 
   private boolean isAvailable(Film pFilm) {
-    if (timeout || System.currentTimeMillis() > (start+timeoutInMS)) {
-      timeout = true;
+    if (timeout.get() || System.currentTimeMillis() > (start+timeoutInMS)) {
+      timeout.set(true);
       return true;
     }
 
@@ -61,21 +63,25 @@ private boolean isAvailable(Film pFilm) {
     if (pFilm.getThema().equalsIgnoreCase("Livestream")) {
       // do not remove livestreams
       return true;
+    } else if (ri == null) {
+      LOG.debug("Film response (null): {} # {} # {} # {} ", normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
+      removedCounter.incrementAndGet();
+      return false;
     } else if (!(ri.getCode() >= 200 && ri.getCode() < 300)) {
       LOG.debug("Film response ({}): {} # {} # {} # {} ", ri.getCode(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-      removedCounter++;
+      removedCounter.incrementAndGet();
       return false;
     } else if (ri.getContentType().equalsIgnoreCase("text/html")) {
       LOG.debug("Film content type({}): {} # {} # {} # {} ", ri.getContentType(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-      removedCounter++;
+      removedCounter.incrementAndGet();
       return false;
     } else if (ri.getSize() < minFileSize && !normalUrl.endsWith("m3u8")) {
       LOG.debug("Film small ({}): {} # {} # {} # {} ", ri.getSize() , normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-      removedCounter++;
+      removedCounter.incrementAndGet();
       return false;
     }  else if (removedVideo(pFilm, ri.getPath())) {
       LOG.debug("Film url ({}): {} # {} # {} # {} ", ri.getPath(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-      removedCounter++;
+      removedCounter.incrementAndGet();
       return false;
     }
     return true;

src/main/java/de/mediathekview/mserver/crawler/dreisat/DreiSatCrawler.java

@@ -26,7 +26,7 @@ public DreiSatCrawler(
       final Collection<MessageListener> aMessageListeners,
       final Collection<SenderProgressListener> aProgressListeners,
       final MServerConfigManager rootConfig) {
-    super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+    super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig, DreisatConstants.PARTNER_TO_SENDER);
   }
 
   /**

src/main/java/de/mediathekview/mserver/crawler/dreisat/DreisatConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.dreisat;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class DreisatConstants {
 
   /** Base url of the 3Sat website. */
@@ -13,5 +18,11 @@ public final class DreisatConstants {
 
   public static final String URL_HTML_DAY = URL_BASE + "/programm?airtimeDate=%s";
 
+  public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+  static {
+    PARTNER_TO_SENDER.put("3sat", Sender.DREISAT);
+  }
+
   private DreisatConstants() {}
 }

src/main/java/de/mediathekview/mserver/crawler/phoenix/PhoenixConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.phoenix;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class PhoenixConstants {
   private PhoenixConstants() {}
 
@@ -10,4 +15,11 @@ private PhoenixConstants() {}
   public static final String URL_FILM_DETAIL_JSON = "/response/id/";
 
   public  static final String URL_VIDEO_DETAILS = "%s/php/mediaplayer/data/beitrags_details.php?id=%s";
+
+  public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+  static {
+    PARTNER_TO_SENDER.put("Phoenix", Sender.PHOENIX);
+  }
+
 }

src/main/java/de/mediathekview/mserver/crawler/phoenix/tasks/PhoenixFilmDetailTask.java

@@ -72,7 +72,7 @@ protected void processRestTarget(final CrawlerUrlDTO aDTO, final WebTarget aTarg
                 this.filmDetailHost,
                 filmDetailDto.getBaseName())));
     final ZdfFilmDetailTask zdfFilmDetailTask =
-        new ZdfFilmDetailTask(this.crawler, "", shows, null);
+        new ZdfFilmDetailTask(this.crawler, "", shows, null, PhoenixConstants.PARTNER_TO_SENDER);
     final Set<Film> films = zdfFilmDetailTask.invoke();
     films.forEach(
         film -> {

src/main/java/de/mediathekview/mserver/crawler/zdf/AbstractZdfCrawler.java

@@ -1,6 +1,7 @@
 package de.mediathekview.mserver.crawler.zdf;
 
 import de.mediathekview.mlib.daten.Film;
+import de.mediathekview.mlib.daten.Sender;
 import de.mediathekview.mlib.messages.listener.MessageListener;
 import de.mediathekview.mserver.base.config.MServerConfigManager;
 import de.mediathekview.mserver.base.messages.ServerMessages;
@@ -19,6 +20,7 @@
 import java.time.temporal.ChronoUnit;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Queue;
 import java.util.Set;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -29,13 +31,16 @@
 public abstract class AbstractZdfCrawler extends AbstractCrawler {
 
   private static final Logger LOG = LogManager.getLogger(AbstractZdfCrawler.class);
+  private final Map<String, Sender> partner2Sender;
 
   protected AbstractZdfCrawler(
       final ForkJoinPool aForkJoinPool,
       final Collection<MessageListener> aMessageListeners,
       final Collection<SenderProgressListener> aProgressListeners,
-      final MServerConfigManager rootConfig) {
+      final MServerConfigManager rootConfig,
+      final Map<String, Sender> partner2Sender) {
     super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+    this.partner2Sender = partner2Sender;
   }
 
   @Override
@@ -60,7 +65,8 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
           this,
           getApiUrlBase(),
           new ConcurrentLinkedQueue<>(shows),
-          configuration.getVideoAuthKey().orElse(null));
+          configuration.getVideoAuthKey().orElse(null),
+          partner2Sender);
     } catch (final InterruptedException ex) {
       LOG.debug("{} crawler interrupted.", getSender().getName(), ex);
       Thread.currentThread().interrupt();

src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfConstants.java

@@ -1,5 +1,10 @@
 package de.mediathekview.mserver.crawler.zdf;
 
+import java.util.HashMap;
+import java.util.Map;
+
+import de.mediathekview.mlib.daten.Sender;
+
 public final class ZdfConstants {
 
   /** Name of the header required for authentification. */
@@ -27,6 +32,16 @@ public final class ZdfConstants {
   /** The language key of german audio description. */
   public static final String LANGUAGE_GERMAN_AD = LANGUAGE_GERMAN + LANGUAGE_SUFFIX_AD;
   public static final String LANGUAGE_GERMAN_DGS = LANGUAGE_GERMAN + LANGUAGE_SUFFIX_DGS;
+
+  public static final Map<String, Sender> PARTNER_TO_SENDER = new HashMap<>();
+
+  static {
+    PARTNER_TO_SENDER.put("ZDFinfo", Sender.ZDF);
+    PARTNER_TO_SENDER.put("ZDFneo", Sender.ZDF);
+    PARTNER_TO_SENDER.put("ZDF", Sender.ZDF); 
+    PARTNER_TO_SENDER.put("EMPTY", Sender.ZDF);
+    // IGNORED Sender [KI.KA, WDR, PHOENIX, one, HR, 3sat, SWR, arte, BR, RBB, ARD, daserste, alpha, MDR, radiobremen, funk, ZDF, NDR, SR]
+  }
 
   private ZdfConstants() {}
 }

src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfCrawler.java

@@ -30,7 +30,7 @@ public ZdfCrawler(
       final Collection<MessageListener> aMessageListeners,
       final Collection<SenderProgressListener> aProgressListeners,
       final MServerConfigManager rootConfig) {
-    super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig);
+    super(aForkJoinPool, aMessageListeners, aProgressListeners, rootConfig, ZdfConstants.PARTNER_TO_SENDER);
   }
 
   @Override

src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfFilmDto.java

@@ -6,14 +6,19 @@
 import java.util.Objects;
 import java.util.Optional;
 
-public class ZdfFilmDto extends CrawlerUrlDTO {
+public class ZdfFilmDto {
 
-  private final Film film;
+  private final Optional<Film> film;
   private final Optional<String> urlSignLanguage;
+  private final Optional<String> videoUrl;
 
-  public ZdfFilmDto(final Film film, final String videoUrl, String urlSignLanguage) {
-    super(videoUrl);
+  public ZdfFilmDto(final Optional<Film> film, final String videoUrl, String urlSignLanguage) {
     this.film = film;
+    if (videoUrl == null) {
+      this.videoUrl = Optional.empty();
+    } else {
+      this.videoUrl = Optional.of(videoUrl);
+    }
 
     if (urlSignLanguage != null && !urlSignLanguage.isEmpty()) {
       this.urlSignLanguage = Optional.of(urlSignLanguage);
@@ -22,7 +27,11 @@ public ZdfFilmDto(final Film film, final String videoUrl, String urlSignLanguage
     }
   }
 
-  public Film getFilm() {
+  public Optional<String> getUrl() {
+    return videoUrl;
+  }
+
+  public Optional<Film> getFilm() {
     return film;
   }
 

src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java

@@ -44,6 +44,7 @@ public class ZdfFilmDetailDeserializer implements JsonDeserializer<Optional<ZdfF
   private static final String JSON_ELEMENT_TITLE = "title";
   private static final String JSON_ELEMENT_TEASER_TEXT = "teasertext";
   private static final String JSON_ATTRIBUTE_TEMPLATE = "http://zdf.de/rels/streams/ptmd-template";
+  private static final String JSON_ELEMENT_TVSERVICE = "tvService";
 
   private static final String PLACEHOLDER_PLAYER_ID = "{playerId}";
   private static final String PLAYER_ID = "android_native_5";
@@ -63,11 +64,11 @@ public class ZdfFilmDetailDeserializer implements JsonDeserializer<Optional<ZdfF
   private static final String[] SEASONNUMBER = {"http://zdf.de/rels/cmdm/season", "seasonNumber"};
 
   private final String apiUrlBase;
-  private final Sender sender;
+  private final Map<String, Sender> partner2Sender;
 
-  public ZdfFilmDetailDeserializer(final String apiUrlBase, final Sender sender) {
+  public ZdfFilmDetailDeserializer(final String apiUrlBase, Map<String, Sender> partner2Sender) {
     this.apiUrlBase = apiUrlBase;
-    this.sender = sender;
+    this.partner2Sender= partner2Sender; 
   }
 
   @Override
@@ -93,7 +94,11 @@ public Optional<ZdfFilmDto> deserialize(
         mainVideoTarget = mainVideo.get(JSON_ELEMENT_TARGET).getAsJsonObject();
       }
     }
-
+    final Optional<String> tvService = JsonUtils.getElementValueAsString(aJsonObject, JSON_ELEMENT_TVSERVICE);
+    if (!partner2Sender.containsKey(tvService.orElse("EMPTY"))) {
+      return Optional.empty();
+    }
+
     final Optional<String> title = parseTitle(rootNode, programItemTarget);
     final Optional<String> topic = parseTopic(rootNode);
     final Optional<String> description = parseDescription(rootNode);
@@ -106,12 +111,8 @@ public Optional<ZdfFilmDto> deserialize(
 
     if (title.isPresent()) {
       final Optional<Film> film =
-          createFilm(topic, title.get(), description, website, time, duration);
-
-      if (film.isPresent() && downloadUrl.containsKey(DOWNLOAD_URL_DEFAULT)) {
-        return Optional.of(new ZdfFilmDto(film.get(), downloadUrl.get(DOWNLOAD_URL_DEFAULT), downloadUrl.get(DOWNLOAD_URL_DGS)));
-      }
-      LOG.error("ZdfFilmDetailDeserializer: no film or downloadUrl: {}, {}", topic, title.get());
+          createFilm(partner2Sender.get(tvService.orElse("EMPTY")), topic, title.get(), description, website, time, duration);
+      return Optional.of(new ZdfFilmDto(film, downloadUrl.get(DOWNLOAD_URL_DEFAULT), downloadUrl.get(DOWNLOAD_URL_DGS)));
     } else {
       LOG.error("ZdfFilmDetailDeserializer: no title found");
     }
@@ -157,6 +158,7 @@ private String finalizeDownloadUrl(final String url) {
   }
 
   private Optional<Film> createFilm(
+      final Sender sender,
       final Optional<String> aTopic,
       final String aTitle,
       final Optional<String> aDescription,