enable more debug info

mediathekview · Jun 3, 2024 · bd3cc4f · bd3cc4f
1 parent b4360ef
commit bd3cc4f
Show file tree

Hide file tree

Showing 5 changed files with 56 additions and 39 deletions.
diff --git a/src/main/java/de/mediathekview/mserver/base/utils/CheckUrlAvailability.java b/src/main/java/de/mediathekview/mserver/base/utils/CheckUrlAvailability.java
@@ -2,6 +2,8 @@
 
 
 import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -19,9 +21,9 @@ public class CheckUrlAvailability {
   private final FileSizeDeterminer fsd;
   private int numberOfThreads = 10;
   private Long minFileSize = 2048L;
-  private int removedCounter = 0;
+  private AtomicInteger removedCounter = new AtomicInteger(0);
   private long timeoutInMS = 1*60*1000L;
-  private boolean timeout = false;
+  private AtomicBoolean timeout = new AtomicBoolean(false);
   private long start = 0;
 
   public CheckUrlAvailability(final long minFileSize, final long timeoutInSec, final int numberOfThreads) {
@@ -45,13 +47,13 @@ public Filmlist getAvaiableFilmlist(final Filmlist importList) {
             .join();
     customThreadPool.shutdown();
     //
-    LOG.debug("checked {} urls and removed {} in {} sec and timeout was reached: {}", importList.getFilms().size(), removedCounter, ((System.currentTimeMillis()-start)/1000), timeout);
+    LOG.debug("checked {} urls and removed {} in {} sec and timeout was reached: {}", importList.getFilms().size(), removedCounter.get(), ((System.currentTimeMillis()-start)/1000), timeout.get());
     return filteredFilmlist;
   }
 
   private boolean isAvailable(Film pFilm) {
-    if (timeout || System.currentTimeMillis() > (start+timeoutInMS)) {
-      timeout = true;
+    if (timeout.get() || System.currentTimeMillis() > (start+timeoutInMS)) {
+      timeout.set(true);
       return true;
     }
 
@@ -61,21 +63,25 @@ private boolean isAvailable(Film pFilm) {
     if (pFilm.getThema().equalsIgnoreCase("Livestream")) {
       // do not remove livestreams
       return true;
+    } else if (ri == null) {
+      LOG.debug("Film response (null): {} # {} # {} # {} ", normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
+      removedCounter.incrementAndGet();
+      return false;
     } else if (!(ri.getCode() >= 200 && ri.getCode() < 300)) {
       LOG.debug("Film response ({}): {} # {} # {} # {} ", ri.getCode(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-      removedCounter++;
+      removedCounter.incrementAndGet();
       return false;
     } else if (ri.getContentType().equalsIgnoreCase("text/html")) {
       LOG.debug("Film content type({}): {} # {} # {} # {} ", ri.getContentType(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-      removedCounter++;
+      removedCounter.incrementAndGet();
       return false;
     } else if (ri.getSize() < minFileSize && !normalUrl.endsWith("m3u8")) {
       LOG.debug("Film small ({}): {} # {} # {} # {} ", ri.getSize() , normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-      removedCounter++;
+      removedCounter.incrementAndGet();
       return false;
     }  else if (removedVideo(pFilm, ri.getPath())) {
       LOG.debug("Film url ({}): {} # {} # {} # {} ", ri.getPath(), normalUrl, pFilm.getSender(), pFilm.getThema(), pFilm.getTitel());
-      removedCounter++;
+      removedCounter.incrementAndGet();
       return false;
     }
     return true;

diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfFilmDto.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/ZdfFilmDto.java
@@ -6,14 +6,19 @@
 import java.util.Objects;
 import java.util.Optional;
 
-public class ZdfFilmDto extends CrawlerUrlDTO {
+public class ZdfFilmDto {
 
-  private final Film film;
+  private final Optional<Film> film;
   private final Optional<String> urlSignLanguage;
+  private final Optional<String> videoUrl;
 
-  public ZdfFilmDto(final Film film, final String videoUrl, String urlSignLanguage) {
-    super(videoUrl);
+  public ZdfFilmDto(final Optional<Film> film, final String videoUrl, String urlSignLanguage) {
     this.film = film;
+    if (videoUrl == null) {
+      this.videoUrl = Optional.empty();
+    } else {
+      this.videoUrl = Optional.of(videoUrl);
+    }
 
     if (urlSignLanguage != null && !urlSignLanguage.isEmpty()) {
       this.urlSignLanguage = Optional.of(urlSignLanguage);
@@ -22,7 +27,11 @@ public ZdfFilmDto(final Film film, final String videoUrl, String urlSignLanguage
     }
   }
 
-  public Film getFilm() {
+  public Optional<String> getUrl() {
+    return videoUrl;
+  }
+
+  public Optional<Film> getFilm() {
     return film;
   }
 

diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java
@@ -95,7 +95,6 @@ public Optional<ZdfFilmDto> deserialize(
       }
     }
     final Optional<String> tvService = JsonUtils.getElementValueAsString(aJsonObject, JSON_ELEMENT_TVSERVICE);
-    //System.out.println(tvService + " " + partner2Sender.get(tvService.orElse("EMPTY")));
     if (!partner2Sender.containsKey(tvService.orElse("EMPTY"))) {
       return Optional.empty();
     }
@@ -113,11 +112,7 @@ public Optional<ZdfFilmDto> deserialize(
     if (title.isPresent()) {
       final Optional<Film> film =
           createFilm(partner2Sender.get(tvService.orElse("EMPTY")), topic, title.get(), description, website, time, duration);
-
-      if (film.isPresent() && downloadUrl.containsKey(DOWNLOAD_URL_DEFAULT)) {
-        return Optional.of(new ZdfFilmDto(film.get(), downloadUrl.get(DOWNLOAD_URL_DEFAULT), downloadUrl.get(DOWNLOAD_URL_DGS)));
-      }
-      LOG.error("ZdfFilmDetailDeserializer: no film or downloadUrl: {}, {}", topic, title.get());
+      return Optional.of(new ZdfFilmDto(film, downloadUrl.get(DOWNLOAD_URL_DEFAULT), downloadUrl.get(DOWNLOAD_URL_DGS)));
     } else {
       LOG.error("ZdfFilmDetailDeserializer: no title found");
     }

diff --git a/src/main/java/de/mediathekview/mserver/crawler/zdf/tasks/ZdfFilmDetailTask.java b/src/main/java/de/mediathekview/mserver/crawler/zdf/tasks/ZdfFilmDetailTask.java
@@ -90,29 +90,36 @@ private static void updateTitle(final String aLanguage, final Film aFilm) {
   protected void processRestTarget(final CrawlerUrlDTO aDto, final WebTarget aTarget) {
     final Optional<ZdfFilmDto> film = deserializeOptional(aTarget, OPTIONAL_FILM_TYPE_TOKEN);
     if (film.isPresent()) {
-      final Optional<DownloadDto> downloadDtoOptional =
-          deserializeOptional(
-              createWebTarget(film.get().getUrl()), OPTIONAL_DOWNLOAD_DTO_TYPE_TOKEN);
-
-      if (downloadDtoOptional.isPresent()) {
-        final DownloadDto downloadDto = downloadDtoOptional.get();
-        appendSignLanguage(downloadDto, film.get().getUrlSignLanguage());
-
-        try {
-          final Film result = film.get().getFilm();
-          if (result.getDuration().isZero() && downloadDto.getDuration().isPresent()) {
-            result.setDuration(downloadDto.getDuration().get());
+      if (film.get().getUrl().isPresent()) {
+        final Optional<DownloadDto> downloadDtoOptional =
+            deserializeOptional(
+                createWebTarget(film.get().getUrl().get()), OPTIONAL_DOWNLOAD_DTO_TYPE_TOKEN);
+
+        if (downloadDtoOptional.isPresent()) {
+          final DownloadDto downloadDto = downloadDtoOptional.get();
+          appendSignLanguage(downloadDto, film.get().getUrlSignLanguage());
+
+          try {
+            final Film result = film.get().getFilm().get();
+            if (result.getDuration().isZero() && downloadDto.getDuration().isPresent()) {
+              result.setDuration(downloadDto.getDuration().get());
+            }
+            addFilm(downloadDto, result);
+
+            crawler.incrementAndGetActualCount();
+            crawler.updateProgress();
+          } catch (final MalformedURLException e) {
+            LOG.error("ZdfFilmDetailTask: url can't be parsed: ", e);
+            crawler.incrementAndGetErrorCount();
+            crawler.updateProgress();
           }
-          addFilm(downloadDto, result);
-
-          crawler.incrementAndGetActualCount();
-          crawler.updateProgress();
-        } catch (final MalformedURLException e) {
-          LOG.error("ZdfFilmDetailTask: url can't be parsed: ", e);
+        } else {
+          LOG.error("ZdfFilmDetailTask: no video {} {} {} in {}",film.get().getFilm().get().getSenderName(), film.get().getFilm().get().getTitel(), film.get().getFilm().get().getThema() , aDto.toString());
           crawler.incrementAndGetErrorCount();
           crawler.updateProgress();
         }
       } else {
+        LOG.error("ZdfFilmDetailTask: no film found in {}", aDto.toString());
         crawler.incrementAndGetErrorCount();
         crawler.updateProgress();
       }

diff --git a/src/test/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializerTest.java b/src/test/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializerTest.java
@@ -186,7 +186,7 @@ public void test() {
     assertThat(actual.isPresent(), equalTo(true));
 
     AssertFilm.assertEquals(
-        actual.get().getFilm(),
+        actual.get().getFilm().get(),
         expectedSender,
         expectedTopic,
         expectedTitle,