Sonar fixes and more logging

mediathekview · Feb 26, 2024 · 2a936d3 · 2a936d3
1 parent bd3fd54
commit 2a936d3
Show file tree

Hide file tree

Showing 7 changed files with 53 additions and 34 deletions.
diff --git a/src/main/java/de/mediathekview/mserver/crawler/orfon/OrfOnCrawler.java b/src/main/java/de/mediathekview/mserver/crawler/orfon/OrfOnCrawler.java
@@ -74,6 +74,7 @@ protected RecursiveTask<Set<Film>> createCrawlerTask() {
       return new OrfOnVideoInfo2FilmTask(this, new ConcurrentLinkedQueue<>(allVideos));
     } catch (final Exception ex) {
       LOG.fatal("Exception in ORFON crawler.", ex);
+      Thread.currentThread().interrupt();
     }
 
     return null;

diff --git a/src/main/java/de/mediathekview/mserver/crawler/orfon/json/OrfOnEpisodeDeserializer.java b/src/main/java/de/mediathekview/mserver/crawler/orfon/json/OrfOnEpisodeDeserializer.java
@@ -20,7 +20,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.EnumMap;
-import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
@@ -85,23 +84,6 @@ public OrfOnVideoInfoDTO deserialize(
         buildOrResolveSubs(jsonElement)
 
         );
-
-    if (aFilm.getVideoUrls().isEmpty()){
-      LOG.debug("#####videoUrlEmpty#######");
-      LOG.debug("{} (id)", aFilm.getId().get());
-      LOG.debug("{} (genre_title)", JsonUtils.getElementValueAsString(jsonElement, "genre_title").get());
-      LOG.debug("{} (headline)", JsonUtils.getElementValueAsString(jsonElement, "headline").get());
-      LOG.debug("{} (profile_title*)", JsonUtils.getElementValueAsString(jsonElement, "profile_title").get());
-      LOG.debug("{} (title*)", JsonUtils.getElementValueAsString(jsonElement, "title").get());
-      LOG.debug("{} (sub_headline)", JsonUtils.getElementValueAsString(jsonElement, "sub_headline").get());
-      LOG.debug("{} (share_subject)", JsonUtils.getElementValueAsString(jsonElement, "share_subject").get());
-      LOG.debug("{} (TAG_RIGHT)", parseGeoLocations(JsonUtils.getElementValueAsString(jsonElement, TAG_RIGHT)));
-      LOG.debug("{} (url)", parseUrl(jsonElement));
-      LOG.debug("{} (segments)", JsonUtils.getElementValueAsString(jsonElement, TAG_SEGMENTS));
-      LOG.debug("{}",jsonElement );
-      LOG.debug("############");
-    }
-    // 
     return aFilm;
   }
 
@@ -119,7 +101,7 @@ private Optional<Set<URL>> buildOrResolveSubs(JsonElement jsonElement) {
           Map.entry("Accept-Encoding", "*"));
       JsonElement newRequestForSubs = null;
       try {
-        newRequestForSubs = crawler.getConnection().requestBodyAsJsonElement(subtitleSource.get().toString(), myMap);
+        newRequestForSubs = crawler.getConnection().requestBodyAsJsonElement(subtitleSource.get(), myMap);
         if (newRequestForSubs != null) {
           setOfSubs = parseSubtitleUrls(newRequestForSubs);
         }

diff --git a/...n/java/de/mediathekview/mserver/crawler/orfon/json/OrfOnHistoryVideoItemDeserializer.java b/...n/java/de/mediathekview/mserver/crawler/orfon/json/OrfOnHistoryVideoItemDeserializer.java
@@ -9,17 +9,14 @@
 import java.lang.reflect.Type;
 import java.util.Optional;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
 public class OrfOnHistoryVideoItemDeserializer implements JsonDeserializer<PagedElementListDTO<OrfOnBreadCrumsUrlDTO>> {
   private static final String[] TAG_NEXT_PAGE = { "next" };
   private static final String[] TAG_ITEM_ARRAY = { "_items" };
   private static final String[] TAG_ITEM_TITLE = {"title"};
   private static final String[] TAG_TARGET_URL = {"_links", "self", "href"};
   private static final String[] TAG_TARGET_URL_EPISODE = {"_links", "episode", "href"};
 
-  protected final Logger LOG = LogManager.getLogger(this.getClass());
+
 
   @Override
   public PagedElementListDTO<OrfOnBreadCrumsUrlDTO> deserialize(

diff --git a/src/main/java/de/mediathekview/mserver/crawler/orfon/task/OrfOnEpisodeTask.java b/src/main/java/de/mediathekview/mserver/crawler/orfon/task/OrfOnEpisodeTask.java
@@ -55,6 +55,16 @@ protected void postProcessing(OrfOnVideoInfoDTO aResponseObj, OrfOnBreadCrumsUrl
       crawler.incrementAndGetErrorCount();
       return;
     }
+    if (aResponseObj.getDuration().isEmpty()) {
+      LOG.warn("Missing duration for {}", aDTO);
+    }
+    if (aResponseObj.getAired().isEmpty()) {
+      LOG.warn("Missing aired date for {}", aDTO);
+    }
+    if (aResponseObj.getWebsite().isEmpty()) {
+      LOG.warn("Missing website for {}", aDTO);
+    }
+
     //LOG.debug(" bread crums {} # {} # {}", String.join("|", aDTO.getBreadCrums()), aResponseObj.getTopic().get(), aResponseObj.getTitle().get());
     taskResults.add(aResponseObj);    
   }

diff --git a/src/main/java/de/mediathekview/mserver/crawler/orfon/task/OrfOnEpisodesTask.java b/src/main/java/de/mediathekview/mserver/crawler/orfon/task/OrfOnEpisodesTask.java
@@ -54,8 +54,34 @@ protected void postProcessing(PagedElementListDTO<OrfOnVideoInfoDTO> aResponseOb
     } else {
       subpageCrawler = Optional.empty();
     }
-
-    taskResults.addAll(aResponseObj.getElements());
+    //
+    for (OrfOnVideoInfoDTO rs : aResponseObj.getElements()) {
+      if (rs.getTitle().isEmpty() && rs.getTitleWithDate().isEmpty()) {
+        LOG.warn("Missing title for {} in {}", rs.getId(), aDTO);
+        crawler.incrementAndGetErrorCount();
+        return;
+      }
+      if (rs.getTopic().isEmpty()) {
+        LOG.warn("Missing topic for {} in {}", rs.getId(), aDTO);
+        crawler.incrementAndGetErrorCount();
+        return;
+      }
+      if (rs.getVideoUrls().isEmpty()) {
+        LOG.warn("Missing videoUrls for {} in {}", rs.getId(), aDTO);
+        crawler.incrementAndGetErrorCount();
+        return;
+      }
+      if (rs.getDuration().isEmpty()) {
+        LOG.warn("Missing duration for {} in {}", rs.getId(), aDTO);
+      }
+      if (rs.getAired().isEmpty()) {
+        LOG.warn("Missing aired date for {} in {}", rs.getId(), aDTO);
+      }
+      if (rs.getWebsite().isEmpty()) {
+        LOG.warn("Missing website for {} in {}", rs.getId(), aDTO);
+      }
+      taskResults.add(rs);
+    }
     subpageCrawler.ifPresent(paginationResults -> taskResults.addAll(paginationResults.join()));
   }
 

diff --git a/src/main/java/de/mediathekview/mserver/crawler/orfon/task/OrfOnPagedTask.java b/src/main/java/de/mediathekview/mserver/crawler/orfon/task/OrfOnPagedTask.java
@@ -22,7 +22,7 @@
 // return T Class from this task, desirialisation of class R , D , Reasearch in this url
 public abstract class OrfOnPagedTask extends AbstractJsonRestTask<OrfOnBreadCrumsUrlDTO, PagedElementListDTO<OrfOnBreadCrumsUrlDTO>, OrfOnBreadCrumsUrlDTO> {
   private static final long serialVersionUID = 1L;
-  protected final Logger LOG = LogManager.getLogger(this.getClass());
+  protected final transient Logger log = LogManager.getLogger(this.getClass());
   protected Optional<AbstractRecursiveConverterTask<OrfOnBreadCrumsUrlDTO, OrfOnBreadCrumsUrlDTO>> nextPageTask = Optional.empty();
 
   protected OrfOnPagedTask(AbstractCrawler crawler, Queue<OrfOnBreadCrumsUrlDTO> urlToCrawlDTOs) {
@@ -37,7 +37,7 @@ protected void postProcessingNextPage(PagedElementListDTO<OrfOnBreadCrumsUrlDTO>
     nextPageLinks.add(new OrfOnBreadCrumsUrlDTO(aDTO.getBreadCrums(), aResponseObj.getNextPage().get()));
     nextPageTask = Optional.of(createNewOwnInstance(nextPageLinks));
     nextPageTask.get().fork();
-    LOG.debug("started paging to url {} for {}", aResponseObj.getNextPage().get(), aDTO.getUrl());
+    log.debug("started paging to url {} for {}", aResponseObj.getNextPage().get(), aDTO.getUrl());
   }
 
   protected void postProcessingElements(Set<OrfOnBreadCrumsUrlDTO> elements, OrfOnBreadCrumsUrlDTO originalDTO) {
@@ -57,7 +57,7 @@ protected void postProcessing(PagedElementListDTO<OrfOnBreadCrumsUrlDTO> aRespon
   @Override
   protected void handleHttpError(OrfOnBreadCrumsUrlDTO dto, URI url, Response response) {
       crawler.printErrorMessage();
-      LOG.fatal(
+      log.fatal(
           "A HTTP error {} occurred when getting REST information from: \"{}\".",
           response.getStatus(),
           url);

diff --git a/src/main/java/de/mediathekview/mserver/crawler/orfon/task/OrfOnVideoInfo2FilmTask.java b/src/main/java/de/mediathekview/mserver/crawler/orfon/task/OrfOnVideoInfo2FilmTask.java
@@ -1,5 +1,7 @@
 package de.mediathekview.mserver.crawler.orfon.task;
 
+import java.time.Duration;
+import java.time.LocalDateTime;
 import java.util.Queue;
 import java.util.UUID;
 
@@ -11,6 +13,7 @@
 
 public class OrfOnVideoInfo2FilmTask extends AbstractRecursiveConverterTask<Film, OrfOnVideoInfoDTO> {
   private static final long serialVersionUID = 1L;
+  private static final String ORF_AUDIODESCRIPTION_PREFIX = "AD | ";
 
   public OrfOnVideoInfo2FilmTask(AbstractCrawler aCrawler, Queue<OrfOnVideoInfoDTO> aUrlToCrawlDTOs) {
     super(aCrawler, aUrlToCrawlDTOs);
@@ -32,8 +35,8 @@ protected void processElement(OrfOnVideoInfoDTO aElement) {
         Sender.ORF,
         buildTitle(aElement.getTitle().get(), aElement.getTopic().get()),
         buildTopic(aElement.getTitle().get(), aElement.getTopic().get(), aElement.getTopicForArchive().orElse("")),
-        aElement.getAired().get(),
-        aElement.getDuration().get()
+        aElement.getAired().orElse(LocalDateTime.of(1970,1,1,00,00,00)),
+        aElement.getDuration().orElse(Duration.ofMinutes(0L))
         );
     aElement.getGeorestriction().ifPresent(aFilm::addAllGeoLocations);
     aElement.getDescription().ifPresent(aFilm::setBeschreibung);
@@ -47,8 +50,8 @@ protected void processElement(OrfOnVideoInfoDTO aElement) {
 
   private String buildTopic(String title, String topic, String archiveTopic) {
     String newTopic = topic;
-    if (newTopic.startsWith("AD | ")) {
-      newTopic = newTopic.replace("AD | ", "");
+    if (newTopic.startsWith(ORF_AUDIODESCRIPTION_PREFIX)) {
+      newTopic = newTopic.replace(ORF_AUDIODESCRIPTION_PREFIX, "");
     }
     if (newTopic.equalsIgnoreCase("archiv")) {
       newTopic = archiveTopic.replace("History | ", "");
@@ -57,8 +60,8 @@ private String buildTopic(String title, String topic, String archiveTopic) {
   }
 
   private String buildTitle(String title, String topic) {
-    if (title.startsWith("AD | ")) {
-      return title.replace("AD | ", "").concat(" (Audiodeskription)");
+    if (title.startsWith(ORF_AUDIODESCRIPTION_PREFIX)) {
+      return title.replace(ORF_AUDIODESCRIPTION_PREFIX, "").concat(" (Audiodeskription)");
     }
     return title;
   }