Feature/955

src/main/java/de/mediathekview/mserver/crawler/orf/OrfCrawler.java

@@ -99,21 +99,24 @@ private Queue<TopicUrlDTO> getLetterEntries() throws InterruptedException, Execu
   @Override
   protected RecursiveTask<Set<Film>> createCrawlerTask() {
     try {
+      boolean processMoreEpisodes = false;
 
       final Queue<TopicUrlDTO> shows = new ConcurrentLinkedQueue<>();
 
       if (Boolean.TRUE.equals(crawlerConfig.getTopicsSearchEnabled())) {
         shows.addAll(getArchiveEntries());
-
         addShows(shows, getLetterEntries());
+        processMoreEpisodes = true;
+      } else {
+        addShows(shows, getDaysEntries());
+        processMoreEpisodes = false;
       }
-      addShows(shows, getDaysEntries());
 
       printMessage(
           ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName(), shows.size());
       getAndSetMaxCount(shows.size());
 
-      return new OrfFilmDetailTask(this, shows);
+      return new OrfFilmDetailTask(this, shows, processMoreEpisodes);
     } catch (final InterruptedException ex) {
       LOG.debug("{} crawler interrupted.", getSender().getName(), ex);
       Thread.currentThread().interrupt();

src/main/java/de/mediathekview/mserver/crawler/orf/json/OrfMoreEpisodesDeserializer.java

@@ -0,0 +1,26 @@
+package de.mediathekview.mserver.crawler.orf.json;
+
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import de.mediathekview.mserver.base.utils.JsonUtils;
+import de.mediathekview.mserver.base.utils.UrlUtils;
+import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
+import de.mediathekview.mserver.crawler.orf.OrfConstants;
+
+import java.lang.reflect.Type;
+import java.util.Optional;
+
+public class OrfMoreEpisodesDeserializer implements JsonDeserializer<CrawlerUrlDTO> {
+
+  private static final String ATTRIBUTE_URL = "url";
+
+  @Override
+  public CrawlerUrlDTO deserialize(
+      JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) {
+
+    final Optional<String> url =
+        JsonUtils.getAttributeAsString(jsonElement.getAsJsonObject(), ATTRIBUTE_URL);
+    return url.map(s -> new CrawlerUrlDTO(UrlUtils.addDomainIfMissing(s, OrfConstants.URL_BASE))).orElse(null);
+  }
+}

src/main/java/de/mediathekview/mserver/crawler/orf/parser/OrfMoreEpisodesParser.java

@@ -0,0 +1,25 @@
+package de.mediathekview.mserver.crawler.orf.parser;
+
+import de.mediathekview.mserver.base.HtmlConsts;
+import de.mediathekview.mserver.crawler.basic.TopicUrlDTO;
+import java.util.ArrayList;
+import java.util.List;
+import org.jsoup.nodes.Document;
+
+public class OrfMoreEpisodesParser {
+  private static final String EPISODES_SELECTOR = "article.b-teaser > a.teaser-link";
+
+  public List<TopicUrlDTO> parse(final Document document, final String topic) {
+    final List<TopicUrlDTO> result = new ArrayList<>();
+
+    document
+        .select(EPISODES_SELECTOR)
+        .forEach(
+            episode -> {
+              final String url = episode.attr(HtmlConsts.ATTRIBUTE_HREF);
+              result.add(new TopicUrlDTO(topic, url));
+            });
+
+    return result;
+  }
+}

src/main/java/de/mediathekview/mserver/crawler/orf/tasks/OrfFilmDetailTask.java

@@ -8,18 +8,18 @@
 import de.mediathekview.mlib.daten.GeoLocations;
 import de.mediathekview.mlib.daten.Resolution;
 import de.mediathekview.mserver.base.utils.HtmlDocumentUtils;
-import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
-import de.mediathekview.mserver.crawler.basic.AbstractDocumentTask;
-import de.mediathekview.mserver.crawler.basic.AbstractUrlTask;
-import de.mediathekview.mserver.crawler.basic.TopicUrlDTO;
+import de.mediathekview.mserver.crawler.basic.*;
 import de.mediathekview.mserver.crawler.orf.OrfEpisodeInfoDTO;
 import de.mediathekview.mserver.crawler.orf.OrfVideoInfoDTO;
+import de.mediathekview.mserver.crawler.orf.json.OrfMoreEpisodesDeserializer;
+import de.mediathekview.mserver.crawler.orf.parser.OrfMoreEpisodesParser;
 import de.mediathekview.mserver.crawler.orf.parser.OrfPlaylistDeserializer;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.jsoup.nodes.Document;
 
+import java.io.IOException;
 import java.lang.reflect.Type;
 import java.net.MalformedURLException;
 import java.net.URL;
@@ -29,6 +29,7 @@
 import java.time.format.DateTimeParseException;
 import java.time.temporal.ChronoUnit;
 import java.util.*;
+import java.util.concurrent.ConcurrentLinkedQueue;
 
 public class OrfFilmDetailTask extends AbstractDocumentTask<Film, TopicUrlDTO> {
 
@@ -40,21 +41,25 @@ public class OrfFilmDetailTask extends AbstractDocumentTask<Film, TopicUrlDTO> {
   private static final String DURATION_SELECTOR = VIDEO_META_DATA_SELECTOR + " span.duration";
   private static final String DESCRIPTION_SELECTOR = ".description-container .description-text";
   private static final String VIDEO_SELECTOR = "div.jsb_VideoPlaylist";
+  private static final String MORE_EPISODES_SELECTOR = "div.more-episodes";
 
   private static final String ATTRIBUTE_DATETIME = "datetime";
   private static final String ATTRIBUTE_DATA_JSB = "data-jsb";
-
   private static final String PREFIX_AUDIO_DESCRIPTION = "AD |";
 
   private static final DateTimeFormatter DATE_TIME_FORMATTER =
       DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
 
+  private static final Type CRAWLER_URL_TYPE_TOKEN = new TypeToken<CrawlerUrlDTO>() {}.getType();
   private static final Type LIST_EPISODEINFO_TYPE_TOKEN =
       new TypeToken<List<OrfEpisodeInfoDTO>>() {}.getType();
+  private final boolean processMoreEpisodes;
 
   public OrfFilmDetailTask(
-      final AbstractCrawler aCrawler, final Queue<TopicUrlDTO> aUrlToCrawlDtos) {
+      final AbstractCrawler aCrawler, final Queue<TopicUrlDTO> aUrlToCrawlDtos, boolean processMoreEpisodes) {
     super(aCrawler, aUrlToCrawlDtos);
+
+    this.processMoreEpisodes = processMoreEpisodes;
   }
 
   private static Optional<LocalDateTime> parseDate(final Document aDocument) {
@@ -147,12 +152,22 @@ protected void processDocument(final TopicUrlDTO aUrlDto, final Document aDocume
             episode.getDuration());
       }
     }
+
+    if (processMoreEpisodes) {
+      final List<TopicUrlDTO> topicUrlDTOS = parseMoreEpisodes(aDocument, aUrlDto.getTopic());
+      topicUrlDTOS.remove(aUrlDto);
+      processMoreEpisodes(topicUrlDTOS);
+    }
   }
 
   @Override
   protected AbstractUrlTask<Film, TopicUrlDTO> createNewOwnInstance(
       final Queue<TopicUrlDTO> aUrlsToCrawl) {
-    return new OrfFilmDetailTask(crawler, aUrlsToCrawl);
+    return createNewOwnInstance(aUrlsToCrawl, processMoreEpisodes);
+  }
+
+  private AbstractUrlTask<Film, TopicUrlDTO> createNewOwnInstance(final Queue<TopicUrlDTO> urlsToCrawl, boolean processMoreEpisodes) {
+    return new OrfFilmDetailTask(crawler, urlsToCrawl, processMoreEpisodes);
   }
 
   private void createFilm(
@@ -255,4 +270,37 @@ private List<OrfEpisodeInfoDTO> parseEpisodes(final Document aDocument) {
 
     return new ArrayList<>();
   }
+
+  private List<TopicUrlDTO> parseMoreEpisodes(final Document document, final String topic) {
+    final Optional<String> json = HtmlDocumentUtils.getElementAttributeString(MORE_EPISODES_SELECTOR, ATTRIBUTE_DATA_JSB, document);
+    if (json.isPresent()) {
+      final Gson gson =
+          new GsonBuilder()
+              .registerTypeAdapter(CRAWLER_URL_TYPE_TOKEN, new OrfMoreEpisodesDeserializer())
+              .create();
+
+      CrawlerUrlDTO moreEpisodesUrl = gson.fromJson(json.get(), CRAWLER_URL_TYPE_TOKEN);
+      if (moreEpisodesUrl != null) {
+        try {
+          final Document moreEpisodesDocument = crawler.requestBodyAsHtmlDocument(moreEpisodesUrl.getUrl());
+          OrfMoreEpisodesParser parser = new OrfMoreEpisodesParser();
+          return parser.parse(moreEpisodesDocument, topic);
+        } catch (IOException e) {
+          LOG.error("OrfFilmDetailTask: loading more episodes url {} failed.", moreEpisodesUrl.getUrl());
+          crawler.incrementAndGetErrorCount();
+        }
+      }
+    }
+
+    return new ArrayList<>();
+  }
+
+  private void processMoreEpisodes(final List<TopicUrlDTO> moreFilms) {
+        if (moreFilms != null && !moreFilms.isEmpty()) {
+      final Queue<TopicUrlDTO> queue = new ConcurrentLinkedQueue<>(moreFilms);
+      final OrfFilmDetailTask task = (OrfFilmDetailTask) createNewOwnInstance(queue, false);
+      task.fork();
+      taskResults.addAll(task.join());
+    }
+  }
 }

src/test/java/de/mediathekview/mserver/crawler/orf/json/OrfMoreEpisodesDeserializerTest.java

@@ -0,0 +1,23 @@
+package de.mediathekview.mserver.crawler.orf.json;
+
+import com.google.gson.JsonElement;
+import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
+import de.mediathekview.mserver.testhelper.JsonFileReader;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class OrfMoreEpisodesDeserializerTest {
+
+  @Test
+  void testDeserialize() {
+    final JsonElement jsonElement = JsonFileReader.readJson("/orf/orf_film_more_episodes.json");
+
+    final OrfMoreEpisodesDeserializer target = new OrfMoreEpisodesDeserializer();
+    final CrawlerUrlDTO actual = target.deserialize(jsonElement, null, null);
+
+    assertNotNull(actual);
+    assertEquals("https://tvthek.orf.at/lane-plus/other_episodes_of_profile?profileId=13895917&profileSlug=Biester", actual.getUrl());
+
+  }
+}

src/test/java/de/mediathekview/mserver/crawler/orf/parser/OrfMoreEpisodesParserTest.java

@@ -0,0 +1,39 @@
+package de.mediathekview.mserver.crawler.orf.parser;
+
+import de.mediathekview.mserver.crawler.basic.TopicUrlDTO;
+import de.mediathekview.mserver.testhelper.FileReader;
+import org.hamcrest.MatcherAssert;
+import org.hamcrest.Matchers;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class OrfMoreEpisodesParserTest {
+  @Test
+  void parseDocumentWithEpisodes() {
+    TopicUrlDTO[] expectedFilms = new TopicUrlDTO[] {
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-9/14207236"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-8/14207235"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-7/14207234"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-6/14207233"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-5/14207232"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-4/14207231"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-3/14207230"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-2/14207229"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Alle-Folgen-jetzt-Biester-1-10/14207227"),
+            new TopicUrlDTO("Biester", "https://tvthek.orf.at/profile/Biester/13895917/Biester-Folge-10/14207252"),
+    };
+
+    final Document document = Jsoup.parse(FileReader.readFile("/orf/orf_film_more_episodes.html"));
+
+    OrfMoreEpisodesParser target = new OrfMoreEpisodesParser();
+    final List<TopicUrlDTO> actual = target.parse(document, "Biester");
+
+    assertEquals(10, actual.size());
+    MatcherAssert.assertThat(actual, Matchers.containsInAnyOrder(expectedFilms));
+  }
+}

src/test/java/de/mediathekview/mserver/crawler/orf/tasks/OrfFilmDetailTaskTestBase.java

@@ -12,7 +12,7 @@ public OrfFilmDetailTaskTestBase() {
     }
 
   protected Set<Film> executeTask(OrfCrawler crawler, String aTheme, String aRequestUrl) {
-      return new OrfFilmDetailTask(crawler, createCrawlerUrlDto(aTheme, aRequestUrl))
+      return new OrfFilmDetailTask(crawler, createCrawlerUrlDto(aTheme, aRequestUrl), false)
               .invoke();
   }
 }