init POC

MServer-Config.yaml

@@ -18,7 +18,7 @@ maximumRequestsPerSecond: 999.0
 # If set only these Sender will be crawled all other will be ignored.
 senderIncluded:
   #- ARD
-  #- ARTE_DE
+  - ARTE_DE
   #- ARGE_FR
   #- ARTE_EN
   #- ARTE_PL
@@ -32,7 +32,7 @@ senderIncluded:
   #- PHOENIX
   #- SRF
   #- SR
-  - ZDF
+  #- ZDF
 
 #SRF,SR,PHONIX,ORF,KIKA,DW,3SAT<
 
@@ -159,9 +159,10 @@ senderConfigurations:
   ORF:
     maximumRequestsPerSecond: 10.0
   ARTE_DE:
-    maximumUrlsPerTask: 1
-    maximumDaysForSendungVerpasstSectionFuture: 0
-    maximumRequestsPerSecond: 2.0
+    maximumSubpages: 2
+    #maximumUrlsPerTask: 1
+    #maximumDaysForSendungVerpasstSectionFuture: 0
+    #maximumRequestsPerSecond: 2.0
   ARTE_FR:
     maximumDaysForSendungVerpasstSectionFuture: 0
     # The maximum amount of URLs to be processed per task.

src/main/java/de/mediathekview/mserver/crawler/artem/ArteMConstants.java

@@ -0,0 +1,15 @@
+package de.mediathekview.mserver.crawler.artem;
+
+public final class ArteMConstants {
+  //
+  public static final int PAGE_LIMIT = 100;
+  //
+  public static final String HOST = "https://api.arte.tv";
+  //
+  public static final String ALL_VIDEOS = HOST + "/api/opa/v3/videos?language=de&sort=-lastModified&limit=" + PAGE_LIMIT;
+  //
+  public static final String AUTH = "Bearer Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA";
+
+  private ArteMConstants() {}
+  //
+}

src/main/java/de/mediathekview/mserver/crawler/artem/ArteMCrawler.java

@@ -0,0 +1,62 @@
+package de.mediathekview.mserver.crawler.artem;
+
+import de.mediathekview.mlib.daten.Film;
+import de.mediathekview.mlib.daten.Sender;
+import de.mediathekview.mlib.messages.listener.MessageListener;
+import de.mediathekview.mserver.base.config.MServerConfigManager;
+import de.mediathekview.mserver.base.messages.ServerMessages;
+import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
+import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
+import de.mediathekview.mserver.crawler.basic.TopicUrlDTO;
+import de.mediathekview.mserver.crawler.kika.json.KikaApiFilmDto;
+import de.mediathekview.mserver.crawler.kika.tasks.*;
+import de.mediathekview.mserver.progress.listeners.SenderProgressListener;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Collection;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ForkJoinPool;
+import java.util.concurrent.RecursiveTask;
+
+public class ArteMCrawler extends AbstractCrawler {
+  private static final Logger LOG = LogManager.getLogger(ArteMCrawler.class);
+
+  public ArteMCrawler(
+      final ForkJoinPool aForkJoinPool,
+      final Collection<MessageListener> aMessageListeners,
+      final Collection<SenderProgressListener> aProgressListeners,
+      final MServerConfigManager aRootConfig) {
+    super(aForkJoinPool, aMessageListeners, aProgressListeners, aRootConfig);
+  }
+
+  @Override
+  public Sender getSender() {
+    return Sender.ARTE_DE;
+  }
+
+  @Override
+  protected RecursiveTask<Set<Film>> createCrawlerTask() {
+
+    try {
+      // get all brands from json doc
+      final Queue<CrawlerUrlDTO> root = new ConcurrentLinkedQueue<>();
+      root.add(new CrawlerUrlDTO(ArteMConstants.ALL_VIDEOS));
+      final ArteMVideoTask arteMVideoTask = new ArteMVideoTask(this, root, ArteMConstants.AUTH, 0);
+      final Queue<ArteMVideoDto> videos = new ConcurrentLinkedQueue<>();
+      videos.addAll(arteMVideoTask.fork().join());
+      //
+      printMessage(ServerMessages.DEBUG_ALL_SENDUNG_FOLGEN_COUNT, getSender().getName(), videos.size());
+      getAndSetMaxCount(videos.size());
+      //
+      return new ArteMStreamTask(this, videos, ArteMConstants.AUTH, 0);
+    } catch (final Exception ex) {
+      LOG.fatal("Exception in ARTE_DE crawler.", ex);
+    }
+
+    return null;
+  }
+
+}

src/main/java/de/mediathekview/mserver/crawler/artem/ArteMSreamDeserializer.java

@@ -0,0 +1,65 @@
+package de.mediathekview.mserver.crawler.artem;
+
+import com.google.gson.*;
+
+import de.mediathekview.mserver.base.utils.JsonUtils;
+import de.mediathekview.mserver.crawler.basic.PagedElementListDTO;
+import java.lang.reflect.Type;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+
+public class ArteMSreamDeserializer implements JsonDeserializer<PagedElementListDTO<ArteMStreamDto>> {
+  private static final String NEXT_PAGE[] = {"meta","videoStreams", "links", "next", "href"};
+  private static final String ELEMENT_STREAMS = "videoStreams";
+  private static final String ATTR_LANGUAGE = "language";
+  private static final String ATTR_QUALITY = "quality";
+  private static final String ATTR_MIMETYPE = "mimeType";
+  private static final String ATTR_AUDIOCODE = "audioCode";
+  private static final String ATTR_URL = "url";
+  private static final String ELEMENT_SUBTITLES = "subtitles";
+  private static final String ATTR_SUBTITLES_VERSION = "version";
+  private static final String ATTR_SUBTITLES_FILENAME = "filename";
+
+  @Override
+  public PagedElementListDTO<ArteMStreamDto> deserialize(
+      final JsonElement jsonElement, final Type typeOfT, final JsonDeserializationContext context)
+      throws JsonParseException {
+    //
+    PagedElementListDTO<ArteMStreamDto> list = new PagedElementListDTO<>();
+    //
+    list.setNextPage(JsonUtils.getElementValueAsString(jsonElement, NEXT_PAGE));
+    //
+    Optional<JsonElement> videos = JsonUtils.getElement(jsonElement, ELEMENT_STREAMS);
+    if (videos.isEmpty()) {
+      return list;
+    }
+    Optional<JsonElement> subtitle = JsonUtils.getElement(jsonElement, ELEMENT_SUBTITLES);
+    Optional<Map<String,String>> subtitleStreams = Optional.empty();
+    if (subtitle.isPresent()) {
+      Map<String,String> subtitleEntries = new HashMap<>();
+      for (JsonElement sub : subtitle.get().getAsJsonArray()) {
+        subtitleEntries.put(
+            JsonUtils.getElementValueAsString(sub, ATTR_SUBTITLES_VERSION).get(),
+            JsonUtils.getElementValueAsString(sub, ATTR_SUBTITLES_FILENAME).get()
+            );
+      }
+      subtitleStreams = Optional.of(subtitleEntries);
+    }
+
+    for (JsonElement stream : videos.get().getAsJsonArray()) {
+      list.addElement(new ArteMStreamDto(
+          JsonUtils.getElementValueAsString(stream, ATTR_LANGUAGE),
+          JsonUtils.getElementValueAsString(stream, ATTR_QUALITY),
+          JsonUtils.getElementValueAsString(stream, ATTR_MIMETYPE),
+          JsonUtils.getElementValueAsString(stream, ATTR_AUDIOCODE),
+          JsonUtils.getElementValueAsString(stream, ATTR_URL),
+          subtitleStreams));
+    }
+
+    return list;
+  }
+
+
+}

src/main/java/de/mediathekview/mserver/crawler/artem/ArteMStreamDto.java

@@ -0,0 +1,44 @@
+package de.mediathekview.mserver.crawler.artem;
+
+import java.util.Map;
+import java.util.Optional;
+
+public class ArteMStreamDto {
+  Optional<String> language;
+  Optional<String> quality;
+  Optional<String> mimeType;
+  Optional<String> audioCode;
+  Optional<String> url;
+  Optional<Map<String,String>> subtitles;
+  public ArteMStreamDto(Optional<String> language, Optional<String> quality, Optional<String> mimeType,
+      Optional<String> audioCode, Optional<String> url, Optional<Map<String, String>> subtitles) {
+    super();
+    this.language = language;
+    this.quality = quality;
+    this.mimeType = mimeType;
+    this.audioCode = audioCode;
+    this.url = url;
+    this.subtitles = subtitles;
+  }
+  public Optional<String> getLanguage() {
+    return language;
+  }
+  public Optional<String> getQuality() {
+    return quality;
+  }
+  public Optional<String> getMimeType() {
+    return mimeType;
+  }
+  public Optional<String> getAudioCode() {
+    return audioCode;
+  }
+  public Optional<String> getUrl() {
+    return url;
+  }
+  public Optional<Map<String, String>> getSubtitles() {
+    return subtitles;
+  }
+
+
+
+}

src/main/java/de/mediathekview/mserver/crawler/artem/ArteMStreamTask.java

@@ -0,0 +1,148 @@
+package de.mediathekview.mserver.crawler.artem;
+
+import java.lang.reflect.Type;
+import java.net.URI;
+import java.net.URL;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.temporal.TemporalUnit;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Queue;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import com.google.gson.reflect.TypeToken;
+
+import de.mediathekview.mlib.daten.Film;
+import de.mediathekview.mlib.daten.GeoLocations;
+import de.mediathekview.mserver.crawler.basic.AbstractCrawler;
+import de.mediathekview.mserver.crawler.basic.AbstractJsonRestTask;
+import de.mediathekview.mserver.crawler.basic.AbstractRecursiveConverterTask;
+import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO;
+import de.mediathekview.mserver.crawler.basic.PagedElementListDTO;
+import jakarta.ws.rs.core.Response;
+
+// <T, R, D extends CrawlerUrlDTO> extends AbstractRestTask<T, D>
+// return T Class from this task, desirialisation of class R , D , Reasearch in this url
+public class ArteMStreamTask extends AbstractJsonRestTask<Film, PagedElementListDTO<ArteMStreamDto>, ArteMVideoDto> {
+  private static final long serialVersionUID = 1L;
+  private static final Logger LOG = LogManager.getLogger(ArteMStreamTask.class);
+  private int subPageIndex = 0;
+
+  protected ArteMStreamTask(AbstractCrawler crawler, Queue<ArteMVideoDto> urlToCrawlDTOs, String authKey, int subPageIndex) {
+    super(crawler, urlToCrawlDTOs, authKey);
+    this.subPageIndex = subPageIndex;
+  }
+
+  @Override
+  protected Type getType() {
+    return new TypeToken<List<ArteMVideoDto>>() {}.getType();
+  }
+
+  @Override
+  protected void handleHttpError(ArteMVideoDto dto, URI url, Response response) {
+    crawler.printErrorMessage();
+    LOG.fatal(
+        "A HTTP error {} occurred when getting REST information from: \"{}\".",
+        response.getStatus(),
+        url);
+  }
+
+  @Override
+  protected void postProcessing(PagedElementListDTO<ArteMStreamDto> aResponseObj, ArteMVideoDto aDTO) {
+    final Optional<AbstractRecursiveConverterTask<Film, ArteMVideoDto>> subpageCrawler;
+    final Optional<String> nextPageLink = aResponseObj.getNextPage();
+    if (nextPageLink.isPresent() && config.getMaximumSubpages() > subPageIndex) {
+      final Queue<ArteMVideoDto> nextPageLinks = new ConcurrentLinkedQueue<>();
+      ArteMVideoDto np = new ArteMVideoDto(aDTO);
+      np.setUrl(nextPageLink.get());
+      nextPageLinks.add(np);
+      subpageCrawler = Optional.of(createNewOwnInstance(nextPageLinks));
+      subpageCrawler.get().fork();
+    } else {
+      subpageCrawler = Optional.empty();
+    }
+    // Trailer
+    if (!aDTO.getPlatform().orElse("").equalsIgnoreCase("EXTRAIT")) {
+      Set<ArteMStreamDto> streams = aResponseObj.getElements();
+      taskResults.add(createFilm(aDTO, streams));
+    }
+    //
+
+
+  }
+
+  @Override
+  protected Object getParser(ArteMVideoDto aDTO) {
+    return new ArteMSreamDeserializer();
+  }
+
+  @Override
+  protected AbstractRecursiveConverterTask<Film, ArteMVideoDto> createNewOwnInstance(
+      Queue<ArteMVideoDto> aElementsToProcess) {
+    return new ArteMStreamTask(crawler, aElementsToProcess, getAuthKey().orElse(""), subPageIndex+1);
+  }
+
+  private Film createFilm(ArteMVideoDto filmData, Set<ArteMStreamDto> streams) {
+    Film film = new Film(
+        UUID.randomUUID(),
+        crawler.getSender(),
+        filmData.getSubtitle().orElse(""),
+        filmData.getTitle().get(),
+        parseDate(filmData.getCreationDate().get()).get(),
+        parseDuration(filmData.getDurationSeconds().get()).get()
+        );
+    film.setBeschreibung(filmData.getShortDescription().get());
+    film.setWebsite(parseWebsite(filmData.getWebsite().get()).get());
+    film.addGeolocation(parseGeo(filmData.getGeoblockingZone().get()));
+    streams.stream().findAny().get().getSubtitles();
+    return film;
+  }
+
+  private Set<URL> parseSubtitle(Optional<Map<String, String>> data) {
+    return null;
+  }
+
+  private GeoLocations parseGeo(String data) {
+    switch(data) {
+      case "ALL":
+        return GeoLocations.GEO_NONE;
+    }
+    return GeoLocations.GEO_NONE;
+  }
+
+  private Optional<LocalDateTime> parseDate(String date) {
+    try {
+      return Optional.of(LocalDateTime.parse(date));
+    } catch (Exception e) {
+
+    }
+    return Optional.empty(); 
+  }
+
+  private Optional<Duration> parseDuration(String data) {
+    try {
+      return Optional.of(Duration.ofSeconds(Long.parseLong(data)));
+    } catch (Exception e) {
+
+    }
+    return Optional.empty(); 
+  }
+
+  private Optional<URL> parseWebsite(String data) {
+    try {
+      return Optional.of(new URL(data));
+    } catch (Exception e) {
+
+    }
+    return Optional.empty(); 
+  }
+
+}