#938 use url optimizers in kika crawler

mediathekview · Nov 12, 2023 · 31fc51f · 31fc51f
1 parent 8fb6640
commit 31fc51f
Show file tree

Hide file tree

Showing 3 changed files with 60 additions and 1 deletion.
diff --git a/src/main/java/mServer/crawler/sender/ard/ArdUrlOptimizer.java b/src/main/java/mServer/crawler/sender/ard/ArdUrlOptimizer.java
@@ -0,0 +1,48 @@
+package mServer.crawler.sender.ard;
+
+import mServer.crawler.sender.base.UrlUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class ArdUrlOptimizer {
+
+  private static final String BR_URL_1280 = "_X.mp4";
+  private static final String BR_URL_1920 = "_HD.mp4";
+  private static final String HR_URL_1280 = "1280x720-50p-3200kbit.mp4";
+  private static final String HR_URL_1920 = "1920x1080-50p-5000kbit.mp4";
+  private static final String NDR_URL_1280 = ".hd.mp4";
+  private static final String NDR_URL_1920 = ".1080.mp4";
+  private static final String RBB_URL_1280 = "hd1080-avc720.mp4";
+  private static final String RBB_URL_1920 = "hd1080-avc1080.mp4";
+  private static final String SR_URL_1280 = "_P.mp4";
+  private static final String SR_URL_1920 = "_H.mp4";
+  private static final String SWR_URL_1280 = ".xl.mp4";
+  private static final String SWR_URL_1920 = ".xxl.mp4";
+
+  private static final Map<String, String[]> HD_OPTIMIZE = new HashMap<>();
+
+  static {
+    HD_OPTIMIZE.put(BR_URL_1280, new String[] {BR_URL_1920});
+    HD_OPTIMIZE.put(HR_URL_1280, new String[] {HR_URL_1920});
+    HD_OPTIMIZE.put(NDR_URL_1280, new String[] {NDR_URL_1920});
+    HD_OPTIMIZE.put(RBB_URL_1280, new String[] {RBB_URL_1920});
+    HD_OPTIMIZE.put(SR_URL_1280, new String[] {SR_URL_1920});
+    HD_OPTIMIZE.put(SWR_URL_1280, new String[] {SWR_URL_1920});
+  }
+
+  public String optimizeHdUrl(final String url) {
+    for (Map.Entry<String, String[]> entry : HD_OPTIMIZE.entrySet()) {
+      if (url.contains(entry.getKey())) {
+        for (String optimizeFragment : entry.getValue()) {
+          final String optimizedUrl = url.replace(entry.getKey(), optimizeFragment);
+          if (UrlUtils.existsUrl(optimizedUrl)) {
+            return optimizedUrl;
+          }
+        }
+      }
+    }
+
+    return url;
+  }
+}
diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java
@@ -9,6 +9,7 @@
 import jakarta.ws.rs.core.Response;
 import mServer.crawler.CrawlerTool;
 import mServer.crawler.sender.MediathekReader;
+import mServer.crawler.sender.ard.ArdUrlOptimizer;
 import mServer.crawler.sender.base.AbstractJsonRestTask;
 import mServer.crawler.sender.base.AbstractRecursivConverterTask;
 import mServer.crawler.sender.base.GeoLocations;
@@ -17,6 +18,7 @@
 import mServer.crawler.sender.kika.KikaApiVideoInfoDto;
 import mServer.crawler.sender.kika.Resolution;
 import mServer.crawler.sender.kika.json.KikaApiVideoInfoPageDeserializer;
+import mServer.crawler.sender.zdf.ZdfVideoUrlOptimizer;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
@@ -36,9 +38,13 @@ public class KikaApiFilmTask extends AbstractJsonRestTask<DatenFilm, KikaApiVide
   private static final long serialVersionUID = 1L;
   private static final Logger LOG = LogManager.getLogger(KikaApiFilmTask.class);
   private static final RateLimiter LIMITER = RateLimiter.create(15);
+  private transient ArdUrlOptimizer ardUrlOptimizer;
+  private transient ZdfVideoUrlOptimizer zdfVideoUrlOptimizer;
 
   public KikaApiFilmTask(MediathekReader crawler, ConcurrentLinkedQueue<KikaApiFilmDto> urlToCrawlDTOs) {
     super(crawler, urlToCrawlDTOs, Optional.empty());
+    ardUrlOptimizer = new ArdUrlOptimizer();
+    zdfVideoUrlOptimizer = new ZdfVideoUrlOptimizer();
   }
 
   @Override
@@ -113,7 +119,10 @@ protected void postProcessing(KikaApiVideoInfoDto aResponseObj, KikaApiFilmDto a
       CrawlerTool.addUrlKlein(aFilm, videoUrls.get(Resolution.SMALL));
     }
     if (videoUrls.containsKey(Resolution.HD)) {
-      CrawlerTool.addUrlHd(aFilm, videoUrls.get(Resolution.HD));
+      String url = videoUrls.get(Resolution.HD);
+      url = ardUrlOptimizer.optimizeHdUrl(url);
+      url = zdfVideoUrlOptimizer.getOptimizedUrlHd(url);
+      CrawlerTool.addUrlHd(aFilm, url);
     }
     //
     getGeo(aDTO).ifPresent(geos -> {

diff --git a/src/main/java/mServer/crawler/sender/zdf/ZdfVideoUrlOptimizer.java b/src/main/java/mServer/crawler/sender/zdf/ZdfVideoUrlOptimizer.java
@@ -32,6 +32,7 @@ public class ZdfVideoUrlOptimizer {
   private static final String HD_3296_15_13 = "3296k_p15v13.mp4";
   private static final String HD_3296_15_14 = "3296k_p15v14.mp4";
   private static final String HD_3328_15_15 = "3328k_p15v15.mp4";
+  private static final String HD_3328_15_17 = "3328k_p15v17.mp4";
   private static final String HD_3328_12 = "3328k_p36v12.mp4";
   private static final String HD_3328_13 = "3328k_p36v13.mp4";
   private static final String HD_3328_14 = "3328k_p36v14.mp4";
@@ -75,6 +76,7 @@ public class ZdfVideoUrlOptimizer {
 
     HD_OPTIMIZE.put(HD_3360_36_17, new String[] {HD_6660_37_17, HD_6628_61_17});
     HD_OPTIMIZE.put(HD_6628_61_17, new String[] {HD_6660_37_17});
+    HD_OPTIMIZE.put(HD_3328_15_17, new String[] {HD_6660_37_17, HD_6628_61_17, HD_3360_36_17});
     HD_OPTIMIZE.put(HD_3328_15_15, new String[] {HD_3360_36_15});
     HD_OPTIMIZE.put(HD_3256, new String[] {HD_3328_12});
     HD_OPTIMIZE.put(HD_3296_15_14, new String[] {HD_3328_36_14});