Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kika: optimize hd urls #946

Merged
merged 2 commits into from
Nov 13, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions src/main/java/mServer/crawler/sender/ard/ArdUrlOptimizer.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package mServer.crawler.sender.ard;

import mServer.crawler.sender.base.UrlUtils;

import java.util.HashMap;
import java.util.Map;

public class ArdUrlOptimizer {

private static final String BR_URL_1280 = "_X.mp4";
private static final String BR_URL_1920 = "_HD.mp4";
private static final String HR_URL_1280 = "1280x720-50p-3200kbit.mp4";
private static final String HR_URL_1920 = "1920x1080-50p-5000kbit.mp4";
private static final String NDR_URL_1280 = ".hd.mp4";
private static final String NDR_URL_1920 = ".1080.mp4";
private static final String RBB_URL_1280 = "hd1080-avc720.mp4";
private static final String RBB_URL_1920 = "hd1080-avc1080.mp4";
private static final String SR_URL_1280 = "_P.mp4";
private static final String SR_URL_1920 = "_H.mp4";
private static final String SWR_URL_1280 = ".xl.mp4";
private static final String SWR_URL_1920 = ".xxl.mp4";

private static final Map<String, String[]> HD_OPTIMIZE = new HashMap<>();

static {
HD_OPTIMIZE.put(BR_URL_1280, new String[] {BR_URL_1920});
HD_OPTIMIZE.put(HR_URL_1280, new String[] {HR_URL_1920});
HD_OPTIMIZE.put(NDR_URL_1280, new String[] {NDR_URL_1920});
HD_OPTIMIZE.put(RBB_URL_1280, new String[] {RBB_URL_1920});
HD_OPTIMIZE.put(SR_URL_1280, new String[] {SR_URL_1920});
HD_OPTIMIZE.put(SWR_URL_1280, new String[] {SWR_URL_1920});
}

public String optimizeHdUrl(final String url) {
for (Map.Entry<String, String[]> entry : HD_OPTIMIZE.entrySet()) {
if (url.contains(entry.getKey())) {
for (String optimizeFragment : entry.getValue()) {
final String optimizedUrl = url.replace(entry.getKey(), optimizeFragment);
if (UrlUtils.existsUrl(optimizedUrl)) {
return optimizedUrl;
}
}
}
}

return url;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
import java.util.stream.StreamSupport;
import mServer.crawler.sender.MediathekReader;
import mServer.crawler.sender.ard.ArdFilmUrlInfoDto;
import mServer.crawler.sender.ard.ArdUrlOptimizer;
import mServer.crawler.sender.base.JsonUtils;
import mServer.crawler.sender.base.UrlUtils;
import mServer.crawler.sender.base.Qualities;
import mServer.crawler.sender.swr.SwrUrlOptimizer;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand All @@ -42,12 +42,12 @@ public class ArdMediaArrayToDownloadUrlsConverter {

private static final String FILE_TYPE_F4M = "f4m";

private final SwrUrlOptimizer ardOptimizer;
private final ArdUrlOptimizer ardOptimizer;
private final Map<Qualities, Set<ArdFilmUrlInfoDto>> urls;
private MediathekReader crawler;

public ArdMediaArrayToDownloadUrlsConverter() {
ardOptimizer = new SwrUrlOptimizer();
ardOptimizer = new ArdUrlOptimizer();
urls = new EnumMap<>(Qualities.class);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import jakarta.ws.rs.core.Response;
import mServer.crawler.CrawlerTool;
import mServer.crawler.sender.MediathekReader;
import mServer.crawler.sender.ard.ArdUrlOptimizer;
import mServer.crawler.sender.base.AbstractJsonRestTask;
import mServer.crawler.sender.base.AbstractRecursivConverterTask;
import mServer.crawler.sender.base.GeoLocations;
Expand All @@ -17,6 +18,7 @@
import mServer.crawler.sender.kika.KikaApiVideoInfoDto;
import mServer.crawler.sender.kika.Resolution;
import mServer.crawler.sender.kika.json.KikaApiVideoInfoPageDeserializer;
import mServer.crawler.sender.zdf.ZdfVideoUrlOptimizer;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

Expand All @@ -36,9 +38,13 @@ public class KikaApiFilmTask extends AbstractJsonRestTask<DatenFilm, KikaApiVide
private static final long serialVersionUID = 1L;
private static final Logger LOG = LogManager.getLogger(KikaApiFilmTask.class);
private static final RateLimiter LIMITER = RateLimiter.create(15);
private transient ArdUrlOptimizer ardUrlOptimizer;
private transient ZdfVideoUrlOptimizer zdfVideoUrlOptimizer;

public KikaApiFilmTask(MediathekReader crawler, ConcurrentLinkedQueue<KikaApiFilmDto> urlToCrawlDTOs) {
super(crawler, urlToCrawlDTOs, Optional.empty());
ardUrlOptimizer = new ArdUrlOptimizer();
zdfVideoUrlOptimizer = new ZdfVideoUrlOptimizer();
}

@Override
Expand Down Expand Up @@ -113,7 +119,10 @@ protected void postProcessing(KikaApiVideoInfoDto aResponseObj, KikaApiFilmDto a
CrawlerTool.addUrlKlein(aFilm, videoUrls.get(Resolution.SMALL));
}
if (videoUrls.containsKey(Resolution.HD)) {
CrawlerTool.addUrlHd(aFilm, videoUrls.get(Resolution.HD));
String url = videoUrls.get(Resolution.HD);
url = ardUrlOptimizer.optimizeHdUrl(url);
url = zdfVideoUrlOptimizer.getOptimizedUrlHd(url);
CrawlerTool.addUrlHd(aFilm, url);
}
//
getGeo(aDTO).ifPresent(geos -> {
Expand Down
20 changes: 0 additions & 20 deletions src/main/java/mServer/crawler/sender/swr/SwrUrlOptimizer.java

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public class ZdfVideoUrlOptimizer {
private static final String HD_3296_15_13 = "3296k_p15v13.mp4";
private static final String HD_3296_15_14 = "3296k_p15v14.mp4";
private static final String HD_3328_15_15 = "3328k_p15v15.mp4";
private static final String HD_3328_15_17 = "3328k_p15v17.mp4";
private static final String HD_3328_12 = "3328k_p36v12.mp4";
private static final String HD_3328_13 = "3328k_p36v13.mp4";
private static final String HD_3328_14 = "3328k_p36v14.mp4";
Expand Down Expand Up @@ -75,6 +76,7 @@ public class ZdfVideoUrlOptimizer {

HD_OPTIMIZE.put(HD_3360_36_17, new String[] {HD_6660_37_17, HD_6628_61_17});
HD_OPTIMIZE.put(HD_6628_61_17, new String[] {HD_6660_37_17});
HD_OPTIMIZE.put(HD_3328_15_17, new String[] {HD_6660_37_17, HD_6628_61_17, HD_3360_36_17});
HD_OPTIMIZE.put(HD_3328_15_15, new String[] {HD_3360_36_15});
HD_OPTIMIZE.put(HD_3256, new String[] {HD_3328_12});
HD_OPTIMIZE.put(HD_3296_15_14, new String[] {HD_3328_36_14});
Expand Down
Loading