Skip to content

Commit

Permalink
fix 992-01-990-989 and remove java preview feature
Browse files Browse the repository at this point in the history
  • Loading branch information
@codingPF
codingPF committed May 24, 2024
1 parent 547e423 commit e846616
Show file tree
Hide file tree
Showing 9 changed files with 1,120 additions and 36 deletions.
12 changes: 12 additions & 0 deletions src/main/java/de/mediathekview/mserver/crawler/ard/ArdConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import java.util.HashMap;
import java.util.Map;

import de.mediathekview.mlib.daten.Resolution;
import de.mediathekview.mlib.daten.Sender;

public class ArdConstants {
Expand Down Expand Up @@ -67,6 +68,17 @@ public class ArdConstants {
public static final String WEBSITE_URL = "https://www.ardmediathek.de/video/%s";

public static final String BASE_URL_SUBTITLES = "https://classic.ardmediathek.de";

// provide the same as master until crawler release
public static Resolution getResolutionFromWidth(final int width) {
if (width > 1280) {
return Resolution.HD;
} else if (width > 640) {
return Resolution.NORMAL;
} else {
return Resolution.SMALL;
}
}

private ArdConstants() {}
}
60 changes: 43 additions & 17 deletions src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import de.mediathekview.mlib.daten.Sender;
import de.mediathekview.mserver.base.utils.GeoLocationGuesser;
import de.mediathekview.mserver.base.utils.JsonUtils;
import de.mediathekview.mserver.base.utils.UrlUtils;
import de.mediathekview.mserver.crawler.ard.ArdConstants;
import de.mediathekview.mserver.crawler.ard.ArdFilmDto;
import de.mediathekview.mserver.crawler.ard.ArdFilmInfoDto;
Expand Down Expand Up @@ -57,13 +58,16 @@ public class ArdFilmDeserializer implements JsonDeserializer<List<ArdFilmDto>> {
private static final String ATTRIBUTE_RESOLUTION_H = "maxHResolutionPx";
private static final String ATTRIBUTE_MIME = "mimeType";
private static final String ATTRIBUTE_KIND = "kind";
private static final String ATTRIBUTE_ADUIO_LANG = "languageCode";

private static final String MARKER_VIDEO_MP4 = "video/mp4";
private static final String MARKER_VIDEO_STANDARD = "standard";
private static final String MARKER_VIDEO_CATEGORY_MAIN = "main";
private static final String MARKER_VIDEO_CATEGORY_MPEG = "application/vnd.apple.mpegurl";
private static final String MARKER_VIDEO_AD = "audio-description";
private static final String MARKER_VIDEO_DGS = "sign-language";
private static final String MARKER_VIDEO_DGS = "sign-language";
private static final String MARKER_VIDEO_OV = "OV";
private static final String MARKER_VIDEO_DE = "deu";

private final ArdVideoInfoJsonDeserializer videoDeserializer;
private final AbstractCrawler crawler;
Expand Down Expand Up @@ -292,7 +296,6 @@ private Film createFilm(
duration == null ? Duration.ofSeconds(0) : duration);

Optional.ofNullable(description).ifPresent(film::setBeschreibung);


film.setGeoLocations(GeoLocationGuesser.getGeoLocations(Sender.ARD, videoInfo.getDefaultVideoUrl()));

Expand Down Expand Up @@ -354,12 +357,19 @@ private void addDGSUrls(final Film film, final Map<Resolution, String> videoUrls
private Optional<ArdVideoInfoDto> parseVideos(final JsonObject playerPageObject, final String title) {
ArdVideoInfoDto allVideoUrls = new ArdVideoInfoDto();
//
final Optional<Map<Resolution, String>> videoInfoStandard = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
final Optional<Map<Resolution, String>> videoInfoAdaptive = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG);
final Optional<Map<Resolution, String>> videoInfoAD = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4);
final Optional<Map<Resolution, String>> videoInfoDGS = parseVideoUrls(playerPageObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
final Optional<Map<Resolution, String>> videoInfoStandard = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Resolution, String>> videoInfoAdaptive = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE);
final Optional<Map<Resolution, String>> videoInfoAD = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Resolution, String>> videoInfoDGS = parseVideoUrls(playerPageObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Resolution, String>> videoInfoOV = parseVideoUrls(playerPageObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV);
final Optional<Set<String>> subtitles = prepareSubtitleUrl(playerPageObject);
//
// OV is a single film but also included in the standard film
if ((title.toUpperCase().contains("(OV)") || title.toUpperCase().contains("(ORIGINALVERSION)")) && videoInfoOV.isPresent()) {
allVideoUrls.putAll(videoInfoOV.get());
return Optional.of(allVideoUrls);
}
//
if (subtitles.isPresent()) {
allVideoUrls.setSubtitleUrl(subtitles);
}
Expand All @@ -385,16 +395,38 @@ private Optional<ArdVideoInfoDto> parseVideos(final JsonObject playerPageObject,
}
return Optional.of(allVideoUrls);
}

private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
Optional<Map<Integer, String>> urls = parseVideoUrlMap(playerPageObject, streamType, aduioType, mimeType, language);
if (urls.isEmpty()) {
return Optional.empty();
}
Map<Resolution, String> videoInfo = new EnumMap<>(Resolution.class);
for (Map.Entry<Integer, String> entry : urls.get().entrySet()) {
Resolution resolution = ArdConstants.getResolutionFromWidth(entry.getKey());
if(!videoInfo.containsKey(resolution)) {
videoInfo.put(resolution, entry.getValue());
}
}
// issue if we do not have normal res
// TODO: FIXME
if (!videoInfo.containsKey(Resolution.NORMAL)) {
Resolution anyResolution = videoInfo.keySet().stream().findFirst().get();
videoInfo.put(Resolution.NORMAL, videoInfo.get(anyResolution));
videoInfo.remove(anyResolution);
}
return Optional.of(videoInfo);
}

private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType) {
private Optional<Map<Integer, String>> parseVideoUrlMap(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
final Optional<JsonObject> mediaCollectionObject = getMediaCollectionObject(playerPageObject);
if (mediaCollectionObject.isEmpty())
return Optional.empty();
final Optional<JsonElement> streams = JsonUtils.getElement(mediaCollectionObject.get(), ELEMENT_STREAMS);
if (streams.isEmpty() || !streams.get().isJsonArray() || (streams.get().getAsJsonArray().size() == 0))
return Optional.empty();
//
Map<Resolution, String> videoInfo = new EnumMap<>(Resolution.class);
Map<Integer, String> videoInfo = new TreeMap<>(Comparator.reverseOrder());
for (JsonElement streamsCategory : streams.get().getAsJsonArray()) {
final Optional<String> streamKind = JsonUtils.getElementValueAsString(streamsCategory, ATTRIBUTE_KIND);
final Optional<JsonElement> media = JsonUtils.getElement(streamsCategory, ELEMENT_MEDIA);
Expand All @@ -409,9 +441,9 @@ private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject player
Optional<String> kind = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_KIND);
Optional<String> resh = JsonUtils.getElementValueAsString(video, ATTRIBUTE_RESOLUTION_H);
Optional<String> url = JsonUtils.getElementValueAsString(video, ATTRIBUTE_URL);
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType)) {
Resolution resolution = Resolution.getResolutionFromWidth(Integer.parseInt(resh.get()));
videoInfo.put(resolution, url.get());
Optional<String> languageCode = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_ADUIO_LANG);
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType) && languageCode.orElse("").equalsIgnoreCase(language)) {
videoInfo.put(Integer.parseInt(resh.get()), UrlUtils.removeParameters(url.get()));
}
}
}
Expand All @@ -421,12 +453,6 @@ private Optional<Map<Resolution, String>> parseVideoUrls(final JsonObject player
if (videoInfo.size() == 0) {
return Optional.empty();
}
// TODO: FIX ME
if (!videoInfo.containsKey(Resolution.NORMAL)) {
Resolution r = videoInfo.keySet().stream().findFirst().get();
videoInfo.put(Resolution.NORMAL, videoInfo.get(r));
videoInfo.remove(r);
}
return Optional.of(videoInfo);
}
}
3 changes: 2 additions & 1 deletion .../java/de/mediathekview/mserver/crawler/ard/json/ArdMediaArrayToDownloadUrlsConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ private static Resolution getQualityForNumber(final int qualityIndicator) {
case 1 -> Resolution.SMALL;
case 3, 4 -> Resolution.HD;
case 5 -> Resolution.UHD;
case 2, default -> Resolution.NORMAL;
case 2 -> Resolution.NORMAL;
default -> Resolution.NORMAL;
};
}

Expand Down
30 changes: 14 additions & 16 deletions ...main/java/de/mediathekview/mserver/crawler/funk/tasks/NexxCloudSessionInitiationTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,24 +37,22 @@ public NexxCloudSessionInitiationTask(final AbstractCrawler aCrawler) {
public Long call() {
final Gson gson = createGson();

try(final Client client = createClient()) {
final WebTarget target =
client.target(FunkUrls.NEXX_CLOUD_SESSION_INIT.getAsString(crawler.getRuntimeConfig()));
final Client client = createClient();
final WebTarget target =
client.target(FunkUrls.NEXX_CLOUD_SESSION_INIT.getAsString(crawler.getRuntimeConfig()));

final MultivaluedHashMap<String, String> formData = createForm();
final MultivaluedHashMap<String, String> formData = createForm();

try(final Response response = target.request().post(Entity.form(formData)))
{
if (response.getStatus() == 201) {
final String jsonOutput = response.readEntity(String.class);
return gson.fromJson(jsonOutput, Long.class);
} else {
crawler.printErrorMessage();
LOG.fatal(
"A HTTP error {} occurred when initialising the Nexx cloud session.",
response.getStatus());
}
}
final Response response = target.request().post(Entity.form(formData));

if (response.getStatus() == 201) {
final String jsonOutput = response.readEntity(String.class);
return gson.fromJson(jsonOutput, Long.class);
} else {
crawler.printErrorMessage();
LOG.fatal(
"A HTTP error {} occurred when initialising the Nexx cloud session.",
response.getStatus());
}
return null;
}
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,9 @@ private Optional<String> formatEpisodeTitle(Optional<Integer> season, Optional<I
if (season.isPresent()) {
result += String.format("S%02d", season.get());
}
if (season.isPresent() && episode.isPresent()) {
result += "/";
}
if (episode.isPresent()) {
result += String.format("E%02d", episode.get());
}
Expand Down
44 changes: 43 additions & 1 deletion src/test/java/de/mediathekview/mserver/crawler/ard/json/ArdFilmDeserializerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ public static Collection<Object[]> data() {
/*DGSsmall */ "",
/*DGSnormal */ "",
/*DGShd */ "",
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:0567b031db73e4b9.vtt",
/*sub*/ "",
/*hd*/ GeoLocations.GEO_DE,
/*related*/ new ArdFilmInfoDto[0],
/*sender*/ Optional.of(Sender.ONE),
Expand Down Expand Up @@ -346,6 +346,48 @@ public static Collection<Object[]> data() {
/*hd*/ GeoLocations.GEO_DE,
/*related*/ new ArdFilmInfoDto[0],
/*sender*/ Optional.of(Sender.SWR),
},
{
/*jsonFile*/ "/ard/ard_item_ignore_OV.json",
/*topic*/ "You shall not lie - Tödliche Geheimnisse",
/*title*/ "Folge 4: Der Verrat (S01/E04)",
/*description*/ "Zwischen Macarena und Iván, der jetzt von ihrer Schwangerschaft weiß, kommt es zu einer dramatischen Auseinandersetzung auf See. Erschöpft schwimmt der 18-Jährige an Land und beschuldigt sie, ihn vom Boot gestoßen zu haben. Die Lehrerin muss nun nicht nur die Folgen ihres Seitensprungs fürchten, sondern auch eine Anzeige wegen versuchten Totschlags. +++ Sechsteilige Thrillerserie, Spanien 2021\n.....",
/*date*/ LocalDateTime.parse("2022-10-15T02:50"),
/*duration*/ Duration.parse("PT42M18S"),
/*small*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461836_sendeton_640x360-50p-1200kbit.mp4",
/*normal*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461836_sendeton_960x540-50p-1600kbit.mp4",
/*hd*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461836_sendeton_1280x720-50p-3200kbit.mp4",
/*ADsmall*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461837_internationalerton_640x360-50p-1200kbit.mp4",
/*ADnormal*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461837_internationalerton_960x540-50p-1600kbit.mp4",
/*ADhd*/ "https://pd-videos.daserste.de/de/2022/10/11/1cb6c5d8-c0f4-4868-ac42-2a06b7a3381f/JOB_461837_internationalerton_1280x720-50p-3200kbit.mp4",
/*DGSsmall */ "",
/*DGSnormal */ "",
/*DGShd */ "",
/*sub*/ "https://api.ardmediathek.de/player-service/subtitle/webvtt/urn:ard:subtitle:efab8bf55007171e.vtt",
/*hd*/ GeoLocations.GEO_DE,
/*related*/ new ArdFilmInfoDto[0],
/*sender*/ Optional.of(Sender.ARD),
},
{
/*jsonFile*/ "/ard/ard_item_OV.json",
/*topic*/ "Murdoch Mysteries",
/*title*/ "Folge 12: Der küssende Bandit (S04/E12) - (Originalversion)",
/*description*/ "Während sich Dr. Ogden auf ihre Hochzeit vorbereitet, muss Murdoch versuchen, den 'Küssenden Banditen' aufzuhalten, einen umstrittenen Bankräuber, der schnell zum Volkshelden aufsteigt.",
/*date*/ LocalDateTime.parse("2024-04-24T22:50"),
/*duration*/ Duration.parse("PT45M44S"),
/*small*/ "https://wdrmedien-a.akamaihd.net/medp/ondemand/de/fsk12/310/3103788/3103788_57016296.mp4",
/*normal*/ "https://wdrmedien-a.akamaihd.net/medp/ondemand/de/fsk12/310/3103788/3103788_57016297.mp4",
/*hd*/ "https://wdrmedien-a.akamaihd.net/medp/ondemand/de/fsk12/310/3103788/3103788_57016298.mp4",
/*ADsmall*/ "",
/*ADnormal*/ "",
/*ADhd*/ "",
/*DGSsmall */ "",
/*DGSnormal */ "",
/*DGShd */ "",
/*sub*/ "",
/*hd*/ GeoLocations.GEO_DE,
/*related*/ new ArdFilmInfoDto[0],
/*sender*/ Optional.of(Sender.ONE),
}
});
}
Expand Down
2 changes: 1 addition & 1 deletion src/test/java/de/mediathekview/mserver/crawler/zdf/json/ZdfFilmDetailDeserializerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ public static Collection<Object[]> data() {
"/zdf/zdf_film_details_with_episodes.json",
Sender.ZDF,
"The Rookie",
"Der Prozess (S05E01)",
"Der Prozess (S05/E01)",
LocalDateTime.of(2024, 4, 11, 20, 15, 0),
Duration.ofMinutes(41).plusSeconds(6),
"Officer John Nolan sieht sich wieder mit der Serienmörderin Rosalind Dyer konfrontiert. Vor Beginn der Gerichtsverhandlung gelingt ihr die Flucht und sie muss erneut gefasst werden.",
Expand Down
Loading

0 comments on commit e846616

Please sign in to comment.