Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

hotfix 992-991-990-989 #993

Merged
merged 2 commits into from
May 25, 2024
Merged

hotfix 992-991-990-989 #993

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6631e31
fix 992-991-990-989
codingPF May 24, 2024
db74e40
Version auf 3.1.235 angehoben
alex1702 May 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17
targetCompatibility = JavaVersion.VERSION_17
group = 'de.mediathekview'
archivesBaseName = "MServer"
version = '3.1.234'
version = '3.1.235'

def jarName = 'MServer.jar'
def mainClass = 'mServer.Main'
Expand Down
76 changes: 60 additions & 16 deletions src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@
import java.time.format.DateTimeFormatter;
import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.EnumMap;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;

import mServer.crawler.CrawlerTool;
Expand All @@ -32,6 +34,8 @@
import mServer.crawler.sender.ard.ArdFilmInfoDto;
import mServer.crawler.sender.base.JsonUtils;
import mServer.crawler.sender.base.Qualities;
import mServer.crawler.sender.base.UrlUtils;

import org.apache.logging.log4j.LogManager;

public class ArdFilmDeserializer implements JsonDeserializer<List<ArdFilmDto>> {
Expand Down Expand Up @@ -66,13 +70,16 @@ public class ArdFilmDeserializer implements JsonDeserializer<List<ArdFilmDto>> {
private static final String ATTRIBUTE_RESOLUTION_H = "maxHResolutionPx";
private static final String ATTRIBUTE_MIME = "mimeType";
private static final String ATTRIBUTE_KIND = "kind";
private static final String ATTRIBUTE_ADUIO_LANG = "languageCode";

private static final String MARKER_VIDEO_MP4 = "video/mp4";
private static final String MARKER_VIDEO_STANDARD = "standard";
private static final String MARKER_VIDEO_CATEGORY_MAIN = "main";
private static final String MARKER_VIDEO_CATEGORY_MPEG = "application/vnd.apple.mpegurl";
private static final String MARKER_VIDEO_AD = "audio-description";
private static final String MARKER_VIDEO_DGS = "sign-language";
private static final String MARKER_VIDEO_OV = "OV";
private static final String MARKER_VIDEO_DE = "deu";

private static final DateTimeFormatter DATE_FORMAT
= DateTimeFormatter.ofPattern("dd.MM.yyyy");
Expand Down Expand Up @@ -222,10 +229,11 @@ public List<ArdFilmDto> deserialize(
final Optional<LocalDateTime> date = parseDate(itemObject);
final Optional<Duration> duration = parseDuration(itemObject);
final Optional<String> partner = parsePartner(itemObject);
final Optional<Map<Qualities, String>> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
final Optional<Map<Qualities, String>> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG);
final Optional<Map<Qualities, String>> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4);
final Optional<Map<Qualities, String>> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4);
final Optional<Map<Qualities, String>> videoInfoStandard = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Qualities, String>> videoInfoAdaptive = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_CATEGORY_MPEG, MARKER_VIDEO_DE);
final Optional<Map<Qualities, String>> videoInfoAD = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_AD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Qualities, String>> videoInfoDGS = parseVideoUrls(itemObject, MARKER_VIDEO_DGS, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_DE);
final Optional<Map<Qualities, String>> videoInfoOV = parseVideoUrls(itemObject, MARKER_VIDEO_CATEGORY_MAIN, MARKER_VIDEO_STANDARD, MARKER_VIDEO_MP4, MARKER_VIDEO_OV);
final Optional<String> subtitles = prepareSubtitleUrl(itemObject);

if (topic.isEmpty() || title.isEmpty() || partner.isEmpty() || ADDITIONAL_SENDER.get(partner.get()) == null) {
Expand All @@ -235,6 +243,24 @@ public List<ArdFilmDto> deserialize(
return films;
}



if ((title.get().toUpperCase().contains("(OV)") || title.get().toUpperCase().contains("(ORIGINALVERSION)")) && videoInfoOV.isPresent()) {
final ArdFilmDto filmDto
= new ArdFilmDto(
createFilm(
ADDITIONAL_SENDER.get(partner.get()),
topic.get(),
title.get(),
description,
date,
duration,
videoInfoOV.get(),
subtitles));
films.add(filmDto);
return films;
}

if(videoInfoStandard.isEmpty() && videoInfoAD.isEmpty() && videoInfoDGS.isEmpty() && videoInfoAdaptive.isPresent()) {
// UUAAAARRGGGG - SAD
Map<Qualities, URL> qualitiesUrls = videoInfoAdaptive.get().entrySet().stream()
Expand Down Expand Up @@ -310,7 +336,7 @@ public List<ArdFilmDto> deserialize(
}
//
if (videoInfoDGS.isPresent() && videoInfoDGS.get().size() > 0) {
// add film standard
// add film dgs
final ArdFilmDto filmDto
= new ArdFilmDto(
createFilm(
Expand Down Expand Up @@ -382,11 +408,7 @@ private DatenFilm createFilm(

String dateValue = time.format(DATE_FORMAT);
String timeValue = time.format(TIME_FORMAT);

String baseUrl = videoUrls.get(Qualities.NORMAL);
baseUrl = baseUrl != null ? baseUrl : videoUrls.get(Qualities.SMALL);
baseUrl = baseUrl != null ? baseUrl : videoUrls.get(Qualities.HD);

DatenFilm film = new DatenFilm(sender, topic, "", title, baseUrl, "",
dateValue, timeValue, duration.orElse(Duration.ZERO).getSeconds(), description.orElse(""));
if (videoUrls.containsKey(Qualities.SMALL)) {
Expand All @@ -402,16 +424,40 @@ private DatenFilm createFilm(
return film;
}

private Optional<Map<Qualities, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
Optional<Map<Integer, String>> urls = parseVideoUrlMap(playerPageObject, streamType, aduioType, mimeType, language);
if (urls.isEmpty()) {
return Optional.empty();
}
Map<Qualities, String> videoInfo = new EnumMap<>(Qualities.class);
for (Map.Entry<Integer, String> entry : urls.get().entrySet()) {
Qualities resolution = Qualities.getResolutionFromWidth(entry.getKey());
if(!videoInfo.containsKey(resolution)) {
videoInfo.put(resolution, entry.getValue());
}
}
// issue if we do not have normal res
if (!videoInfo.containsKey(Qualities.NORMAL)) {
if (videoInfo.containsKey(Qualities.HD)) {
videoInfo.put(Qualities.NORMAL, videoInfo.get(Qualities.HD));
videoInfo.remove(Qualities.HD);
} else {
videoInfo.put(Qualities.NORMAL, videoInfo.get(Qualities.SMALL));
videoInfo.remove(Qualities.SMALL);
}
}
return Optional.of(videoInfo);
}

private Optional<Map<Qualities, String>> parseVideoUrls(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType) {
private Optional<Map<Integer, String>> parseVideoUrlMap(final JsonObject playerPageObject, String streamType, String aduioType, String mimeType, String language) {
final Optional<JsonObject> mediaCollectionObject = getMediaCollectionObject(playerPageObject);
if (mediaCollectionObject.isEmpty())
return Optional.empty();
final Optional<JsonElement> streams = JsonUtils.getElement(mediaCollectionObject.get(), ELEMENT_STREAMS);
if (streams.isEmpty() || !streams.get().isJsonArray() || (streams.get().getAsJsonArray().size() == 0))
return Optional.empty();
//
Map<Qualities, String> videoInfo = new EnumMap<>(Qualities.class);
Map<Integer, String> videoInfo = new TreeMap<>(Comparator.reverseOrder());
for (JsonElement streamsCategory : streams.get().getAsJsonArray()) {
final Optional<String> streamKind = JsonUtils.getElementValueAsString(streamsCategory, ATTRIBUTE_KIND);
final Optional<JsonElement> media = JsonUtils.getElement(streamsCategory, ELEMENT_MEDIA);
Expand All @@ -426,11 +472,9 @@ private Optional<Map<Qualities, String>> parseVideoUrls(final JsonObject playerP
Optional<String> kind = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_KIND);
Optional<String> resh = JsonUtils.getElementValueAsString(video, ATTRIBUTE_RESOLUTION_H);
Optional<String> url = JsonUtils.getElementValueAsString(video, ATTRIBUTE_URL);
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType)) {
Qualities resolution = Qualities.getResolutionFromWidth(Integer.parseInt(resh.get()));
if(!videoInfo.containsKey(resolution)) { // do not overwrite 1920 with 1280 res
videoInfo.put(resolution, url.get());
}
Optional<String> languageCode = JsonUtils.getElementValueAsString(audios.get().getAsJsonArray().get(0), ATTRIBUTE_ADUIO_LANG);
if (url.isPresent() && resh.isPresent() && kind.isPresent() && kind.get().equalsIgnoreCase(aduioType) && languageCode.orElse("").equalsIgnoreCase(language)) {
videoInfo.put(Integer.parseInt(resh.get()), UrlUtils.removeParameters(url.get()));
}
}
}
Expand Down
13 changes: 5 additions & 8 deletions src/main/java/mServer/crawler/sender/base/Qualities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,12 @@ public String getDescription()
}

public static Qualities getResolutionFromWidth(final int width) {
if (width >= 2160) {
return Qualities.UHD;
}
if (width >= 1280) {
if (width > 1280) {
return Qualities.HD;
}
if (width >= 720) {
} else if (width > 640) {
return Qualities.NORMAL;
} else {
return Qualities.SMALL;
}
return Qualities.SMALL;
}
}
}
3 changes: 3 additions & 0 deletions src/main/java/mServer/crawler/sender/zdf/json/ZdfFilmDetailDeserializer.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,9 @@ private Optional<String> formatEpisodeTitle(Optional<Integer> season, Optional<I
if (season.isPresent()) {
result += String.format("S%02d", season.get());
}
if (season.isPresent() && episode.isPresent()) {
result += "/";
}
if (episode.isPresent()) {
result += String.format("E%02d", episode.get());
}
Expand Down
Loading