diff --git a/build.gradle b/build.gradle index 42b721c0f..d53390155 100644 --- a/build.gradle +++ b/build.gradle @@ -27,7 +27,7 @@ sourceCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17 group = 'de.mediathekview' archivesBaseName = "MServer" -version = '3.1.220' +version = '3.1.221' def jarName = 'MServer.jar' def mainClass = 'mServer.Main' diff --git a/src/main/java/mServer/crawler/FilmeSuchen.java b/src/main/java/mServer/crawler/FilmeSuchen.java index 073d59c18..d79d6876d 100644 --- a/src/main/java/mServer/crawler/FilmeSuchen.java +++ b/src/main/java/mServer/crawler/FilmeSuchen.java @@ -28,7 +28,6 @@ import mServer.crawler.sender.*; import mServer.crawler.sender.ard.ArdCrawler; import mServer.crawler.sender.arte.MediathekArte; -import mServer.crawler.sender.br.BrCrawler; import mServer.crawler.sender.dreisat.DreiSatCrawler; import mServer.crawler.sender.dw.DwCrawler; import mServer.crawler.sender.funk.FunkCrawler; @@ -81,7 +80,6 @@ public FilmeSuchen() { mediathekListe.add(new FunkCrawler(this, 0)); // Spalte 2 mediathekListe.add(new SrCrawler(this, 1)); - mediathekListe.add(new BrCrawler(this, 0)); mediathekListe.add(new SrfCrawler(this, 1)); mediathekListe.add(new MediathekSrfPod(this, 1)); mediathekListe.add(new OrfCrawler(this, 1)); diff --git a/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java b/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java index 10e713fb8..c6bf04ace 100644 --- a/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java +++ b/src/main/java/mServer/crawler/sender/ard/ArdCrawler.java @@ -44,6 +44,7 @@ protected synchronized void meldungThreadUndFertig() { mlibFilmeSuchen.meldenFertig(Const.NDR); mlibFilmeSuchen.meldenFertig(Const.WDR); mlibFilmeSuchen.meldenFertig(Const.HR); + mlibFilmeSuchen.meldenFertig(Const.BR); mlibFilmeSuchen.meldenFertig("rbtv"); } @@ -74,9 +75,6 @@ private void addDayUrls(ConcurrentLinkedQueue dayUrlsToCrawl, Loc private void addSpecialDays( ConcurrentLinkedQueue dayUrlsToCrawl) { final LocalDateTime[] specialDates = new LocalDateTime[] { - LocalDateTime.of(2021, 2, 7, 23, 59, 59), - LocalDateTime.of(2021, 2, 8, 23, 59, 59), - LocalDateTime.of(2021, 2, 9, 23, 59, 59) }; final LocalDateTime minDayOnline = LocalDateTime.now().minusDays(MAX_DAYS_PAST_AVAILABLE); diff --git a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java index e9d52ca08..93e74e1ba 100644 --- a/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java +++ b/src/main/java/mServer/crawler/sender/ard/json/ArdFilmDeserializer.java @@ -66,6 +66,7 @@ public class ArdFilmDeserializer implements JsonDeserializer> { ADDITIONAL_SENDER.put("ndr", Const.NDR); ADDITIONAL_SENDER.put("wdr", Const.WDR); ADDITIONAL_SENDER.put("hr", Const.HR); + ADDITIONAL_SENDER.put("br", Const.BR); ADDITIONAL_SENDER.put("radio_bremen", "rbtv"); } diff --git a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java index 645575900..b52a1ca68 100644 --- a/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java +++ b/src/main/java/mServer/crawler/sender/ard/tasks/ArdTopicPageTask.java @@ -66,13 +66,15 @@ public class ArdTopicPageTask extends ArdTaskBase TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydA"); // Filme im Ersten TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL2Zlcm5zZWhmaWxtZSBpbSBlcnN0ZW4"); - // temporary load all => remove if old entries exists - // Unter unserem Himmel - TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2JyLmRlL2Jyb2FkY2FzdFNlcmllcy9icm9hZGNhc3RTZXJpZXM6L2JyZGUvZmVybnNlaGVuL2JheWVyaXNjaGVzLWZlcm5zZWhlbi9zZW5kdW5nZW4vdW50ZXItdW5zZXJlbS1oaW1tZWw"); - // in aller freundschaft die jungen ärzte - TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL2luLWFsbGVyLWZyZXVuZHNjaGFmdC1kaWUtanVuZ2VuLWFlcnp0ZQ"); // Sportschau TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3Nwb3J0c2NoYXU"); + // temporary load all => remove if old entries exists + // Morden im Norden + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL21vcmRlbi1pbS1ub3JkZW4"); + // Babylon Berlin + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL2JhYnlsb24tYmVybGlu"); + // Watzmann ermittelt + TOPICS_LOAD_ALL_PAGES.add("Y3JpZDovL2Rhc2Vyc3RlLmRlL3dhdHptYW5uLWVybWl0dGVsdA"); } public ArdTopicPageTask(MediathekReader aCrawler, diff --git a/src/main/java/mServer/crawler/sender/br/BrAllSendungenTask.java b/src/main/java/mServer/crawler/sender/br/BrAllSendungenTask.java deleted file mode 100644 index 83bbb2db0..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrAllSendungenTask.java +++ /dev/null @@ -1,78 +0,0 @@ -package mServer.crawler.sender.br; - -import java.util.ArrayList; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ForkJoinPool; -import java.util.concurrent.ForkJoinTask; -import java.util.concurrent.RecursiveTask; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import java.net.URL; -import mServer.crawler.FilmeSuchen; -import mServer.crawler.RunSender; -import mServer.crawler.sender.MediathekReader; - -public class BrAllSendungenTask extends RecursiveTask> { - - private static final Logger LOG = LogManager.getLogger(BrAllSendungenTask.class); - private static final long serialVersionUID = 8178190311832356223L; - - private static final String QUERY = - "{\"query\":\"query SeriesIndexRefetchQuery(\\n $seriesFilter: SeriesFilter\\n) {\\n viewer {\\n ...SeriesIndex_viewer_19SNIy\\n id\\n }\\n}\\n\\nfragment SeriesIndex_viewer_19SNIy on Viewer {\\n seriesIndexAllSeries: allSeries(first: 1000, orderBy: TITLE_ASC, filter: $seriesFilter) {\\n edges {\\n node {\\n __typename\\n id\\n title\\n ...SeriesTeaserBox_node\\n ...TeaserListItem_node\\n }\\n }\\n }\\n}\\n\\nfragment SeriesTeaserBox_node on Node {\\n __typename\\n id\\n ... on CreativeWorkInterface {\\n ...TeaserImage_creativeWorkInterface\\n }\\n ... on SeriesInterface {\\n ...SubscribeAction_series\\n subscribed\\n title\\n }\\n}\\n\\nfragment TeaserListItem_node on Node {\\n __typename\\n id\\n ... on CreativeWorkInterface {\\n ...TeaserImage_creativeWorkInterface\\n }\\n ... on ClipInterface {\\n title\\n }\\n}\\n\\nfragment TeaserImage_creativeWorkInterface on CreativeWorkInterface {\\n id\\n kicker\\n title\\n }\\n\\nfragment SubscribeAction_series on SeriesInterface {\\n id\\n subscribed\\n}\\n\",\"variables\":{\"seriesFilter\":{\"title\":{\"startsWith\":\"*\"},\"audioOnly\":{\"eq\":false},\"status\":{\"id\":{\"eq\":\"Status:http://ard.de/ontologies/lifeCycle#published\"}}}}}"; - - private final transient ForkJoinPool forkJoinPool; - private final transient MediathekReader crawler; - - private BrIdsDTO allSendungen; - - public BrAllSendungenTask(final MediathekReader aCrawler, final ForkJoinPool aForkJoinPool) { - crawler = aCrawler; - forkJoinPool = aForkJoinPool; - } - - private Set getAllSendungenIds() { - allSendungen = new BrIdsDTO(); - - BrWebAccessHelper.handleWebAccessExecution(LOG, crawler, () -> { - final Gson gson = new GsonBuilder() - .registerTypeAdapter(BrIdsDTO.class, new BrSendungenIdsDeserializer()).create(); - final String response = WebAccessHelper.getJsonResultFromPostAccess(new URL(Consts.BR_API_URL), QUERY); - - allSendungen = gson.fromJson(response, BrIdsDTO.class); - }); - - LOG.debug("BR Anzahl Sendungen: " + allSendungen.getIds().size()); - return allSendungen.getIds(); - } - - @Override - protected Set compute() { - final Set results = ConcurrentHashMap.newKeySet(); - - try { - final Set sendungenIds = getAllSendungenIds(); - - final List>> futureSendungsfolgenTasks = new ArrayList<>(); - for (final String sendungsId : sendungenIds) { - futureSendungsfolgenTasks - .add(forkJoinPool.submit(new BrSendungsFolgenTask(crawler, sendungsId))); - } - - for (final ForkJoinTask> featureSendungsfolgenTask : futureSendungsfolgenTasks) { - results.addAll(featureSendungsfolgenTask.get()); - } - } catch (InterruptedException | ExecutionException exception) { - LOG.error("Something wen't terrible wrong while getting the Folgen for a BR Sendung.", - exception); - FilmeSuchen.listeSenderLaufen.inc(crawler.getSendername(), RunSender.Count.FEHLER); - } - - return results; - } - -} diff --git a/src/main/java/mServer/crawler/sender/br/BrCrawler.java b/src/main/java/mServer/crawler/sender/br/BrCrawler.java deleted file mode 100644 index 97f9c4763..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrCrawler.java +++ /dev/null @@ -1,72 +0,0 @@ -package mServer.crawler.sender.br; - -import de.mediathekview.mlib.Const; -import de.mediathekview.mlib.daten.DatenFilm; -import de.mediathekview.mlib.tool.Log; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Future; -import java.util.concurrent.RecursiveTask; -import mServer.crawler.CrawlerTool; -import mServer.crawler.FilmeSuchen; -import mServer.crawler.sender.MediathekCrawler; - -public class BrCrawler extends MediathekCrawler { - - public static final String SENDERNAME = Const.BR; - public static final String BASE_URL = "https://www.br.de/mediathek/"; - - public BrCrawler(FilmeSuchen ssearch, int startPrio) { - super(ssearch, SENDERNAME, 0, 100, startPrio); - } - - private RecursiveTask> createAllSendungenOverviewCrawler() { - return new BrAllSendungenTask(this, forkJoinPool); - } - - private Callable> createMissedFilmsCrawler() { - int maximumDays; - if (CrawlerTool.loadLongMax()) { - maximumDays = 21; - } else { - maximumDays = 7; - } - - return new BrMissedSendungsFolgenTask(this, maximumDays); - } - - @Override - protected RecursiveTask> createCrawlerTask() { - final Callable> missedFilmsTask = createMissedFilmsCrawler(); - final Future> missedFilmIds = forkJoinPool.submit(missedFilmsTask); - - final ConcurrentLinkedQueue brFilmIds = new ConcurrentLinkedQueue<>(); - try { - brFilmIds.addAll(missedFilmIds.get()); - Log.sysLog("BR Anzahl verpasste Sendungen: " + missedFilmIds.get().size()); - } catch (Exception exception) { - Log.errorLog(782346382, exception); - } - - if (CrawlerTool.loadLongMax()) { - try { - final RecursiveTask> sendungenFilmsTask = createAllSendungenOverviewCrawler(); - - forkJoinPool.execute(sendungenFilmsTask); - Set allSendungen = sendungenFilmsTask.join(); - brFilmIds.addAll(allSendungen); - Log.sysLog("BR Anzahl alle Sendungen: " + allSendungen.size()); - } catch (Exception exception) { - Log.errorLog(782346383, exception); - } - } - Log.sysLog("BR Anzahl: " + brFilmIds.size()); - - int max = (brFilmIds.size() / BrSendungDetailsTask.MAXIMUM_URLS_PER_TASK) + 1; - meldungAddMax(max); - - return new BrSendungDetailsTask(this, brFilmIds); - } - -} diff --git a/src/main/java/mServer/crawler/sender/br/BrFilmDeserializer.java b/src/main/java/mServer/crawler/sender/br/BrFilmDeserializer.java deleted file mode 100644 index bbd090c68..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrFilmDeserializer.java +++ /dev/null @@ -1,402 +0,0 @@ -package mServer.crawler.sender.br; - -import java.lang.reflect.Type; -import java.time.Duration; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import com.google.gson.JsonArray; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import de.mediathekview.mlib.daten.DatenFilm; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.HashMap; -import java.util.Map; -import mServer.crawler.CrawlerTool; -import mServer.crawler.FilmeSuchen; -import mServer.crawler.RunSender; -import static mServer.crawler.sender.MediathekBr.SENDERNAME; -import mServer.crawler.sender.MediathekReader; - -public class BrFilmDeserializer implements JsonDeserializer> { - - private static final String ERROR_NO_START_TEMPLATE - = "The BR film \"%s - %s\" has no broadcast start so it will using the actual date and time."; - private static final String HD = "HD"; - private static final String FILM_WEBSITE_TEMPLATE = "%s/video/%s"; - private static final String ERROR_MISSING_DETAIL_TEMPLATE - = "A BR film can't be created because of missing details. The JSON element \"%s\" is missing."; - - private static final Logger LOG = LogManager.getLogger(BrFilmDeserializer.class); - - private static final String JSON_ELEMENT_DATA = "data"; - private static final String JSON_ELEMENT_VIEWER = "viewer"; - private static final String JSON_ELEMENT_CLIP = "clip"; - private static final String JSON_ELEMENT_VIDEO_FILES = "videoFiles"; - private static final String JSON_ELEMENT_EDGES = "edges"; - private static final String JSON_ELEMENT_NODE = "node"; - private static final String JSON_ELEMENT_ID = "id"; - private static final String JSON_ELEMENT_CAPTION_FILES = "captionFiles"; - private static final String JSON_ELEMENT_EPISODEOF = "episodeOf"; - - private static final String JSON_ELEMENT_DETAIL_CLIP = "detailClip"; - private static final String JSON_ELEMENT_TITLE = "title"; - private static final String JSON_ELEMENT_KICKER = "kicker"; - private static final String JSON_ELEMENT_DURATION = "duration"; - private static final String JSON_ELEMENT_INITIAL_SCREENING = "initialScreening"; - private static final String JSON_ELEMENT_START = "start"; - private static final String JSON_ELEMENT_SHORT_DESCRIPTION = "shortDescription"; - private static final String JSON_ELEMENT_DESCRIPTION = "description"; - private static final String JSON_ELEMENT_PUBLIC_LOCATION = "publicLocation"; - - private static final String JSON_ELEMENT_VIDEO_PROFILE = "videoProfile"; - private static final String JSON_ELEMENT_WIDTH = "width"; - - private static final ZoneId ZONE_ID = ZoneId.of("Europe/Berlin"); - - private final MediathekReader crawler; - private final String filmId; - - private final DateTimeFormatter dateFormatDatenFilm = DateTimeFormatter.ofPattern("dd.MM.yyyy"); - private final DateTimeFormatter timeFormatDatenFilm = DateTimeFormatter.ofPattern("HH:mm:ss"); - - public BrFilmDeserializer(final MediathekReader aCrawler, final String aFilmId) { - crawler = aCrawler; - filmId = aFilmId; - } - - /** - * Resolves the Film details which and creates a Film of it.
- * The data has this structure: - * data -> viewer -> clip -> videoFiles -> edges[] -> node -> id
- * data -> viewer -> detailClip -> title
- * data -> viewer -> detailClip -> kicker
- * data -> viewer -> detailClip -> duration
- * data -> viewer -> detailClip -> broadcasts -> edges[0] -> node -> start
- * Optional: data -> viewer -> detailClip -> shortDescription
- * Optional: data -> viewer -> detailClip -> description - */ - @Override - public Optional deserialize(final JsonElement aElement, final Type aType, - final JsonDeserializationContext aContext) { - try { - final Optional viewer = getViewer(aElement.getAsJsonObject()); - if (viewer.isPresent()) { - - final Optional detailClip = getDetailClip(viewer.get()); - - return buildFilm(detailClip, viewer.get()); - - } else { - printMissingDetails(JSON_ELEMENT_VIEWER); - } - } catch (final UnsupportedOperationException unsupportedOperationException) { - // This will happen when a element is JsonNull. - LOG.error("BR: A needed JSON element is JsonNull.", unsupportedOperationException); - FilmeSuchen.listeSenderLaufen.inc(crawler.getSendername(), RunSender.Count.FEHLER); - } - - return Optional.empty(); - } - - private String getDescriptions(final JsonObject aDetailClip) { - String description = ""; - - if (aDetailClip.has(JSON_ELEMENT_DESCRIPTION) - && !aDetailClip.get(JSON_ELEMENT_DESCRIPTION).isJsonNull()) { - description = aDetailClip.get(JSON_ELEMENT_DESCRIPTION).getAsString(); - } else if (aDetailClip.has(JSON_ELEMENT_SHORT_DESCRIPTION) - && !aDetailClip.get(JSON_ELEMENT_SHORT_DESCRIPTION).isJsonNull()) { - description = aDetailClip.get(JSON_ELEMENT_SHORT_DESCRIPTION).getAsString(); - } - - return description; - } - - private Map getUrls(final JsonObject viewer) { - Map urlMap = new HashMap<>(); - - final Set urls = edgesToUrls(viewer); - if (!urls.isEmpty()) { - // Sorts the urls by width descending, then it limits the amount to three to get the three - // best. - final List bestUrls - = urls.stream().sorted(Comparator.comparingInt(BrUrlDTO::getWidth).reversed()).limit(3) - .collect(Collectors.toList()); - - for (int id = 0; id < bestUrls.size(); id++) { - final Resolution resolution = Resolution.getResolutionFromArdAudioVideoOrdinalsByProfileName(bestUrls.get(id).getVideoProfile()); - final String url = bestUrls.get(id).getUrl(); - - if (url != null && !url.isEmpty() && !urlMap.containsKey(resolution)) { - urlMap.put(resolution, url); - } - } - } - - return urlMap; - } - - private Optional buildFilm(final Optional detailClip, final JsonObject viewer) { - final Optional newFilm; - if (detailClip.isPresent()) { - String description = getDescriptions(detailClip.get()); - Map urls = getUrls(viewer); - - if (urls.containsKey(Resolution.NORMAL) && MediathekReader.urlExists(urls.get(Resolution.NORMAL))) { - Optional subTitle = getSubtitleUrl(viewer); - newFilm = createFilm(detailClip.get(), description, subTitle, urls); - return newFilm; - } - } else { - printMissingDetails(JSON_ELEMENT_DETAIL_CLIP); - } - return Optional.empty(); - } - - private Optional getSubtitleUrl(JsonObject viewer) { - String subtitle = ""; - - if (viewer.has(JSON_ELEMENT_CLIP)) { - JsonObject clip = viewer.getAsJsonObject(JSON_ELEMENT_CLIP); - if (clip.has(JSON_ELEMENT_CAPTION_FILES)) { - JsonObject captionFiles = clip.getAsJsonObject(JSON_ELEMENT_CAPTION_FILES); - if (captionFiles.has(JSON_ELEMENT_EDGES)) { - JsonArray edges = captionFiles.getAsJsonArray(JSON_ELEMENT_EDGES); - if (edges.size() > 0) { - - for (JsonElement edge : edges) { - if (edge.getAsJsonObject().has(JSON_ELEMENT_NODE)) { - JsonObject node = edge.getAsJsonObject().getAsJsonObject(JSON_ELEMENT_NODE); - if (node.has(JSON_ELEMENT_PUBLIC_LOCATION)) { - String value = node.get(JSON_ELEMENT_PUBLIC_LOCATION).getAsString(); - if (subtitle.isEmpty()) { - subtitle = value; - } else { - // ttml anderen Formaten vorziehen - if (value.endsWith(".ttml")) { - subtitle = value; - } - } - } - } - } - } - } - } - } - - if (!subtitle.isEmpty()) { - return Optional.of(subtitle); - } - - return Optional.empty(); - } - - private String getTheme(final JsonObject aDetailClip) { - - String theme = ""; - - if (aDetailClip.has(JSON_ELEMENT_EPISODEOF)) { - JsonElement element = aDetailClip.get(JSON_ELEMENT_EPISODEOF); - if (!element.isJsonNull()) { - JsonObject episodeOf = aDetailClip.getAsJsonObject(JSON_ELEMENT_EPISODEOF); - if (episodeOf.has(JSON_ELEMENT_TITLE)) { - theme = episodeOf.get(JSON_ELEMENT_TITLE).getAsString(); - } - } - } - - if (theme.isEmpty()) { - theme = aDetailClip.get(JSON_ELEMENT_KICKER).getAsString(); - } - - return theme; - } - - private Optional createFilm(final JsonObject aDetailClip, String aDescription, Optional aSubTitle, Map aUrls) { - final Optional start = getDate(aDetailClip); - if (aDetailClip.has(JSON_ELEMENT_TITLE) && aDetailClip.has(JSON_ELEMENT_KICKER) - && aDetailClip.has(JSON_ELEMENT_DURATION)) { - final String title = aDetailClip.get(JSON_ELEMENT_TITLE).getAsString(); - final String thema = getTheme(aDetailClip); - - final String dateValue; - final String timeValue; - if (start.isPresent()) { - final LocalDateTime time = toTime(start.get().getAsString()); - dateValue = time.format(dateFormatDatenFilm); - timeValue = time.format(timeFormatDatenFilm); - } else { - LOG.debug(String.format(ERROR_NO_START_TEMPLATE, thema, title)); - dateValue = ""; - timeValue = ""; - } - - final Duration duration = toDuration(aDetailClip.get(JSON_ELEMENT_DURATION).getAsLong()); - - final String website = String.format(FILM_WEBSITE_TEMPLATE, BrCrawler.BASE_URL, filmId); - DatenFilm film = new DatenFilm(SENDERNAME, thema, website, title, aUrls.get(Resolution.NORMAL), "", - dateValue, timeValue, duration.getSeconds(), aDescription); - - if (aUrls.containsKey(Resolution.SMALL)) { - CrawlerTool.addUrlKlein(film, aUrls.get(Resolution.SMALL)); - } - if (aUrls.containsKey(Resolution.HD)) { - CrawlerTool.addUrlHd(film, aUrls.get(Resolution.HD)); - } - if (aSubTitle.isPresent()) { - CrawlerTool.addUrlSubtitle(film, aSubTitle.get()); - } - - return Optional.of(film); - } else { - if (!aDetailClip.has(JSON_ELEMENT_TITLE)) { - printMissingDetails(JSON_ELEMENT_TITLE); - } - - if (!aDetailClip.has(JSON_ELEMENT_KICKER)) { - printMissingDetails(JSON_ELEMENT_KICKER); - } - - if (!aDetailClip.has(JSON_ELEMENT_DURATION)) { - printMissingDetails(JSON_ELEMENT_DURATION); - } - } - return Optional.empty(); - } - - private Set edgesToUrls(final JsonObject viewer) { - final Set urls = new HashSet<>(); - final Optional edges = getVideoFileEdges(viewer); - if (edges.isPresent()) { - for (final JsonElement edge : edges.get()) { - final JsonObject ebdgeObj = edge.getAsJsonObject(); - if (ebdgeObj.has(JSON_ELEMENT_NODE)) { - final JsonObject node = ebdgeObj.getAsJsonObject(JSON_ELEMENT_NODE); - final Optional url = nodeToUrl(node); - if (url.isPresent()) { - urls.add(url.get()); - } - } - } - } - return urls; - } - - private Optional getDate(final JsonObject aDetailClip) { - if (!aDetailClip.has(JSON_ELEMENT_INITIAL_SCREENING)) { - return Optional.empty(); - } - - final JsonElement initialScreeningElement = aDetailClip.get(JSON_ELEMENT_INITIAL_SCREENING); - if (initialScreeningElement.isJsonNull()) { - return Optional.empty(); - } - - final JsonObject initialScreening = initialScreeningElement.getAsJsonObject(); - if (!initialScreening.has(JSON_ELEMENT_START)) { - return Optional.empty(); - } - - return Optional.of(initialScreening.get(JSON_ELEMENT_START)); - } - - private Optional getDetailClip(final JsonObject aViewer) { - if (!aViewer.has(JSON_ELEMENT_DETAIL_CLIP)) { - return Optional.empty(); - } - - return Optional.of(aViewer.getAsJsonObject(JSON_ELEMENT_DETAIL_CLIP)); - } - - private Optional getVideoFileEdges(final JsonObject aViewer) { - if (!aViewer.has(JSON_ELEMENT_CLIP)) { - return Optional.empty(); - } - - final JsonObject clip = aViewer.getAsJsonObject(JSON_ELEMENT_CLIP); - if (!clip.has(JSON_ELEMENT_VIDEO_FILES)) { - return Optional.empty(); - } - - final JsonObject videoFiles = clip.getAsJsonObject(JSON_ELEMENT_VIDEO_FILES); - if (!videoFiles.has(JSON_ELEMENT_EDGES)) { - return Optional.empty(); - } - - return Optional.of(videoFiles.getAsJsonArray(JSON_ELEMENT_EDGES)); - } - - private Optional getViewer(final JsonObject aBaseObject) { - if (!aBaseObject.has(JSON_ELEMENT_DATA)) { - return Optional.empty(); - } - - final JsonObject data = aBaseObject.getAsJsonObject(JSON_ELEMENT_DATA); - if (!data.has(JSON_ELEMENT_VIEWER)) { - return Optional.empty(); - } - - return Optional.of(data.getAsJsonObject(JSON_ELEMENT_VIEWER)); - } - - private Optional nodeToUrl(final JsonObject aNode) { - if (aNode.has(JSON_ELEMENT_PUBLIC_LOCATION)) { - if (aNode.has(JSON_ELEMENT_VIDEO_PROFILE)) { - final JsonObject videoProfile = aNode.getAsJsonObject(JSON_ELEMENT_VIDEO_PROFILE); - if (videoProfile.has(JSON_ELEMENT_ID)) { - if (videoProfile.has(JSON_ELEMENT_WIDTH)) { - if (!videoProfile.get(JSON_ELEMENT_WIDTH).isJsonNull() - && !videoProfile.get(JSON_ELEMENT_ID).isJsonNull()) { - return Optional.of(new BrUrlDTO(aNode.get(JSON_ELEMENT_PUBLIC_LOCATION).getAsString(), - videoProfile.get(JSON_ELEMENT_WIDTH).getAsInt(), - videoProfile.get(JSON_ELEMENT_ID).getAsString())); - } - - } else { - printMissingDetails(JSON_ELEMENT_VIDEO_PROFILE + " -> " + JSON_ELEMENT_WIDTH); - } - - } else { - printMissingDetails(JSON_ELEMENT_VIDEO_PROFILE + " -> " + JSON_ELEMENT_ID); - } - - } else { - printMissingDetails(JSON_ELEMENT_VIDEO_PROFILE); - } - - } else { - printMissingDetails(JSON_ELEMENT_PUBLIC_LOCATION); - } - return Optional.empty(); - } - - private void printMissingDetails(final String aMissingJsonElement) { - LOG.error(String.format(ERROR_MISSING_DETAIL_TEMPLATE, aMissingJsonElement)); - } - - private Duration toDuration(final long aSeconds) { - return Duration.of(aSeconds, ChronoUnit.SECONDS); - } - - private LocalDateTime toTime(final String aStart) { - LocalDateTime local = LocalDateTime.parse(aStart, DateTimeFormatter.ISO_OFFSET_DATE_TIME); - ZonedDateTime zoned = local.atZone(ZONE_ID); - int hoursToAdd = zoned.getOffset().getTotalSeconds() / 3600; - return local.plusHours(hoursToAdd); - } - -} diff --git a/src/main/java/mServer/crawler/sender/br/BrIdsDTO.java b/src/main/java/mServer/crawler/sender/br/BrIdsDTO.java deleted file mode 100644 index 51ea77133..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrIdsDTO.java +++ /dev/null @@ -1,22 +0,0 @@ -package mServer.crawler.sender.br; - -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentSkipListSet; - -public class BrIdsDTO { - private final Set ids; - - public BrIdsDTO() { - super(); - ids = ConcurrentHashMap.newKeySet(); - } - - public boolean add(final String aId) { - return ids.add(aId); - } - - public Set getIds() { - return new ConcurrentSkipListSet<>(ids); - } -} diff --git a/src/main/java/mServer/crawler/sender/br/BrMissedSendungsFolgenDeserializer.java b/src/main/java/mServer/crawler/sender/br/BrMissedSendungsFolgenDeserializer.java deleted file mode 100644 index 3c6b80b51..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrMissedSendungsFolgenDeserializer.java +++ /dev/null @@ -1,87 +0,0 @@ -package mServer.crawler.sender.br; - -import java.lang.reflect.Type; -import java.util.Optional; -import com.google.gson.JsonArray; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import mServer.crawler.sender.MediathekReader; - -public class BrMissedSendungsFolgenDeserializer implements JsonDeserializer { - - private static final String JSON_ELEMENT_CONTAINER_TODAY = "containerToday"; - private static final String JSON_ELEMENT_BROADCAST_SERVICE = "broadcastService"; - private static final String JSON_ELEMENT_VIEWER = "viewer"; - private static final String JSON_ELEMENT_DATA = "data"; - private static final String JSON_ELEMENT_ID = "id"; - private static final String JSON_ELEMENT_BROADCAST_EVENT = "broadcastEvent"; - private static final String JSON_ELEMENT_PUBLICATION = "publicationOf"; - - private final MediathekReader crawler; - - public BrMissedSendungsFolgenDeserializer(final MediathekReader aCrawler) { - crawler = aCrawler; - } - - /** - * Resolves the Sendung ids which are needed to get the Sendung details.
- * The data has this structure: - * data -> viewer -> broadcastService -> containerToday -> edges[] -> node -> id - * - * @param aElement - * @param aType - * @param aContext - * @return - */ - @Override - public BrIdsDTO deserialize(final JsonElement aElement, final Type aType, - final JsonDeserializationContext aContext) { - final BrIdsDTO results = new BrIdsDTO(); - - final JsonObject baseObject = aElement.getAsJsonObject(); - final Optional dayEntries = getDayEntries(baseObject); - if (dayEntries.isPresent()) { - for (final JsonElement dayEntry : dayEntries.get()) { - final JsonObject dayEntryObject = dayEntry.getAsJsonObject(); - if (dayEntryObject.has(JSON_ELEMENT_BROADCAST_EVENT) - && !dayEntryObject.get(JSON_ELEMENT_BROADCAST_EVENT).isJsonNull()) { - final JsonObject broadcastEvent = dayEntryObject.getAsJsonObject(JSON_ELEMENT_BROADCAST_EVENT); - if (broadcastEvent.has(JSON_ELEMENT_PUBLICATION)) { - final JsonObject publication = broadcastEvent.getAsJsonObject(JSON_ELEMENT_PUBLICATION); - if (publication.has(JSON_ELEMENT_ID)) { - results.add(publication.get(JSON_ELEMENT_ID).getAsString()); - } - } - } - } - } - - return results; - } - - private Optional getDayEntries(final JsonObject aBaseObject) { - if (!aBaseObject.has(JSON_ELEMENT_DATA)) { - return Optional.empty(); - } - - final JsonObject data = aBaseObject.getAsJsonObject(JSON_ELEMENT_DATA); - if (!data.has(JSON_ELEMENT_VIEWER)) { - return Optional.empty(); - } - - final JsonObject viewer = data.getAsJsonObject(JSON_ELEMENT_VIEWER); - if (!viewer.has(JSON_ELEMENT_BROADCAST_SERVICE)) { - return Optional.empty(); - } - - final JsonObject broadcastService = viewer.getAsJsonObject(JSON_ELEMENT_BROADCAST_SERVICE); - if (!broadcastService.has(JSON_ELEMENT_CONTAINER_TODAY)) { - return Optional.empty(); - } - - return Optional.of(broadcastService.getAsJsonArray(JSON_ELEMENT_CONTAINER_TODAY)); - } - -} diff --git a/src/main/java/mServer/crawler/sender/br/BrMissedSendungsFolgenTask.java b/src/main/java/mServer/crawler/sender/br/BrMissedSendungsFolgenTask.java deleted file mode 100644 index 92abd9ea5..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrMissedSendungsFolgenTask.java +++ /dev/null @@ -1,75 +0,0 @@ -package mServer.crawler.sender.br; - -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; -import java.time.temporal.ChronoUnit; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.Callable; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import java.net.URL; -import mServer.crawler.sender.MediathekReader; -import org.glassfish.jersey.uri.UriComponent; - -import static org.glassfish.jersey.uri.UriComponent.Type.QUERY_PARAM; - -public class BrMissedSendungsFolgenTask implements Callable> { - - private static final Logger LOG = LogManager.getLogger(BrMissedSendungsFolgenTask.class); - - private static final String QUERY_TEMPLATE = Consts.BR_API_URL + - "?operationName=ProgrammeCalendarPageQuery&variables=%s&extensions=%s"; - - private static final String SENDER_BR ="BR_Fernsehen"; - private static final String SENDER_ALPHA ="ARD_alpha"; - - private static final String JSON_VARIABLES = "{\"day\":\"%s\",\"slots\":[\"MORNING\",\"NOON\",\"EVENING\",\"NIGHT\"],\"broadcasterId\":\"av:http://ard.de/ontologies/ard#%s\"}"; - private static final String JSON_EXTENSION = "{\"persistedQuery\":{\"version\":1,\"sha256Hash\":\"ec36d20a87979c32146b32bea1953fe074e62c42d868483638f147a774984ed1\"}}"; - - private final MediathekReader crawler; - - private final int maximumDays; - - private final Gson gson; - - public BrMissedSendungsFolgenTask(final MediathekReader aCrawler, int aMaximumDays) { - crawler = aCrawler; - maximumDays = aMaximumDays; - - gson = new GsonBuilder() - .registerTypeAdapter(BrIdsDTO.class, new BrMissedSendungsFolgenDeserializer(crawler)) - .create(); - } - - private static String buildUrl(final LocalDateTime date, final String sender) { - final String variables = String.format(JSON_VARIABLES, date.format(DateTimeFormatter.ISO_LOCAL_DATE), sender); - return String.format(QUERY_TEMPLATE, UriComponent.encode(variables, QUERY_PARAM), UriComponent.encode(JSON_EXTENSION, QUERY_PARAM)); - - } - - @Override - public Set call() { - Set filmIds = new HashSet<>(); - - getFilms(filmIds, SENDER_BR); - getFilms(filmIds, SENDER_ALPHA); - - return filmIds; - } - - private void getFilms(final Set filmIds, final String sender) { - for (int i = 0; i < maximumDays + 3; i++) { - LocalDateTime day = LocalDateTime.now().plus(3, ChronoUnit.DAYS).minusDays(i); - - BrWebAccessHelper.handleWebAccessExecution(LOG, crawler, () -> { - - final String response = WebAccessHelper.getJsonResultFromGetAccess(new URL(buildUrl(day, sender))); - final BrIdsDTO missedSendungsFolgen = gson.fromJson(response, BrIdsDTO.class); - filmIds.addAll(missedSendungsFolgen.getIds()); - }); - } - } -} diff --git a/src/main/java/mServer/crawler/sender/br/BrSendungDetailsTask.java b/src/main/java/mServer/crawler/sender/br/BrSendungDetailsTask.java deleted file mode 100644 index f31c51c11..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrSendungDetailsTask.java +++ /dev/null @@ -1,97 +0,0 @@ -package mServer.crawler.sender.br; - -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import com.google.gson.reflect.TypeToken; -import de.mediathekview.mlib.Config; -import de.mediathekview.mlib.daten.DatenFilm; -import de.mediathekview.mlib.tool.Log; -import java.lang.reflect.Type; -import java.net.URL; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.RecursiveTask; -import mServer.crawler.sender.MediathekReader; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class BrSendungDetailsTask extends RecursiveTask> { - - public static final int MAXIMUM_URLS_PER_TASK = 50; - private static final long serialVersionUID = 5682617879894452978L; - private static final Logger LOG = LogManager.getLogger(BrSendungDetailsTask.class); - private static final String QUERY_TEMPLATE - = "{\"query\":\"query DetailPageRendererQuery( $clipId: ID! $isClip: Boolean! $isLivestream: Boolean! $livestream: ID!) { viewer { ...DetailPage_viewer_22r5xP id }}fragment DetailPage_viewer_22r5xP on Viewer { ...VideoPlayer_viewer_22r5xP ...ClipActions_viewer detailClip: clip(id: $clipId) { __typename id title ...ClipActions_clip ...ClipInfo_clip ...ChildContentRedirect_creativeWork }}fragment VideoPlayer_viewer_22r5xP on Viewer { id clip(id: $clipId) @include(if: $isClip) { __typename id ageRestriction videoFiles(first: 100) { edges { node { __typename id mimetype publicLocation videoProfile { __typename id width } } } } captionFiles(first: 2) { edges { node { __typename publicLocation id } } } ...Error_clip title } livestream(id: $livestream) @include(if: $isLivestream) { __typename id streamingUrls(first: 10, filter: {accessibleIn: {contains: \\\"GeoZone:http://ard.de/ontologies/coreConcepts#GeoZone_World\\\"}, hasEmbeddedSubtitles: {eq: false}}) { edges { node { __typename id publicLocation } } } }}fragment ClipActions_viewer on Viewer { me { __typename bookmarks(first: 12) { ...BookmarkAction_bookmarks } id }}fragment ClipActions_clip on ClipInterface { id bookmarked downloadable ...BookmarkAction_clip ...Rate_clip ...Share_clip ...Download_clip}fragment ClipInfo_clip on ClipInterface { __typename id title kicker shortDescription description availableUntil ...Duration_clip ... on ProgrammeInterface { publications(first: 1) { edges { node { __typename publishedBy { __typename name id } id } } } initialScreening { start id } episodeOf { __typename id title scheduleInfo subscribed ...SubscribeAction_series ... on CreativeWorkInterface { ...TeaserImage_creativeWorkInterface } } } ... on ItemInterface { itemOf(first: 1) { edges { node { __typename publications(first: 1) { edges { node { __typename publishedBy { __typename name id } id } } } broadcasts(first: 1) { edges { node { __typename start id } } } episodeOf { __typename id title scheduleInfo subscribed ...SubscribeAction_series ... on CreativeWorkInterface { ...TeaserImage_creativeWorkInterface } } id } } } }}fragment ChildContentRedirect_creativeWork on CreativeWorkInterface { categories(first: 100) { edges { node { __typename id } } }}fragment Duration_clip on ClipInterface { duration}fragment SubscribeAction_series on SeriesInterface { id subscribed}fragment TeaserImage_creativeWorkInterface on CreativeWorkInterface { id kicker title teaserImages(first: 1) { edges { node { __typename shortDescription id } } } defaultTeaserImage { __typename imageFiles(first: 1) { edges { node { __typename id publicLocation crops(first: 10) { count edges { node { __typename publicLocation width height id } } } } } } id }}fragment BookmarkAction_clip on ClipInterface { id}fragment Rate_clip on ClipInterface { id reactions { likes dislikes } myInteractions { __typename reaction { __typename id } id }}fragment Share_clip on ClipInterface { title id}fragment Download_clip on ClipInterface { videoFiles(first: 100) { edges { node { __typename publicLocation videoProfile { __typename height id } id } } }}fragment BookmarkAction_bookmarks on ClipRemoteConnection { count ...TeaserSlider_clipRemoteConnection}fragment TeaserSlider_clipRemoteConnection on ClipRemoteConnection { edges { node { __typename ...SmallTeaserBox_node id } }}fragment SmallTeaserBox_node on Node { id ... on CreativeWorkInterface { ...TeaserImage_creativeWorkInterface } ... on ClipInterface { id title kicker ...Duration_clip } ... on ProgrammeInterface { broadcasts(first: 1, orderBy: START_DESC) { edges { node { __typename start id } } } }}fragment Error_clip on ClipInterface { ageRestriction}\",\"variables\":{\"clipId\":\"%s\",\"isClip\":true,\"isLivestream\":false,\"livestream\":\"Livestream:\"}}"; - - private final transient MediathekReader crawler; - private final ConcurrentLinkedQueue filmIds; - private final transient Set convertedFilms; - - public BrSendungDetailsTask(final MediathekReader aCrawler, - final ConcurrentLinkedQueue aBrFilmIds) { - crawler = aCrawler; - filmIds = aBrFilmIds; - convertedFilms = ConcurrentHashMap.newKeySet(); - } - - private ConcurrentLinkedQueue createSubSet( - final ConcurrentLinkedQueue aBaseQueue) { - final int halfSize = aBaseQueue.size() / 2; - final ConcurrentLinkedQueue urlsToCrawlSubset = new ConcurrentLinkedQueue<>(); - for (int i = 0; i < halfSize; i++) { - urlsToCrawlSubset.offer(aBaseQueue.poll()); - } - return urlsToCrawlSubset; - } - - private void filmIdsToFilms(final ConcurrentLinkedQueue aFilmIds) { - for (final String filmId : aFilmIds) { - filmIdToFilm(filmId); - } - crawler.meldungProgress(""); - } - - private void filmIdToFilm(final String aFilmId) { - if (Config.getStop()) { - return; - } - - BrWebAccessHelper.handleWebAccessExecution(LOG, crawler, () -> { - - final Type optionalFilmType = new TypeToken>() { - }.getType(); - final Gson gson = new GsonBuilder() - .registerTypeAdapter(optionalFilmType, new BrFilmDeserializer(crawler, aFilmId)).create(); - - try { - final String response = WebAccessHelper.getJsonResultFromPostAccess(new URL(Consts.BR_API_URL), String.format(QUERY_TEMPLATE, aFilmId)); - - final Optional film = gson.fromJson(response, optionalFilmType); - if (film.isPresent()) { - convertedFilms.add(film.get()); - } - } catch (Exception e) { - Log.errorLog(1615561215, e, aFilmId); - } - }); - - } - - @Override - protected Set compute() { - if (filmIds.size() <= MAXIMUM_URLS_PER_TASK) { - filmIdsToFilms(filmIds); - } else { - final BrSendungDetailsTask rightTask - = new BrSendungDetailsTask(crawler, createSubSet(filmIds)); - final BrSendungDetailsTask leftTask = new BrSendungDetailsTask(crawler, filmIds); - leftTask.fork(); - convertedFilms.addAll(rightTask.compute()); - convertedFilms.addAll(leftTask.join()); - } - return convertedFilms; - } - -} diff --git a/src/main/java/mServer/crawler/sender/br/BrSendungenIdsDeserializer.java b/src/main/java/mServer/crawler/sender/br/BrSendungenIdsDeserializer.java deleted file mode 100644 index 618c2b36b..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrSendungenIdsDeserializer.java +++ /dev/null @@ -1,70 +0,0 @@ -package mServer.crawler.sender.br; - -import java.lang.reflect.Type; -import java.util.Optional; -import com.google.gson.JsonArray; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -public class BrSendungenIdsDeserializer implements JsonDeserializer { - private static final String JSON_ELEMENT_EDGES = "edges"; - private static final String JSON_ELEMENT_SERIES_INDEX_ALL_SERIES = "seriesIndexAllSeries"; - private static final String JSON_ELEMENT_VIEWER = "viewer"; - private static final String JSON_ELEMENT_DATA = "data"; - private static final String JSON_ELEMENT_ID = "id"; - private static final String JSON_ELEMENT_NODE = "node"; - - /** - * Resolves the Sendung ids which are needed to get the Sendung details.
- * The data has this structure: - * data -> viewer -> seriesIndexAllSeries -> edges[] -> node -> id - */ - @Override - public BrIdsDTO deserialize(final JsonElement aElement, final Type aType, - final JsonDeserializationContext aContext) { - final BrIdsDTO results = new BrIdsDTO(); - - final JsonObject baseObject = aElement.getAsJsonObject(); - final Optional edges = getEdges(baseObject); - if (edges.isPresent()) { - for (final JsonElement edge : edges.get()) { - final JsonObject ebdgeObj = edge.getAsJsonObject(); - if (ebdgeObj.has(JSON_ELEMENT_NODE)) { - final JsonObject node = ebdgeObj.getAsJsonObject(JSON_ELEMENT_NODE); - if (node.has(JSON_ELEMENT_ID)) { - results.add(node.get(JSON_ELEMENT_ID).getAsString()); - } - } - } - } - - return results; - } - - private Optional getEdges(final JsonObject aBaseObject) { - if (!aBaseObject.has(JSON_ELEMENT_DATA)) { - return Optional.empty(); - } - - final JsonObject data = aBaseObject.getAsJsonObject(JSON_ELEMENT_DATA); - if (!data.has(JSON_ELEMENT_VIEWER)) { - return Optional.empty(); - } - - final JsonObject viewer = data.getAsJsonObject(JSON_ELEMENT_VIEWER); - if (!viewer.has(JSON_ELEMENT_SERIES_INDEX_ALL_SERIES)) { - return Optional.empty(); - } - - final JsonObject seriesIndexAllSeries = - viewer.getAsJsonObject(JSON_ELEMENT_SERIES_INDEX_ALL_SERIES); - if (!seriesIndexAllSeries.has(JSON_ELEMENT_EDGES)) { - return Optional.empty(); - } - - return Optional.of(seriesIndexAllSeries.getAsJsonArray(JSON_ELEMENT_EDGES)); - } - -} diff --git a/src/main/java/mServer/crawler/sender/br/BrSendungsFolgenDeserializer.java b/src/main/java/mServer/crawler/sender/br/BrSendungsFolgenDeserializer.java deleted file mode 100644 index c43005e86..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrSendungsFolgenDeserializer.java +++ /dev/null @@ -1,106 +0,0 @@ -package mServer.crawler.sender.br; - -import java.lang.reflect.Type; -import java.util.Optional; -import com.google.gson.JsonArray; -import com.google.gson.JsonDeserializationContext; -import com.google.gson.JsonDeserializer; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; -import mServer.crawler.sender.MediathekReader; - -public class BrSendungsFolgenDeserializer implements JsonDeserializer { - private static final String JSON_ELEMENT_EDGES = "edges"; - private static final String JSON_ELEMENT_CLIPS_ONLY = "clipsOnly"; - private static final String JSON_ELEMENT_PREVIOUS_EPISODES = "previousEpisodes"; - private static final String JSON_ELEMENT_SERIES = "series"; - private static final String JSON_ELEMENT_VIEWER = "viewer"; - private static final String JSON_ELEMENT_DATA = "data"; - private static final String JSON_ELEMENT_ID = "id"; - private static final String JSON_ELEMENT_NODE = "node"; - private final MediathekReader crawler; - - public BrSendungsFolgenDeserializer(final MediathekReader aCrawler) { - crawler = aCrawler; - } - - /** - * Resolves the Sendung ids which are needed to get the Sendung details.
- * The data has these two structures: - * data -> viewer -> series -> clipsOnly -> edges[] -> node -> id
- * data -> viewer -> series -> previousEpisodes -> edges[] -> node -> id - */ - @Override - public BrIdsDTO deserialize(final JsonElement aElement, final Type aType, - final JsonDeserializationContext aContext) { - final BrIdsDTO results = new BrIdsDTO(); - - final JsonObject baseObject = aElement.getAsJsonObject(); - - final Optional series = getSeries(baseObject); - - if (series.isPresent()) { - final Optional clipsEdges = getClipsEdges(series.get()); - addToResult(results, clipsEdges); - - final Optional previosEpisodesEdges = getPreviousEpisodesEdges(series.get()); - addToResult(results, previosEpisodesEdges); - } - - return results; - } - - private void addToResult(final BrIdsDTO results, final Optional aEdges) { - if (aEdges.isPresent()) { - for (final JsonElement edge : aEdges.get()) { - final JsonObject ebdgeObj = edge.getAsJsonObject(); - if (ebdgeObj.has(JSON_ELEMENT_NODE)) { - final JsonObject node = ebdgeObj.getAsJsonObject(JSON_ELEMENT_NODE); - if (node.has(JSON_ELEMENT_ID)) { - results.add(node.get(JSON_ELEMENT_ID).getAsString()); - } - } - } - } - } - - private Optional getClipsEdges(final JsonObject aSeries) { - return getEdges(aSeries, JSON_ELEMENT_CLIPS_ONLY); - } - - private Optional getEdges(final JsonObject aSeries, - final String aEdgesParentElementId) { - if (!aSeries.has(aEdgesParentElementId)) { - return Optional.empty(); - } - - final JsonObject edgesParent = aSeries.getAsJsonObject(aEdgesParentElementId); - if (!edgesParent.has(JSON_ELEMENT_EDGES)) { - return Optional.empty(); - } - return Optional.of(edgesParent.getAsJsonArray(JSON_ELEMENT_EDGES)); - } - - private Optional getPreviousEpisodesEdges(final JsonObject aSeries) { - return getEdges(aSeries, JSON_ELEMENT_PREVIOUS_EPISODES); - } - - private Optional getSeries(final JsonObject aBaseObject) { - if (!aBaseObject.has(JSON_ELEMENT_DATA)) { - return Optional.empty(); - } - - final JsonObject data = aBaseObject.getAsJsonObject(JSON_ELEMENT_DATA); - if (!data.has(JSON_ELEMENT_VIEWER)) { - return Optional.empty(); - } - - final JsonObject viewer = data.getAsJsonObject(JSON_ELEMENT_VIEWER); - if (!viewer.has(JSON_ELEMENT_SERIES)) { - return Optional.empty(); - } - - return Optional.of(viewer.getAsJsonObject(JSON_ELEMENT_SERIES)); - - } -} diff --git a/src/main/java/mServer/crawler/sender/br/BrSendungsFolgenTask.java b/src/main/java/mServer/crawler/sender/br/BrSendungsFolgenTask.java deleted file mode 100644 index 4e23c595a..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrSendungsFolgenTask.java +++ /dev/null @@ -1,51 +0,0 @@ -package mServer.crawler.sender.br; - -import java.time.LocalDateTime; -import java.util.Set; -import java.util.concurrent.Callable; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import com.google.gson.Gson; -import com.google.gson.GsonBuilder; -import java.net.URL; -import mServer.crawler.sender.MediathekReader; - -public class BrSendungsFolgenTask implements Callable> { - - private static final int MAXIMUM_SUB_PAGES = 3; - private static final Logger LOG = LogManager.getLogger(BrSendungsFolgenTask.class); - private static final int VIRTUAL_PAGE_COUNT = 25; - private static final String QUERY_TEMPLATE - = "{\"query\":\"query SeriesPageRendererQuery( $id: ID! $itemCount: Int $clipCount: Int $previousEpisodesFilter: ProgrammeFilter $clipsOnlyFilter: ProgrammeFilter) { viewer { ...SeriesPage_viewer_2PDDaq id }}fragment SeriesPage_viewer_2PDDaq on Viewer { series(id: $id) { __typename ...TeaserImage_creativeWorkInterface ...SeriesBrandBanner_series clipsOnly: episodes(orderBy: VERSIONFROM_DESC, first: $clipCount, filter: $clipsOnlyFilter) { ...ProgrammeSlider_programmes } previousEpisodes: episodes(first: $itemCount, orderBy: BROADCASTS_START_DESC, filter: $previousEpisodesFilter) { ...ProgrammeSlider_programmes edges { node { __typename ...SmallTeaserBox_node id } } } id }}fragment TeaserImage_creativeWorkInterface on CreativeWorkInterface { id kicker title }fragment SeriesBrandBanner_series on SeriesInterface { ...SubscribeAction_series title shortDescription externalURLS(first: 1) { edges { node { __typename id url label } } } }fragment ProgrammeSlider_programmes on ProgrammeConnection { edges { node { __typename ...SmallTeaserBox_node id } }}fragment SmallTeaserBox_node on Node { id ... on CreativeWorkInterface { ...TeaserImage_creativeWorkInterface } ... on ClipInterface { id title kicker ...Bookmark_clip ...Duration_clip ...Progress_clip } ... on ProgrammeInterface { broadcasts(first: 1, orderBy: START_DESC) { edges { node { __typename start id } } } }}fragment Bookmark_clip on ClipInterface { id bookmarked title}fragment Duration_clip on ClipInterface { duration}fragment Progress_clip on ClipInterface { myInteractions { __typename progress completed id }}fragment SubscribeAction_series on SeriesInterface { id subscribed}\",\"variables\":{\"id\":\"%s\",\"itemCount\":%d,\"clipCount\":%d,\"previousEpisodesFilter\":{\"essences\":{\"empty\":{\"eq\":false}},\"broadcasts\":{\"empty\":{\"eq\":false}}},\"clipsOnlyFilter\":{\"broadcasts\":{\"empty\":{\"eq\":true}},\"essences\":{\"empty\":{\"eq\":false}}}}}"; - private final String sendungsReihenId; - private final MediathekReader crawler; - - private BrIdsDTO sendungsFolgen; - - public BrSendungsFolgenTask(final MediathekReader aCrawler, final String aSendungsReihenId) { - crawler = aCrawler; - sendungsReihenId = aSendungsReihenId; - } - - @Override - public Set call() { - sendungsFolgen = new BrIdsDTO(); - - BrWebAccessHelper.handleWebAccessExecution(LOG, crawler, () -> { - - final Gson gson = new GsonBuilder() - .registerTypeAdapter(BrIdsDTO.class, new BrSendungsFolgenDeserializer(crawler)).create(); - - // 2017-09-19T16:52:25.559Z - final String todayDateString = LocalDateTime.now().format(Consts.BR_FORMATTER); - final int folgenCount = MAXIMUM_SUB_PAGES + 1 * VIRTUAL_PAGE_COUNT; - - final String response = WebAccessHelper.getJsonResultFromPostAccess(new URL(Consts.BR_API_URL), String.format(QUERY_TEMPLATE, sendungsReihenId, folgenCount, - folgenCount, todayDateString)); - - sendungsFolgen = gson.fromJson(response, BrIdsDTO.class); - }); - - return sendungsFolgen.getIds(); - } -} diff --git a/src/main/java/mServer/crawler/sender/br/BrUrlDTO.java b/src/main/java/mServer/crawler/sender/br/BrUrlDTO.java deleted file mode 100644 index 52321a64d..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrUrlDTO.java +++ /dev/null @@ -1,70 +0,0 @@ -package mServer.crawler.sender.br; - -public class BrUrlDTO { - private final String url; - private final int width; - private final String videoProfile; - - public BrUrlDTO(final String aUrl, final int aWidth, final String aVideoProfile) { - super(); - url = aUrl; - width = aWidth; - videoProfile = aVideoProfile; - } - - @Override - public boolean equals(final Object obj) { - if (this == obj) { - return true; - } - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - final BrUrlDTO other = (BrUrlDTO) obj; - if (url == null) { - if (other.url != null) { - return false; - } - } else if (!url.equals(other.url)) { - return false; - } - if (videoProfile == null) { - if (other.videoProfile != null) { - return false; - } - } else if (!videoProfile.equals(other.videoProfile)) { - return false; - } - if (width != other.width) { - return false; - } - return true; - } - - public String getUrl() { - return url; - } - - public String getVideoProfile() { - return videoProfile; - } - - public int getWidth() { - return width; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + (url == null ? 0 : url.hashCode()); - result = prime * result + (videoProfile == null ? 0 : videoProfile.hashCode()); - result = prime * result + width; - return result; - } - - -} diff --git a/src/main/java/mServer/crawler/sender/br/BrWebAccessExecution.java b/src/main/java/mServer/crawler/sender/br/BrWebAccessExecution.java deleted file mode 100644 index 907aebe0b..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrWebAccessExecution.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * BrWebAccessExecution.java - * - * Projekt : MServer - * erstellt am: 07.10.2017 - * Autor : Sascha - * - * (c) 2017 by Sascha Wiegandt - */ -package mServer.crawler.sender.br; - -import java.net.MalformedURLException; - -import com.google.gson.JsonSyntaxException; - -@FunctionalInterface -public interface BrWebAccessExecution { - void run() throws JsonSyntaxException, - MalformedURLException; -} \ No newline at end of file diff --git a/src/main/java/mServer/crawler/sender/br/BrWebAccessHelper.java b/src/main/java/mServer/crawler/sender/br/BrWebAccessHelper.java deleted file mode 100644 index e8de9ca6a..000000000 --- a/src/main/java/mServer/crawler/sender/br/BrWebAccessHelper.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * BrWebAccessHelper.java - * - * Projekt : MServer - * erstellt am: 07.10.2017 - * Autor : Sascha - * - * (c) 2017 by Sascha Wiegandt - */ -package mServer.crawler.sender.br; - -import java.net.MalformedURLException; - -import org.apache.logging.log4j.Logger; - -import com.google.gson.JsonSyntaxException; -import mServer.crawler.FilmeSuchen; -import mServer.crawler.RunSender; -import mServer.crawler.sender.MediathekReader; - -public class BrWebAccessHelper { - - private BrWebAccessHelper() {} - - public static void handleWebAccessExecution(Logger log, MediathekReader crawler, BrWebAccessExecution action) { - try { - action.run(); - } catch (final JsonSyntaxException jsonSyntaxException) { - handleError("The json syntax for the BR task to get all Sendungen has an error.", jsonSyntaxException, log, crawler); - } catch (final MalformedURLException malformedURLException) { - handleError("The URL given to WebAccess is not valid.", malformedURLException, log, crawler); - } catch (final IllegalArgumentException illegalArgumentException) { - handleError("URL given to the WebAccess was Null", illegalArgumentException, log, crawler); - } - } - - private static void handleError(String errorMessage, Exception e, Logger log, MediathekReader crawler) { - log.error(errorMessage, e); - FilmeSuchen.listeSenderLaufen.inc(crawler.getSendername(), RunSender.Count.FEHLER); - } -} diff --git a/src/main/java/mServer/crawler/sender/br/Consts.java b/src/main/java/mServer/crawler/sender/br/Consts.java deleted file mode 100644 index 132dc994c..000000000 --- a/src/main/java/mServer/crawler/sender/br/Consts.java +++ /dev/null @@ -1,15 +0,0 @@ -package mServer.crawler.sender.br; - -import java.time.format.DateTimeFormatter; - -public final class Consts { - public static final String BR_API_URL = "https://api.mediathek.br.de/graphql"; - public static final DateTimeFormatter BR_FORMATTER = DateTimeFormatter.ISO_DATE; - public static final String JSON_SYNTAX_ERROR = - "The site \"%s\" for the \"%s\" crawler in't a valid JSON page."; - public static final String ATTRIBUTE_HREF = "href"; - - private Consts() { - super(); - } -} \ No newline at end of file diff --git a/src/main/java/mServer/crawler/sender/br/WebAccessHelper.java b/src/main/java/mServer/crawler/sender/br/WebAccessHelper.java deleted file mode 100644 index 0b321b97e..000000000 --- a/src/main/java/mServer/crawler/sender/br/WebAccessHelper.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * WebAccessHelper.java - * - * Projekt : MLib - * erstellt am: 04.10.2017 - * Autor : Sascha - * - */ -// Ist eigentlich aus MLib in der neuen Architektur, aber für den neuen BR-Crawler erstmal hierher kopiert -package mServer.crawler.sender.br; - -import de.mediathekview.mlib.Const; -import java.net.URL; - -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.WebTarget; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; - -import mServer.crawler.FilmeSuchen; -import mServer.crawler.RunSender; - -public class WebAccessHelper { - - private WebAccessHelper() {} - - public static String getJsonResultFromGetAccess(URL serverUrl) { - if(null == serverUrl) - throw new IllegalArgumentException("Es wurde keine gültige ServerURL angegeben"); - - WebTarget target = WebTargetDS.getInstance(serverUrl.toString()); - Response response = target.request(MediaType.APPLICATION_JSON_TYPE).get(); - String result = response.readEntity(String.class); - - FilmeSuchen.listeSenderLaufen.inc(Const.BR, RunSender.Count.ANZAHL); - - return result; - } - - public static String getJsonResultFromPostAccess(URL serverUrl, String request) { - if(null == serverUrl) - throw new IllegalArgumentException("Es wurde keine gültige ServerURL angegeben"); - - WebTarget target = WebTargetDS.getInstance(serverUrl.toString()); - - Response response = target.request(MediaType.APPLICATION_JSON_TYPE) - .post(Entity.entity(request, MediaType.APPLICATION_JSON_TYPE)); - String result = response.readEntity(String.class); - - FilmeSuchen.listeSenderLaufen.inc(Const.BR, RunSender.Count.ANZAHL); - - return result; - } - -} diff --git a/src/main/java/mServer/crawler/sender/br/WebTargetDS.java b/src/main/java/mServer/crawler/sender/br/WebTargetDS.java deleted file mode 100644 index d4c5a7ac1..000000000 --- a/src/main/java/mServer/crawler/sender/br/WebTargetDS.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * WebTargetDS.java - * - * Projekt : MServer - * erstellt am: 05.10.2017 - * Autor : Sascha - * - * (c) 2017 by Sascha Wiegandt - */ -// Ist eigentlich aus MLib in der neuen Architektur, aber für den neuen BR-Crawler erstmal hierher kopiert -package mServer.crawler.sender.br; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import jakarta.ws.rs.client.Client; -import jakarta.ws.rs.client.ClientBuilder; -import jakarta.ws.rs.client.WebTarget; - -public class WebTargetDS { - - private static final Map connectionPool = new ConcurrentHashMap<>(); - - private WebTargetDS() { - - } - - public static WebTarget getInstance(String url) { - if (connectionPool.containsKey(url)) { - return connectionPool.get(url); - } else { - Client client = ClientBuilder.newClient(); - connectionPool.put(url, client.target(url)); - - return connectionPool.get(url); - } - } - -} diff --git a/src/main/java/mServer/crawler/sender/kika/KikaApiVideoInfoDto.java b/src/main/java/mServer/crawler/sender/kika/KikaApiVideoInfoDto.java index 8bcd66e6b..8f65501d1 100644 --- a/src/main/java/mServer/crawler/sender/kika/KikaApiVideoInfoDto.java +++ b/src/main/java/mServer/crawler/sender/kika/KikaApiVideoInfoDto.java @@ -6,10 +6,6 @@ import java.util.Optional; import java.util.Set; -import mServer.crawler.sender.br.Resolution; - - - public class KikaApiVideoInfoDto { // private Optional errorMesssage = Optional.empty(); diff --git a/src/main/java/mServer/crawler/sender/br/Resolution.java b/src/main/java/mServer/crawler/sender/kika/Resolution.java similarity index 87% rename from src/main/java/mServer/crawler/sender/br/Resolution.java rename to src/main/java/mServer/crawler/sender/kika/Resolution.java index b0cdc75f6..f1563c343 100644 --- a/src/main/java/mServer/crawler/sender/br/Resolution.java +++ b/src/main/java/mServer/crawler/sender/kika/Resolution.java @@ -1,4 +1,4 @@ -package mServer.crawler.sender.br; +package mServer.crawler.sender.kika; import java.util.Arrays; import java.util.Comparator; @@ -37,8 +37,8 @@ private Resolution(final int resolutionSize, final String description) { public static List getFromBestToLowest() { return Arrays.asList(Resolution.values()).stream() - .sorted(Comparator.comparing(Resolution::getResolutionSize).reversed()) - .collect(Collectors.toList()); + .sorted(Comparator.comparing(Resolution::getResolutionSize).reversed()) + .collect(Collectors.toList()); } public static Resolution getHighestResolution() { @@ -72,7 +72,7 @@ public static Resolution getNextLowerResolution(final Resolution startingResolut * @return */ public static Resolution getResolutionFromArdAudioVideoOrdinalsByProfileName( - final String profileName) { + final String profileName) { if (profileName.endsWith("HD")) { return Resolution.HD; } @@ -89,7 +89,7 @@ public static Resolution getResolutionFromArdAudioVideoOrdinalsByProfileName( return Resolution.VERY_SMALL; } if (profileName.endsWith("Mobile_S")) { - return Resolution.VERY_SMALL; + return Resolution.VERY_SMALL; } return Resolution.VERY_SMALL; @@ -120,12 +120,12 @@ public static Resolution getResolutionFromWidth(final int width) { } return Resolution.VERY_SMALL; } - + static Resolution getNextResolutionByDirection(final Resolution startingResolution, - final CountingDirection direction) { + final CountingDirection direction) { try { return getResoultionByResolutionSize( - startingResolution.getResolutionSize() + direction.getDirectionValue()); + startingResolution.getResolutionSize() + direction.getDirectionValue()); } catch (final NoSuchElementException nsee) { return startingResolution; } @@ -140,7 +140,7 @@ static Resolution getResoultionByResolutionSize(final int searchedResolutionSize } throw new NoSuchElementException( - String.format("Resolution with ResolutionIndex %d not found", searchedResolutionSize)); + String.format("Resolution with ResolutionIndex %d not found", searchedResolutionSize)); } public String getDescription() { diff --git a/src/main/java/mServer/crawler/sender/kika/json/KikaApiVideoInfoPageDeserializer.java b/src/main/java/mServer/crawler/sender/kika/json/KikaApiVideoInfoPageDeserializer.java index 37d35cd0b..c3edc9285 100644 --- a/src/main/java/mServer/crawler/sender/kika/json/KikaApiVideoInfoPageDeserializer.java +++ b/src/main/java/mServer/crawler/sender/kika/json/KikaApiVideoInfoPageDeserializer.java @@ -4,8 +4,8 @@ import mServer.crawler.sender.base.JsonUtils; import mServer.crawler.sender.base.UrlUtils; -import mServer.crawler.sender.br.Resolution; import mServer.crawler.sender.kika.KikaApiVideoInfoDto; +import mServer.crawler.sender.kika.Resolution; import java.lang.reflect.Type; import java.util.ArrayList; diff --git a/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java b/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java index 7e9c82475..1bd8e1893 100644 --- a/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java +++ b/src/main/java/mServer/crawler/sender/kika/tasks/KikaApiFilmTask.java @@ -13,9 +13,9 @@ import mServer.crawler.sender.base.AbstractRecursivConverterTask; import mServer.crawler.sender.base.GeoLocations; import mServer.crawler.sender.base.UrlUtils; -import mServer.crawler.sender.br.Resolution; import mServer.crawler.sender.kika.KikaApiFilmDto; import mServer.crawler.sender.kika.KikaApiVideoInfoDto; +import mServer.crawler.sender.kika.Resolution; import mServer.crawler.sender.kika.json.KikaApiVideoInfoPageDeserializer; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger;