Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Hotfix/#191 #196

Merged
merged 8 commits into from
Jul 12, 2017
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.JsonElement;
import com.google.gson.JsonNull;
import com.google.gson.JsonObject;

import de.mediathekview.mlib.daten.DatenFilm;
import de.mediathekview.mlib.tool.MVHttpClient;
import java.util.Calendar;
import mServer.crawler.CrawlerTool;
import mServer.crawler.sender.newsearch.Qualities;
import mServer.tool.MserverDatumZeit;
Expand All @@ -36,20 +36,20 @@ public class ArteJsonObjectToDatenFilmCallable implements Callable<DatenFilm>
private static final String ARTE_VIDEO_INFORMATION_URL_PATTERN = "https://api.arte.tv/api/player/v1/config/%s/%s?platform=ARTE_NEXT";
private static final String ARTE_VIDEO_INFORMATION_URL_PATTERN_2 = "https://api.arte.tv/api/opa/v3/programs/%s/%s"; // Für broadcastBeginRounded
private static final String JSON_ELEMENT_KEY_SHORT_DESCRIPTION = "shortDescription";
private static final String JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1 = "programs";
private static final String JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2 = "broadcastProgrammings";
private static final String JSON_ELEMENT_BROADCAST = "broadcastBeginRounded";

private final JsonObject jsonObject;
private final String langCode;
private final String senderName;

private final FastDateFormat broadcastDate = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ssX");//2016-10-29T16:15:00Z
private final Calendar today;

private final FastDateFormat broadcastDateFormat = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ssX");//2016-10-29T16:15:00Z

public ArteJsonObjectToDatenFilmCallable(JsonObject aJsonObjec, String aLangCode, String aSenderName) {
jsonObject = aJsonObjec;
langCode = aLangCode;
senderName = aSenderName;
today = Calendar.getInstance();
}

@Override
Expand All @@ -69,7 +69,10 @@ public DatenFilm call() {
String programId = getElementValue(jsonObject, JSON_ELEMENT_KEY_PROGRAM_ID);
String videosUrl = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN, langCode, programId);

Gson gson = new GsonBuilder().registerTypeAdapter(ArteVideoDTO.class, new ArteVideoDeserializer()).create();
Gson gson = new GsonBuilder()
.registerTypeAdapter(ArteVideoDTO.class, new ArteVideoDeserializer())
.registerTypeAdapter(ArteVideoDetailsDTO.class, new ArteVideoDetailsDeserializer(today))
.create();

try(Response responseVideoDetails = executeRequest(videosUrl))
{
Expand All @@ -82,51 +85,9 @@ public DatenFilm call() {

if (video.getVideoUrls().containsKey(Qualities.NORMAL))
{
String broadcastBegin = "";

OkHttpClient httpClient = MVHttpClient.getInstance().getHttpClient();
//https://api.arte.tv/api/opa/v3/programs/[language:de/fr]/[programId]
String videosUrlVideoDetails2 = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN_2, langCode, programId);
Request request = new Request.Builder()
.addHeader(MediathekArte_de.AUTH_HEADER, MediathekArte_de.AUTH_TOKEN)
.url(videosUrlVideoDetails2).build();

try(Response responseVideoDetails2 = httpClient.newCall(request).execute())
{
if(responseVideoDetails2.isSuccessful())
{
/*
* Grobe Struktur des Json's:
* {
* "meta": {}
* "programs": [{ (JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1)
"broadcastProgrammings": [{ (JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2)
"broadcastBeginRounded": "2016-07-06T02:40:00Z", (JSON_ELEMENT_BROADCAST)
]}
]}
* }
*/
JsonObject jsonObjectVideoDetails2 = gson.fromJson(responseVideoDetails2.body().string(), JsonObject.class);
if(jsonObjectVideoDetails2.isJsonObject() &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().size() > 0 &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().size() > 0 &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().get(0).getAsJsonObject()
.has(JSON_ELEMENT_BROADCAST))
{
JsonElement jsonBegin = jsonObjectVideoDetails2
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST);
if(jsonBegin != JsonNull.INSTANCE) {
broadcastBegin = jsonBegin.getAsString();
}
}
}
}

film = createFilm(thema, urlWeb, titel, video, broadcastBegin, durationAsTime, beschreibung);
ArteVideoDetailsDTO details = getVideoDetails(gson, programId);

film = createFilm(thema, urlWeb, titel, video, details.getBroadcastBegin(), durationAsTime, beschreibung);
} else {
LOG.debug(String.format("Keine \"normale\" Video URL für den Film \"%s\" mit der URL \"%s\". Video Details URL:\"%s\" ", titel, urlWeb, videosUrl));
}
Expand All @@ -144,15 +105,43 @@ public DatenFilm call() {
return film;
}

private ArteVideoDetailsDTO getVideoDetails(Gson gson, String programId) throws IOException {
ArteVideoDetailsDTO details = null;

OkHttpClient httpClient = MVHttpClient.getInstance().getHttpClient();
//https://api.arte.tv/api/opa/v3/programs/[language:de/fr]/[programId]
String videosUrlVideoDetails2 = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN_2, langCode, programId);
Request request = new Request.Builder()
.addHeader(MediathekArte_de.AUTH_HEADER, MediathekArte_de.AUTH_TOKEN)
.url(videosUrlVideoDetails2).build();

try(Response responseVideoDetails2 = httpClient.newCall(request).execute())
{
if(responseVideoDetails2.isSuccessful())
{
details = gson.fromJson(responseVideoDetails2.body().string(), ArteVideoDetailsDTO.class);
}
}
return details;
}

private static String getSubject(JsonObject programObject) {
String category = "";
String subcategory = "";
String subject;

JsonObject catObject = programObject.get(JSON_ELEMENT_KEY_CATEGORY).getAsJsonObject();
JsonObject subcatObject = programObject.get(JSON_ELEMENT_KEY_SUBCATEGORY).getAsJsonObject();

String category = catObject != null ? getElementValue(catObject, JSON_ELEMENT_KEY_NAME) : "";
String subcategory = subcatObject != null ? getElementValue(subcatObject, JSON_ELEMENT_KEY_NAME) : "";
JsonElement catElement = programObject.get(JSON_ELEMENT_KEY_CATEGORY);
if(!catElement.isJsonNull()) {
JsonObject catObject = catElement.getAsJsonObject();
category = catObject != null ? getElementValue(catObject, JSON_ELEMENT_KEY_NAME) : "";
}

JsonElement subcatElement = programObject.get(JSON_ELEMENT_KEY_SUBCATEGORY);
if(!subcatElement.isJsonNull()) {
JsonObject subcatObject = subcatElement.getAsJsonObject();
subcategory = subcatObject != null ? getElementValue(subcatObject, JSON_ELEMENT_KEY_NAME) : "";
}

if(!category.equals(subcategory) && !subcategory.isEmpty()) {
subject = category + " - " + subcategory;
} else {
Expand All @@ -161,7 +150,7 @@ private static String getSubject(JsonObject programObject) {

return subject;
}

private static String getTitle(JsonObject programObject) {
String title = getElementValue(programObject, JSON_ELEMENT_KEY_TITLE);
String subtitle = getElementValue(programObject, JSON_ELEMENT_KEY_SUBTITLE);
Expand All @@ -188,8 +177,8 @@ private static String getElementValue(JsonObject jsonObject, String elementName)

private DatenFilm createFilm(String thema, String urlWeb, String titel, ArteVideoDTO video, String broadcastBegin, LocalTime durationAsTime, String beschreibung) {

String date = MserverDatumZeit.formatDate(broadcastBegin, broadcastDate);
String time = MserverDatumZeit.formatTime(broadcastBegin, broadcastDate);
String date = MserverDatumZeit.formatDate(broadcastBegin, broadcastDateFormat);
String time = MserverDatumZeit.formatTime(broadcastBegin, broadcastDateFormat);

DatenFilm film = new DatenFilm(senderName, thema, urlWeb, titel, video.getUrl(Qualities.NORMAL), "" /*urlRtmp*/,
date, time, durationAsTime.toSecondOfDay(), beschreibung);
Expand Down
16 changes: 16 additions & 0 deletions src/main/java/mServer/crawler/sender/arte/ArteVideoDetailsDTO.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package mServer.crawler.sender.arte;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MINOR Rename this package name to match the regular expression '^[a-z]+(.[a-z][a-z0-9])$'. rule


public class ArteVideoDetailsDTO {
private String broadcastBegin = "";

public ArteVideoDetailsDTO() {
}

public String getBroadcastBegin() {
return this.broadcastBegin;
}

public void setBroadcastBegin(String aBroadcastBegin) {
this.broadcastBegin = aBroadcastBegin;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,194 @@
package mServer.crawler.sender.arte;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

MINOR Rename this package name to match the regular expression '^[a-z]+(.[a-z][a-z0-9])$'. rule


import com.google.gson.JsonArray;
import com.google.gson.JsonDeserializationContext;
import com.google.gson.JsonDeserializer;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonParseException;
import java.lang.reflect.Type;
import java.text.ParseException;
import java.util.Calendar;
import mServer.tool.DateWithoutTimeComparer;
import org.apache.commons.lang3.time.FastDateFormat;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

public class ArteVideoDetailsDeserializer implements JsonDeserializer<ArteVideoDetailsDTO> {

/*
* Grobe Struktur des Json's:
* {
* "meta": {}
* "programs": [{ (JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1)
"broadcastProgrammings": [{ (JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2)
"broadcastBeginRounded": "2016-07-06T02:40:00Z", (JSON_ELEMENT_BROADCAST)
"broadcastType": MAJOR_REBROADCAST, (JSON_ELEMENT_BROADCAST_TYPE)
]}
]}
* }
*/
private static final String JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1 = "programs";
private static final String JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2 = "broadcastProgrammings";
private static final String JSON_ELEMENT_BROADCAST = "broadcastBeginRounded";
private static final String JSON_ELEMENT_BROADCASTTYPE = "broadcastType";
private static final String JSON_ELEMENT_BROADCAST_CATCHUPRIGHTS_BEGIN = "catchupRightsBegin";
private static final String JSON_ELEMENT_BROADCAST_CATCHUPRIGHTS_END = "catchupRightsEnd";
private static final String BROADCASTTTYPE_FIRST = "FIRST_BROADCAST";
private static final String BROADCASTTTYPE_MINOR_RE = "MINOR_REBROADCAST";
private static final String BROADCASTTTYPE_MAJOR_RE = "MAJOR_REBROADCAST";

private static final Logger LOG = LogManager.getLogger(ArteVideoDeserializer.class);

private final FastDateFormat broadcastDateFormat = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ssX");//2016-10-29T16:15:00Z

private final Calendar today;

public ArteVideoDetailsDeserializer(Calendar aToday) {
today = aToday;
}

@Override
public ArteVideoDetailsDTO deserialize(JsonElement aJsonElement, Type aType, JsonDeserializationContext aContext) throws JsonParseException {
ArteVideoDetailsDTO detailsDTO = new ArteVideoDetailsDTO();

if(aJsonElement.isJsonObject() &&
aJsonElement.getAsJsonObject().get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().size() > 0) {

JsonObject programElement = aJsonElement.getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject();
JsonArray broadcastArray = programElement.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray();

if(broadcastArray.size() > 0) {
String broadcastBeginFirst = "";
String broadcastBeginMajor = "";
String broadcastBeginMinor = "";

// nach Priorität der BroadcastTypen den relevanten Eintrag suchen
// FIRST_BROADCAST => MAJOR_REBROADCAST => MINOR_REBROADCAST
// dabei die "aktuellste" Ausstrahlung verwenden
for(int i = 0; i < broadcastArray.size(); i++) {
JsonObject broadcastObject = broadcastArray.get(i).getAsJsonObject();

if(broadcastObject.has(JSON_ELEMENT_BROADCASTTYPE) &&
broadcastObject.has(JSON_ELEMENT_BROADCAST)) {
String value = this.getBroadcastDate(broadcastObject);

if(!value.isEmpty()) {
String type = broadcastObject.get(JSON_ELEMENT_BROADCASTTYPE).getAsString();
switch(type) {
case BROADCASTTTYPE_FIRST:
broadcastBeginFirst = value;
break;
case BROADCASTTTYPE_MAJOR_RE:
broadcastBeginMajor = value;
break;
case BROADCASTTTYPE_MINOR_RE:
broadcastBeginMinor = value;
break;
default:
LOG.debug("New broadcasttype: " + type);
}
}
}
}

if(!broadcastBeginFirst.isEmpty()) {
detailsDTO.setBroadcastBegin(broadcastBeginFirst);
} else if(!broadcastBeginMajor.isEmpty()) {
detailsDTO.setBroadcastBegin(broadcastBeginMajor);
} else if(!broadcastBeginMinor.isEmpty()) {
detailsDTO.setBroadcastBegin(broadcastBeginMinor);
}

// wenn kein Ausstrahlungsdatum vorhanden, dann die erste Ausstrahlung nehmen
// egal, wann die CatchupRights liegen, damit ein "sinnvolles" Datum vorhanden ist
if(detailsDTO.getBroadcastBegin().isEmpty()) {
detailsDTO.setBroadcastBegin(getFirstBroadcastDateIgnoringCatchupRights(broadcastArray));
}
} else {
// keine Ausstrahlungen verfügbar => catchupRightsBegin verwenden
JsonElement elementBegin = programElement.get(JSON_ELEMENT_BROADCAST_CATCHUPRIGHTS_BEGIN);

if(!elementBegin.isJsonNull()) {
detailsDTO.setBroadcastBegin(elementBegin.getAsString());
}
}
}

return detailsDTO;
}

/**
* Liefert den Beginn der Ausstrahlung,
* wenn
* - heute im Zeitraum von CatchUpRights liegt
* - oder heute vor dem Zeitraum liegt
* - oder CatchUpRights nicht gesetzt ist und die Ausstrahlung in der Vergangenheit liegt
* @param broadcastObject
* @return der Beginn der Ausstrahlung oder ""
*/
private String getBroadcastDate(JsonObject broadcastObject) {
String broadcastDate = "";

JsonElement elementBegin = broadcastObject.get(JSON_ELEMENT_BROADCAST_CATCHUPRIGHTS_BEGIN);
JsonElement elementEnd = broadcastObject.get(JSON_ELEMENT_BROADCAST_CATCHUPRIGHTS_END);

if (!elementBegin.isJsonNull() && !elementEnd.isJsonNull()) {
String begin = elementBegin.getAsString();
String end = elementEnd.getAsString();

try {
Calendar beginDate = Calendar.getInstance();
beginDate.setTime(broadcastDateFormat.parse(begin));
Calendar endDate = Calendar.getInstance();
endDate.setTime(broadcastDateFormat.parse(end));

if(DateWithoutTimeComparer.compare(today, beginDate) >= 0 && DateWithoutTimeComparer.compare(today, endDate) <= 0) {
// wenn das heutige Datum zwischen begin und end liegt,
// dann ist es die aktuelle Ausstrahlung
broadcastDate = broadcastObject.get(JSON_ELEMENT_BROADCAST).getAsString();
} else if(DateWithoutTimeComparer.compare(today, beginDate) < 0) {
// ansonsten die zukünftige verwenden
broadcastDate = broadcastObject.get(JSON_ELEMENT_BROADCAST).getAsString();
}

} catch (ParseException ex) {
LOG.debug(ex);
}
} else {
String broadcast = broadcastObject.get(JSON_ELEMENT_BROADCAST).getAsString();

try {
Calendar broadcastCal = Calendar.getInstance();
broadcastCal.setTime(broadcastDateFormat.parse(broadcast));
broadcastDate = broadcast;

} catch (ParseException ex) {
LOG.debug(ex);
}
}
return broadcastDate;
}

private static String getFirstBroadcastDateIgnoringCatchupRights(JsonArray broadcastArray) {
String broadcastDate = "";

for(int i = 0; i < broadcastArray.size(); i++) {
JsonObject broadcastObject = broadcastArray.get(i).getAsJsonObject();

if(broadcastObject.has(JSON_ELEMENT_BROADCASTTYPE) &&
broadcastObject.has(JSON_ELEMENT_BROADCAST)) {
String type = broadcastObject.get(JSON_ELEMENT_BROADCASTTYPE).getAsString();

switch(type) {
case BROADCASTTTYPE_FIRST:
broadcastDate = (broadcastObject.get(JSON_ELEMENT_BROADCAST).getAsString());
break;
}
}
}

return broadcastDate;
}
}
Loading