Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/hotfix/#191' into develop
Browse files Browse the repository at this point in the history 
Conflicts:
	src/main/java/mServer/crawler/sender/arte/ArteDatenFilmDeserializer.java
	src/main/java/mServer/crawler/sender/arte/ArteJsonObjectToDatenFilmCallable.java
	src/main/java/mServer/crawler/sender/arte/MediathekArte_de.java
  • Loading branch information
Nicklas2751 committed Jul 6, 2017
2 parents 3b291f5 + 7cf6838 commit 4bbe025
Show file tree
Hide file tree
Showing 3 changed files with 129 additions and 62 deletions.
4 changes: 3 additions & 1 deletion src/main/java/mServer/crawler/sender/arte/ArteFilmDeserializer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

public class ArteFilmDeserializer implements JsonDeserializer<ListeFilme>
{
private static final String JSON_ELEMENT_VIDEOS = "videos";
private static final Logger LOG = LogManager.getLogger(ArteFilmDeserializer.class);

private final String langCode;
Expand All @@ -33,7 +34,8 @@ public ListeFilme deserialize(JsonElement aJsonElement, Type aType, JsonDeserial
ListeFilme listeFilme = new ListeFilme();

Collection<Future<Film>> futureFilme = new ArrayList<>();
for (JsonElement jsonElement : aJsonElement.getAsJsonArray())

for (JsonElement jsonElement : aJsonElement.getAsJsonObject().get(JSON_ELEMENT_VIDEOS).getAsJsonArray())
{
ExecutorService executor = Executors.newCachedThreadPool();
futureFilme.add(executor.submit(new ArteJsonObjectToFilmCallable(jsonElement.getAsJsonObject(), langCode, sender)));
Expand Down
149 changes: 97 additions & 52 deletions src/main/java/mServer/crawler/sender/arte/ArteJsonObjectToFilmCallable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
public class ArteJsonObjectToFilmCallable implements Callable<Film>
{
private static final Logger LOG = LogManager.getLogger(ArteJsonObjectToFilmCallable.class);
private static final String JSON_OBJECT_KEY_PROGRAM = "program";
private static final String JSON_ELEMENT_KEY_CATEGORY = "category";
private static final String JSON_ELEMENT_KEY_SUBCATEGORY = "subcategory";
private static final String JSON_ELEMENT_KEY_NAME = "name";
Expand All @@ -39,7 +38,10 @@ public class ArteJsonObjectToFilmCallable implements Callable<Film>
private static final String JSON_ELEMENT_KEY_URL = "url";
private static final String JSON_ELEMENT_KEY_PROGRAM_ID = "programId";
private static final String ARTE_VIDEO_INFORMATION_URL_PATTERN = "https://api.arte.tv/api/player/v1/config/%s/%s?platform=ARTE_NEXT";
private static final String ARTE_VIDEO_INFORMATION_URL_PATTERN_2 = "https://api.arte.tv/api/opa/v3/programs/%s/%s"; // Für broadcastBeginRounded
private static final String JSON_ELEMENT_KEY_SHORT_DESCRIPTION = "shortDescription";
private static final String JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1 = "programs";
private static final String JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2 = "broadcastProgrammings";
private static final String JSON_ELEMENT_BROADCAST = "broadcastBeginRounded";

private final JsonObject jsonObject;
Expand All @@ -59,61 +61,104 @@ public ArteJsonObjectToFilmCallable(JsonObject aJsonObjec, String aLangCode, Sen
public Film call() throws CantCreateFilmException
{
Film film = null;
if (jsonObject != null && jsonObject.has(JSON_OBJECT_KEY_PROGRAM))
{
JsonObject programObject = jsonObject.get(JSON_OBJECT_KEY_PROGRAM).getAsJsonObject();
if (isValidProgramObject(programObject))
try {
if(isValidProgramObject(jsonObject))
{
String titel = getTitle(programObject);
String thema = getSubject(programObject);

String beschreibung = getElementValue(programObject, JSON_ELEMENT_KEY_SHORT_DESCRIPTION);

String urlWeb = getElementValue(programObject, JSON_ELEMENT_KEY_URL);

//https://api.arte.tv/api/player/v1/config/[language:de/fr]/[programId]
String programId = getElementValue(programObject, JSON_ELEMENT_KEY_PROGRAM_ID);
String videosUrl = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN, langCode, programId);

Gson gson = new GsonBuilder().registerTypeAdapter(ArteVideoDTO.class, new ArteVideoDeserializer()).create();

try (Response responseVideoDetails = executeRequest(videosUrl))
JsonObject programObject = jsonObject.get(JSON_OBJECT_KEY_PROGRAM).getAsJsonObject();
if (isValidProgramObject(programObject))
{
if (responseVideoDetails.isSuccessful())
String titel = getTitle(jsonObject);
String thema = getSubject(jsonObject);

String beschreibung = getElementValue(jsonObject, JSON_ELEMENT_KEY_SHORT_DESCRIPTION);

String urlWeb = getElementValue(jsonObject, JSON_ELEMENT_KEY_URL);

//https://api.arte.tv/api/player/v1/config/[language:de/fr]/[programId]?platform=ARTE_NEXT
String programId = getElementValue(jsonObject, JSON_ELEMENT_KEY_PROGRAM_ID);
String videosUrl = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN, langCode, programId);


Gson gson = new GsonBuilder().registerTypeAdapter(ArteVideoDTO.class, new ArteVideoDeserializer()).create();

try (Response responseVideoDetails = executeRequest(videosUrl))
{
ArteVideoDTO video = gson.fromJson(responseVideoDetails.body().string(), ArteVideoDTO.class);

//The duration as time so it can be formatted and co.
Duration duration = Duration.of(video.getDurationInSeconds(), ChronoUnit.SECONDS);

if (video.getVideoUrls().containsKey(Qualities.NORMAL))
if (responseVideoDetails.isSuccessful())
{
String broadcastBegin = "";

if (jsonObject.has(JSON_ELEMENT_BROADCAST))
ArteVideoDTO video = gson.fromJson(responseVideoDetails.body().string(), ArteVideoDTO.class);

//The duration as time so it can be formatted and co.
Duration duration = Duration.of(video.getDurationInSeconds(), ChronoUnit.SECONDS);

if (video.getVideoUrls().containsKey(Qualities.NORMAL))
{
String broadcastBegin = "";

OkHttpClient httpClient = MVHttpClient.getInstance().getHttpClient();
//https://api.arte.tv/api/opa/v3/programs/[language:de/fr]/[programId]
String videosUrlVideoDetails2 = String.format(ARTE_VIDEO_INFORMATION_URL_PATTERN_2, langCode, programId);
Request request = new Request.Builder()
.addHeader(MediathekArte_de.AUTH_HEADER, MediathekArte_de.AUTH_TOKEN)
.url(videosUrlVideoDetails2).build();

try(Response responseVideoDetails2 = httpClient.newCall(request).execute())
{
broadcastBegin = jsonObject.get(JSON_ELEMENT_BROADCAST).getAsString();
if(responseVideoDetails2.isSuccessful())
{
/*
* Grobe Struktur des Json's:
* {
* "meta": {}
* "programs": [{ (JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1)
"broadcastProgrammings": [{ (JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2)
"broadcastBeginRounded": "2016-07-06T02:40:00Z", (JSON_ELEMENT_BROADCAST)
]}
]}
* }
*/
JsonObject jsonObjectVideoDetails2 = gson.fromJson(responseVideoDetails2.body().string(), JsonObject.class);
if(jsonObjectVideoDetails2.isJsonObject() &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().size() > 0 &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().size() > 0 &&
jsonObjectVideoDetails2.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().get(0).getAsJsonObject()
.has(JSON_ELEMENT_BROADCAST))
{
JsonElement jsonBegin = jsonObjectVideoDetails2
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_1).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST_ELTERNKNOTEN_2).getAsJsonArray().get(0).getAsJsonObject()
.get(JSON_ELEMENT_BROADCAST);
if(jsonBegin != JsonNull.INSTANCE) {
broadcastBegin = jsonBegin.getAsString();
}
}
}
}

film = createFilm(thema, urlWeb, titel, video, LocalDateTime.parse(broadcastBegin, broadcastDateFormat), duration, beschreibung);
} else {
LOG.debug(String.format("Keine \"normale\" Video URL für den Film \"%s\" mit der URL \"%s\". Video Details URL:\"%s\" ", titel, urlWeb, videosUrl));
}
film = createFilm(thema, urlWeb, titel, video, LocalDateTime.parse(broadcastBegin, broadcastDateFormat), duration, beschreibung);
} else {
LOG.debug(String.format("Keine \"normale\" Video URL für den Film \"%s\" mit der URL \"%s\". Video Details URL:\"%s\" ", titel, urlWeb, videosUrl));
}
} catch (IOException ioException)
{
LOG.fatal("Beim laden der Informationen eines Filmes für Arte kam es zu Verbindungsproblemen.", ioException);
throw new CantCreateFilmException(ioException);
}catch (URISyntaxException uriSyntaxEception)
{
throw new CantCreateFilmException(uriSyntaxEception);
}
} catch (IOException ioException)
{
LOG.fatal("Beim laden der Informationen eines Filmes für Arte kam es zu Verbindungsproblemen.", ioException);
throw new CantCreateFilmException(ioException);
}catch (URISyntaxException uriSyntaxEception)
{
throw new CantCreateFilmException(uriSyntaxEception);
}

}
} catch (Exception exception)
{
throw new CantCreateFilmException(exception);
}
return film;
}

private static String getSubject(JsonObject programObject) {
private String getSubject(JsonObject programObject) {
String subject;

JsonObject catObject = programObject.get(JSON_ELEMENT_KEY_CATEGORY).getAsJsonObject();
Expand All @@ -131,7 +176,7 @@ private static String getSubject(JsonObject programObject) {
return subject;
}

private static String getTitle(JsonObject programObject) {
private String getTitle(JsonObject programObject) {
String title = getElementValue(programObject, JSON_ELEMENT_KEY_TITLE);
String subtitle = getElementValue(programObject, JSON_ELEMENT_KEY_SUBTITLE);

Expand All @@ -142,19 +187,19 @@ private static String getTitle(JsonObject programObject) {
return title;
}

private static boolean isValidProgramObject(JsonObject programObject)
private boolean isValidProgramObject(JsonObject programObject)
{
return programObject.has(JSON_ELEMENT_KEY_TITLE) &&
programObject.has(JSON_ELEMENT_KEY_PROGRAM_ID) &&
programObject.has(JSON_ELEMENT_KEY_URL) &&
!programObject.get(JSON_ELEMENT_KEY_TITLE).isJsonNull() &&
!programObject.get(JSON_ELEMENT_KEY_PROGRAM_ID).isJsonNull() &&
!programObject.get(JSON_ELEMENT_KEY_URL).isJsonNull();
return programObject.has(JSON_ELEMENT_KEY_TITLE) &&
programObject.has(JSON_ELEMENT_KEY_PROGRAM_ID) &&
programObject.has(JSON_ELEMENT_KEY_URL) &&
!programObject.get(JSON_ELEMENT_KEY_TITLE).isJsonNull() &&
!programObject.get(JSON_ELEMENT_KEY_PROGRAM_ID).isJsonNull() &&
!programObject.get(JSON_ELEMENT_KEY_URL).isJsonNull();
}

private static String getElementValue(JsonObject jsonObject, String elementName)
private String getElementValue(JsonObject jsonObject, String elementName)
{
return !jsonObject.get(elementName).isJsonNull() ? jsonObject.get(elementName).getAsString() : "";
return !jsonObject.get(elementName).isJsonNull() ? jsonObject.get(elementName).getAsString() : "";
}

private Film createFilm(final String aThema,
Expand Down
38 changes: 29 additions & 9 deletions src/main/java/mServer/crawler/sender/arte/MediathekArte_de.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,9 +45,28 @@

public class MediathekArte_de extends MediathekReader
{
/*
* Informationen zu den ARTE-URLs:
* {} sind nur Makierungen, dass es Platzhalter sind, sie gehören nicht zur URL.
*
* Allgemeine URL eines Films: (050169-002-A = ID des Films); (die-spur-der-steine = Titel)
* http://www.arte.tv/de/videos/{050169-002-A}/{die-spur-der-steine}
*
* Alle Sendungen: (Deutsch = DE; Französisch = FR)
* https://api.arte.tv/api/opa/v3/videos?channel={DE}
*
* Informationen zum Film: (050169-002-A = ID des Films); (de für deutsch / fr für französisch)
* https://api.arte.tv/api/player/v1/config/{de}/{050169-002-A}?platform=ARTE_NEXT
*
* Zweite Quelle für Informationen zum Film: (050169-002-A = ID des Films); (de für deutsch / fr für französisch)
* https://api.arte.tv/api/opa/v3/programs/{de}/{050169-002-A}
*
*/
private static final Logger LOG = LogManager.getLogger(MediathekArte_de.class);
private static final String ARTE_API_TAG_URL_PATTERN = "http://www.arte.tv/guide/api/api/program/%s/scheduled/%s";
private static final DateTimeFormatter ARTE_API_DATEFORMATTER = DateTimeFormatter.ofPattern("yy-MM-dd");
private static final String ARTE_API_TAG_URL_PATTERN = "https://api.arte.tv/api/opa/v3/videos?channel=%s&arteSchedulingDay=%s";
private static final DateTimeFormatter ARTE_API_DATEFORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd");
public static final String AUTH_HEADER = "Authorization";
public static final String AUTH_TOKEN = "Bearer Nzc1Yjc1ZjJkYjk1NWFhN2I2MWEwMmRlMzAzNjI5NmU3NWU3ODg4ODJjOWMxNTMxYzEzZGRjYjg2ZGE4MmIwOA";
protected String LANG_CODE = "de";
protected String URL_CONCERT = "http://concert.arte.tv/de/videos/all";
protected String URL_CONCERT_NOT_CONTAIN = "-STF";
Expand Down Expand Up @@ -118,9 +137,8 @@ private void addConcert()
private void addTage()
{
// http://www.arte.tv/guide/de/plus7/videos?day=-2&page=1&isLoading=true&sort=newest&country=DE
for (int i = 0; i <= 14; ++i)
{
String u = String.format(ARTE_API_TAG_URL_PATTERN, LANG_CODE, LocalDate.now().minusDays(i).format(ARTE_API_DATEFORMATTER));
for (int i = 0; i <= 14; ++i) {
String u = String.format(ARTE_API_TAG_URL_PATTERN,LANG_CODE.toUpperCase(),LocalDate.now().minusDays(i).format(ARTE_API_DATEFORMATTER));
listeThemen.add(new String[]{u});
}
}
Expand Down Expand Up @@ -335,10 +353,12 @@ private void addFilmeForTag(String aUrl)

MVHttpClient mvhttpClient = MVHttpClient.getInstance();
OkHttpClient httpClient = mvhttpClient.getHttpClient();
Request request = new Request.Builder().url(aUrl).build();
try
{
Response response = httpClient.newCall(request).execute();
Request request = new Request.Builder()
.addHeader(AUTH_HEADER, AUTH_TOKEN)
.url(aUrl).build();
try
{
Response response = httpClient.newCall(request).execute();

if (response.isSuccessful())
{
Expand Down

0 comments on commit 4bbe025

Please sign in to comment.