-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
arte: use recent list instead of categories
ard: use experimental topic search
- Loading branch information
Showing
20 changed files
with
1,565 additions
and
175 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
src/main/java/mServer/crawler/sender/ard/PaginationUrlDto.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package mServer.crawler.sender.ard; | ||
|
|
||
| import mServer.crawler.sender.base.CrawlerUrlDTO; | ||
|
|
||
| import java.util.HashSet; | ||
| import java.util.Set; | ||
|
|
||
| public class PaginationUrlDto { | ||
| private final Set<CrawlerUrlDTO> urls = new HashSet<>(); | ||
| private int actualPage; | ||
| private int maxPages; | ||
|
|
||
| public void addUrl(CrawlerUrlDTO url) { | ||
| urls.add(url); | ||
| } | ||
|
|
||
| public void addAll(Set<CrawlerUrlDTO> urls) { | ||
| this.urls.addAll(urls); | ||
| } | ||
|
|
||
| public Set<CrawlerUrlDTO> getUrls() { | ||
| return urls; | ||
| } | ||
|
|
||
| public int getActualPage() { | ||
| return actualPage; | ||
| } | ||
|
|
||
| public int getMaxPages() { | ||
| return maxPages; | ||
| } | ||
|
|
||
| public void setActualPage(int actualPage) { | ||
| this.actualPage = actualPage; | ||
| } | ||
|
|
||
| public void setMaxPages(int maxPages) { | ||
| this.maxPages = maxPages; | ||
| } | ||
| } |
62 changes: 62 additions & 0 deletions
62
src/main/java/mServer/crawler/sender/ard/json/ArdTopicsDeserializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| package mServer.crawler.sender.ard.json; | ||
|
|
||
|
|
||
| import com.google.gson.JsonArray; | ||
| import com.google.gson.JsonDeserializationContext; | ||
| import com.google.gson.JsonDeserializer; | ||
| import com.google.gson.JsonElement; | ||
| import mServer.crawler.sender.ard.ArdConstants; | ||
| import mServer.crawler.sender.base.CrawlerUrlDTO; | ||
| import mServer.crawler.sender.base.JsonUtils; | ||
|
|
||
| import java.lang.reflect.Type; | ||
| import java.util.HashSet; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
|
|
||
| public class ArdTopicsDeserializer implements JsonDeserializer<Set<CrawlerUrlDTO>> { | ||
| private static final String ELEMENT_WIDGETS = "widgets"; | ||
| private static final String ELEMENT_LINKS = "links"; | ||
| private static final String ELEMENT_SELF = "self"; | ||
|
|
||
| private static final String ATTRIBUTE_ID = "id"; | ||
|
|
||
| private final String sender; | ||
|
|
||
| public ArdTopicsDeserializer(String sender) { | ||
| this.sender = sender; | ||
| } | ||
|
|
||
| @Override | ||
| public Set<CrawlerUrlDTO> deserialize( | ||
| JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) { | ||
| final Set<CrawlerUrlDTO> result = new HashSet<>(); | ||
|
|
||
| if (JsonUtils.hasElements(jsonElement, ELEMENT_WIDGETS)) { | ||
| final JsonArray widgets = jsonElement.getAsJsonObject().getAsJsonArray(ELEMENT_WIDGETS); | ||
| widgets.forEach(widget -> parseWidget(widget.getAsJsonObject()).ifPresent(result::add)); | ||
| } | ||
|
|
||
| return result; | ||
| } | ||
|
|
||
| private Optional<CrawlerUrlDTO> parseWidget(final JsonElement compilation) { | ||
| if (JsonUtils.hasElements(compilation, ELEMENT_LINKS)) { | ||
| final JsonElement selfLink = | ||
| compilation.getAsJsonObject().get(ELEMENT_LINKS).getAsJsonObject().get(ELEMENT_SELF); | ||
| final Optional<String> id = | ||
| JsonUtils.getAttributeAsString(selfLink.getAsJsonObject(), ATTRIBUTE_ID); | ||
| if (id.isPresent()) { | ||
| return Optional.of( | ||
| new CrawlerUrlDTO( | ||
| String.format( | ||
| ArdConstants.TOPICS_COMPILATION_URL, | ||
| sender, | ||
| id.get(), | ||
| ArdConstants.TOPICS_COMPILATION_PAGE_SIZE))); | ||
| } | ||
| } | ||
|
|
||
| return Optional.empty(); | ||
| } | ||
| } |
91 changes: 91 additions & 0 deletions
91
src/main/java/mServer/crawler/sender/ard/json/ArdTopicsLetterDeserializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| package mServer.crawler.sender.ard.json; | ||
|
|
||
| import com.google.gson.JsonDeserializationContext; | ||
| import com.google.gson.JsonDeserializer; | ||
| import com.google.gson.JsonElement; | ||
| import com.google.gson.JsonObject; | ||
| import mServer.crawler.sender.ard.ArdConstants; | ||
| import mServer.crawler.sender.ard.PaginationUrlDto; | ||
| import mServer.crawler.sender.base.CrawlerUrlDTO; | ||
| import mServer.crawler.sender.base.JsonUtils; | ||
|
|
||
| import java.lang.reflect.Type; | ||
| import java.util.HashSet; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
|
|
||
| public class ArdTopicsLetterDeserializer implements JsonDeserializer<PaginationUrlDto> { | ||
|
|
||
| private static final String ELEMENT_TEASERS = "teasers"; | ||
| private static final String ELEMENT_LINKS = "links"; | ||
| private static final String ELEMENT_TARGET = "target"; | ||
| private static final String ELEMENT_PAGE_NUMBER = "pageNumber"; | ||
| private static final String ELEMENT_TOTAL_ELEMENTS = "totalElements"; | ||
| private static final String ELEMENT_PAGE_SIZE = "pageSize"; | ||
| private static final String ELEMENT_PAGINATION = "pagination"; | ||
|
|
||
| private static final String ATTRIBUTE_ID = "id"; | ||
|
|
||
| @Override | ||
| public PaginationUrlDto deserialize( | ||
| final JsonElement jsonElement, final Type type, final JsonDeserializationContext context) { | ||
| final PaginationUrlDto results = new PaginationUrlDto(); | ||
|
|
||
| if (!jsonElement.getAsJsonObject().has(ELEMENT_TEASERS) | ||
| || !jsonElement.getAsJsonObject().get(ELEMENT_TEASERS).isJsonArray() | ||
| || jsonElement.getAsJsonObject().getAsJsonArray(ELEMENT_TEASERS).isEmpty()) { | ||
| return results; | ||
| } | ||
|
|
||
| jsonElement.getAsJsonObject().getAsJsonArray(ELEMENT_TEASERS).forEach(teaser -> results.addAll(parseTeaser(teaser.getAsJsonObject()))); | ||
|
|
||
| final JsonElement paginationElement = jsonElement.getAsJsonObject().get(ELEMENT_PAGINATION); | ||
| results.setActualPage(getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER)); | ||
| final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS); | ||
| final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE); | ||
| int maxPageSize = pageSize == 0 ? 0 : | ||
| (totalElements+pageSize-1)/pageSize; | ||
| results.setMaxPages(maxPageSize); | ||
|
|
||
| return results; | ||
| } | ||
|
|
||
| private int getChildElementAsIntOrNullIfNotExist( | ||
| final JsonElement parentElement, final String childElementName) { | ||
| if (parentElement == null || parentElement.isJsonNull()) { | ||
| return 0; | ||
| } | ||
| return getJsonElementAsIntOrNullIfNotExist( | ||
| parentElement.getAsJsonObject().get(childElementName)); | ||
| } | ||
|
|
||
| private int getJsonElementAsIntOrNullIfNotExist(final JsonElement element) { | ||
| if (element.isJsonNull()) { | ||
| return 0; | ||
| } | ||
| return element.getAsInt(); | ||
| } | ||
|
|
||
| private Set<CrawlerUrlDTO> parseTeaser(final JsonObject teaserObject) { | ||
| final Set<CrawlerUrlDTO> results = new HashSet<>(); | ||
|
|
||
| final Optional<String> id; | ||
|
|
||
| if (JsonUtils.checkTreePath(teaserObject, ELEMENT_LINKS, ELEMENT_TARGET)) { | ||
| final JsonObject targetObject = | ||
| teaserObject.get(ELEMENT_LINKS).getAsJsonObject().get(ELEMENT_TARGET).getAsJsonObject(); | ||
| id = JsonUtils.getAttributeAsString(targetObject, ATTRIBUTE_ID); | ||
| } else { | ||
| id = JsonUtils.getAttributeAsString(teaserObject, ATTRIBUTE_ID); | ||
| } | ||
|
|
||
| id.ifPresent( | ||
| nonNullId -> | ||
| results.add( | ||
| new CrawlerUrlDTO( | ||
| String.format( | ||
| ArdConstants.TOPIC_URL, nonNullId, ArdConstants.TOPIC_PAGE_SIZE)))); | ||
|
|
||
| return results; | ||
| } | ||
| } |
Oops, something went wrong.