-
-
Notifications
You must be signed in to change notification settings - Fork 2.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Feature/enable paginated fetchers (#7082)
- Loading branch information
1 parent
3d5f35a
commit 00e3409
Showing
30 changed files
with
381 additions
and
300 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
37 changes: 34 additions & 3 deletions
37
src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,21 +1,52 @@ | ||
| package org.jabref.logic.importer; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.Collections; | ||
| import java.util.List; | ||
| import java.util.Optional; | ||
|
|
||
| import org.jabref.logic.importer.fetcher.ComplexSearchQuery; | ||
| import org.jabref.model.entry.BibEntry; | ||
| import org.jabref.model.paging.Page; | ||
|
|
||
| public interface PagedSearchBasedFetcher extends SearchBasedFetcher { | ||
|
|
||
| /** | ||
| * @param query search query send to endpoint | ||
| * @param pageNumber requested site number | ||
| * @param complexSearchQuery the complex query defining all fielded search parameters | ||
| * @param pageNumber requested site number indexed from 0 | ||
| * @return Page with search results | ||
| */ | ||
| Page<BibEntry> performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException; | ||
|
|
||
| /** | ||
| * @param complexSearchQuery query string that can be parsed into a complex search query | ||
| * @param pageNumber requested site number indexed from 0 | ||
| * @return Page with search results | ||
| */ | ||
| Page<BibEntry> performSearchPaged(String query, int pageNumber) throws FetcherException; | ||
| default Page<BibEntry> performSearchPaged(String complexSearchQuery, int pageNumber) throws FetcherException { | ||
| if (complexSearchQuery.isBlank()) { | ||
| return new Page<>(complexSearchQuery, pageNumber, Collections.emptyList()); | ||
| } | ||
| QueryParser queryParser = new QueryParser(); | ||
| Optional<ComplexSearchQuery> generatedQuery = queryParser.parseQueryStringIntoComplexQuery(complexSearchQuery); | ||
| // Otherwise just use query as a default term | ||
| return this.performSearchPaged(generatedQuery.orElse(ComplexSearchQuery.builder().defaultFieldPhrase(complexSearchQuery).build()), pageNumber); | ||
| } | ||
|
|
||
| /** | ||
| * @return default pageSize | ||
| */ | ||
| default int getPageSize() { | ||
| return 20; | ||
| } | ||
|
|
||
| @Override | ||
| default List<BibEntry> performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { | ||
| return new ArrayList<>(performSearchPaged(complexSearchQuery, 0).getContent()); | ||
| } | ||
|
|
||
| @Override | ||
| default List<BibEntry> performSearch(String complexSearchQuery) throws FetcherException { | ||
| return new ArrayList<>(performSearchPaged(complexSearchQuery, 0).getContent()); | ||
| } | ||
| } |
66 changes: 61 additions & 5 deletions
66
src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,16 +1,72 @@ | ||
| package org.jabref.logic.importer; | ||
|
|
||
| import java.io.IOException; | ||
| import java.io.InputStream; | ||
| import java.net.MalformedURLException; | ||
| import java.net.URISyntaxException; | ||
| import java.net.URL; | ||
| import java.util.List; | ||
|
|
||
| import org.jabref.logic.importer.fetcher.ComplexSearchQuery; | ||
| import org.jabref.model.entry.BibEntry; | ||
| import org.jabref.model.paging.Page; | ||
|
|
||
| public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher { | ||
|
|
||
| @Override | ||
| default Page<BibEntry> performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { | ||
| // ADR-0014 | ||
| URL urlForQuery; | ||
| try { | ||
| urlForQuery = getComplexQueryURL(complexSearchQuery, pageNumber); | ||
| } catch (URISyntaxException | MalformedURLException e) { | ||
| throw new FetcherException("Search URI crafted from complex search query is malformed", e); | ||
| } | ||
| return new Page<>(complexSearchQuery.toString(), pageNumber, getBibEntries(urlForQuery)); | ||
| } | ||
|
|
||
| private List<BibEntry> getBibEntries(URL urlForQuery) throws FetcherException { | ||
| try (InputStream stream = getUrlDownload(urlForQuery).asInputStream()) { | ||
| List<BibEntry> fetchedEntries = getParser().parseEntries(stream); | ||
| fetchedEntries.forEach(this::doPostCleanup); | ||
| return fetchedEntries; | ||
| } catch (IOException e) { | ||
| throw new FetcherException("A network error occurred while fetching from " + urlForQuery, e); | ||
| } catch (ParseException e) { | ||
| throw new FetcherException("An internal parser error occurred while fetching from " + urlForQuery, e); | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Constructs a URL based on the query, size and page number. | ||
| * @param query the search query | ||
| * @param size the size of the page | ||
| * @param pageNumber the number of the page | ||
| * */ | ||
| URL getURLForQuery(String query, int size, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException; | ||
| * | ||
| * @param query the search query | ||
| * @param pageNumber the number of the page indexed from 0 | ||
| */ | ||
| URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException; | ||
|
|
||
| /** | ||
| * Constructs a URL based on the query, size and page number. | ||
| * | ||
| * @param complexSearchQuery the search query | ||
| * @param pageNumber the number of the page indexed from 0 | ||
| */ | ||
| default URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery, int pageNumber) throws URISyntaxException, MalformedURLException { | ||
| return getURLForQuery(complexSearchQuery.toString(), pageNumber); | ||
| } | ||
|
|
||
| @Override | ||
| default List<BibEntry> performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { | ||
| return SearchBasedParserFetcher.super.performSearch(complexSearchQuery); | ||
| } | ||
|
|
||
| @Override | ||
| default URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { | ||
| return getURLForQuery(query, 0); | ||
| } | ||
|
|
||
| @Override | ||
| default URL getURLForQuery(ComplexSearchQuery query) throws URISyntaxException, MalformedURLException, FetcherException { | ||
| return getComplexQueryURL(query, 0); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.