-
-
Notifications
You must be signed in to change notification settings - Fork 2.6k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Enable automated cross library search using a cross library query lan… (
#7124) * Enable automated cross library search using a cross library query language. Signed-off-by: Dominik Voigt <[email protected]> * Pull Global upward through constructor. * Pull Globals and ImportFormatPreferences up through constructor Signed-off-by: Dominik Voigt <[email protected]> * Integrate requested changes and fix architecture tests by correcting test classes Signed-off-by: Dominik Voigt <[email protected]> * Remove unused imports Signed-off-by: Dominik Voigt <[email protected]>
- Loading branch information
1 parent
5ca3d0d
commit b19c3e4
Showing
29 changed files
with
1,636 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
81 changes: 81 additions & 0 deletions
81
src/main/java/org/jabref/gui/StartLiteratureReviewAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| package org.jabref.gui; | ||
|
|
||
| import java.io.IOException; | ||
| import java.nio.file.Path; | ||
| import java.util.Optional; | ||
|
|
||
| import org.jabref.gui.actions.SimpleCommand; | ||
| import org.jabref.gui.importer.actions.OpenDatabaseAction; | ||
| import org.jabref.gui.util.BackgroundTask; | ||
| import org.jabref.gui.util.FileDialogConfiguration; | ||
| import org.jabref.gui.util.TaskExecutor; | ||
| import org.jabref.logic.crawler.Crawler; | ||
| import org.jabref.logic.importer.ParseException; | ||
| import org.jabref.logic.l10n.Localization; | ||
| import org.jabref.model.entry.BibEntryTypesManager; | ||
| import org.jabref.model.util.FileUpdateMonitor; | ||
| import org.jabref.preferences.JabRefPreferences; | ||
|
|
||
| import org.eclipse.jgit.api.errors.GitAPIException; | ||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| public class StartLiteratureReviewAction extends SimpleCommand { | ||
| private static final Logger LOGGER = LoggerFactory.getLogger(StartLiteratureReviewAction.class); | ||
| private final JabRefFrame frame; | ||
| private final DialogService dialogService; | ||
| private final FileUpdateMonitor fileUpdateMonitor; | ||
| private final Path workingDirectory; | ||
| private final TaskExecutor taskExecutor; | ||
|
|
||
| public StartLiteratureReviewAction(JabRefFrame frame, FileUpdateMonitor fileUpdateMonitor, Path standardWorkingDirectory, TaskExecutor taskExecutor) { | ||
| this.frame = frame; | ||
| this.dialogService = frame.getDialogService(); | ||
| this.fileUpdateMonitor = fileUpdateMonitor; | ||
| this.workingDirectory = getInitialDirectory(standardWorkingDirectory); | ||
| this.taskExecutor = taskExecutor; | ||
| } | ||
|
|
||
| @Override | ||
| public void execute() { | ||
| FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() | ||
| .withInitialDirectory(workingDirectory) | ||
| .build(); | ||
|
|
||
| Optional<Path> studyDefinitionFile = dialogService.showFileOpenDialog(fileDialogConfiguration); | ||
| if (studyDefinitionFile.isEmpty()) { | ||
| // Do nothing if selection was canceled | ||
| return; | ||
| } | ||
| final Crawler crawler; | ||
| try { | ||
| crawler = new Crawler(studyDefinitionFile.get(), fileUpdateMonitor, JabRefPreferences.getInstance().getImportFormatPreferences(), JabRefPreferences.getInstance().getSavePreferences(), new BibEntryTypesManager()); | ||
| } catch (IOException | ParseException | GitAPIException e) { | ||
| LOGGER.error("Error during reading of study definition file.", e); | ||
| dialogService.showErrorDialogAndWait(Localization.lang("Error during reading of study definition file."), e); | ||
| return; | ||
| } | ||
| BackgroundTask.wrap(() -> { | ||
| crawler.performCrawl(); | ||
| return 0; // Return any value to make this a callable instead of a runnable. This allows throwing exceptions. | ||
| }) | ||
| .onFailure(e -> { | ||
| LOGGER.error("Error during persistence of crawling results."); | ||
| dialogService.showErrorDialogAndWait(Localization.lang("Error during persistence of crawling results."), e); | ||
| }) | ||
| .onSuccess(unused -> new OpenDatabaseAction(frame).openFile(Path.of(studyDefinitionFile.get().getParent().toString(), "studyResult.bib"), true)) | ||
| .executeWith(taskExecutor); | ||
| } | ||
|
|
||
| /** | ||
| * @return Path of current panel database directory or the standard working directory | ||
| */ | ||
| private Path getInitialDirectory(Path standardWorkingDirectory) { | ||
| if (frame.getBasePanelCount() == 0) { | ||
| return standardWorkingDirectory; | ||
| } else { | ||
| Optional<Path> databasePath = frame.getCurrentLibraryTab().getBibDatabaseContext().getDatabasePath(); | ||
| return databasePath.map(Path::getParent).orElse(standardWorkingDirectory); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| package org.jabref.logic.crawler; | ||
|
|
||
| import java.io.IOException; | ||
| import java.nio.file.Path; | ||
| import java.util.List; | ||
|
|
||
| import org.jabref.logic.crawler.git.GitHandler; | ||
| import org.jabref.logic.exporter.SavePreferences; | ||
| import org.jabref.logic.importer.ImportFormatPreferences; | ||
| import org.jabref.logic.importer.ParseException; | ||
| import org.jabref.model.entry.BibEntryTypesManager; | ||
| import org.jabref.model.study.QueryResult; | ||
| import org.jabref.model.study.Study; | ||
| import org.jabref.model.util.FileUpdateMonitor; | ||
|
|
||
| import org.eclipse.jgit.api.errors.GitAPIException; | ||
|
|
||
| /** | ||
| * This class provides a service for SLR support by conducting an automated search and persistance | ||
| * of studies using the queries and E-Libraries specified in the provided study definition file. | ||
| * | ||
| * It composes a StudyRepository for repository management, | ||
| * and a StudyFetcher that manages the crawling over the selected E-Libraries. | ||
| */ | ||
| public class Crawler { | ||
| private final StudyRepository studyRepository; | ||
| private final StudyFetcher studyFetcher; | ||
|
|
||
| /** | ||
| * Creates a crawler for retrieving studies from E-Libraries | ||
| * | ||
| * @param studyDefinitionFile The path to the study definition file that contains the list of targeted E-Libraries and used cross-library queries | ||
| */ | ||
| public Crawler(Path studyDefinitionFile, FileUpdateMonitor fileUpdateMonitor, ImportFormatPreferences importFormatPreferences, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IllegalArgumentException, IOException, ParseException, GitAPIException { | ||
| Path studyRepositoryRoot = studyDefinitionFile.getParent(); | ||
| studyRepository = new StudyRepository(studyRepositoryRoot, new GitHandler(studyRepositoryRoot), importFormatPreferences, fileUpdateMonitor, savePreferences, bibEntryTypesManager); | ||
| Study study = studyRepository.getStudy(); | ||
| LibraryEntryToFetcherConverter libraryEntryToFetcherConverter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); | ||
| this.studyFetcher = new StudyFetcher(libraryEntryToFetcherConverter.getActiveFetchers(), study.getSearchQueryStrings()); | ||
| } | ||
|
|
||
| /** | ||
| * This methods performs the crawling of the active libraries defined in the study definition file. | ||
| * This method also persists the results in the same folder the study definition file is stored in. | ||
| * | ||
| * @throws IOException Thrown if a problem occurred during the persistence of the result. | ||
| */ | ||
| public void performCrawl() throws IOException, GitAPIException { | ||
| List<QueryResult> results = studyFetcher.crawl(); | ||
| studyRepository.persist(results); | ||
| } | ||
| } |
67 changes: 67 additions & 0 deletions
67
src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,67 @@ | ||
| package org.jabref.logic.crawler; | ||
|
|
||
| import java.util.List; | ||
| import java.util.Objects; | ||
| import java.util.Set; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import org.jabref.logic.importer.ImportFormatPreferences; | ||
| import org.jabref.logic.importer.SearchBasedFetcher; | ||
| import org.jabref.logic.importer.WebFetchers; | ||
| import org.jabref.model.entry.BibEntry; | ||
| import org.jabref.model.entry.field.UnknownField; | ||
|
|
||
| import static org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY; | ||
|
|
||
| /** | ||
| * Converts library entries from the given study into their corresponding fetchers. | ||
| */ | ||
| class LibraryEntryToFetcherConverter { | ||
| private final List<BibEntry> libraryEntries; | ||
| private final ImportFormatPreferences importFormatPreferences; | ||
|
|
||
| public LibraryEntryToFetcherConverter(List<BibEntry> libraryEntries, ImportFormatPreferences importFormatPreferences) { | ||
| this.libraryEntries = libraryEntries; | ||
| this.importFormatPreferences = importFormatPreferences; | ||
| } | ||
|
|
||
| /** | ||
| * Returns a list of instances of all active library fetchers. | ||
| * | ||
| * A fetcher is considered active if there exists an library entry of the library the fetcher is associated with that is enabled. | ||
| * | ||
| * @return Instances of all active fetchers defined in the study definition. | ||
| */ | ||
| public List<SearchBasedFetcher> getActiveFetchers() { | ||
| return getFetchersFromLibraryEntries(this.libraryEntries); | ||
| } | ||
|
|
||
| /** | ||
| * Transforms a list of libraryEntries into a list of SearchBasedFetcher instances. | ||
| * | ||
| * @param libraryEntries List of entries | ||
| * @return List of fetcher instances | ||
| */ | ||
| private List<SearchBasedFetcher> getFetchersFromLibraryEntries(List<BibEntry> libraryEntries) { | ||
| return libraryEntries.parallelStream() | ||
| .filter(bibEntry -> bibEntry.getType().getName().equals(LIBRARY_ENTRY.getName())) | ||
| .map(this::createFetcherFromLibraryEntry) | ||
| .filter(Objects::nonNull) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| /** | ||
| * Transforms a library entry into a SearchBasedFetcher instance. This only works if the library entry specifies a supported fetcher. | ||
| * | ||
| * @param libraryEntry the entry that will be converted | ||
| * @return An instance of the fetcher defined by the library entry. | ||
| */ | ||
| private SearchBasedFetcher createFetcherFromLibraryEntry(BibEntry libraryEntry) { | ||
| Set<SearchBasedFetcher> searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences); | ||
| String libraryNameFromFetcher = libraryEntry.getField(new UnknownField("name")).orElse(""); | ||
| return searchBasedFetchers.stream() | ||
| .filter(searchBasedFetcher -> searchBasedFetcher.getName().toLowerCase().equals(libraryNameFromFetcher.toLowerCase())) | ||
| .findAny() | ||
| .orElse(null); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| package org.jabref.logic.crawler; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Objects; | ||
| import java.util.stream.Collectors; | ||
|
|
||
| import org.jabref.logic.importer.FetcherException; | ||
| import org.jabref.logic.importer.PagedSearchBasedFetcher; | ||
| import org.jabref.logic.importer.SearchBasedFetcher; | ||
| import org.jabref.model.database.BibDatabase; | ||
| import org.jabref.model.entry.BibEntry; | ||
| import org.jabref.model.study.FetchResult; | ||
| import org.jabref.model.study.QueryResult; | ||
|
|
||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| /** | ||
| * Delegates the search of the provided set of targeted E-Libraries with the provided queries to the E-Library specific fetchers, | ||
| * and aggregates the results returned by the fetchers by query and E-Library. | ||
| */ | ||
| class StudyFetcher { | ||
| private static final Logger LOGGER = LoggerFactory.getLogger(StudyFetcher.class); | ||
| private static final int MAX_AMOUNT_OF_RESULTS_PER_FETCHER = 100; | ||
|
|
||
| private final List<SearchBasedFetcher> activeFetchers; | ||
| private final List<String> searchQueries; | ||
|
|
||
| StudyFetcher(List<SearchBasedFetcher> activeFetchers, List<String> searchQueries) throws IllegalArgumentException { | ||
| this.searchQueries = searchQueries; | ||
| this.activeFetchers = activeFetchers; | ||
| } | ||
|
|
||
| /** | ||
| * Each Map Entry contains the results for one search term for all libraries. | ||
| * Each entry of the internal map contains the results for a given library. | ||
| * If any library API is not available, its corresponding entry is missing from the internal map. | ||
| */ | ||
| public List<QueryResult> crawl() { | ||
| return searchQueries.parallelStream() | ||
| .map(this::getQueryResult) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| private QueryResult getQueryResult(String searchQuery) { | ||
| return new QueryResult(searchQuery, performSearchOnQuery(searchQuery)); | ||
| } | ||
|
|
||
| /** | ||
| * Queries all Databases on the given searchQuery. | ||
| * | ||
| * @param searchQuery The query the search is performed for. | ||
| * @return Mapping of each fetcher by name and all their retrieved publications as a BibDatabase | ||
| */ | ||
| private List<FetchResult> performSearchOnQuery(String searchQuery) { | ||
| return activeFetchers.parallelStream() | ||
| .map(fetcher -> performSearchOnQueryForFetcher(searchQuery, fetcher)) | ||
| .filter(Objects::nonNull) | ||
| .collect(Collectors.toList()); | ||
| } | ||
|
|
||
| private FetchResult performSearchOnQueryForFetcher(String searchQuery, SearchBasedFetcher fetcher) { | ||
| try { | ||
| List<BibEntry> fetchResult = new ArrayList<>(); | ||
| if (fetcher instanceof PagedSearchBasedFetcher) { | ||
| int pages = ((int) Math.ceil(((double) MAX_AMOUNT_OF_RESULTS_PER_FETCHER) / ((PagedSearchBasedFetcher) fetcher).getPageSize())); | ||
| for (int page = 0; page < pages; page++) { | ||
| fetchResult.addAll(((PagedSearchBasedFetcher) fetcher).performSearchPaged(searchQuery, page).getContent()); | ||
| } | ||
| } else { | ||
| fetchResult = fetcher.performSearch(searchQuery); | ||
| } | ||
| return new FetchResult(fetcher.getName(), new BibDatabase(fetchResult)); | ||
| } catch (FetcherException e) { | ||
| LOGGER.warn(String.format("%s API request failed", fetcher.getName()), e); | ||
| return null; | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.