From b19c3e4700f82d90a804a5cf3467aedf00385dad Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 25 Nov 2020 17:41:30 +0100 Subject: [PATCH] =?UTF-8?q?Enable=20automated=20cross=20library=20search?= =?UTF-8?q?=20using=20a=20cross=20library=20query=20lan=E2=80=A6=20(#7124)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Enable automated cross library search using a cross library query language. Signed-off-by: Dominik Voigt * Pull Global upward through constructor. * Pull Globals and ImportFormatPreferences up through constructor Signed-off-by: Dominik Voigt * Integrate requested changes and fix architecture tests by correcting test classes Signed-off-by: Dominik Voigt * Remove unused imports Signed-off-by: Dominik Voigt --- CHANGELOG.md | 1 + build.gradle | 2 + src/main/java/module-info.java | 1 + src/main/java/org/jabref/gui/JabRefFrame.java | 8 +- .../gui/StartLiteratureReviewAction.java | 81 +++++ .../jabref/gui/actions/StandardActions.java | 1 + .../org/jabref/logic/crawler/Crawler.java | 52 +++ .../LibraryEntryToFetcherConverter.java | 67 ++++ .../jabref/logic/crawler/StudyFetcher.java | 80 ++++ .../jabref/logic/crawler/StudyRepository.java | 344 ++++++++++++++++++ .../jabref/logic/crawler/git/GitHandler.java | 83 +++++ .../importer/fetcher/SpringerFetcher.java | 2 +- .../model/entry/types/EntryTypeFactory.java | 1 + ...tematicLiteratureReviewStudyEntryType.java | 33 ++ ...ratureReviewStudyEntryTypeDefinitions.java | 60 +++ .../org/jabref/model/study/FetchResult.java | 24 ++ .../org/jabref/model/study/QueryResult.java | 24 ++ .../java/org/jabref/model/study/Study.java | 98 +++++ .../model/study/StudyMetaDataField.java | 24 ++ src/main/resources/l10n/JabRef_en.properties | 19 +- .../org/jabref/logic/crawler/CrawlerTest.java | 105 ++++++ .../LibraryEntryToFetcherConverterTest.java | 69 ++++ .../logic/crawler/StudyRepositoryTest.java | 312 ++++++++++++++++ .../SearchBasedFetcherCapabilityTest.java | 2 +- .../org/jabref/model/study/StudyTest.java | 94 +++++ .../jabref/logic/crawler/ArXivQuantumMock.bib | 15 + .../crawler/SpringerCloud ComputingMock.bib | 9 + .../logic/crawler/SpringerQuantumMock.bib | 9 + .../org/jabref/logic/crawler/study.bib | 37 ++ 29 files changed, 1636 insertions(+), 21 deletions(-) create mode 100644 src/main/java/org/jabref/gui/StartLiteratureReviewAction.java create mode 100644 src/main/java/org/jabref/logic/crawler/Crawler.java create mode 100644 src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java create mode 100644 src/main/java/org/jabref/logic/crawler/StudyFetcher.java create mode 100644 src/main/java/org/jabref/logic/crawler/StudyRepository.java create mode 100644 src/main/java/org/jabref/logic/crawler/git/GitHandler.java create mode 100644 src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java create mode 100644 src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java create mode 100644 src/main/java/org/jabref/model/study/FetchResult.java create mode 100644 src/main/java/org/jabref/model/study/QueryResult.java create mode 100644 src/main/java/org/jabref/model/study/Study.java create mode 100644 src/main/java/org/jabref/model/study/StudyMetaDataField.java create mode 100644 src/test/java/org/jabref/logic/crawler/CrawlerTest.java create mode 100644 src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java create mode 100644 src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java create mode 100644 src/test/java/org/jabref/model/study/StudyTest.java create mode 100644 src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/study.bib diff --git a/CHANGELOG.md b/CHANGELOG.md index 38c9e5bd2dc..bfcd7cb3ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ to the page field for cases where the page numbers are missing. [#7019](https:// - We added a new formatter to output shorthand month format. [#6579](https://github.com/JabRef/jabref/issues/6579) - We added support for the new Microsoft Edge browser in all platforms. [#7056](https://github.com/JabRef/jabref/pull/7056) - We reintroduced emacs/bash-like keybindings. [#6017](https://github.com/JabRef/jabref/issues/6017) +- We added a feature to provide automated cross library search using a cross library query language. This provides support for the search step of systematic literature reviews (SLRs). [koppor#369](https://github.com/koppor/jabref/issues/369) ### Changed diff --git a/build.gradle b/build.gradle index 6e04f5b0b22..6e2326393b8 100644 --- a/build.gradle +++ b/build.gradle @@ -139,6 +139,8 @@ dependencies { exclude group: 'org.apache.lucene', module: 'lucene-sandbox' } + implementation group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.9.0.202009080501-r' + implementation group: 'org.mariadb.jdbc', name: 'mariadb-java-client', version: '2.7.0' implementation 'org.postgresql:postgresql:42.2.18' diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index eb7e0102e92..c080fccc99e 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -91,4 +91,5 @@ requires com.h2database.mvstore; requires lucene.queryparser; requires lucene.core; + requires org.eclipse.jgit; } diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 963df97bb57..d44f6307084 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -815,7 +815,9 @@ private MenuBar createMenu() { new SeparatorMenuItem(), factory.createMenuItem(StandardActions.SEND_AS_EMAIL, new SendAsEMailAction(dialogService, stateManager)), - pushToApplicationMenuItem + pushToApplicationMenuItem, + new SeparatorMenuItem(), + factory.createMenuItem(StandardActions.START_SYSTEMATIC_LITERATURE_REVIEW, new StartLiteratureReviewAction(this, Globals.getFileUpdateMonitor(), Globals.prefs.getWorkingDir(), Globals.TASK_EXECUTOR)) ); SidePaneComponent webSearch = sidePaneManager.getComponent(SidePaneType.WEB_SEARCH); @@ -992,7 +994,7 @@ public void addParserResult(ParserResult parserResult, boolean focusPanel) { * This method causes all open LibraryTabs to set up their tables anew. When called from PreferencesDialogViewModel, * this updates to the new settings. * We need to notify all tabs about the changes to avoid problems when changing the column set. - * */ + */ public void setupAllTables() { tabbedPane.getTabs().forEach(tab -> { LibraryTab libraryTab = (LibraryTab) tab; @@ -1013,7 +1015,7 @@ private ContextMenu createTabContextMenu(KeyBindingRepository keyBindingReposito new SeparatorMenuItem(), factory.createMenuItem(StandardActions.OPEN_DATABASE_FOLDER, new OpenDatabaseFolder()), factory.createMenuItem(StandardActions.OPEN_CONSOLE, new OpenConsoleAction(stateManager)) - ); + ); return contextMenu; } diff --git a/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java new file mode 100644 index 00000000000..d05d0f817f5 --- /dev/null +++ b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java @@ -0,0 +1,81 @@ +package org.jabref.gui; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; + +import org.jabref.gui.actions.SimpleCommand; +import org.jabref.gui.importer.actions.OpenDatabaseAction; +import org.jabref.gui.util.BackgroundTask; +import org.jabref.gui.util.FileDialogConfiguration; +import org.jabref.gui.util.TaskExecutor; +import org.jabref.logic.crawler.Crawler; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.l10n.Localization; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StartLiteratureReviewAction extends SimpleCommand { + private static final Logger LOGGER = LoggerFactory.getLogger(StartLiteratureReviewAction.class); + private final JabRefFrame frame; + private final DialogService dialogService; + private final FileUpdateMonitor fileUpdateMonitor; + private final Path workingDirectory; + private final TaskExecutor taskExecutor; + + public StartLiteratureReviewAction(JabRefFrame frame, FileUpdateMonitor fileUpdateMonitor, Path standardWorkingDirectory, TaskExecutor taskExecutor) { + this.frame = frame; + this.dialogService = frame.getDialogService(); + this.fileUpdateMonitor = fileUpdateMonitor; + this.workingDirectory = getInitialDirectory(standardWorkingDirectory); + this.taskExecutor = taskExecutor; + } + + @Override + public void execute() { + FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() + .withInitialDirectory(workingDirectory) + .build(); + + Optional studyDefinitionFile = dialogService.showFileOpenDialog(fileDialogConfiguration); + if (studyDefinitionFile.isEmpty()) { + // Do nothing if selection was canceled + return; + } + final Crawler crawler; + try { + crawler = new Crawler(studyDefinitionFile.get(), fileUpdateMonitor, JabRefPreferences.getInstance().getImportFormatPreferences(), JabRefPreferences.getInstance().getSavePreferences(), new BibEntryTypesManager()); + } catch (IOException | ParseException | GitAPIException e) { + LOGGER.error("Error during reading of study definition file.", e); + dialogService.showErrorDialogAndWait(Localization.lang("Error during reading of study definition file."), e); + return; + } + BackgroundTask.wrap(() -> { + crawler.performCrawl(); + return 0; // Return any value to make this a callable instead of a runnable. This allows throwing exceptions. + }) + .onFailure(e -> { + LOGGER.error("Error during persistence of crawling results."); + dialogService.showErrorDialogAndWait(Localization.lang("Error during persistence of crawling results."), e); + }) + .onSuccess(unused -> new OpenDatabaseAction(frame).openFile(Path.of(studyDefinitionFile.get().getParent().toString(), "studyResult.bib"), true)) + .executeWith(taskExecutor); + } + + /** + * @return Path of current panel database directory or the standard working directory + */ + private Path getInitialDirectory(Path standardWorkingDirectory) { + if (frame.getBasePanelCount() == 0) { + return standardWorkingDirectory; + } else { + Optional databasePath = frame.getCurrentLibraryTab().getBibDatabaseContext().getDatabasePath(); + return databasePath.map(Path::getParent).orElse(standardWorkingDirectory); + } + } +} diff --git a/src/main/java/org/jabref/gui/actions/StandardActions.java b/src/main/java/org/jabref/gui/actions/StandardActions.java index 2b316dca80a..0767d4021bb 100644 --- a/src/main/java/org/jabref/gui/actions/StandardActions.java +++ b/src/main/java/org/jabref/gui/actions/StandardActions.java @@ -88,6 +88,7 @@ public enum StandardActions implements Action { PARSE_LATEX(Localization.lang("Search for citations in LaTeX files..."), IconTheme.JabRefIcons.LATEX_CITATIONS), NEW_SUB_LIBRARY_FROM_AUX(Localization.lang("New sublibrary based on AUX file") + "...", Localization.lang("New BibTeX sublibrary") + Localization.lang("This feature generates a new library based on which entries are needed in an existing LaTeX document."), IconTheme.JabRefIcons.NEW), WRITE_XMP(Localization.lang("Write XMP metadata to PDFs"), Localization.lang("Will write XMP metadata to the PDFs linked from selected entries."), KeyBinding.WRITE_XMP), + START_SYSTEMATIC_LITERATURE_REVIEW(Localization.lang("Start systematic literature review")), OPEN_DATABASE_FOLDER(Localization.lang("Reveal in file explorer")), OPEN_FOLDER(Localization.lang("Open folder"), Localization.lang("Open folder"), KeyBinding.OPEN_FOLDER), OPEN_FILE(Localization.lang("Open file"), Localization.lang("Open file"), IconTheme.JabRefIcons.FILE, KeyBinding.OPEN_FILE), diff --git a/src/main/java/org/jabref/logic/crawler/Crawler.java b/src/main/java/org/jabref/logic/crawler/Crawler.java new file mode 100644 index 00000000000..eade3b55a59 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/Crawler.java @@ -0,0 +1,52 @@ +package org.jabref.logic.crawler; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParseException; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.util.FileUpdateMonitor; + +import org.eclipse.jgit.api.errors.GitAPIException; + +/** + * This class provides a service for SLR support by conducting an automated search and persistance + * of studies using the queries and E-Libraries specified in the provided study definition file. + * + * It composes a StudyRepository for repository management, + * and a StudyFetcher that manages the crawling over the selected E-Libraries. + */ +public class Crawler { + private final StudyRepository studyRepository; + private final StudyFetcher studyFetcher; + + /** + * Creates a crawler for retrieving studies from E-Libraries + * + * @param studyDefinitionFile The path to the study definition file that contains the list of targeted E-Libraries and used cross-library queries + */ + public Crawler(Path studyDefinitionFile, FileUpdateMonitor fileUpdateMonitor, ImportFormatPreferences importFormatPreferences, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IllegalArgumentException, IOException, ParseException, GitAPIException { + Path studyRepositoryRoot = studyDefinitionFile.getParent(); + studyRepository = new StudyRepository(studyRepositoryRoot, new GitHandler(studyRepositoryRoot), importFormatPreferences, fileUpdateMonitor, savePreferences, bibEntryTypesManager); + Study study = studyRepository.getStudy(); + LibraryEntryToFetcherConverter libraryEntryToFetcherConverter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); + this.studyFetcher = new StudyFetcher(libraryEntryToFetcherConverter.getActiveFetchers(), study.getSearchQueryStrings()); + } + + /** + * This methods performs the crawling of the active libraries defined in the study definition file. + * This method also persists the results in the same folder the study definition file is stored in. + * + * @throws IOException Thrown if a problem occurred during the persistence of the result. + */ + public void performCrawl() throws IOException, GitAPIException { + List results = studyFetcher.crawl(); + studyRepository.persist(results); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java b/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java new file mode 100644 index 00000000000..cadf5b2978e --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java @@ -0,0 +1,67 @@ +package org.jabref.logic.crawler; + +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.WebFetchers; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.UnknownField; + +import static org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY; + +/** + * Converts library entries from the given study into their corresponding fetchers. + */ +class LibraryEntryToFetcherConverter { + private final List libraryEntries; + private final ImportFormatPreferences importFormatPreferences; + + public LibraryEntryToFetcherConverter(List libraryEntries, ImportFormatPreferences importFormatPreferences) { + this.libraryEntries = libraryEntries; + this.importFormatPreferences = importFormatPreferences; + } + + /** + * Returns a list of instances of all active library fetchers. + * + * A fetcher is considered active if there exists an library entry of the library the fetcher is associated with that is enabled. + * + * @return Instances of all active fetchers defined in the study definition. + */ + public List getActiveFetchers() { + return getFetchersFromLibraryEntries(this.libraryEntries); + } + + /** + * Transforms a list of libraryEntries into a list of SearchBasedFetcher instances. + * + * @param libraryEntries List of entries + * @return List of fetcher instances + */ + private List getFetchersFromLibraryEntries(List libraryEntries) { + return libraryEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().getName().equals(LIBRARY_ENTRY.getName())) + .map(this::createFetcherFromLibraryEntry) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + /** + * Transforms a library entry into a SearchBasedFetcher instance. This only works if the library entry specifies a supported fetcher. + * + * @param libraryEntry the entry that will be converted + * @return An instance of the fetcher defined by the library entry. + */ + private SearchBasedFetcher createFetcherFromLibraryEntry(BibEntry libraryEntry) { + Set searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences); + String libraryNameFromFetcher = libraryEntry.getField(new UnknownField("name")).orElse(""); + return searchBasedFetchers.stream() + .filter(searchBasedFetcher -> searchBasedFetcher.getName().toLowerCase().equals(libraryNameFromFetcher.toLowerCase())) + .findAny() + .orElse(null); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java new file mode 100644 index 00000000000..c39ba7efe52 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java @@ -0,0 +1,80 @@ +package org.jabref.logic.crawler; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Delegates the search of the provided set of targeted E-Libraries with the provided queries to the E-Library specific fetchers, + * and aggregates the results returned by the fetchers by query and E-Library. + */ +class StudyFetcher { + private static final Logger LOGGER = LoggerFactory.getLogger(StudyFetcher.class); + private static final int MAX_AMOUNT_OF_RESULTS_PER_FETCHER = 100; + + private final List activeFetchers; + private final List searchQueries; + + StudyFetcher(List activeFetchers, List searchQueries) throws IllegalArgumentException { + this.searchQueries = searchQueries; + this.activeFetchers = activeFetchers; + } + + /** + * Each Map Entry contains the results for one search term for all libraries. + * Each entry of the internal map contains the results for a given library. + * If any library API is not available, its corresponding entry is missing from the internal map. + */ + public List crawl() { + return searchQueries.parallelStream() + .map(this::getQueryResult) + .collect(Collectors.toList()); + } + + private QueryResult getQueryResult(String searchQuery) { + return new QueryResult(searchQuery, performSearchOnQuery(searchQuery)); + } + + /** + * Queries all Databases on the given searchQuery. + * + * @param searchQuery The query the search is performed for. + * @return Mapping of each fetcher by name and all their retrieved publications as a BibDatabase + */ + private List performSearchOnQuery(String searchQuery) { + return activeFetchers.parallelStream() + .map(fetcher -> performSearchOnQueryForFetcher(searchQuery, fetcher)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private FetchResult performSearchOnQueryForFetcher(String searchQuery, SearchBasedFetcher fetcher) { + try { + List fetchResult = new ArrayList<>(); + if (fetcher instanceof PagedSearchBasedFetcher) { + int pages = ((int) Math.ceil(((double) MAX_AMOUNT_OF_RESULTS_PER_FETCHER) / ((PagedSearchBasedFetcher) fetcher).getPageSize())); + for (int page = 0; page < pages; page++) { + fetchResult.addAll(((PagedSearchBasedFetcher) fetcher).performSearchPaged(searchQuery, page).getContent()); + } + } else { + fetchResult = fetcher.performSearch(searchQuery); + } + return new FetchResult(fetcher.getName(), new BibDatabase(fetchResult)); + } catch (FetcherException e) { + LOGGER.warn(String.format("%s API request failed", fetcher.getName()), e); + return null; + } + } +} diff --git a/src/main/java/org/jabref/logic/crawler/StudyRepository.java b/src/main/java/org/jabref/logic/crawler/StudyRepository.java new file mode 100644 index 00000000000..b302e065946 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/StudyRepository.java @@ -0,0 +1,344 @@ +package org.jabref.logic.crawler; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import org.jabref.logic.citationkeypattern.CitationKeyGenerator; +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.database.DatabaseMerger; +import org.jabref.logic.exporter.BibtexDatabaseWriter; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.OpenDatabase; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class manages all aspects of the study process related to the repository. + * + * It includes the parsing of the study definition file (study.bib) into a Study instance, + * the structured persistence of the crawling results for the study within the file based repository, + * as well as the sharing, and versioning of results using git. + */ +class StudyRepository { + // Tests work with study.bib + private static final String STUDY_DEFINITION_FILE_NAME = "study.bib"; + private static final Logger LOGGER = LoggerFactory.getLogger(StudyRepository.class); + private static final Pattern MATCHCOLON = Pattern.compile(":"); + private static final Pattern MATCHILLEGALCHARACTERS = Pattern.compile("[^A-Za-z0-9_.\\s=-]"); + + private final Path repositoryPath; + private final Path studyDefinitionBib; + private final GitHandler gitHandler; + private final Study study; + private final ImportFormatPreferences importFormatPreferences; + private final FileUpdateMonitor fileUpdateMonitor; + private final SavePreferences savePreferences; + private final BibEntryTypesManager bibEntryTypesManager; + + /** + * Creates a study repository. + * + * @param pathToRepository Where the repository root is located. + * @param gitHandler The git handler that managages any interaction with the remote repository + * @throws IllegalArgumentException If the repository root directory does not exist, or the root directory does not contain the study definition file. + * @throws IOException Thrown if the given repository does not exists, or the study definition file does not exist + * @throws ParseException Problem parsing the study definition file. + */ + public StudyRepository(Path pathToRepository, GitHandler gitHandler, ImportFormatPreferences importFormatPreferences, FileUpdateMonitor fileUpdateMonitor, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IOException, ParseException, GitAPIException { + this.repositoryPath = pathToRepository; + this.gitHandler = gitHandler; + try { + gitHandler.updateLocalRepository(); + } catch (GitAPIException e) { + LOGGER.error("Updating repository from remote failed"); + } + this.importFormatPreferences = importFormatPreferences; + this.fileUpdateMonitor = fileUpdateMonitor; + this.studyDefinitionBib = Path.of(repositoryPath.toString(), STUDY_DEFINITION_FILE_NAME); + this.savePreferences = savePreferences; + this.bibEntryTypesManager = bibEntryTypesManager; + + if (Files.notExists(repositoryPath)) { + throw new IOException("The given repository does not exists."); + } else if (Files.notExists(studyDefinitionBib)) { + throw new IOException("The study definition file does not exist in the given repository."); + } + study = parseStudyFile(); + this.setUpRepositoryStructure(); + } + + /** + * Returns entries stored in the repository for a certain query and fetcher + */ + public BibDatabaseContext getFetcherResultEntries(String query, String fetcherName) throws IOException { + return OpenDatabase.loadDatabase(getPathToFetcherResultFile(query, fetcherName), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * Returns the merged entries stored in the repository for a certain query + */ + public BibDatabaseContext getQueryResultEntries(String query) throws IOException { + return OpenDatabase.loadDatabase(getPathToQueryResultFile(query), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * Returns the merged entries stored in the repository for all queries + */ + public BibDatabaseContext getStudyResultEntries() throws IOException { + return OpenDatabase.loadDatabase(getPathToStudyResultFile(), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * The study definition file contains all the definitions of a study. This method extracts the BibEntries from the study BiB file. + * + * @return Returns the BibEntries parsed from the study definition file. + * @throws IOException Problem opening the input stream. + * @throws ParseException Problem parsing the study definition file. + */ + private Study parseStudyFile() throws IOException, ParseException { + BibtexParser parser = new BibtexParser(importFormatPreferences, fileUpdateMonitor); + List parsedEntries = new ArrayList<>(); + try (InputStream inputStream = Files.newInputStream(studyDefinitionBib)) { + parsedEntries.addAll(parser.parseEntries(inputStream)); + } + + BibEntry studyEntry = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY)).findAny() + .orElseThrow(() -> new ParseException("Study definition file does not contain a study entry")); + List queryEntries = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY)) + .collect(Collectors.toList()); + List libraryEntries = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY)) + .collect(Collectors.toList()); + + return new Study(studyEntry, queryEntries, libraryEntries); + } + + public Study getStudy() { + return study; + } + + public void persist(List crawlResults) throws IOException, GitAPIException { + try { + gitHandler.updateLocalRepository(); + } catch (GitAPIException e) { + LOGGER.error("Updating repository from remote failed"); + } + persistResults(crawlResults); + study.setLastSearchDate(LocalDate.now()); + persistStudy(); + try { + gitHandler.updateRemoteRepository("Conducted search " + LocalDate.now()); + } catch (GitAPIException e) { + LOGGER.error("Updating remote repository failed"); + } + } + + private void persistStudy() throws IOException { + writeResultToFile(studyDefinitionBib, new BibDatabase(study.getAllEntries())); + } + + /** + * Create for each query a folder, and for each fetcher a bib file in the query folder to store its results. + */ + private void setUpRepositoryStructure() throws IOException { + // Cannot use stream here since IOException has to be thrown + LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); + for (String query : study.getSearchQueryStrings()) { + createQueryResultFolder(query); + converter.getActiveFetchers() + .forEach(searchBasedFetcher -> createFetcherResultFile(query, searchBasedFetcher)); + createQueryResultFile(query); + } + createStudyResultFile(); + } + + /** + * Creates a folder using the query and its corresponding query id. + * This folder name is unique for each query, as long as the query id in the study definition is unique for each query. + * + * @param query The query the folder is created for + */ + private void createQueryResultFolder(String query) throws IOException { + Path queryResultFolder = getPathToQueryDirectory(query); + createFolder(queryResultFolder); + } + + private void createFolder(Path folder) throws IOException { + if (Files.notExists(folder)) { + Files.createDirectory(folder); + } + } + + private void createFetcherResultFile(String query, SearchBasedFetcher searchBasedFetcher) { + String fetcherName = searchBasedFetcher.getName(); + Path fetcherResultFile = getPathToFetcherResultFile(query, fetcherName); + createBibFile(fetcherResultFile); + } + + private void createQueryResultFile(String query) { + Path queryResultFile = getPathToFetcherResultFile(query, "result"); + createBibFile(queryResultFile); + } + + private void createStudyResultFile() { + createBibFile(getPathToStudyResultFile()); + } + + private void createBibFile(Path file) { + if (Files.notExists(file)) { + try { + Files.createFile(file); + } catch (IOException e) { + throw new IllegalStateException("Error during creation of repository structure.", e); + } + } + } + + /** + * Returns a string that can be used as a folder name. + * This removes all characters from the query that are illegal for directory names. + * Structure: ID-trimmed query + * + * Examples: + * Input: '(title: test-title AND abstract: Test)' as a query entry with id 1 + * Output: '1 - title= test-title AND abstract= Test' + * + * Input: 'abstract: Test*' as a query entry with id 1 + * Output: '1 - abstract= Test' + * + * Input: '"test driven"' as a query entry with id 1 + * Output: '1 - test driven' + * + * @param query that is trimmed and combined with its query id + * @return a unique folder name for any query. + */ + private String trimNameAndAddID(String query) { + // Replace all field: with field= for folder name + String trimmedNamed = MATCHCOLON.matcher(query).replaceAll("="); + trimmedNamed = MATCHILLEGALCHARACTERS.matcher(trimmedNamed).replaceAll(""); + if (query.length() > 240) { + trimmedNamed = query.substring(0, 240); + } + String id = findQueryIDByQueryString(query); + return id + " - " + trimmedNamed; + } + + /** + * Helper to find the query id for folder name creation. + * Returns the id of the first SearchQuery BibEntry with a query field that matches the given query. + * + * @param query The query whose ID is searched + * @return ID of the query defined in the study definition. + */ + private String findQueryIDByQueryString(String query) { + String queryField = "query"; + return study.getSearchQueryEntries() + .parallelStream() + .filter(bibEntry -> bibEntry.getField(new UnknownField(queryField)).orElse("").equals(query)) + .map(BibEntry::getCitationKey) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst() + .orElseThrow() + .replaceFirst(queryField, ""); + } + + /** + * Persists the crawling results in the local file based repository. + * + * @param crawlResults The results that shall be persisted. + */ + private void persistResults(List crawlResults) throws IOException { + DatabaseMerger merger = new DatabaseMerger(); + BibDatabase newStudyResultEntries = new BibDatabase(); + + for (QueryResult result : crawlResults) { + BibDatabase queryResultEntries = new BibDatabase(); + for (FetchResult fetcherResult : result.getResultsPerFetcher()) { + BibDatabase fetcherEntries = fetcherResult.getFetchResult(); + BibDatabaseContext existingFetcherResult = getFetcherResultEntries(result.getQuery(), fetcherResult.getFetcherName()); + + // Create citation keys for all entries that do not have one + generateCiteKeys(existingFetcherResult, fetcherEntries); + + // Merge new entries into fetcher result file + merger.merge(existingFetcherResult.getDatabase(), fetcherEntries); + // Aggregate each fetcher result into the query result + merger.merge(queryResultEntries, fetcherEntries); + + writeResultToFile(getPathToFetcherResultFile(result.getQuery(), fetcherResult.getFetcherName()), existingFetcherResult.getDatabase()); + } + BibDatabase existingQueryEntries = getQueryResultEntries(result.getQuery()).getDatabase(); + + // Merge new entries into query result file + merger.merge(existingQueryEntries, queryResultEntries); + // Aggregate all new entries for every query into the study result + merger.merge(newStudyResultEntries, queryResultEntries); + + writeResultToFile(getPathToQueryResultFile(result.getQuery()), existingQueryEntries); + } + BibDatabase existingStudyResultEntries = getStudyResultEntries().getDatabase(); + + // Merge new entries into study result file + merger.merge(existingStudyResultEntries, newStudyResultEntries); + + writeResultToFile(getPathToStudyResultFile(), existingStudyResultEntries); + } + + private void generateCiteKeys(BibDatabaseContext existingEntries, BibDatabase targetEntries) { + CitationKeyGenerator citationKeyGenerator = new CitationKeyGenerator(existingEntries, JabRefPreferences.getInstance().getCitationKeyPatternPreferences()); + targetEntries.getEntries().stream().filter(bibEntry -> !bibEntry.hasCitationKey()).forEach(citationKeyGenerator::generateAndSetKey); + } + + private void writeResultToFile(Path pathToFile, BibDatabase entries) throws IOException { + try (Writer fileWriter = new FileWriter(pathToFile.toFile())) { + BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(fileWriter, savePreferences, bibEntryTypesManager); + databaseWriter.saveDatabase(new BibDatabaseContext(entries)); + } + } + + private Path getPathToFetcherResultFile(String query, String fetcherName) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query), fetcherName + ".bib"); + } + + private Path getPathToQueryResultFile(String query) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query), "result.bib"); + } + + private Path getPathToStudyResultFile() { + return Path.of(repositoryPath.toString(), "studyResult.bib"); + } + + private Path getPathToQueryDirectory(String query) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query)); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/git/GitHandler.java b/src/main/java/org/jabref/logic/crawler/git/GitHandler.java new file mode 100644 index 00000000000..439f08dfccd --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/git/GitHandler.java @@ -0,0 +1,83 @@ +package org.jabref.logic.crawler.git; + +import java.io.IOException; +import java.nio.file.Path; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.RmCommand; +import org.eclipse.jgit.api.Status; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class handles the updating of the local and remote git repository that is located at the repository path + */ +public class GitHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(GitHandler.class); + private final Path repositoryPath; + private final CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(System.getenv("GIT_EMAIL"), System.getenv("GIT_PW")); + + /** + * Initialize the handler for the given repository + * + * @param repositoryPath The root of the intialized git repository + */ + public GitHandler(Path repositoryPath) { + this.repositoryPath = repositoryPath; + } + + /** + * Updates the local repository based on the main branch of the original remote repository + */ + public void updateLocalRepository() throws IOException, GitAPIException { + try (Git git = Git.open(this.repositoryPath.toFile())) { + git.pull() + .setRemote("origin") + .setRemoteBranchName("main") + .setCredentialsProvider(credentialsProvider) + .call(); + } + } + + /** + * Adds all the added, changed, and removed files to the index and updates the remote origin repository + * If pushiong to remote fails it fails silently + * + * @param commitMessage The commit message used for the commit to the remote repository + */ + public void updateRemoteRepository(String commitMessage) throws IOException, GitAPIException { + // First get up to date + this.updateLocalRepository(); + try (Git git = Git.open(this.repositoryPath.toFile())) { + Status status = git.status().call(); + if (!status.isClean()) { + // Add new and changed files to index + git.add() + .addFilepattern(".") + .call(); + // Add all removed files to index + if (!status.getMissing().isEmpty()) { + RmCommand removeCommand = git.rm() + .setCached(true); + status.getMissing().forEach(removeCommand::addFilepattern); + removeCommand.call(); + } + git.commit() + .setAllowEmpty(false) + .setMessage(commitMessage) + .call(); + try { + + git.push() + .setCredentialsProvider(credentialsProvider) + .call(); + } catch (GitAPIException e) { + LOGGER.info("Failed to push"); + } + } + } + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index a547dbe2175..1064a7f272e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -178,7 +178,7 @@ private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery complexSearchQuery.getTitlePhrases().forEach(title -> searchTerms.add("title:" + title)); complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("journal:" + journal)); // Since Springer API does not support year range search, we ignore formYear and toYear and use "singleYear" only - complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add("year:" + year.toString())); + complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add("date:" + year.toString() + "*")); searchTerms.addAll(complexSearchQuery.getDefaultFieldPhrases()); return String.join(" AND ", searchTerms); } diff --git a/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java b/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java index 1ecf238382f..29422891f4e 100644 --- a/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java +++ b/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java @@ -50,6 +50,7 @@ public static EntryType parse(String typeName) { List types = new ArrayList<>(Arrays.asList(StandardEntryType.values())); types.addAll(Arrays.asList(IEEETranEntryType.values())); + types.addAll(Arrays.asList(SystematicLiteratureReviewStudyEntryType.values())); return types.stream().filter(type -> type.getName().equals(typeName.toLowerCase(Locale.ENGLISH))).findFirst().orElse(new UnknownEntryType(typeName)); } diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java new file mode 100644 index 00000000000..1d9bd4be112 --- /dev/null +++ b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java @@ -0,0 +1,33 @@ +package org.jabref.model.entry.types; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Optional; + +public enum SystematicLiteratureReviewStudyEntryType implements EntryType { + STUDY_ENTRY("Study"), + SEARCH_QUERY_ENTRY("SearchQuery"), + LIBRARY_ENTRY("Library"); + + private final String displayName; + + SystematicLiteratureReviewStudyEntryType(String displayName) { + this.displayName = displayName; + } + + public static Optional fromName(String name) { + return Arrays.stream(SystematicLiteratureReviewStudyEntryType.values()) + .filter(field -> field.getName().equalsIgnoreCase(name)) + .findAny(); + } + + @Override + public String getName() { + return displayName.toLowerCase(Locale.ENGLISH); + } + + @Override + public String getDisplayName() { + return displayName; + } +} diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java new file mode 100644 index 00000000000..5d1bf665bfe --- /dev/null +++ b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java @@ -0,0 +1,60 @@ +package org.jabref.model.entry.types; + +import java.util.Arrays; +import java.util.List; + +import org.jabref.model.entry.BibEntryType; +import org.jabref.model.entry.BibEntryTypeBuilder; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; + +/** + * This class represents all supported entry types used in a study definition file + */ +public class SystematicLiteratureReviewStudyEntryTypeDefinitions { + + /** + * Entry type used for study meta data within a study definition file + * + *
    + *
  • Required fields: author, lastsearchdate, name, enabled
    • + *
  • Optional fields:
    • + *
+ */ + private static final BibEntryType STUDY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) + .withRequiredFields(StandardField.AUTHOR, new UnknownField("lastsearchdate"), new UnknownField("name"), new UnknownField("researchquestions")) + .build(); + + /** + * Entry type for the queries within the study definition file + * + *
    + *
  • Required fields: query
    • + *
  • Optional fields:
    • + *
+ */ + private static final BibEntryType SEARCH_QUERY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY) + .withRequiredFields(new UnknownField("query")) + .build(); + + /** + * Entry type for the targeted libraries within a study definition file + * + *
    + *
  • Required fields: name, enabled
    • + *
  • Optional fields: comment
    • + *
+ */ + private static final BibEntryType LIBRARY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) + .withRequiredFields(new UnknownField("name"), new UnknownField("enabled")) + .withImportantFields(StandardField.COMMENT) + .build(); + + public static final List ALL = Arrays.asList(STUDY_ENTRY, SEARCH_QUERY_ENTRY, LIBRARY_ENTRY); + + private SystematicLiteratureReviewStudyEntryTypeDefinitions() { + } +} diff --git a/src/main/java/org/jabref/model/study/FetchResult.java b/src/main/java/org/jabref/model/study/FetchResult.java new file mode 100644 index 00000000000..80637feb4ab --- /dev/null +++ b/src/main/java/org/jabref/model/study/FetchResult.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import org.jabref.model.database.BibDatabase; + +/** + * Represents the result of fetching the results for a query for a specific library + */ +public class FetchResult { + private final String fetcherName; + private final BibDatabase fetchResult; + + public FetchResult(String fetcherName, BibDatabase fetcherResult) { + this.fetcherName = fetcherName; + this.fetchResult = fetcherResult; + } + + public String getFetcherName() { + return fetcherName; + } + + public BibDatabase getFetchResult() { + return fetchResult; + } +} diff --git a/src/main/java/org/jabref/model/study/QueryResult.java b/src/main/java/org/jabref/model/study/QueryResult.java new file mode 100644 index 00000000000..2976b5224fe --- /dev/null +++ b/src/main/java/org/jabref/model/study/QueryResult.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import java.util.List; + +/** + * Represents the result of fetching the results from all active fetchers for a specific query. + */ +public class QueryResult { + private final String query; + private final List resultsPerLibrary; + + public QueryResult(String query, List resultsPerLibrary) { + this.query = query; + this.resultsPerLibrary = resultsPerLibrary; + } + + public String getQuery() { + return query; + } + + public List getResultsPerFetcher() { + return resultsPerLibrary; + } +} diff --git a/src/main/java/org/jabref/model/study/Study.java b/src/main/java/org/jabref/model/study/Study.java new file mode 100644 index 00000000000..37ed6e2328a --- /dev/null +++ b/src/main/java/org/jabref/model/study/Study.java @@ -0,0 +1,98 @@ +package org.jabref.model.study; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.UnknownField; + +/** + * This class represents a scientific study. + * + * This class defines all aspects of a scientific study relevant to the application. It is a proxy for the file based study definition. + */ +public class Study { + private static final String SEARCH_QUERY_FIELD_NAME = "query"; + + private final BibEntry studyEntry; + private final List queryEntries; + private final List libraryEntries; + + public Study(BibEntry studyEntry, List queryEntries, List libraryEntries) { + this.studyEntry = studyEntry; + this.queryEntries = queryEntries; + this.libraryEntries = libraryEntries; + } + + public List getAllEntries() { + List allEntries = new ArrayList<>(); + allEntries.add(studyEntry); + allEntries.addAll(queryEntries); + allEntries.addAll(libraryEntries); + return allEntries; + } + + /** + * Returns all query strings + * + * @return List of all queries as Strings. + */ + public List getSearchQueryStrings() { + return queryEntries.parallelStream() + .map(bibEntry -> bibEntry.getField(new UnknownField(SEARCH_QUERY_FIELD_NAME))) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + /** + * This method returns the SearchQuery entries. + * This is required when the BibKey of the search term entry is required in combination with the search query (e.g. + * for the creation of the study repository structure). + */ + public List getSearchQueryEntries() { + return queryEntries; + } + + /** + * Returns a meta data entry of the first study entry found in the study definition file of the provided type. + * + * @param metaDataField The type of requested meta-data + * @return returns the requested meta data type of the first found study entry + * @throws IllegalArgumentException If the study file does not contain a study entry. + */ + public Optional getStudyMetaDataField(StudyMetaDataField metaDataField) throws IllegalArgumentException { + return studyEntry.getField(metaDataField.toField()); + } + + /** + * Sets the lastSearchDate field of the study entry + * + * @param date date the last time a search was conducted + */ + public void setLastSearchDate(LocalDate date) { + studyEntry.setField(StudyMetaDataField.STUDY_LAST_SEARCH.toField(), date.toString()); + } + + /** + * Extracts all active LibraryEntries from the BibEntries. + * + * @return List of BibEntries of type Library + * @throws IllegalArgumentException If a transformation from Library entry to LibraryDefinition fails + */ + public List getActiveLibraryEntries() throws IllegalArgumentException { + return libraryEntries + .parallelStream() + .filter(bibEntry -> { + // If enabled is not defined, the fetcher is active. + return bibEntry.getField(new UnknownField("enabled")) + .map(enabled -> enabled.equals("true")) + .orElse(true); + }) + .collect(Collectors.toList()); + } +} + diff --git a/src/main/java/org/jabref/model/study/StudyMetaDataField.java b/src/main/java/org/jabref/model/study/StudyMetaDataField.java new file mode 100644 index 00000000000..6dbea2a2dc8 --- /dev/null +++ b/src/main/java/org/jabref/model/study/StudyMetaDataField.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import org.jabref.model.entry.field.Field; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; + +/** + * This enum represents the different fields in the study entry + */ +public enum StudyMetaDataField { + STUDY_NAME(new UnknownField("name")), STUDY_RESEARCH_QUESTIONS(new UnknownField("researchQuestions")), + STUDY_AUTHORS(StandardField.AUTHOR), STUDY_GIT_REPOSITORY(new UnknownField("gitRepositoryURL")), + STUDY_LAST_SEARCH(new UnknownField("lastSearchDate")); + + private final Field field; + + StudyMetaDataField(Field field) { + this.field = field; + } + + public Field toField() { + return this.field; + } +} diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index b58510e6d5c..d66d978e315 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -295,32 +295,21 @@ Entry\ owner=Entry owner Entry\ preview=Entry preview Entry\ table=Entry table - Entry\ table\ columns=Entry table columns Entry\ Title\ (Required\ to\ deliver\ recommendations.)=Entry Title (Required to deliver recommendations.) - Entry\ type=Entry type - Error=Error - Error\ occurred\ when\ parsing\ entry=Error occurred when parsing entry - Error\ opening\ file=Error opening file - Error\ while\ writing=Error while writing - +Error\ during\ persistence\ of\ crawling\ results.=Error during persistence of crawling results. +Error\ during\ reading\ of\ study\ definition\ file.=Error during reading of study definition file. '%0'\ exists.\ Overwrite\ file?='%0' exists. Overwrite file? - Export=Export - Export\ preferences=Export preferences - Export\ preferences\ to\ file=Export preferences to file - Export\ to\ clipboard=Export to clipboard - Export\ to\ text\ file.=Export to text file. - Exporting=Exporting Extension=Extension @@ -644,11 +633,9 @@ Previous\ preview\ layout=Previous preview layout Available=Available Selected=Selected Selected\ Layouts\ can\ not\ be\ empty=Selected Layouts can not be empty - +Start\ systematic\ literature\ review=Start systematic literature review Reset\ default\ preview\ style=Reset default preview style - Previous\ entry=Previous entry - Primary\ sort\ criterion=Primary sort criterion Problem\ with\ parsing\ entry=Problem with parsing entry Processing\ %0=Processing %0 diff --git a/src/test/java/org/jabref/logic/crawler/CrawlerTest.java b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java new file mode 100644 index 00000000000..7c6b53e85a2 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java @@ -0,0 +1,105 @@ +package org.jabref.logic.crawler; + +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.jabref.logic.bibtex.FieldContentFormatterPreferences; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.eclipse.jgit.api.Git; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Integration test of the components used for SLR support + */ +class CrawlerTest { + @TempDir + Path tempRepositoryDirectory; + ImportFormatPreferences importFormatPreferences; + SavePreferences savePreferences; + BibEntryTypesManager entryTypesManager; + + @Test + public void testWhetherAllFilesAreCreated() throws Exception { + setUp(); + Crawler testCrawler = new Crawler(getPathToStudyDefinitionFile(), + new DummyFileUpdateMonitor(), + importFormatPreferences, + savePreferences, + entryTypesManager + ); + + testCrawler.performCrawl(); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "ArXiv.bib"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "Springer.bib"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "studyResult.bib"))); + } + + private Path getPathToStudyDefinitionFile() { + return tempRepositoryDirectory.resolve("study.bib"); + } + + /** + * Set up mocks and copies the study definition file into the test repository + */ + private void setUp() throws Exception { + setUpRepository(); + importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); + savePreferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(savePreferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(savePreferences.getEncoding()).thenReturn(null); + when(savePreferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + when(importFormatPreferences.getKeywordSeparator()).thenReturn(','); + when(importFormatPreferences.getFieldContentFormatterPreferences()).thenReturn(new FieldContentFormatterPreferences()); + when(importFormatPreferences.isKeywordSyncEnabled()).thenReturn(false); + when(importFormatPreferences.getEncoding()).thenReturn(StandardCharsets.UTF_8); + entryTypesManager = new BibEntryTypesManager(); + } + + private void setUpRepository() throws Exception { + Git git = Git.init() + .setDirectory(tempRepositoryDirectory.toFile()) + .call(); + setUpTestStudyDefinitionFile(); + git.add() + .addFilepattern(".") + .call(); + git.commit() + .setMessage("Initialize") + .call(); + git.close(); + } + + private void setUpTestStudyDefinitionFile() throws Exception { + Path destination = tempRepositoryDirectory.resolve("study.bib"); + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } +} diff --git a/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java b/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java new file mode 100644 index 00000000000..629fad93ec4 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java @@ -0,0 +1,69 @@ +package org.jabref.logic.crawler; + +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.List; + +import org.jabref.logic.bibtex.FieldContentFormatterPreferences; +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.study.Study; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class LibraryEntryToFetcherConverterTest { + ImportFormatPreferences importFormatPreferences; + SavePreferences savePreferences; + BibEntryTypesManager entryTypesManager; + GitHandler gitHandler; + @TempDir + Path tempRepositoryDirectory; + + @BeforeEach + void setUpMocks() { + importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); + savePreferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(savePreferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(savePreferences.getEncoding()).thenReturn(null); + when(savePreferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + when(importFormatPreferences.getKeywordSeparator()).thenReturn(','); + when(importFormatPreferences.getFieldContentFormatterPreferences()).thenReturn(new FieldContentFormatterPreferences()); + when(importFormatPreferences.isKeywordSyncEnabled()).thenReturn(false); + when(importFormatPreferences.getEncoding()).thenReturn(StandardCharsets.UTF_8); + entryTypesManager = new BibEntryTypesManager(); + gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + } + + @Test + public void getActiveFetcherInstances() throws Exception { + Path studyDefinition = tempRepositoryDirectory.resolve("study.bib"); + copyTestStudyDefinitionFileIntoDirectory(studyDefinition); + + Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager).getStudy(); + LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); + List result = converter.getActiveFetchers(); + + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(result.get(0).getName(), "Springer"); + Assertions.assertEquals(result.get(1).getName(), "ArXiv"); + } + + private void copyTestStudyDefinitionFileIntoDirectory(Path destination) throws Exception { + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } +} diff --git a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java new file mode 100644 index 00000000000..8a69c6d7a01 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java @@ -0,0 +1,312 @@ +package org.jabref.logic.crawler; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.jabref.logic.bibtex.FieldContentFormatterPreferences; +import org.jabref.logic.citationkeypattern.CitationKeyGenerator; +import org.jabref.logic.citationkeypattern.CitationKeyPatternPreferences; +import org.jabref.logic.citationkeypattern.GlobalCitationKeyPattern; +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.database.DatabaseMerger; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.study.StudyMetaDataField; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.jabref.logic.citationkeypattern.CitationKeyGenerator.DEFAULT_UNWANTED_CHARACTERS; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StudyRepositoryTest { + private static final String NON_EXISTING_DIRECTORY = "nonExistingTestRepositoryDirectory"; + CitationKeyPatternPreferences citationKeyPatternPreferences; + ImportFormatPreferences importFormatPreferences; + SavePreferences savePreferences; + BibEntryTypesManager entryTypesManager; + @TempDir + Path tempRepositoryDirectory; + StudyRepository studyRepository; + GitHandler gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + + /** + * Set up mocks + */ + @BeforeEach + public void setUpMocks() { + savePreferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); + citationKeyPatternPreferences = new CitationKeyPatternPreferences( + false, + false, + false, + CitationKeyPatternPreferences.KeySuffix.SECOND_WITH_A, + "", + "", + DEFAULT_UNWANTED_CHARACTERS, + GlobalCitationKeyPattern.fromPattern("[auth][year]"), + ','); + when(savePreferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(savePreferences.getEncoding()).thenReturn(null); + when(savePreferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + when(importFormatPreferences.getKeywordSeparator()).thenReturn(','); + when(importFormatPreferences.getFieldContentFormatterPreferences()).thenReturn(new FieldContentFormatterPreferences()); + when(importFormatPreferences.isKeywordSyncEnabled()).thenReturn(false); + when(importFormatPreferences.getEncoding()).thenReturn(StandardCharsets.UTF_8); + entryTypesManager = new BibEntryTypesManager(); + } + + @Test + void providePathToNonExistentRepositoryThrowsException() { + Path nonExistingRepositoryDirectory = tempRepositoryDirectory.resolve(NON_EXISTING_DIRECTORY); + + assertThrows(IOException.class, () -> new StudyRepository(nonExistingRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager)); + } + + @Test + void providePathToExistentRepositoryWithOutStudyDefinitionFileThrowsException() { + assertThrows(IOException.class, () -> new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager)); + } + + /** + * Tests whether the StudyRepository correctly imports the study file. + */ + @Test + void studyFileCorrectlyImported() throws Exception { + setUpTestStudyDefinitionFile(); + List expectedSearchterms = List.of("Quantum", "Cloud Computing", "TestSearchQuery3"); + List expectedActiveFetchersByName = List.of("Springer", "ArXiv"); + + Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager).getStudy(); + + assertEquals(expectedSearchterms, study.getSearchQueryStrings()); + assertEquals("TestStudyName", study.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); + assertEquals("Jab Ref", study.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); + assertEquals("Question1; Question2", study.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); + assertEquals(expectedActiveFetchersByName, study.getActiveLibraryEntries() + .stream() + .filter(bibEntry -> bibEntry.getType().getName().equals("library")) + .map(bibEntry -> bibEntry.getField(new UnknownField("name")).orElse("")) + .collect(Collectors.toList()) + ); + } + + /** + * Tests whether the file structure of the repository is created correctly from the study definitions file. + */ + @Test + void repositoryStructureCorrectlyCreated() throws Exception { + // When repository is instantiated the directory structure is created + getTestStudyRepository(); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "Springer.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "IEEEXplore.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "IEEEXplore.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "IEEEXplore.bib"))); + } + + /** + * This tests whether the repository returns the stored bib entries correctly. + */ + @Test + void bibEntriesCorrectlyStored() throws Exception { + StudyRepository repository = getTestStudyRepository(); + setUpTestResultFile(); + List result = repository.getFetcherResultEntries("Quantum", "ArXiv").getEntries(); + assertEquals(getArXivQuantumMockResults(), result); + } + + @Test + void fetcherResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(getArXivQuantumMockResults(), getTestStudyRepository().getFetcherResultEntries("Quantum", "ArXiv").getEntries()); + assertEquals(getSpringerQuantumMockResults(), getTestStudyRepository().getFetcherResultEntries("Quantum", "Springer").getEntries()); + assertEquals(getSpringerCloudComputingMockResults(), getTestStudyRepository().getFetcherResultEntries("Cloud Computing", "Springer").getEntries()); + } + + @Test + void mergedResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + List expected = new ArrayList<>(); + expected.addAll(getArXivQuantumMockResults()); + expected.add(getSpringerQuantumMockResults().get(1)); + expected.add(getSpringerQuantumMockResults().get(2)); + + getTestStudyRepository().persist(mockResults); + + // All Springer results are duplicates for "Quantum" + assertEquals(expected, getTestStudyRepository().getQueryResultEntries("Quantum").getEntries()); + assertEquals(getSpringerCloudComputingMockResults(), getTestStudyRepository().getQueryResultEntries("Cloud Computing").getEntries()); + } + + @Test + void setsLastSearchDatePersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(LocalDate.now().toString(), getTestStudyRepository().getStudy().getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); + } + + @Test + void studyResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(new HashSet<>(getNonDuplicateBibEntryResult().getEntries()), new HashSet<>(getTestStudyRepository().getStudyResultEntries().getEntries())); + } + + private StudyRepository getTestStudyRepository() throws Exception { + if (Objects.isNull(studyRepository)) { + setUpTestStudyDefinitionFile(); + studyRepository = new StudyRepository(tempRepositoryDirectory, gitHandler, importFormatPreferences, new DummyFileUpdateMonitor(), savePreferences, entryTypesManager); + } + return studyRepository; + } + + /** + * Copies the study definition file into the test repository + */ + private void setUpTestStudyDefinitionFile() throws Exception { + Path destination = tempRepositoryDirectory.resolve("study.bib"); + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } + + /** + * This overwrites the existing result file in the repository with a result file containing multiple BibEntries. + * The repository has to exist before this method is called. + */ + private void setUpTestResultFile() throws Exception { + Path queryDirectory = Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"); + Path resultFileLocation = Path.of(queryDirectory.toString(), "ArXiv" + ".bib"); + URL resultFile = this.getClass().getResource("ArXivQuantumMock.bib"); + FileUtil.copyFile(Path.of(resultFile.toURI()), resultFileLocation, true); + resultFileLocation = Path.of(queryDirectory.toString(), "Springer" + ".bib"); + resultFile = this.getClass().getResource("SpringerQuantumMock.bib"); + FileUtil.copyFile(Path.of(resultFile.toURI()), resultFileLocation, true); + } + + private BibDatabase getNonDuplicateBibEntryResult() { + BibDatabase mockResults = new BibDatabase(getSpringerCloudComputingMockResults()); + DatabaseMerger merger = new DatabaseMerger(); + merger.merge(mockResults, new BibDatabase(getSpringerQuantumMockResults())); + merger.merge(mockResults, new BibDatabase(getArXivQuantumMockResults())); + return mockResults; + } + + private List getMockResults() { + QueryResult resultQuantum = + new QueryResult("Quantum", List.of( + new FetchResult("ArXiv", new BibDatabase(stripCitationKeys(getArXivQuantumMockResults()))), + new FetchResult("Springer", new BibDatabase(stripCitationKeys(getSpringerQuantumMockResults()))))); + QueryResult resultCloudComputing = new QueryResult("Cloud Computing", List.of(new FetchResult("Springer", new BibDatabase(getSpringerCloudComputingMockResults())))); + return List.of(resultQuantum, resultCloudComputing); + } + + /** + * Strips the citation key from fetched entries as these normally do not have a citation key + */ + private List stripCitationKeys(List entries) { + entries.forEach(bibEntry -> bibEntry.setCitationKey("")); + return entries; + } + + private List getArXivQuantumMockResults() { + BibEntry entry1 = new BibEntry() + .withCitationKey("Blaha") + .withField(StandardField.AUTHOR, "Stephen Blaha") + .withField(StandardField.TITLE, "Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Kaye") + .withField(StandardField.AUTHOR, "Phillip Kaye and Michele Mosca") + .withField(StandardField.TITLE, "Quantum Networks for Generating Arbitrary Quantum States"); + entry2.setType(StandardEntryType.Article); + BibEntry entry3 = new BibEntry() + .withCitationKey("Watrous") + .withField(StandardField.AUTHOR, "John Watrous") + .withField(StandardField.TITLE, "Quantum Computational Complexity"); + entry3.setType(StandardEntryType.Article); + + return List.of(entry1, entry2, entry3); + } + + private List getSpringerQuantumMockResults() { + // This is a duplicate of entry 1 of ArXiv + BibEntry entry1 = new BibEntry() + .withCitationKey("Blaha") + .withField(StandardField.AUTHOR, "Stephen Blaha") + .withField(StandardField.TITLE, "Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Kroeger") + .withField(StandardField.AUTHOR, "H. Kröger") + .withField(StandardField.TITLE, "Nonlinear Dynamics In Quantum Physics -- Quantum Chaos and Quantum Instantons"); + entry2.setType(StandardEntryType.Article); + BibEntry entry3 = new BibEntry() + .withField(StandardField.AUTHOR, "Zieliński, Cezary") + .withField(StandardField.TITLE, "Automatic Control, Robotics, and Information Processing"); + entry3.setType(StandardEntryType.Article); + + CitationKeyGenerator citationKeyGenerator = new CitationKeyGenerator(new BibDatabaseContext(), citationKeyPatternPreferences); + citationKeyGenerator.generateAndSetKey(entry3); + + return List.of(entry1, entry2, entry3); + } + + private List getSpringerCloudComputingMockResults() { + BibEntry entry1 = new BibEntry() + .withCitationKey("Gritzalis") + .withField(StandardField.AUTHOR, "Gritzalis, Dimitris and Stergiopoulos, George and Vasilellis, Efstratios and Anagnostopoulou, Argiro") + .withField(StandardField.TITLE, "Readiness Exercises: Are Risk Assessment Methodologies Ready for the Cloud?"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Rangras") + .withField(StandardField.AUTHOR, "Rangras, Jimit and Bhavsar, Sejal") + .withField(StandardField.TITLE, "Design of Framework for Disaster Recovery in Cloud Computing"); + entry2.setType(StandardEntryType.Article); + return List.of(entry1, entry2); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index 73afa6d63c9..c31bd348b0c 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -21,7 +21,7 @@ /** * Defines the set of capability tests that each tests a given search capability, e.g. author based search. * The idea is to code the capabilities of a fetcher into Java code. - * This way, a) the capbilities of a fetcher are checked automatically (because they can change from time-to-time by the provider) + * This way, a) the capabilities of a fetcher are checked automatically (because they can change from time-to-time by the provider) * and b) the queries sent to the fetchers can be debugged directly without a route through to some fetcher code. */ interface SearchBasedFetcherCapabilityTest { diff --git a/src/test/java/org/jabref/model/study/StudyTest.java b/src/test/java/org/jabref/model/study/StudyTest.java new file mode 100644 index 00000000000..9ab34fcd55e --- /dev/null +++ b/src/test/java/org/jabref/model/study/StudyTest.java @@ -0,0 +1,94 @@ +package org.jabref.model.study; + +import java.time.LocalDate; +import java.util.List; + +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class StudyTest { + Study testStudy; + + @BeforeEach + public void setUpTestStudy() { + BibEntry studyEntry = new BibEntry() + .withField(new UnknownField("name"), "TestStudyName") + .withField(StandardField.AUTHOR, "Jab Ref") + .withField(new UnknownField("researchQuestions"), "Question1; Question2") + .withField(new UnknownField("gitRepositoryURL"), "https://github.com/eclipse/jgit.git"); + studyEntry.setType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY); + + // Create three SearchTerm entries. + BibEntry searchQuery1 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery1"); + searchQuery1.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery1.setCitationKey("query1"); + + BibEntry searchQuery2 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery2"); + searchQuery2.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery2.setCitationKey("query2"); + + BibEntry searchQuery3 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery3"); + searchQuery3.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery3.setCitationKey("query3"); + + // Create two Library entries + BibEntry library1 = new BibEntry() + .withField(new UnknownField("name"), "acm") + .withField(new UnknownField("enabled"), "false") + .withField(new UnknownField("comment"), "disabled, because no good results"); + library1.setType(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY); + library1.setCitationKey("library1"); + + BibEntry library2 = new BibEntry() + .withField(new UnknownField("name"), "arxiv") + .withField(new UnknownField("enabled"), "true") + .withField(new UnknownField("Comment"), ""); + library2.setType(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY); + library2.setCitationKey("library2"); + + testStudy = new Study(studyEntry, List.of(searchQuery1, searchQuery2, searchQuery3), List.of(library1, library2)); + } + + @Test + void getSearchTermsAsStrings() { + List expectedSearchTerms = List.of("TestSearchQuery1", "TestSearchQuery2", "TestSearchQuery3"); + assertEquals(expectedSearchTerms, testStudy.getSearchQueryStrings()); + } + + @Test + void setLastSearchTime() { + LocalDate date = LocalDate.now(); + testStudy.setLastSearchDate(date); + assertEquals(date.toString(), testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); + } + + @Test + void getStudyName() { + assertEquals("TestStudyName", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); + } + + @Test + void getStudyAuthor() { + assertEquals("Jab Ref", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); + } + + @Test + void getResearchQuestions() { + assertEquals("Question1; Question2", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); + } + + @Test + void getGitRepositoryURL() { + assertEquals("https://github.com/eclipse/jgit.git", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_GIT_REPOSITORY).get()); + } +} diff --git a/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib b/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib new file mode 100644 index 00000000000..85df0f1060b --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib @@ -0,0 +1,15 @@ + +@Article{Blaha, + author = {Stephen Blaha}, + title = {Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language}, +} + +@Article{Kaye, + author = {Phillip Kaye and Michele Mosca}, + title = {Quantum Networks for Generating Arbitrary Quantum States}, +} + +@Article{Watrous, + author = {John Watrous}, + title = {Quantum Computational Complexity}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib b/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib new file mode 100644 index 00000000000..627166213fa --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib @@ -0,0 +1,9 @@ +@InCollection{Gritzalis, + author = {Gritzalis, Dimitris and Stergiopoulos, George and Vasilellis, Efstratios and Anagnostopoulou, Argiro}, + title = {Readiness Exercises: Are Risk Assessment Methodologies Ready for the Cloud?}, +} + +@InCollection{Rangras, + author = {Rangras, Jimit and Bhavsar, Sejal}, + title = {Design of Framework for Disaster Recovery in Cloud Computing}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib b/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib new file mode 100644 index 00000000000..3cfa2f88487 --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib @@ -0,0 +1,9 @@ +@Article{Zielinski, + author = {Zieliński, Cezary}, + title = {Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language}, +} + +@Article{Kaye, + author = {H. Kröger}, + title = {Quantum Networks for Generating Arbitrary Quantum States}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/study.bib b/src/test/resources/org/jabref/logic/crawler/study.bib new file mode 100644 index 00000000000..3f9809a82e5 --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/study.bib @@ -0,0 +1,37 @@ +% Encoding: UTF-8 + +@Study{v10, + name={TestStudyName}, + author={Jab Ref}, + researchQuestions={Question1; Question2}, +} + +@SearchQuery{query1, + query={Quantum}, +} + +@SearchQuery{query2, + query={Cloud Computing}, +} + +@SearchQuery{query3, + query={TestSearchQuery3}, +} + +@Library{library1, + name = {Springer}, + enabled = {true}, + comment = {}, +} + +@Library{library2, + name = {ArXiv}, + enabled = {true}, + comment = {}, +} + +@Library{library3, + name = {IEEEXplore}, + enabled = {false}, + comment = {}, +}