Skip to content

Commit

Permalink
Add logic for parsing references from last page of PDF (#11156)
Browse files Browse the repository at this point in the history 
* Add logic for parsing references from last page of PDF (BibliopgraphyFromPdfImporter)

- Support more date formats
- Increase log level for issues for date parsing

* Remove unused method

* Wire in ExtractReferencesAction

* Add CHANGELOG.md entry

* Fix reviewdog

* Switch "... (online)" and "... (offline)" in context meno

* Fix filename

* Make Patterns static

* Refine AuthorListParser for IEEE formatting

Example: "I. Podadera, J. M. Carmona, A. Ibarra, and J. Molla"

* Fix authorlist

* Complete test for tua3i2refpage()

* Use "Optional" instead of null

* Rewrite

* Remove empty lines

* Replace magic number by "constant"

* Revert "Rewrite"

This reverts commit 7adb334.

* Add more comments

* Update CHANGELOG.md
  • Loading branch information
@koppor
koppor authored Apr 8, 2024
1 parent cfeb27b commit a0080ba
Show file tree
Hide file tree
Showing 17 changed files with 816 additions and 76 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv

### Added

- We added support for offline extracting refereferences from PDFs following the IEEE format. [#11156](https://github.com/JabRef/jabref/pull/11156)
- We added a new keyboard shortcut <kbd>ctrl</kbd> + <kbd>,</kbd> to open the preferences. [#11154](https://github.com/JabRef/jabref/pull/11154)

### Changed
Expand Down
7 changes: 4 additions & 3 deletions src/main/java/org/jabref/gui/actions/ActionFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,11 @@ private static Label getAssociatedNode(MenuItem menuItem) {
public MenuItem configureMenuItem(Action action, Command command, MenuItem menuItem) {
ActionUtils.configureMenuItem(new JabRefAction(action, command, keyBindingRepository, Sources.FromMenu), menuItem);
setGraphic(menuItem, action);
enableTooltips(command, menuItem);
return menuItem;
}

// Show tooltips
private static void enableTooltips(Command command, MenuItem menuItem) {
if (command instanceof SimpleCommand simpleCommand) {
EasyBind.subscribe(
simpleCommand.statusMessageProperty(),
Expand All @@ -96,8 +99,6 @@ public MenuItem configureMenuItem(Action action, Command command, MenuItem menuI
}
);
}

return menuItem;
}

public MenuItem createMenuItem(Action action, Command command) {
Expand Down
4 changes: 0 additions & 4 deletions src/main/java/org/jabref/gui/actions/SimpleCommand.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,6 @@ public abstract class SimpleCommand extends CommandBase {

protected ReadOnlyStringWrapper statusMessage = new ReadOnlyStringWrapper("");

public String getStatusMessage() {
return statusMessage.get();
}

public ReadOnlyStringProperty statusMessageProperty() {
return statusMessage.getReadOnlyProperty();
}
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/jabref/gui/actions/StandardActions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ public enum StandardActions implements Action {
REBUILD_FULLTEXT_SEARCH_INDEX(Localization.lang("Rebuild fulltext search index"), IconTheme.JabRefIcons.FILE),
REDOWNLOAD_MISSING_FILES(Localization.lang("Redownload missing files"), IconTheme.JabRefIcons.DOWNLOAD),
OPEN_EXTERNAL_FILE(Localization.lang("Open file"), IconTheme.JabRefIcons.FILE, KeyBinding.OPEN_FILE),
EXTRACT_FILE_REFERENCES(Localization.lang("Extract references from file"), IconTheme.JabRefIcons.FILE_STAR),
EXTRACT_FILE_REFERENCES_ONLINE(Localization.lang("Extract references from file (online)"), IconTheme.JabRefIcons.FILE_STAR),
EXTRACT_FILE_REFERENCES_OFFLINE(Localization.lang("Extract references from file (offline)"), IconTheme.JabRefIcons.FILE_STAR),
OPEN_URL(Localization.lang("Open URL or DOI"), IconTheme.JabRefIcons.WWW, KeyBinding.OPEN_URL_OR_DOI),
SEARCH_SHORTSCIENCE(Localization.lang("Search ShortScience")),
MERGE_WITH_FETCHED_ENTRY(Localization.lang("Get bibliographic data from %0", "DOI/ISBN/...")),
Expand Down
164 changes: 141 additions & 23 deletions src/main/java/org/jabref/gui/maintable/ExtractReferencesAction.java
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package org.jabref.gui.maintable;

import java.nio.file.Path;
import java.util.LinkedList;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.StringJoiner;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jabref.gui.DialogService;
import org.jabref.gui.StateManager;
Expand All @@ -13,48 +17,78 @@
import org.jabref.gui.util.BackgroundTask;
import org.jabref.gui.util.TaskExecutor;
import org.jabref.logic.importer.ParserResult;
import org.jabref.logic.importer.fetcher.GrobidPreferences;
import org.jabref.logic.importer.fileformat.BibliographyFromPdfImporter;
import org.jabref.logic.importer.util.GrobidService;
import org.jabref.logic.l10n.Localization;
import org.jabref.logic.util.io.FileUtil;
import org.jabref.model.database.BibDatabaseContext;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.LinkedFile;
import org.jabref.model.entry.field.StandardField;
import org.jabref.preferences.PreferencesService;

import org.jspecify.annotations.NonNull;
import org.jspecify.annotations.Nullable;

/**
* SIDE EFFECT: Sets the "cites" field of the entry having the linked files
*
* Mode choice A: online or offline
* Mode choice B: complete entry or single file (the latter is not implemented)
*
* The different modes should be implemented as sub classes. However, this was too complicated, thus we use variables at the constructor to parameterize this class.
*/
public class ExtractReferencesAction extends SimpleCommand {
private final int FILES_LIMIT = 10;

private final boolean online;
private final DialogService dialogService;
private final StateManager stateManager;
private final PreferencesService preferencesService;
private final BibEntry entry;
private final LinkedFile linkedFile;
private final TaskExecutor taskExecutor;

public ExtractReferencesAction(DialogService dialogService,
private final BibliographyFromPdfImporter bibliographyFromPdfImporter;

public ExtractReferencesAction(boolean online,
DialogService dialogService,
StateManager stateManager,
PreferencesService preferencesService,
TaskExecutor taskExecutor) {
this(dialogService, stateManager, preferencesService, null, null, taskExecutor);
this(online, dialogService, stateManager, preferencesService, null, null, taskExecutor);
}

public ExtractReferencesAction(DialogService dialogService,
StateManager stateManager,
PreferencesService preferencesService,
BibEntry entry,
LinkedFile linkedFile,
TaskExecutor taskExecutor) {
/**
* Can be used to bind the action on a context menu in the linked file view (future work)
*
* @param entry the entry to handle (can be null)
* @param linkedFile the linked file (can be null)
*/
private ExtractReferencesAction(boolean online,
@NonNull DialogService dialogService,
@NonNull StateManager stateManager,
@NonNull PreferencesService preferencesService,
@Nullable BibEntry entry,
@Nullable LinkedFile linkedFile,
@NonNull TaskExecutor taskExecutor) {
this.online = online;
this.dialogService = dialogService;
this.stateManager = stateManager;
this.preferencesService = preferencesService;
this.entry = entry;
this.linkedFile = linkedFile;
this.taskExecutor = taskExecutor;
bibliographyFromPdfImporter = new BibliographyFromPdfImporter(preferencesService.getCitationKeyPatternPreferences());

String text;
GrobidPreferences grobidPreferences = preferencesService.getGrobidPreferences();

if (this.linkedFile == null) {
this.executable.bind(
ActionHelper.needsEntriesSelected(stateManager)
.and(ActionHelper.hasLinkedFileForSelectedEntries(stateManager))
.and(this.preferencesService.getGrobidPreferences().grobidEnabledProperty())
);
} else {
this.setExecutable(true);
Expand All @@ -68,34 +102,118 @@ public void execute() {

private void extractReferences() {
stateManager.getActiveDatabase().ifPresent(databaseContext -> {
List<BibEntry> selectedEntries = new LinkedList<>();
assert online == this.preferencesService.getGrobidPreferences().isGrobidEnabled();

List<BibEntry> selectedEntries;
if (entry == null) {
selectedEntries = stateManager.getSelectedEntries();
} else {
selectedEntries.add(entry);
selectedEntries = List.of(entry);
}

List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
if (fileList.size() > FILES_LIMIT) {
boolean continueOpening = dialogService.showConfirmationDialogAndWait(Localization.lang("Processing a large number of files"),
Localization.lang("You are about to process %0 files. Continue?", fileList.size()),
Localization.lang("Continue"), Localization.lang("Cancel"));
if (!continueOpening) {
Callable<ParserResult> parserResultCallable;
if (online) {
Optional<Callable<ParserResult>> parserResultCallableOnline = getParserResultCallableOnline(databaseContext, selectedEntries);
if (parserResultCallableOnline.isEmpty()) {
return;
}
parserResultCallable = parserResultCallableOnline.get();
} else {
parserResultCallable = getParserResultCallableOffline(databaseContext, selectedEntries);
}

Callable<ParserResult> parserResultCallable = () -> new ParserResult(
new GrobidService(this.preferencesService.getGrobidPreferences()).processReferences(fileList, preferencesService.getImportFormatPreferences())
);
BackgroundTask<ParserResult> task = BackgroundTask.wrap(parserResultCallable)
.withInitialMessage(Localization.lang("Processing PDF(s)"));

task.onFailure(dialogService::showErrorDialogAndWait);

ImportEntriesDialog dialog = new ImportEntriesDialog(stateManager.getActiveDatabase().get(), task);
dialog.setTitle(Localization.lang("Extract References"));
String title;
if (online) {
title = Localization.lang("Extract References (online)");
} else {
title = Localization.lang("Extract References (offline)");
}
dialog.setTitle(title);
dialogService.showCustomDialogAndWait(dialog);
});
}

private @NonNull Callable<ParserResult> getParserResultCallableOffline(BibDatabaseContext databaseContext, List<BibEntry> selectedEntries) {
return () -> {
BibEntry currentEntry = selectedEntries.getFirst();
List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));

// We need to have ParserResult handled at the importer, because it imports the meta data (library type, encoding, ...)
ParserResult result = bibliographyFromPdfImporter.importDatabase(fileList.getFirst());

// subsequent files are just appended to result
Iterator<Path> fileListIterator = fileList.iterator();
fileListIterator.next(); // skip first file
extractReferences(fileListIterator, result, currentEntry);

// handle subsequent entries
Iterator<BibEntry> selectedEntriesIterator = selectedEntries.iterator();
selectedEntriesIterator.next(); // skip first entry
while (selectedEntriesIterator.hasNext()) {
currentEntry = selectedEntriesIterator.next();
fileList = FileUtil.getListOfLinkedFiles(List.of(currentEntry), databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
fileListIterator = fileList.iterator();
extractReferences(fileListIterator, result, currentEntry);
}

return result;
};
}

private void extractReferences(Iterator<Path> fileListIterator, ParserResult result, BibEntry currentEntry) {
while (fileListIterator.hasNext()) {
result.getDatabase().insertEntries(bibliographyFromPdfImporter.importDatabase(fileListIterator.next()).getDatabase().getEntries());
}

StringJoiner cites = new StringJoiner(",");
int count = 0;
for (BibEntry importedEntry : result.getDatabase().getEntries()) {
count++;
Optional<String> citationKey = importedEntry.getCitationKey();
String citationKeyToAdd;
if (citationKey.isPresent()) {
citationKeyToAdd = citationKey.get();
} else {
// No key present -> generate one based on
// the citation key of the entry holding the files and
// the number of the current entry (extracted from the reference; fallback: current number of the entry (count variable))

String sourceCitationKey = currentEntry.getCitationKey().orElse("unknown");
String newCitationKey;
// Could happen if no author and no year is present
// We use the number of the comment field (because there is no other way to get the number reliable)
Pattern pattern = Pattern.compile("^\\[(\\d+)\\]");
Matcher matcher = pattern.matcher(importedEntry.getField(StandardField.COMMENT).orElse(""));
if (matcher.hasMatch()) {
newCitationKey = sourceCitationKey + "-" + matcher.group(1);
} else {
newCitationKey = sourceCitationKey + "-" + count;
}
importedEntry.setCitationKey(newCitationKey);
citationKeyToAdd = newCitationKey;
}
cites.add(citationKeyToAdd);
}
currentEntry.setField(StandardField.CITES, cites.toString());
}

private Optional<Callable<ParserResult>> getParserResultCallableOnline(BibDatabaseContext databaseContext, List<BibEntry> selectedEntries) {
List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
if (fileList.size() > FILES_LIMIT) {
boolean continueOpening = dialogService.showConfirmationDialogAndWait(Localization.lang("Processing a large number of files"),
Localization.lang("You are about to process %0 files. Continue?", fileList.size()),
Localization.lang("Continue"), Localization.lang("Cancel"));
if (!continueOpening) {
return Optional.empty();
}
}
return Optional.of(() -> new ParserResult(
new GrobidService(this.preferencesService.getGrobidPreferences()).processReferences(fileList, preferencesService.getImportFormatPreferences())
));
}
}
14 changes: 13 additions & 1 deletion src/main/java/org/jabref/gui/maintable/RightClickMenu.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import javafx.scene.control.ContextMenu;
import javafx.scene.control.Menu;
import javafx.scene.control.MenuItem;
import javafx.scene.control.SeparatorMenuItem;

import org.jabref.gui.ClipBoardManager;
Expand Down Expand Up @@ -34,6 +35,8 @@
import org.jabref.preferences.PreferencesService;
import org.jabref.preferences.PreviewPreferences;

import com.tobiasdiez.easybind.EasyBind;

public class RightClickMenu {

public static ContextMenu create(BibEntryTableViewModel entry,
Expand All @@ -50,6 +53,9 @@ public static ContextMenu create(BibEntryTableViewModel entry,
ActionFactory factory = new ActionFactory(keyBindingRepository);
ContextMenu contextMenu = new ContextMenu();

MenuItem extractFileReferencesOnline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_ONLINE, new ExtractReferencesAction(true, dialogService, stateManager, preferencesService, taskExecutor));
MenuItem extractFileReferencesOffline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_OFFLINE, new ExtractReferencesAction(false, dialogService, stateManager, preferencesService, taskExecutor));

contextMenu.getItems().addAll(
factory.createMenuItem(StandardActions.COPY, new EditAction(StandardActions.COPY, () -> libraryTab, stateManager, undoManager)),
createCopySubMenu(factory, dialogService, stateManager, preferencesService, clipBoardManager, abbreviationRepository, taskExecutor),
Expand All @@ -75,7 +81,8 @@ public static ContextMenu create(BibEntryTableViewModel entry,
factory.createMenuItem(StandardActions.ATTACH_FILE_FROM_URL, new AttachFileFromURLAction(dialogService, stateManager, taskExecutor, preferencesService)),
factory.createMenuItem(StandardActions.OPEN_FOLDER, new OpenFolderAction(dialogService, stateManager, preferencesService, taskExecutor)),
factory.createMenuItem(StandardActions.OPEN_EXTERNAL_FILE, new OpenExternalFileAction(dialogService, stateManager, preferencesService, taskExecutor)),
factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES, new ExtractReferencesAction(dialogService, stateManager, preferencesService, taskExecutor)),
extractFileReferencesOnline,
extractFileReferencesOffline,

factory.createMenuItem(StandardActions.OPEN_URL, new OpenUrlAction(dialogService, stateManager, preferencesService)),
factory.createMenuItem(StandardActions.SEARCH_SHORTSCIENCE, new SearchShortScienceAction(dialogService, stateManager, preferencesService)),
Expand All @@ -86,6 +93,11 @@ public static ContextMenu create(BibEntryTableViewModel entry,
factory.createMenuItem(StandardActions.MERGE_WITH_FETCHED_ENTRY, new MergeWithFetchedEntryAction(dialogService, stateManager, taskExecutor, preferencesService, undoManager))
);

EasyBind.subscribe(preferencesService.getGrobidPreferences().grobidEnabledProperty(), enabled -> {
extractFileReferencesOnline.setVisible(enabled);
extractFileReferencesOffline.setVisible(!enabled);
});

return contextMenu;
}

Expand Down
11 changes: 0 additions & 11 deletions src/main/java/org/jabref/gui/menus/ChangeEntryTypeAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

import javax.swing.undo.UndoManager;

import javafx.beans.property.ReadOnlyStringProperty;
import javafx.beans.property.ReadOnlyStringWrapper;

import org.jabref.gui.EntryTypeView;
Expand Down Expand Up @@ -36,14 +35,4 @@ public void execute() {
.ifPresent(change -> compound.addEdit(new UndoableChangeType(change))));
undoManager.addEdit(compound);
}

@Override
public String getStatusMessage() {
return statusMessage.get();
}

@Override
public ReadOnlyStringProperty statusMessageProperty() {
return statusMessageProperty.getReadOnlyProperty();
}
}
Loading

0 comments on commit a0080ba

Please sign in to comment.