Add logic for parsing references from last page of PDF

CHANGELOG.md

@@ -11,8 +11,10 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 
 ### Added
 
+- We added support for parsing the "References" section of IEEE papers offline. [#11156](https://github.com/JabRef/jabref/pull/11156)
 - We added a new keyboard shortcut  <kbd>ctrl</kbd> + <kbd>,</kbd> to open the preferences. [#11154](https://github.com/JabRef/jabref/pull/11154)
 
+
 ### Changed
 
 - We replaced the word "Key bindings" with "Keyboard shortcuts" in the Preferences tab. [#11153](https://github.com/JabRef/jabref/pull/11153)

src/main/java/org/jabref/gui/actions/ActionFactory.java

@@ -78,8 +78,11 @@ private static Label getAssociatedNode(MenuItem menuItem) {
     public MenuItem configureMenuItem(Action action, Command command, MenuItem menuItem) {
         ActionUtils.configureMenuItem(new JabRefAction(action, command, keyBindingRepository, Sources.FromMenu), menuItem);
         setGraphic(menuItem, action);
+        enableTooltips(command, menuItem);
+        return menuItem;
+    }
 
-        // Show tooltips
+    private static void enableTooltips(Command command, MenuItem menuItem) {
         if (command instanceof SimpleCommand simpleCommand) {
             EasyBind.subscribe(
                     simpleCommand.statusMessageProperty(),
@@ -96,8 +99,6 @@ public MenuItem configureMenuItem(Action action, Command command, MenuItem menuI
                     }
             );
         }
-
-        return menuItem;
     }
 
     public MenuItem createMenuItem(Action action, Command command) {

src/main/java/org/jabref/gui/actions/SimpleCommand.java

@@ -15,10 +15,6 @@ public abstract class SimpleCommand extends CommandBase {
 
     protected ReadOnlyStringWrapper statusMessage = new ReadOnlyStringWrapper("");
 
-    public String getStatusMessage() {
-        return statusMessage.get();
-    }
-
     public ReadOnlyStringProperty statusMessageProperty() {
         return statusMessage.getReadOnlyProperty();
     }

src/main/java/org/jabref/gui/actions/StandardActions.java

@@ -32,7 +32,8 @@ public enum StandardActions implements Action {
     REBUILD_FULLTEXT_SEARCH_INDEX(Localization.lang("Rebuild fulltext search index"), IconTheme.JabRefIcons.FILE),
     REDOWNLOAD_MISSING_FILES(Localization.lang("Redownload missing files"), IconTheme.JabRefIcons.DOWNLOAD),
     OPEN_EXTERNAL_FILE(Localization.lang("Open file"), IconTheme.JabRefIcons.FILE, KeyBinding.OPEN_FILE),
-    EXTRACT_FILE_REFERENCES(Localization.lang("Extract references from file"), IconTheme.JabRefIcons.FILE_STAR),
+    EXTRACT_FILE_REFERENCES_ONLINE(Localization.lang("Extract references from file (online)"), IconTheme.JabRefIcons.FILE_STAR),
+    EXTRACT_FILE_REFERENCES_OFFLINE(Localization.lang("Extract references from file (offline)"), IconTheme.JabRefIcons.FILE_STAR),
     OPEN_URL(Localization.lang("Open URL or DOI"), IconTheme.JabRefIcons.WWW, KeyBinding.OPEN_URL_OR_DOI),
     SEARCH_SHORTSCIENCE(Localization.lang("Search ShortScience")),
     MERGE_WITH_FETCHED_ENTRY(Localization.lang("Get bibliographic data from %0", "DOI/ISBN/...")),

src/main/java/org/jabref/gui/maintable/ExtractReferencesAction.java

@@ -1,9 +1,13 @@
 package org.jabref.gui.maintable;
 
 import java.nio.file.Path;
-import java.util.LinkedList;
+import java.util.Iterator;
 import java.util.List;
+import java.util.Optional;
+import java.util.StringJoiner;
 import java.util.concurrent.Callable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.jabref.gui.DialogService;
 import org.jabref.gui.StateManager;
@@ -13,48 +17,78 @@
 import org.jabref.gui.util.BackgroundTask;
 import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.importer.ParserResult;
+import org.jabref.logic.importer.fetcher.GrobidPreferences;
+import org.jabref.logic.importer.fileformat.BibliographyFromPdfImporter;
 import org.jabref.logic.importer.util.GrobidService;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.util.io.FileUtil;
+import org.jabref.model.database.BibDatabaseContext;
 import org.jabref.model.entry.BibEntry;
 import org.jabref.model.entry.LinkedFile;
+import org.jabref.model.entry.field.StandardField;
 import org.jabref.preferences.PreferencesService;
 
+import org.jspecify.annotations.NonNull;
+import org.jspecify.annotations.Nullable;
+
+/**
+ * SIDE EFFECT: Sets the "cites" field of the entry having the linked files
+ *
+ * Mode choice A: online or offline
+ * Mode choice B: complete entry or single file (the latter is not implemented)
+ *
+ * The different modes should be implemented as sub classes. However, this was too complicated, thus we use variables at the constructor to parameterize this class.
+ */
 public class ExtractReferencesAction extends SimpleCommand {
     private final int FILES_LIMIT = 10;
 
+    private final boolean online;
     private final DialogService dialogService;
     private final StateManager stateManager;
     private final PreferencesService preferencesService;
     private final BibEntry entry;
     private final LinkedFile linkedFile;
     private final TaskExecutor taskExecutor;
 
-    public ExtractReferencesAction(DialogService dialogService,
+    private final BibliographyFromPdfImporter bibliographyFromPdfImporter;
+
+    public ExtractReferencesAction(boolean online,
+                                   DialogService dialogService,
                                    StateManager stateManager,
                                    PreferencesService preferencesService,
                                    TaskExecutor taskExecutor) {
-        this(dialogService, stateManager, preferencesService, null, null, taskExecutor);
+        this(online, dialogService, stateManager, preferencesService, null, null, taskExecutor);
     }
 
-    public ExtractReferencesAction(DialogService dialogService,
-                                   StateManager stateManager,
-                                   PreferencesService preferencesService,
-                                   BibEntry entry,
-                                   LinkedFile linkedFile,
-                                   TaskExecutor taskExecutor) {
+    /**
+     * Can be used to bind the action on a context menu in the linked file view (future work)
+     *
+     * @param entry the entry to handle (can be null)
+     * @param linkedFile the linked file (can be null)
+     */
+    private ExtractReferencesAction(boolean online,
+                                    @NonNull DialogService dialogService,
+                                    @NonNull StateManager stateManager,
+                                    @NonNull PreferencesService preferencesService,
+                                    @Nullable BibEntry entry,
+                                    @Nullable LinkedFile linkedFile,
+                                    @NonNull TaskExecutor taskExecutor) {
+        this.online = online;
         this.dialogService = dialogService;
         this.stateManager = stateManager;
         this.preferencesService = preferencesService;
         this.entry = entry;
         this.linkedFile = linkedFile;
         this.taskExecutor = taskExecutor;
+        bibliographyFromPdfImporter = new BibliographyFromPdfImporter(preferencesService.getCitationKeyPatternPreferences());
+
+        String text;
+        GrobidPreferences grobidPreferences = preferencesService.getGrobidPreferences();
 
         if (this.linkedFile == null) {
             this.executable.bind(
                     ActionHelper.needsEntriesSelected(stateManager)
                                 .and(ActionHelper.hasLinkedFileForSelectedEntries(stateManager))
-                                .and(this.preferencesService.getGrobidPreferences().grobidEnabledProperty())
             );
         } else {
             this.setExecutable(true);
@@ -68,34 +102,113 @@ public void execute() {
 
     private void extractReferences() {
         stateManager.getActiveDatabase().ifPresent(databaseContext -> {
-            List<BibEntry> selectedEntries = new LinkedList<>();
+            assert online == this.preferencesService.getGrobidPreferences().isGrobidEnabled();
+
+            List<BibEntry> selectedEntries;
             if (entry == null) {
                 selectedEntries = stateManager.getSelectedEntries();
             } else {
-                selectedEntries.add(entry);
+                selectedEntries = List.of(entry);
             }
 
-            List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
-            if (fileList.size() > FILES_LIMIT) {
-                boolean continueOpening = dialogService.showConfirmationDialogAndWait(Localization.lang("Processing a large number of files"),
-                        Localization.lang("You are about to process %0 files. Continue?", fileList.size()),
-                        Localization.lang("Continue"), Localization.lang("Cancel"));
-                if (!continueOpening) {
+            Callable<ParserResult> parserResultCallable;
+            if (online) {
+                Optional<Callable<ParserResult>> parserResultCallableOnline = getParserResultCallableOnline(databaseContext, selectedEntries);
+                if (parserResultCallableOnline.isEmpty()) {
                     return;
                 }
+                parserResultCallable = parserResultCallableOnline.get();
+            } else {
+                parserResultCallable = getParserResultCallableOffline(databaseContext, selectedEntries);
             }
-
-            Callable<ParserResult> parserResultCallable = () -> new ParserResult(
-                    new GrobidService(this.preferencesService.getGrobidPreferences()).processReferences(fileList, preferencesService.getImportFormatPreferences())
-            );
             BackgroundTask<ParserResult> task = BackgroundTask.wrap(parserResultCallable)
                                                               .withInitialMessage(Localization.lang("Processing PDF(s)"));
 
             task.onFailure(dialogService::showErrorDialogAndWait);
 
             ImportEntriesDialog dialog = new ImportEntriesDialog(stateManager.getActiveDatabase().get(), task);
-            dialog.setTitle(Localization.lang("Extract References"));
+            String title;
+            if (online) {
+                title = Localization.lang("Extract References (online)");
+            } else {
+                title = Localization.lang("Extract References (offline)");
+            }
+            dialog.setTitle(title);
             dialogService.showCustomDialogAndWait(dialog);
         });
     }
+
+    private @NonNull Callable<ParserResult> getParserResultCallableOffline(BibDatabaseContext databaseContext, List<BibEntry> selectedEntries) {
+        return () -> {
+            BibEntry currentEntry = selectedEntries.getFirst();
+            List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
+
+            // We need to have ParserResult handled at the importer, because it imports the meta data (library type, encoding, ...)
+            ParserResult result = bibliographyFromPdfImporter.importDatabase(fileList.getFirst());
+
+            // subsequent files are just appended to result
+            Iterator<Path> fileListIterator = fileList.iterator();
+            fileListIterator.next(); // skip first file
+            extractReferences(fileListIterator, result, currentEntry);
+
+            // handle subsequent entries
+            Iterator<BibEntry> selectedEntriesIterator = selectedEntries.iterator();
+            selectedEntriesIterator.next(); // skip first entry
+            while (selectedEntriesIterator.hasNext()) {
+                currentEntry = selectedEntriesIterator.next();
+                fileList = FileUtil.getListOfLinkedFiles(List.of(currentEntry), databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
+                fileListIterator = fileList.iterator();
+                extractReferences(fileListIterator, result, currentEntry);
+            }
+
+            return result;
+        };
+    }
+
+    private void extractReferences(Iterator<Path> fileListIterator, ParserResult result, BibEntry currentEntry) {
+        while (fileListIterator.hasNext()) {
+            result.getDatabase().insertEntries(bibliographyFromPdfImporter.importDatabase(fileListIterator.next()).getDatabase().getEntries());
+        }
+
+        StringJoiner cites = new StringJoiner(",");
+        // required because of "orElseGet"
+        var ref = new Object() {
+            int count = 0;
+        };
+        for (BibEntry importedEntry : result.getDatabase().getEntries()) {
+            ref.count++;
+            Optional<String> citationKey = importedEntry.getCitationKey();
+            cites.add(citationKey.orElseGet(() -> {
+                String sourceCitationKey = currentEntry.getCitationKey().orElse("unknown");
+                String newCitationKey;
+                // Could happen if no author and no year is present
+                // We use the number of the comment field (because there is no other way to get the number reliable)
+                Pattern pattern = Pattern.compile("^\\[(\\d+)\\]");
+                Matcher matcher = pattern.matcher(importedEntry.getField(StandardField.COMMENT).orElse(""));
+                if (matcher.hasMatch()) {
+                    newCitationKey = sourceCitationKey + "-" + matcher.group(1);
+                } else {
+                    newCitationKey = sourceCitationKey + "-" + ref.count;
+                }
+                importedEntry.setCitationKey(newCitationKey);
+                return newCitationKey;
+            }));
+        }
+        currentEntry.setField(StandardField.CITES, cites.toString());
+    }
+
+    private Optional<Callable<ParserResult>> getParserResultCallableOnline(BibDatabaseContext databaseContext, List<BibEntry> selectedEntries) {
+        List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
+        if (fileList.size() > FILES_LIMIT) {
+            boolean continueOpening = dialogService.showConfirmationDialogAndWait(Localization.lang("Processing a large number of files"),
+                    Localization.lang("You are about to process %0 files. Continue?", fileList.size()),
+                    Localization.lang("Continue"), Localization.lang("Cancel"));
+            if (!continueOpening) {
+                return Optional.empty();
+            }
+        }
+        return Optional.of(() -> new ParserResult(
+                new GrobidService(this.preferencesService.getGrobidPreferences()).processReferences(fileList, preferencesService.getImportFormatPreferences())
+        ));
+    }
 }

src/main/java/org/jabref/gui/maintable/RightClickMenu.java

@@ -4,6 +4,7 @@
 
 import javafx.scene.control.ContextMenu;
 import javafx.scene.control.Menu;
+import javafx.scene.control.MenuItem;
 import javafx.scene.control.SeparatorMenuItem;
 
 import org.jabref.gui.ClipBoardManager;
@@ -34,6 +35,8 @@
 import org.jabref.preferences.PreferencesService;
 import org.jabref.preferences.PreviewPreferences;
 
+import com.tobiasdiez.easybind.EasyBind;
+
 public class RightClickMenu {
 
     public static ContextMenu create(BibEntryTableViewModel entry,
@@ -50,6 +53,9 @@ public static ContextMenu create(BibEntryTableViewModel entry,
         ActionFactory factory = new ActionFactory(keyBindingRepository);
         ContextMenu contextMenu = new ContextMenu();
 
+        MenuItem extractFileReferencesOnline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_ONLINE, new ExtractReferencesAction(true, dialogService, stateManager, preferencesService, taskExecutor));
+        MenuItem extractFileReferencesOffline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_OFFLINE, new ExtractReferencesAction(false, dialogService, stateManager, preferencesService, taskExecutor));
+
         contextMenu.getItems().addAll(
                 factory.createMenuItem(StandardActions.COPY, new EditAction(StandardActions.COPY, () -> libraryTab, stateManager, undoManager)),
                 createCopySubMenu(factory, dialogService, stateManager, preferencesService, clipBoardManager, abbreviationRepository, taskExecutor),
@@ -75,7 +81,8 @@ public static ContextMenu create(BibEntryTableViewModel entry,
                 factory.createMenuItem(StandardActions.ATTACH_FILE_FROM_URL, new AttachFileFromURLAction(dialogService, stateManager, taskExecutor, preferencesService)),
                 factory.createMenuItem(StandardActions.OPEN_FOLDER, new OpenFolderAction(dialogService, stateManager, preferencesService, taskExecutor)),
                 factory.createMenuItem(StandardActions.OPEN_EXTERNAL_FILE, new OpenExternalFileAction(dialogService, stateManager, preferencesService, taskExecutor)),
-                factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES, new ExtractReferencesAction(dialogService, stateManager, preferencesService, taskExecutor)),
+                extractFileReferencesOnline,
+                extractFileReferencesOffline,
 
                 factory.createMenuItem(StandardActions.OPEN_URL, new OpenUrlAction(dialogService, stateManager, preferencesService)),
                 factory.createMenuItem(StandardActions.SEARCH_SHORTSCIENCE, new SearchShortScienceAction(dialogService, stateManager, preferencesService)),
@@ -86,6 +93,11 @@ public static ContextMenu create(BibEntryTableViewModel entry,
                 factory.createMenuItem(StandardActions.MERGE_WITH_FETCHED_ENTRY, new MergeWithFetchedEntryAction(dialogService, stateManager, taskExecutor, preferencesService, undoManager))
         );
 
+        EasyBind.subscribe(preferencesService.getGrobidPreferences().grobidEnabledProperty(), enabled -> {
+            extractFileReferencesOnline.setVisible(enabled);
+            extractFileReferencesOffline.setVisible(!enabled);
+        });
+
         return contextMenu;
     }
 

src/main/java/org/jabref/gui/menus/ChangeEntryTypeAction.java

@@ -4,7 +4,6 @@
 
 import javax.swing.undo.UndoManager;
 
-import javafx.beans.property.ReadOnlyStringProperty;
 import javafx.beans.property.ReadOnlyStringWrapper;
 
 import org.jabref.gui.EntryTypeView;
@@ -36,14 +35,4 @@ public void execute() {
                               .ifPresent(change -> compound.addEdit(new UndoableChangeType(change))));
         undoManager.addEdit(compound);
     }
-
-    @Override
-    public String getStatusMessage() {
-        return statusMessage.get();
-    }
-
-    @Override
-    public ReadOnlyStringProperty statusMessageProperty() {
-        return statusMessageProperty.getReadOnlyProperty();
-    }
 }