Implement an interface to import PDF metadata from multiple sources (XMP, Grobid, ...)

src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java

@@ -23,9 +23,10 @@
 
 public class GrobidCitationFetcher implements SearchBasedFetcher {
 
+    public static final String GROBID_URL = "http://grobid.jabref.org:8070";
+
     private static final Logger LOGGER = LoggerFactory.getLogger(GrobidCitationFetcher.class);
 
-    private static final String GROBID_URL = "http://grobid.jabref.org:8070";
     private ImportFormatPreferences importFormatPreferences;
     private GrobidService grobidService;
 

src/main/java/org/jabref/logic/importer/fileformat/GrobidPdfMetadataImporter.java

@@ -0,0 +1,95 @@
+package org.jabref.logic.importer.fileformat;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Path;
+import java.util.Objects;
+
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.Importer;
+import org.jabref.logic.importer.ParserResult;
+import org.jabref.logic.importer.util.GrobidService;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.util.StandardFileType;
+
+/**
+ * Wraps the GrobidService function to be used as an Importer.
+ */
+public class GrobidPdfMetadataImporter extends Importer {
+
+    private final GrobidService grobidService;
+    private final ImportFormatPreferences importFormatPreferences;
+
+    public GrobidPdfMetadataImporter(String grobidServerURL, ImportFormatPreferences importFormatPreferences) {
+        this.grobidService = new GrobidService(grobidServerURL);
+        this.importFormatPreferences = importFormatPreferences;
+    }
+
+    @Override
+    public String getName() {
+        return Localization.lang("Grobid PDF metadata");
+    }
+
+    @Override
+    public StandardFileType getFileType() {
+        return StandardFileType.PDF;
+    }
+
+    @Override
+    public ParserResult importDatabase(BufferedReader reader) throws IOException {
+        Objects.requireNonNull(reader);
+        throw new UnsupportedOperationException(
+                "PdfXmpImporter does not support importDatabase(BufferedReader reader)."
+                        + "Instead use importDatabase(Path filePath, Charset defaultEncoding).");
+    }
+
+    @Override
+    public ParserResult importDatabase(String data) throws IOException {
+        Objects.requireNonNull(data);
+        throw new UnsupportedOperationException(
+                "PdfXmpImporter does not support importDatabase(String data)."
+                        + "Instead use importDatabase(Path filePath, Charset defaultEncoding).");
+    }
+
+    @Override
+    public ParserResult importDatabase(Path filePath, Charset defaultEncoding) {
+        Objects.requireNonNull(filePath);
+        try {
+            return new ParserResult(grobidService.processPDF(filePath, importFormatPreferences));
+        } catch (Exception exception) {
+            return ParserResult.fromError(exception);
+        }
+    }
+
+    @Override
+    public boolean isRecognizedFormat(BufferedReader reader) throws IOException {
+        Objects.requireNonNull(reader);
+        return false;
+    }
+
+    /**
+     * Returns whether the given stream contains data that is a.) a pdf and b.)
+     * contains at least one BibEntry.
+     */
+    @Override
+    public boolean isRecognizedFormat(Path filePath, Charset defaultEncoding) throws IOException {
+        Objects.requireNonNull(filePath);
+        String[] splittedFileName = filePath.getFileName().toString().split("\\.");
+        if (splittedFileName.length <= 1) {
+            return false;
+        }
+        String extension = splittedFileName[splittedFileName.length - 1];
+        return getFileType().getExtensions().contains(extension);
+    }
+
+    @Override
+    public String getId() {
+        return "grobidPdf";
+    }
+
+    @Override
+    public String getDescription() {
+        return "Wraps the GrobidService function to be used as an Importer.";
+    }
+}

src/main/java/org/jabref/logic/importer/util/GrobidService.java

@@ -3,9 +3,17 @@
 import java.io.IOException;
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.Duration;
+import java.util.List;
 
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.ParseException;
+import org.jabref.logic.importer.fileformat.BibtexParser;
 import org.jabref.logic.net.URLDownload;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.util.DummyFileUpdateMonitor;
 
 /**
  * Implements an API to a GROBID server, as described at
@@ -19,6 +27,8 @@
  */
 public class GrobidService {
 
+    public static String HTTP_REQUEST_BOUNDARY = "---------------------------JabRefRequest";
+
     public enum ConsolidateCitations {
         NO(0), WITH_METADATA(1), WITH_DOI_ONLY(2);
         private int code;
@@ -59,4 +69,52 @@ public String processCitation(String rawCitation, ConsolidateCitations consolida
 
         return httpResponse;
     }
+
+    public List<BibEntry> processPDF(Path filePath, ImportFormatPreferences importFormatPreferences) throws IOException, ParseException {
+        URLDownload urlDownload = new URLDownload(grobidServerURL
+                + "/api/processHeaderDocument"); // shall we use processFulltextDocument?
+        urlDownload.setConnectTimeout(Duration.ofSeconds(150));
+        urlDownload.addHeader("Accept", MediaTypes.APPLICATION_BIBTEX);
+        urlDownload.addHeader("Content-Type", "multipart/form-data; boundary=" + HTTP_REQUEST_BOUNDARY);
+        urlDownload.setPostData(readPdf(filePath));
+        String httpResponse = urlDownload.asString();
+
+        if (httpResponse == null || httpResponse.equals("@misc{-1,\n  author = {}\n}\n")) { // This filters empty BibTeX entries
+            throw new IOException("The GROBID server response does not contain anything.");
+        }
+
+        BibtexParser parser = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor());
+        return parser.parseEntries(httpResponse);
+    }
+
+    private byte[] readPdf(Path filePath) throws IOException {
+        StringBuilder preFile = new StringBuilder();
+        preFile.append("--");
+        preFile.append(HTTP_REQUEST_BOUNDARY);
+        preFile.append("\r\n");
+        preFile.append("Content-Disposition: form-data; name=\"consolidateHeader\"\r\n\r\n1\r\n--");
+        preFile.append(HTTP_REQUEST_BOUNDARY);
+        preFile.append("\r\n");
+        preFile.append("Content-Disposition: form-data; name=\"input\"; filename=\"");
+        preFile.append(filePath.getFileName().toString());
+        preFile.append("\"\r\nContent-Type: application/pdf\r\n\r\n");
+        byte[] preFileBytes = preFile.toString().getBytes();
+
+        byte[] fileContent = Files.readAllBytes(filePath);
+
+        StringBuilder postFile = new StringBuilder();
+        postFile.append("\r\n--");
+        postFile.append(HTTP_REQUEST_BOUNDARY);
+        postFile.append("\r\n");
+        postFile.append("Content-Disposition: form-data; name=\"input\"\r\n\r\n\r\n--");
+        postFile.append(HTTP_REQUEST_BOUNDARY);
+        postFile.append("--\r\n");
+        byte[] postFileBytes = postFile.toString().getBytes();
+
+        byte[] post = new byte[preFileBytes.length + fileContent.length + postFileBytes.length];
+        System.arraycopy(preFileBytes, 0, post, 0, preFileBytes.length);
+        System.arraycopy(fileContent, 0, post, preFileBytes.length, fileContent.length);
+        System.arraycopy(postFileBytes, 0, post, preFileBytes.length + fileContent.length, postFileBytes.length);
+        return post;
+    }
 }

src/main/java/org/jabref/logic/net/URLDownload.java

@@ -69,7 +69,7 @@ public class URLDownload {
 
     private final URL source;
     private final Map<String, String> parameters = new HashMap<>();
-    private String postData = "";
+    private byte[] postData = null;
     private Duration connectTimeout = DEFAULT_CONNECT_TIMEOUT;
 
     /**
@@ -222,6 +222,12 @@ public void addHeader(String key, String value) {
     }
 
     public void setPostData(String postData) {
+        if (postData != null) {
+            this.postData = postData.getBytes();
+        }
+    }
+
+    public void setPostData(byte[] postData) {
         if (postData != null) {
             this.postData = postData;
         }
@@ -339,10 +345,10 @@ private URLConnection openConnection() throws IOException {
         for (Entry<String, String> entry : this.parameters.entrySet()) {
             connection.setRequestProperty(entry.getKey(), entry.getValue());
         }
-        if (!this.postData.isEmpty()) {
+        if (this.postData != null) {
             connection.setDoOutput(true);
             try (DataOutputStream wr = new DataOutputStream(connection.getOutputStream())) {
-                wr.writeBytes(this.postData);
+                wr.write(this.postData);
             }
         }
 

src/main/resources/l10n/JabRef_en.properties

@@ -2363,3 +2363,5 @@ Rebuild\ fulltext\ search\ index\ for\ current\ library?=Rebuild fulltext search
 Rebuilding\ fulltext\ search\ index...=Rebuilding fulltext search index...
 Failed\ to\ access\ fulltext\ search\ index=Failed to access fulltext search index
 Found\ match\ in\ %0=Found match in %0
+
+Grobid\ PDF\ metadata=Grobid PDF metadata

src/test/java/org/jabref/logic/importer/fileformat/GrobidPdfMetadataImporterTest.java

@@ -0,0 +1,71 @@
+package org.jabref.logic.importer.fileformat;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Optional;
+
+import org.jabref.logic.importer.ImportFormatPreferences;
+import org.jabref.logic.importer.fetcher.GrobidCitationFetcher;
+import org.jabref.logic.util.StandardFileType;
+import org.jabref.model.entry.BibEntry;
+import org.jabref.model.entry.field.StandardField;
+
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.mockito.Answers;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class GrobidPdfMetadataImporterTest {
+
+    private GrobidPdfMetadataImporter importer;
+    private ImportFormatPreferences importFormatPreferences;
+
+    @BeforeEach
+    public void setUp() {
+        importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS);
+        when(importFormatPreferences.getKeywordSeparator()).thenReturn(',');
+        importer = new GrobidPdfMetadataImporter(GrobidCitationFetcher.GROBID_URL, importFormatPreferences);
+    }
+
+    @Test
+    public void testsGetExtensions() {
+        assertEquals(StandardFileType.PDF, importer.getFileType());
+    }
+
+    @Test
+    public void testImportEntries() throws URISyntaxException {
+        Path file = Path.of(GrobidPdfMetadataImporterTest.class.getResource("LNCS-minimal.pdf").toURI());
+        List<BibEntry> bibEntries = importer.importDatabase(file, StandardCharsets.UTF_8).getDatabase().getEntries();
+
+        assertEquals(1, bibEntries.size());
+
+        BibEntry be0 = bibEntries.get(0);
+        assertEquals(Optional.of("Lastname, Firstname"), be0.getField(StandardField.AUTHOR));
+        assertEquals(Optional.of("Paper Title"), be0.getField(StandardField.TITLE));
+    }
+
+    @Test
+    public void testIsRecognizedFormat() throws IOException, URISyntaxException {
+        Path file = Path.of(GrobidPdfMetadataImporterTest.class.getResource("annotated.pdf").toURI());
+        assertTrue(importer.isRecognizedFormat(file, StandardCharsets.UTF_8));
+    }
+
+    @Test
+    public void testIsRecognizedFormatReject() throws IOException, URISyntaxException {
+        Path file = Path.of(PdfXmpImporterTest.class.getResource("BibtexImporter.examples.bib").toURI());
+        assertFalse(importer.isRecognizedFormat(file, StandardCharsets.UTF_8));
+    }
+
+    @Test
+    public void testGetCommandLineId() {
+        assertEquals("grobidPdf", importer.getId());
+    }
+}