From a6029c93f4c4b380eb7eb4aefe5de967af3c6edc Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Tue, 26 Nov 2019 16:11:13 +0100 Subject: [PATCH 01/12] [WIP] Fix fetcher tests Fix some wrong data Fix help pages Use jsoup for parsing pre content in inspire fetcher TODO: Find out why the bibtex parser does not work for the second and third returned entry --- .../importer/fetcher/INSPIREFetcher.java | 19 ++++++++++++------- .../importer/fetcher/GoogleScholarTest.java | 2 +- .../importer/fetcher/GvkFetcherTest.java | 2 +- .../logic/importer/fetcher/IEEETest.java | 3 +++ .../importer/fetcher/MedlineFetcherTest.java | 2 +- .../importer/fetcher/RfcFetcherTest.java | 2 +- .../importer/fetcher/SpringerFetcherTest.java | 10 ++++++---- .../importer/fetcher/TitleFetcherTest.java | 2 +- .../logic/importer/fetcher/ZbMATHTest.java | 2 +- 9 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index 87ca334ec5e..e70c3fbed00 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -8,8 +8,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.jabref.logic.formatter.bibtexfields.ClearFormatter; @@ -28,6 +26,10 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; /** * Fetches data from the INSPIRE database. @@ -73,11 +75,14 @@ public Parser getParser() { List entries = new ArrayList<>(); BibtexParser bibtexParser = new BibtexParser(preferences, new DummyFileUpdateMonitor()); - Pattern pattern = Pattern.compile("
(?s)(.*)
"); - Matcher matcher = pattern.matcher(response); - while (matcher.find()) { - String bibtexEntryString = matcher.group(1); - entries.addAll(bibtexParser.parseEntries(bibtexEntryString)); + + Document doc = Jsoup.parse(response); + Elements preElements = doc.getElementsByTag("pre"); + + for (Element elem : preElements) { + //TODO: Second and third entry are not parsed + List entry = bibtexParser.parseEntries(elem.wholeText()); + entries.addAll(entry); } return entries; }; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java index a9b7ced5928..930dbbb2ba7 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java @@ -77,6 +77,6 @@ void findSingleEntry() throws FetcherException { void find20Entries() throws FetcherException { List foundEntries = finder.performSearch("random test string"); - assertEquals(20, foundEntries.size()); + assertEquals(10, foundEntries.size()); } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java index c97574620a7..4f675184af3 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java @@ -65,7 +65,7 @@ public void testGetName() { @Test public void testGetHelpPage() { - assertEquals("GVK", fetcher.getHelpPage().get().getPageName()); + assertEquals("import-using-online-bibliographic-database/gvk", fetcher.getHelpPage().get().getPageName()); } @Test diff --git a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java index 3303aab4669..b2a45b5fa42 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java @@ -101,6 +101,7 @@ void searchResultHasNoKeywordTerms() throws FetcherException { expected.setField(StandardField.DOI, "10.1049/iet-rpg.2018.5648"); expected.setField(StandardField.FILE, ":https\\://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8636659:PDF"); expected.setField(StandardField.ISSUE, "3"); + expected.setField(StandardField.ISSN, "1752-1424"); expected.setField(StandardField.JOURNALTITLE, "IET Renewable Power Generation"); expected.setField(StandardField.PAGES, "418--426"); expected.setField(StandardField.PUBLISHER, "IET"); @@ -125,6 +126,8 @@ void searchByQueryFindsEntry() throws Exception { expected.setField(StandardField.DOI, "10.1145/2884781.2884806"); expected.setField(StandardField.JOURNALTITLE, "2016 IEEE/ACM 38th International Conference on Software Engineering (ICSE)"); expected.setField(StandardField.PAGES, "273--284"); + expected.setField(StandardField.ISBN, "978-1-5090-2071-3"); + expected.setField(StandardField.ISSN, "1558-1225"); expected.setField(StandardField.PUBLISHER, "IEEE"); expected.setField(StandardField.KEYWORDS, "Portals, Documentation, Computer bugs, Joining processes, Industries, Open source software, Newcomers, Newbies, Novices, Beginners, Open Source Software, Barriers, Obstacles, Onboarding, Joining Process"); expected.setField(StandardField.TITLE, "Overcoming Open Source Project Entry Barriers with a Portal for Newcomers"); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java index 578e8cdb569..f3c2fd6d052 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java @@ -130,7 +130,7 @@ public void testGetName() { @Test public void testGetHelpPage() { - assertEquals("Medline", fetcher.getHelpPage().get().getPageName()); + assertEquals("import-using-online-bibliographic-database/medline", fetcher.getHelpPage().get().getPageName()); } @Test diff --git a/src/test/java/org/jabref/logic/importer/fetcher/RfcFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/RfcFetcherTest.java index 2cea53e900f..0acd38b68d8 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/RfcFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/RfcFetcherTest.java @@ -49,7 +49,7 @@ public void getNameReturnsEqualIdName() { @Test public void getHelpPageReturnsEqualHelpPage() { - assertEquals("RFCtoBibTeX", fetcher.getHelpPage().get().getPageName()); + assertEquals("import-using-publication-identifiers/rfctobibtex", fetcher.getHelpPage().get().getPageName()); } @Test diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java index c4db21ec256..cb4d188e5e0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java @@ -29,15 +29,17 @@ void setUp() { void searchByQueryFindsEntry() throws Exception { BibEntry expected = new BibEntry(StandardEntryType.Article); expected.setField(StandardField.AUTHOR, "Steinmacher, Igor and Gerosa, Marco and Conte, Tayana U. and Redmiles, David F."); - expected.setField(StandardField.DATE, "2018-06-14"); + expected.setField(StandardField.DATE, "2019-04-15"); expected.setField(StandardField.DOI, "10.1007/s10606-018-9335-z"); expected.setField(StandardField.ISSN, "0925-9724"); expected.setField(StandardField.JOURNAL, "Computer Supported Cooperative Work (CSCW)"); - expected.setField(StandardField.MONTH, "#jun#"); - expected.setField(StandardField.PAGES, "1--44"); + expected.setField(StandardField.MONTH, "#apr#"); + expected.setField(StandardField.PAGES, "247--290"); + expected.setField(StandardField.NUMBER, "1-2"); + expected.setField(StandardField.VOLUME, "28"); expected.setField(StandardField.PUBLISHER, "Springer"); expected.setField(StandardField.TITLE, "Overcoming Social Barriers When Contributing to Open Source Software Projects"); - expected.setField(StandardField.YEAR, "2018"); + expected.setField(StandardField.YEAR, "2019"); expected.setField(StandardField.FILE, "online:http\\://link.springer.com/openurl/pdf?id=doi\\:10.1007/s10606-018-9335-z:PDF"); expected.setField(StandardField.ABSTRACT, "An influx of newcomers is critical to the survival, long-term success, and continuity of many Open Source Software (OSS) community-based projects. However, newcomers face many barriers when making their first contribution, leading in many cases to dropouts. Due to the collaborative nature of community-based OSS projects, newcomers may be susceptible to social barriers, such as communication breakdowns and reception issues. In this article, we report a two-phase study aimed at better understanding social barriers faced by newcomers. In the first phase, we qualitatively analyzed the literature and data collected from practitioners to identify barriers that hinder newcomers’ first contribution. We designed a model composed of 58 barriers, including 13 social barriers. In the second phase, based on the barriers model, we developed FLOSScoach, a portal to support newcomers making their first contribution. We evaluated the portal in a diary-based study and found that the portal guided the newcomers and reduced the need for communication. Our results provide insights for communities that want to support newcomers and lay a foundation for building better onboarding tools. The contributions of this paper include identifying and gathering empirical evidence of social barriers faced by newcomers; understanding how social barriers can be reduced or avoided by using a portal that organizes proper information for newcomers (FLOSScoach); presenting guidelines for communities and newcomers on how to reduce or avoid social barriers; and identifying new streams of research."); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/TitleFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/TitleFetcherTest.java index aadd223f5cf..7ba2435fe62 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/TitleFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/TitleFetcherTest.java @@ -44,7 +44,7 @@ public void testGetName() { @Test public void testGetHelpPage() { - assertEquals("TitleToBibTeX", fetcher.getHelpPage().get().getPageName()); + assertEquals("import-using-publication-identifiers/titletobibtex", fetcher.getHelpPage().get().getPageName()); } @Test diff --git a/src/test/java/org/jabref/logic/importer/fetcher/ZbMATHTest.java b/src/test/java/org/jabref/logic/importer/fetcher/ZbMATHTest.java index 55a77da28ca..35e387fe926 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/ZbMATHTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/ZbMATHTest.java @@ -34,7 +34,7 @@ void setUp() throws Exception { donaldsonEntry = new BibEntry(); donaldsonEntry.setType(StandardEntryType.Article); donaldsonEntry.setCiteKey("zbMATH03800580"); - donaldsonEntry.setField(StandardField.AUTHOR, "S.K. {Donaldson}"); + donaldsonEntry.setField(StandardField.AUTHOR, "S. K. {Donaldson}"); donaldsonEntry.setField(StandardField.JOURNAL, "Journal of Differential Geometry"); donaldsonEntry.setField(StandardField.ISSN, "0022-040X; 1945-743X/e"); donaldsonEntry.setField(StandardField.LANGUAGE, "English"); From e851e4c2e78248dd922fe7b71b3506a170c87387 Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Wed, 27 Nov 2019 09:26:47 +0100 Subject: [PATCH 02/12] fix ads abstract, replace html char --- .../jabref/logic/importer/fetcher/AstrophysicsDataSystem.java | 4 ++++ .../org/jabref/logic/importer/fetcher/DoiFetcherTest.java | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index b1523a495a7..3415597a690 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -151,6 +151,10 @@ public void doPostCleanup(BibEntry entry) { entry.getField(StandardField.ABSTRACT) .filter(abstractText -> abstractText.equals("Not Available

")) .ifPresent(abstractText -> entry.clearField(StandardField.ABSTRACT)); + + entry.getField(StandardField.ABSTRACT) + .map(abstractText -> abstractText.replace("

", "").trim()) + .ifPresent(abstractText-> entry.setField(StandardField.ABSTRACT,abstractText)); // The fetcher adds some garbage (number of found entries etc before) entry.setCommentsBeforeEntry(""); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java index 88411f1f08d..ebff8b6985d 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java @@ -56,7 +56,7 @@ public void testGetName() { @Test public void testGetHelpPage() { - assertEquals("DOItoBibTeX", fetcher.getHelpPage().get().getPageName()); + assertEquals("import-using-publication-identifiers/doitobibtex>", fetcher.getHelpPage().get().getPageName()); } @Test From 10ef00a734456f0a50da80e468430363d6a53e40 Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Wed, 27 Nov 2019 09:41:32 +0100 Subject: [PATCH 03/12] fix wrong json --- .../jabref/logic/importer/fetcher/DOAJFetcherTest.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java index 68ea36fafa0..2cad38a3f17 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/DOAJFetcherTest.java @@ -51,15 +51,7 @@ void searchByQueryFindsEntry() throws Exception { @Test void testBibJSONConverter() { - String jsonString = "{\n\"title\": \"Design of Finite Word Length Linear-Phase FIR Filters in the Logarithmic Number System Domain\",\n" - + "\"journal\": {\n\"publisher\": \"Hindawi Publishing Corporation\",\n\"language\": [" - + "\"English\"],\n\"title\": \"VLSI Design\",\"country\": \"US\",\"volume\": \"2014\"" - + "},\"author\":[{\"name\": \"Syed Asad Alam\"},{\"name\": \"Oscar Gustafsson\"" - + "}\n],\n\"link\":[{\"url\": \"http://dx.doi.org/10.1155/2014/217495\"," - + "\"type\": \"fulltext\"}],\"year\":\"2014\",\"identifier\":[{" - + "\"type\": \"pissn\",\"id\": \"1065-514X\"},\n{\"type\": \"eissn\"," - + "\"id\": \"1563-5171\"},{\"type\": \"doi\",\"id\": \"10.1155/2014/217495\"" - + "}],\"created_date\":\"2014-05-09T19:38:31Z\"}\""; + String jsonString = "{\"title\":\"Design of Finite Word Length Linear-Phase FIR Filters in the Logarithmic Number System Domain\",\"journal\":{\"publisher\":\"Hindawi Publishing Corporation\",\"language\":[\"English\"],\"title\":\"VLSI Design\",\"country\":\"US\",\"volume\":\"2014\"},\"author\":[{\"name\":\"Syed Asad Alam\"},{\"name\":\"Oscar Gustafsson\"}],\"link\":[{\"url\":\"http://dx.doi.org/10.1155/2014/217495\",\"type\":\"fulltext\"}],\"year\":\"2014\",\"identifier\":[{\"type\":\"pissn\",\"id\":\"1065-514X\"},{\"type\":\"eissn\",\"id\":\"1563-5171\"},{\"type\":\"doi\",\"id\":\"10.1155/2014/217495\"}],\"created_date\":\"2014-05-09T19:38:31Z\"}"; JSONObject jsonObject = new JSONObject(jsonString); BibEntry bibEntry = DOAJFetcher.parseBibJSONtoBibtex(jsonObject, ','); From 3f6bf7e9442a53e135fadc1f6574e03c3005dc27 Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Wed, 27 Nov 2019 09:45:26 +0100 Subject: [PATCH 04/12] fix openaccessdoi url --- .../org/jabref/logic/importer/fetcher/OpenAccessDoiTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/OpenAccessDoiTest.java b/src/test/java/org/jabref/logic/importer/fetcher/OpenAccessDoiTest.java index db4c1271a5f..b7e6c7533bd 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/OpenAccessDoiTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/OpenAccessDoiTest.java @@ -29,7 +29,7 @@ void setUp() { void findByDOI() throws IOException { entry.setField(StandardField.DOI, "10.1038/nature12373"); - assertEquals(Optional.of(new URL("https://dash.harvard.edu/bitstream/1/12285462/Nanometer-Scale%20Thermometry.pdf?sequence=1")), finder.findFullText(entry)); + assertEquals(Optional.of(new URL("https://dash.harvard.edu/bitstream/1/12285462/1/Nanometer-Scale%20Thermometry.pdf")), finder.findFullText(entry)); } @Test From 55220b75f65e6ee1352da5ed5a127edd32b954b6 Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Wed, 27 Nov 2019 12:00:23 +0100 Subject: [PATCH 05/12] update medline --- .../jabref/logic/importer/fetcher/MedlineFetcherTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java index f3c2fd6d052..c8bcfd12946 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/MedlineFetcherTest.java @@ -29,7 +29,7 @@ public void setUp() { entryWijedasa = new BibEntry(); entryWijedasa.setType(StandardEntryType.Article); - entryWijedasa.setField(StandardField.AUTHOR, "Wijedasa, Lahiru S. and Jauhiainen, Jyrki and Könönen, Mari and Lampela, Maija and Vasander, Harri and Leblanc, Marie-Claire and Evers, Stephanie and Smith, Thomas E. L. and Yule, Catherine M. and Varkkey, Helena and Lupascu, Massimo and Parish, Faizal and Singleton, Ian and Clements, Gopalasamy R. and Aziz, Sheema Abdul and Harrison, Mark E. and Cheyne, Susan and Anshari, Gusti Z. and Meijaard, Erik and Goldstein, Jenny E. and Waldron, Susan and Hergoualc'h, Kristell and Dommain, Rene and Frolking, Steve and Evans, Christopher D. and Posa, Mary Rose C. and Glaser, Paul H. and Suryadiputra, Nyoman and Lubis, Reza and Santika, Truly and Padfield, Rory and Kurnianto, Sofyan and Hadisiswoyo, Panut and Lim, Teck Wyn and Page, Susan E. and Gauci, Vincent and Van Der Meer, Peter J. and Buckland, Helen and Garnier, Fabien and Samuel, Marshall K. and Choo, Liza Nuriati Lim Kim and O'Reilly, Patrick and Warren, Matthew and Suksuwan, Surin and Sumarga, Elham and Jain, Anuj and Laurance, William F. and Couwenberg, John and Joosten, Hans and Vernimmen, Ronald and Hooijer, Aljosja and Malins, Chris and Cochrane, Mark A. and Perumal, Balu and Siegert, Florian and Peh, Kelvin S.-H. and Comeau, Louis-Pierre and Verchot, Louis and Harvey, Charles F. and Cobb, Alex and Jaafar, Zeehan and Wösten, Henk and Manuri, Solichin and Müller, Moritz and Giesen, Wim and Phelps, Jacob and Yong, Ding Li and Silvius, Marcel and Wedeux, Béatrice M. M. and Hoyt, Alison and Osaki, Mitsuru and Hirano, Takashi and Takahashi, Hidenori and Kohyama, Takashi S. and Haraguchi, Akira and Nugroho, Nunung P. and Coomes, David A. and Quoi, Le Phat and Dohong, Alue and Gunawan, Haris and Gaveau, David L. A. and Langner, Andreas and Lim, Felix K. S. and Edwards, David P. and Giam, Xingli and Van Der Werf, Guido and Carmenta, Rachel and Verwer, Caspar C. and Gibson, Luke and Gandois, Laure and Graham, Laura Linda Bozena and Regalino, Jhanson and Wich, Serge A. and Rieley, Jack and Kettridge, Nicholas and Brown, Chloe and Pirard, Romain and Moore, Sam and Capilla, B. Ripoll and Ballhorn, Uwe and Ho, Hua Chew and Hoscilo, Agata and Lohberger, Sandra and Evans, Theodore A. and Yulianti, Nina and Blackham, Grace and Onrizal and Husson, Simon and Murdiyarso, Daniel and Pangala, Sunita and Cole, Lydia E. S. and Tacconi, Luca and Segah, Hendrik and Tonoto, Prayoto and Lee, Janice S. H. and Schmilewski, Gerald and Wulffraat, Stephan and Putra, Erianto Indra and Cattau, Megan E. and Clymo, R. S. and Morrison, Ross and Mujahid, Aazani and Miettinen, Jukka and Liew, Soo Chin and Valpola, Samu and Wilson, David and D'Arcy, Laura and Gerding, Michiel and Sundari, Siti and Thornton, Sara A. and Kalisz, Barbara and Chapman, Stephen J. and Su, Ahmad Suhaizi Mat and Basuki, Imam and Itoh, Masayuki and Traeholt, Carl and Sloan, Sean and Sayok, Alexander K. and Andersen, Roxane"); + entryWijedasa.setField(StandardField.AUTHOR, "Wijedasa, Lahiru S. and Jauhiainen, Jyrki and Könönen, Mari and Lampela, Maija and Vasander, Harri and Leblanc, Marie-Claire and Evers, Stephanie and Smith, Thomas E. L. and Yule, Catherine M. and Varkkey, Helena and Lupascu, Massimo and Parish, Faizal and Singleton, Ian and Clements, Gopalasamy R. and Aziz, Sheema Abdul and Harrison, Mark E. and Cheyne, Susan and Anshari, Gusti Z. and Meijaard, Erik and Goldstein, Jenny E. and Waldron, Susan and Hergoualc'h, Kristell and Dommain, Rene and Frolking, Steve and Evans, Christopher D. and Posa, Mary Rose C. and Glaser, Paul H. and Suryadiputra, Nyoman and Lubis, Reza and Santika, Truly and Padfield, Rory and Kurnianto, Sofyan and Hadisiswoyo, Panut and Lim, Teck Wyn and Page, Susan E. and Gauci, Vincent and Van Der Meer, Peter J. and Buckland, Helen and Garnier, Fabien and Samuel, Marshall K. and Choo, Liza Nuriati Lim Kim and O'Reilly, Patrick and Warren, Matthew and Suksuwan, Surin and Sumarga, Elham and Jain, Anuj and Laurance, William F. and Couwenberg, John and Joosten, Hans and Vernimmen, Ronald and Hooijer, Aljosja and Malins, Chris and Cochrane, Mark A. and Perumal, Balu and Siegert, Florian and Peh, Kelvin S.-H. and Comeau, Louis-Pierre and Verchot, Louis and Harvey, Charles F. and Cobb, Alex and Jaafar, Zeehan and Wösten, Henk and Manuri, Solichin and Müller, Moritz and Giesen, Wim and Phelps, Jacob and Yong, Ding Li and Silvius, Marcel and Wedeux, Béatrice M. M. and Hoyt, Alison and Osaki, Mitsuru and Hirano, Takashi and Takahashi, Hidenori and Kohyama, Takashi S. and Haraguchi, Akira and Nugroho, Nunung P. and Coomes, David A. and Quoi, Le Phat and Dohong, Alue and Gunawan, Haris and Gaveau, David L. A. and Langner, Andreas and Lim, Felix K. S. and Edwards, David P. and Giam, Xingli and Van Der Werf, Guido and Carmenta, Rachel and Verwer, Caspar C. and Gibson, Luke and Gandois, Laure and Graham, Laura Linda Bozena and Regalino, Jhanson and Wich, Serge A. and Rieley, Jack and Kettridge, Nicholas and Brown, Chloe and Pirard, Romain and Moore, Sam and Capilla, B. Ripoll and Ballhorn, Uwe and Ho, Hua Chew and Hoscilo, Agata and Lohberger, Sandra and Evans, Theodore A. and Yulianti, Nina and Blackham, Grace and Onrizal and Husson, Simon and Murdiyarso, Daniel and Pangala, Sunita and Cole, Lydia E. S. and Tacconi, Luca and Segah, Hendrik and Tonoto, Prayoto and Lee, Janice S. H. and Schmilewski, Gerald and Wulffraat, Stephan and Putra, Erianto Indra and Cattau, Megan E. and Clymo, R. S. and Morrison, Ross and Mujahid, Aazani and Miettinen, Jukka and Liew, Soo Chin and Valpola, Samu and Wilson, David and D'Arcy, Laura and Gerding, Michiel and Sundari, Siti and Thornton, Sara A. and Kalisz, Barbara and Chapman, Stephen J. and Su, Ahmad Suhaizi Mat and Basuki, Imam and Itoh, Masayuki and Traeholt, Carl and Sloan, Sean and Sayok, Alexander K. and Andersen, Roxane"); entryWijedasa.setField(new UnknownField("country"), "England"); entryWijedasa.setField(StandardField.DOI, "10.1111/gcb.13516"); entryWijedasa.setField(StandardField.ISSN, "1365-2486"); @@ -43,7 +43,7 @@ public void setUp() { entryWijedasa.setField(StandardField.PMID, "27670948"); entryWijedasa.setField(new UnknownField("pubmodel"), "Print-Electronic"); entryWijedasa.setField(StandardField.PUBSTATE, "ppublish"); - entryWijedasa.setField(new UnknownField("revised"), "2018-01-23"); + entryWijedasa.setField(new UnknownField("revised"), "2019-11-20"); entryWijedasa.setField(StandardField.TITLE, "Denial of long-term issues with agriculture on tropical peatlands will have devastating consequences."); entryWijedasa.setField(StandardField.VOLUME, "23"); entryWijedasa.setField(StandardField.YEAR, "2017"); @@ -68,7 +68,7 @@ public void setUp() { entryEndharti.setField(StandardField.PMID, "27670445"); entryEndharti.setField(new UnknownField("pubmodel"), "Electronic"); entryEndharti.setField(StandardField.PUBSTATE, "epublish"); - entryEndharti.setField(new UnknownField("revised"), "2018-11-13"); + entryEndharti.setField(new UnknownField("revised"), "2019-11-20"); entryEndharti.setField(StandardField.VOLUME, "16"); entryEndharti.setField(StandardField.YEAR, "2016"); From 0af0ca3b8b8f35fcafed90ac5578ab9be27a2913 Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Fri, 29 Nov 2019 08:29:00 +0100 Subject: [PATCH 06/12] throw parseException when input is empty --- src/main/java/org/jabref/logic/importer/util/JsonReader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/org/jabref/logic/importer/util/JsonReader.java b/src/main/java/org/jabref/logic/importer/util/JsonReader.java index 7bfd02986b1..d2a3e0d064e 100644 --- a/src/main/java/org/jabref/logic/importer/util/JsonReader.java +++ b/src/main/java/org/jabref/logic/importer/util/JsonReader.java @@ -24,6 +24,9 @@ public static JSONObject toJsonObject(InputStreamReader input) throws ParseExcep while ((inputStr = streamReader.readLine()) != null) { responseStrBuilder.append(inputStr); } + if (responseStrBuilder.toString().isBlank()) { + throw new ParseException("Empty input!"); + } return new JSONObject(responseStrBuilder.toString()); } catch (IOException e) { throw new ParseException(e); From 8eba91594fffaa1877ca50d6674f04e1f43b6790 Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Fri, 29 Nov 2019 09:08:29 +0100 Subject: [PATCH 07/12] fix inspire fetcher --- .../importer/fetcher/INSPIREFetcher.java | 6 +-- .../importer/fetcher/INSPIREFetcherTest.java | 45 ++++++++++++++----- 2 files changed, 36 insertions(+), 15 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index e70c3fbed00..7a676021eff 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -74,14 +74,14 @@ public Parser getParser() { String response = new BufferedReader(new InputStreamReader(inputStream)).lines().collect(Collectors.joining(OS.NEWLINE)); List entries = new ArrayList<>(); - BibtexParser bibtexParser = new BibtexParser(preferences, new DummyFileUpdateMonitor()); Document doc = Jsoup.parse(response); Elements preElements = doc.getElementsByTag("pre"); for (Element elem : preElements) { - //TODO: Second and third entry are not parsed - List entry = bibtexParser.parseEntries(elem.wholeText()); + //We have to use a new instance here, because otherwise only the first entry gets parsed + BibtexParser bibtexParser = new BibtexParser(preferences, new DummyFileUpdateMonitor()); + List entry = bibtexParser.parseEntries(elem.text()); entries.addAll(entry); } return entries; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java index 239955ca8dd..5b83cad5fe6 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java @@ -1,6 +1,6 @@ package org.jabref.logic.importer.fetcher; -import java.util.Collections; +import java.util.Arrays; import java.util.List; import org.jabref.logic.bibtex.FieldContentParserPreferences; @@ -20,6 +20,7 @@ @FetcherTest class INSPIREFetcherTest { + private INSPIREFetcher fetcher; @BeforeEach @@ -31,18 +32,38 @@ void setUp() { @Test void searchByQueryFindsEntry() throws Exception { - BibEntry expected = new BibEntry(StandardEntryType.MastersThesis); - expected.setCiteKey("Diez:2014ppa"); - expected.setField(StandardField.AUTHOR, "Diez, Tobias"); - expected.setField(StandardField.TITLE, "Slice theorem for Fr\\'echet group actions and covariant symplectic field theory"); - expected.setField(StandardField.SCHOOL, "Leipzig U."); - expected.setField(StandardField.YEAR, "2013"); - expected.setField(StandardField.URL, "https://inspirehep.net/record/1295621/files/arXiv:1405.2249.pdf"); - expected.setField(StandardField.EPRINT, "1405.2249"); - expected.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - expected.setField(new UnknownField("primaryClass"), "math-ph"); + + BibEntry phd = new BibEntry(StandardEntryType.PhdThesis); + phd.setCiteKey("Diez:2019pkg"); + phd.setField(StandardField.AUTHOR, "Diez, Tobias"); + phd.setField(StandardField.TITLE, "Normal Form of Equivariant Maps and Singular Symplectic Reduction in Infinite Dimensions with Applications to Gauge Field Theory"); + phd.setField(StandardField.YEAR, "2019"); + phd.setField(StandardField.EPRINT, "1909.00744"); + phd.setField(new UnknownField("reportnumber"), "urn:nbn:de:bsz:15-qucosa2-352179"); + phd.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + phd.setField(new UnknownField("primaryClass"), "math.SG"); + + BibEntry article = new BibEntry(StandardEntryType.Article); + article.setCiteKey("Diez:2018gjz"); + article.setField(StandardField.AUTHOR, "Diez, Tobias and Rudolph, Gerd"); + article.setField(StandardField.TITLE, "Singular symplectic cotangent bundle reduction of gauge field theory"); + article.setField(StandardField.YEAR, "2018"); + article.setField(StandardField.EPRINT, "1812.04707"); + article.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + article.setField(new UnknownField("primaryClass"), "math-ph"); + + BibEntry master = new BibEntry(StandardEntryType.MastersThesis); + master.setCiteKey("Diez:2014ppa"); + master.setField(StandardField.AUTHOR, "Diez, Tobias"); + master.setField(StandardField.TITLE, "Slice theorem for Fr\\'echet group actions and covariant symplectic field theory"); + master.setField(StandardField.SCHOOL, "Leipzig U."); + master.setField(StandardField.YEAR, "2013"); + master.setField(StandardField.EPRINT, "1405.2249"); + master.setField(StandardField.ARCHIVEPREFIX, "arXiv"); + master.setField(new UnknownField("primaryClass"), "math-ph"); List fetchedEntries = fetcher.performSearch("Fr\\'echet group actions field"); - assertEquals(Collections.singletonList(expected), fetchedEntries); + + assertEquals(Arrays.asList(phd, article, master), fetchedEntries); } } From c5940b0f5a98c062b1ab7b681d3cd3c1de9e8e67 Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Fri, 29 Nov 2019 14:53:45 +0100 Subject: [PATCH 08/12] apparently primaryclass is a standard field --- .../jabref/logic/importer/fetcher/INSPIREFetcherTest.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java index 5b83cad5fe6..bf38c3be756 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java @@ -41,7 +41,7 @@ void searchByQueryFindsEntry() throws Exception { phd.setField(StandardField.EPRINT, "1909.00744"); phd.setField(new UnknownField("reportnumber"), "urn:nbn:de:bsz:15-qucosa2-352179"); phd.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - phd.setField(new UnknownField("primaryClass"), "math.SG"); + phd.setField(StandardField.PRIMARYCLASS, "math.SG"); BibEntry article = new BibEntry(StandardEntryType.Article); article.setCiteKey("Diez:2018gjz"); @@ -50,7 +50,7 @@ void searchByQueryFindsEntry() throws Exception { article.setField(StandardField.YEAR, "2018"); article.setField(StandardField.EPRINT, "1812.04707"); article.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - article.setField(new UnknownField("primaryClass"), "math-ph"); + article.setField(StandardField.PRIMARYCLASS, "math-ph"); BibEntry master = new BibEntry(StandardEntryType.MastersThesis); master.setCiteKey("Diez:2014ppa"); @@ -60,7 +60,7 @@ void searchByQueryFindsEntry() throws Exception { master.setField(StandardField.YEAR, "2013"); master.setField(StandardField.EPRINT, "1405.2249"); master.setField(StandardField.ARCHIVEPREFIX, "arXiv"); - master.setField(new UnknownField("primaryClass"), "math-ph"); + master.setField(StandardField.PRIMARYCLASS, "math-ph"); List fetchedEntries = fetcher.performSearch("Fr\\'echet group actions field"); From 941a26a259d8c784ab66446e00b91833da0cd8b7 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 16:02:04 +0100 Subject: [PATCH 09/12] Update GoogleScholarTest.java --- .../org/jabref/logic/importer/fetcher/GoogleScholarTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java index 930dbbb2ba7..0c17f8935c2 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GoogleScholarTest.java @@ -74,7 +74,7 @@ void findSingleEntry() throws FetcherException { @Test @DisabledOnCIServer("CI server is blocked by Google") - void find20Entries() throws FetcherException { + void findManyEntries() throws FetcherException { List foundEntries = finder.performSearch("random test string"); assertEquals(10, foundEntries.size()); From 94b311ffbb44092c741c7cc5b8a0448c8a3512c9 Mon Sep 17 00:00:00 2001 From: Tobias Diez Date: Fri, 29 Nov 2019 16:02:48 +0100 Subject: [PATCH 10/12] Update INSPIREFetcherTest.java --- .../org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java index bf38c3be756..e9508e936d4 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java @@ -32,7 +32,6 @@ void setUp() { @Test void searchByQueryFindsEntry() throws Exception { - BibEntry phd = new BibEntry(StandardEntryType.PhdThesis); phd.setCiteKey("Diez:2019pkg"); phd.setField(StandardField.AUTHOR, "Diez, Tobias"); From 99f9d48c00270dfe5c8fbed7374b0e5fcc0e5a19 Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Fri, 29 Nov 2019 16:17:23 +0100 Subject: [PATCH 11/12] fix acs fetcher --- src/main/java/org/jabref/logic/importer/fetcher/ACS.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ACS.java b/src/main/java/org/jabref/logic/importer/fetcher/ACS.java index 801627d9d57..2fa068e1eb4 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ACS.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ACS.java @@ -46,7 +46,7 @@ public Optional findFullText(BibEntry entry) throws IOException { String source = String.format(SOURCE, doi.get().getDOI()); // Retrieve PDF link Document html = Jsoup.connect(source).ignoreHttpErrors(true).get(); - Element link = html.select(".pdf-high-res a").first(); + Element link = html.select("a.button_primary").first(); if (link != null) { LOGGER.info("Fulltext PDF found @ ACS."); From 609b60cd823a33473c3d6f4d77491f562848b2bd Mon Sep 17 00:00:00 2001 From: Siedlerchr Date: Fri, 29 Nov 2019 16:41:46 +0100 Subject: [PATCH 12/12] fix help pag test --- .../java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java index ebff8b6985d..0484fb2af4b 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/DoiFetcherTest.java @@ -56,7 +56,7 @@ public void testGetName() { @Test public void testGetHelpPage() { - assertEquals("import-using-publication-identifiers/doitobibtex>", fetcher.getHelpPage().get().getPageName()); + assertEquals("import-using-publication-identifiers/doitobibtex", fetcher.getHelpPage().get().getPageName()); } @Test