From dbcf61c6f7d053a53c9d6b179cdc7bb957b9f216 Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 28 Mar 2021 12:53:05 +0200 Subject: [PATCH 1/3] [Bandcamp] Load more featured pages --- .../extractors/BandcampFeaturedExtractor.java | 40 +++++++++++++++++-- .../BandcampFeaturedExtractorTest.java | 19 ++++++++- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 93b1b0297a..019c5aba37 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -25,6 +25,7 @@ public class BandcampFeaturedExtractor extends KioskExtractor public static final String KIOSK_FEATURED = "Featured"; public static final String FEATURED_API_URL = BASE_API_URL + "/mobile/24/bootstrap_data"; + public static final String MORE_FEATURED_API_URL = BASE_API_URL + "/mobile/24/feed_older_logged_out"; private JsonObject json; @@ -56,12 +57,18 @@ public String getName() throws ParsingException { @Override public InfoItemsPage getInitialPage() throws IOException, ExtractionException { - final PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); final JsonArray featuredStories = json.getObject("feed_content") .getObject("stories") .getArray("featured"); + return extractItems(featuredStories); + } + + private InfoItemsPage extractItems(JsonArray featuredStories) { + + final PlaylistInfoItemsCollector c = new PlaylistInfoItemsCollector(getServiceId()); + for (int i = 0; i < featuredStories.size(); i++) { final JsonObject featuredStory = featuredStories.getObject(i); @@ -73,12 +80,37 @@ public InfoItemsPage getInitialPage() throws IOException, Extr c.commit(new BandcampPlaylistInfoItemFeaturedExtractor(featuredStory)); } - return new InfoItemsPage<>(c, null); + final JsonObject lastFeaturedStory = featuredStories.getObject(featuredStories.size() - 1); + + return new InfoItemsPage<>(c, makeNextPage(lastFeaturedStory)); + } + /** + * The query for more featured stories needs parameters from the last featured + * story. + */ + private Page makeNextPage(JsonObject lastFeaturedStory) { + final long lastStoryDate = lastFeaturedStory.getLong("story_date"); + final long lastStoryId = lastFeaturedStory.getLong("ntid"); + final String lastStoryType = lastFeaturedStory.getString("story_type"); + return new Page( + MORE_FEATURED_API_URL + "?story_groups=featured" + + ':' + lastStoryDate + ':' + lastStoryType + ':' + lastStoryId + ); } @Override - public InfoItemsPage getPage(Page page) { - return null; + public InfoItemsPage getPage(Page page) throws IOException, ExtractionException { + + JsonObject response; + try { + response = JsonParser.object().from( + getDownloader().get(page.getUrl()).responseBody() + ); + } catch (final JsonParserException e) { + throw new ParsingException("Could not parse Bandcamp featured API response", e); + } + + return extractItems(response.getObject("stories").getArray("featured")); } } diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java index 1be966dea2..7f4f7b1f56 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/bandcamp/BandcampFeaturedExtractorTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import org.schabi.newpipe.downloader.DownloaderTestImpl; import org.schabi.newpipe.extractor.NewPipe; +import org.schabi.newpipe.extractor.Page; import org.schabi.newpipe.extractor.exceptions.ExtractionException; import org.schabi.newpipe.extractor.playlist.PlaylistInfoItem; import org.schabi.newpipe.extractor.services.BaseListExtractorTest; @@ -16,6 +17,7 @@ import java.util.List; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; import static org.schabi.newpipe.extractor.ServiceList.Bandcamp; @@ -37,7 +39,7 @@ public static void setUp() throws ExtractionException, IOException { @Test public void testFeaturedCount() throws ExtractionException, IOException { final List list = extractor.getInitialPage().getItems(); - assertTrue(list.size() > 1); + assertTrue(list.size() > 5); } @Test @@ -46,6 +48,21 @@ public void testHttps() throws ExtractionException, IOException { assertTrue(list.get(0).getUrl().contains("https://")); } + @Test + public void testMorePages() throws ExtractionException, IOException { + + final Page page2 = extractor.getInitialPage().getNextPage(); + final Page page3 = extractor.getPage(page2).getNextPage(); + + assertTrue(extractor.getPage(page2).getItems().size() > 5); + + // Compare first item of second page with first item of third page + assertNotEquals( + extractor.getPage(page2).getItems().get(0), + extractor.getPage(page3).getItems().get(0) + ); + } + @Override public void testRelatedItems() throws Exception { DefaultTests.defaultTestRelatedItems(extractor); From d663be5a786f1af1d6162f778fefe3103b1d753f Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Sun, 28 Mar 2021 13:12:22 +0200 Subject: [PATCH 2/3] [Bandcamp] Fix featured albums not showing thumbnails --- .../extractors/BandcampPlaylistInfoItemFeaturedExtractor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java index 65872aab34..060d461fcf 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampPlaylistInfoItemFeaturedExtractor.java @@ -35,6 +35,7 @@ public String getUrl() { @Override public String getThumbnailUrl() { - return featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) : ""; + return featuredStory.has("art_id") ? getImageUrl(featuredStory.getLong("art_id"), true) + : getImageUrl(featuredStory.getLong("item_art_id"), true); } } From 705f6c6e33bcc2fc989872433b247e462ba8cdcf Mon Sep 17 00:00:00 2001 From: Fynn Godau Date: Wed, 31 Mar 2021 10:43:13 +0200 Subject: [PATCH 3/3] Apply review --- .../bandcamp/extractors/BandcampFeaturedExtractor.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java index 019c5aba37..69f0813912 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java @@ -82,14 +82,13 @@ private InfoItemsPage extractItems(JsonArray featuredStories) final JsonObject lastFeaturedStory = featuredStories.getObject(featuredStories.size() - 1); - return new InfoItemsPage<>(c, makeNextPage(lastFeaturedStory)); + return new InfoItemsPage<>(c, getNextPageFrom(lastFeaturedStory)); } /** - * The query for more featured stories needs parameters from the last featured - * story. + * Next Page can be generated from metadata of last featured story */ - private Page makeNextPage(JsonObject lastFeaturedStory) { + private Page getNextPageFrom(JsonObject lastFeaturedStory) { final long lastStoryDate = lastFeaturedStory.getLong("story_date"); final long lastStoryId = lastFeaturedStory.getLong("ntid"); final String lastStoryType = lastFeaturedStory.getString("story_type");