Fix YouTube tracks parsing through avoiding sponsored

s-frei · May 12, 2024 · 7b9aae5 · 7b9aae5
1 parent 3286bed
commit 7b9aae5
Show file tree

Hide file tree

Showing 8 changed files with 78 additions and 64 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,10 +1,16 @@
 Changelog
 =========
 
-0.9.1 - unrelesed
------------------
+0.9.1 - unreleased
+------------------
 
-Nothing to see here yet
+**Features:**
+
+- Updated dependencies
+
+**Bugfixes:**
+
+- Fix YouTube tracks parsing through avoiding sponsored
 
 0.9.0
 -----

diff --git a/src/main/java/io/sfrei/tracksearch/clients/soundcloud/SoundCloudUtility.java b/src/main/java/io/sfrei/tracksearch/clients/soundcloud/SoundCloudUtility.java
@@ -113,7 +113,7 @@ protected GenericTrackList<SoundCloudTrack> extractSoundCloudTracks(final String
 
         final JsonElement responseElement = JsonElement.readTreeCatching(MAPPER, json)
                 .orElseThrow(() -> new SoundCloudException("Cannot parse SoundCloudTracks JSON"))
-                .path("collection");
+                .paths("collection");
 
         final List<SoundCloudTrack> scTracks = responseElement.elements()
                 .map(element -> element.mapCatching(MAPPER, SoundCloudTrack.SoundCloudTrackBuilder.class))

diff --git a/src/main/java/io/sfrei/tracksearch/clients/youtube/YouTubeUtility.java b/src/main/java/io/sfrei/tracksearch/clients/youtube/YouTubeUtility.java
@@ -101,41 +101,40 @@ protected GenericTrackList<YouTubeTrack> extractYouTubeTracks(final String json,
         final JsonElement rootElement = JsonElement.readTreeCatching(MAPPER, json)
                 .orElseThrow(() -> new YouTubeException("Cannot parse YouTubeTracks JSON"));
 
-        final JsonElement responseElement = rootElement.path("response").orElse(rootElement).elementAtIndex(1).path("response");
+        final JsonElement responseElement = rootElement.paths("response").orElse(rootElement).elementAtIndex(1).paths("response");
 
         final JsonElement defaultElement = responseElement.asUnresolved()
-                .path("contents", "twoColumnSearchResultsRenderer", "primaryContents", "sectionListRenderer", "contents");
+                .paths("contents", "twoColumnSearchResultsRenderer", "primaryContents", "sectionListRenderer", "contents");
 
         final JsonElement contentHolder = defaultElement
-                .firstElement()
-                .path("itemSectionRenderer")
+                .lastForPath("itemSectionRenderer") // Avoid sponsored
                 .orElse(responseElement)
-                .path("onResponseReceivedCommands")
+                .paths("onResponseReceivedCommands")
                 .firstElement()
-                .path("appendContinuationItemsAction", "continuationItems")
+                .paths("appendContinuationItemsAction", "continuationItems")
                 .firstElement()
-                .path("itemSectionRenderer")
+                .paths("itemSectionRenderer")
                 .orElse(responseElement)
-                .path("onResponseReceivedCommands")
+                .paths("onResponseReceivedCommands")
                 .firstElement()
-                .path("appendContinuationItemsAction", "continuationItems")
+                .paths("appendContinuationItemsAction", "continuationItems")
                 .firstElement()
-                .path("itemSectionRenderer")
+                .paths("itemSectionRenderer")
                 .orElse(responseElement)
-                .path("continuationContents", "itemSectionContinuation", "itemSectionContinuation")
+                .paths("continuationContents", "itemSectionContinuation", "itemSectionContinuation")
                 .orElse(responseElement)
-                .path("continuationContents", "sectionListContinuation", "contents")
+                .paths("continuationContents", "sectionListContinuation", "contents")
                 .firstElement()
-                .path("itemSectionRenderer");
+                .paths("itemSectionRenderer");
 
         final String cToken = extractCToken(responseElement, defaultElement, contentHolder);
 
-        final JsonElement contents = contentHolder.asUnresolved().path("contents");
+        final JsonElement contents = contentHolder.asUnresolved().paths("contents");
         final List<YouTubeTrack> ytTracks = contents.elements()
-                .filter(content -> content.path("videoRenderer", "upcomingEventData").isNull()) // Avoid premieres
-                .filter(content -> content.path("promotedSparklesWebRenderer").isNull()) // Avoid ads
-                .map(content -> content.path("videoRenderer").orElse(content).path("searchPyvRenderer", "ads").firstElement().path("promotedVideoRenderer"))
-                .filter(renderer -> renderer.asUnresolved().path("lengthText").isPresent()) // Avoid live streams
+                .filter(content -> content.paths("videoRenderer", "upcomingEventData").isNull()) // Avoid premieres
+                .filter(content -> content.paths("promotedSparklesWebRenderer").isNull()) // Avoid ads
+                .map(content -> content.paths("videoRenderer").orElse(content).paths("searchPyvRenderer", "ads").firstElement().paths("promotedVideoRenderer"))
+                .filter(renderer -> renderer.asUnresolved().paths("lengthText").isPresent()) // Avoid live streams
                 .map(renderer -> renderer.mapCatching(MAPPER, YouTubeTrack.ListYouTubeTrackBuilder.class))
                 .filter(Objects::nonNull)
                 .map(YouTubeTrack.ListYouTubeTrackBuilder::getBuilder)
@@ -156,20 +155,20 @@ protected GenericTrackList<YouTubeTrack> extractYouTubeTracks(final String json,
     private static String extractCToken(JsonElement responseElement, JsonElement defaultElement, JsonElement contentHolder) {
         if (contentHolder.nodePresent("continuations")) {
             return contentHolder.asUnresolved()
-                    .path("continuations")
+                    .paths("continuations")
                     .firstElement()
-                    .path("nextContinuationData")
+                    .paths("nextContinuationData")
                     .asString("continuation");
         }
         return responseElement.asUnresolved()
-                .path("onResponseReceivedCommands")
+                .paths("onResponseReceivedCommands")
                 .firstElement()
-                .path("appendContinuationItemsAction", "continuationItems")
+                .paths("appendContinuationItemsAction", "continuationItems")
                 .elementAtIndex(1)
-                .path("continuationItemRenderer", "continuationEndpoint", "continuationCommand")
+                .paths("continuationItemRenderer", "continuationEndpoint", "continuationCommand")
                 .orElse(defaultElement)
                 .findElement("continuationItemRenderer")
-                .path("continuationEndpoint", "continuationCommand")
+                .paths("continuationEndpoint", "continuationCommand")
                 .asString("token");
     }
 
@@ -181,7 +180,7 @@ protected YouTubeTrackInfo extractTrackInfo(final String json, final String trac
 
             final JsonElement playerElement;
             if (jsonElement.isArray()) {
-                playerElement = jsonElement.elementAtIndex(2).path("player");
+                playerElement = jsonElement.elementAtIndex(2).paths("player");
             } else {
                 playerElement = jsonElement.findElement("player");
             }
@@ -190,26 +189,26 @@ protected YouTubeTrackInfo extractTrackInfo(final String json, final String trac
 
             final JsonElement streamingData;
 
-            final JsonElement playerArgs = playerElement.path("args");
+            final JsonElement playerArgs = playerElement.paths("args");
             if (playerElement.isPresent() && playerArgs.isPresent()) {
 
-                scriptUrl.set(playerElement.path("assets").asString("js"));
+                scriptUrl.set(playerElement.paths("assets").asString("js"));
 
-                streamingData = playerArgs.path("player_response")
+                streamingData = playerArgs.paths("player_response")
                         .reReadTree(MAPPER)
-                        .path("streamingData");
+                        .paths("streamingData");
 
             } else {
                 final JsonElement playerResponse = playerResponseFromTrackJSON(jsonElement);
 
-                streamingData = playerResponse.asUnresolved().path("streamingData");
+                streamingData = playerResponse.asUnresolved().paths("streamingData");
             }
 
-            final JsonElement formatsElement = streamingData.path("formats");
+            final JsonElement formatsElement = streamingData.paths("formats");
             final Stream<YouTubeTrackFormat> formats = formatsElement.isPresent() ?
                     getFormatsFromStream(formatsElement.arrayElements()) : Stream.empty();
 
-            final Stream<JsonElement> adaptiveFormatsStream = streamingData.path("adaptiveFormats").arrayElements();
+            final Stream<JsonElement> adaptiveFormatsStream = streamingData.paths("adaptiveFormats").arrayElements();
             final Stream<YouTubeTrackFormat> adaptiveFormats = getFormatsFromStream(adaptiveFormatsStream);
 
             final List<YouTubeTrackFormat> trackFormats = Stream.concat(formats, adaptiveFormats).collect(Collectors.toList());
@@ -236,9 +235,9 @@ protected YouTubeTrackInfo extractTrackInfo(final String json, final String trac
 
     private static JsonElement playerResponseFromTrackJSON(JsonElement jsonElement) {
         return jsonElement.elementAtIndex(2)
-                .path("playerResponse")
+                .paths("playerResponse")
                 .orElse(jsonElement)
-                .path("playerResponse");
+                .paths("playerResponse");
     }
 
     private Stream<YouTubeTrackFormat> getFormatsFromStream(final Stream<JsonElement> formats) {
@@ -248,9 +247,9 @@ private Stream<YouTubeTrackFormat> getFormatsFromStream(final Stream<JsonElement
             final String audioQuality = format.asString("audioQuality");
             final String audioSampleRate = format.asString("audioSampleRate");
 
-            final JsonElement cipherElement = format.path("cipher")
+            final JsonElement cipherElement = format.paths("cipher")
                     .orElse(format)
-                    .path("signatureCipher");
+                    .paths("signatureCipher");
 
             if (cipherElement.isNull()) {
                 final String url = format.asString("url");

diff --git a/...java/io/sfrei/tracksearch/tracks/deserializer/soundcloud/SoundCloudTrackDeserializer.java b/...java/io/sfrei/tracksearch/tracks/deserializer/soundcloud/SoundCloudTrackDeserializer.java
@@ -54,7 +54,7 @@ public SoundCloudTrackBuilder deserialize(final JsonParser p, final Deserializat
 
         // Metadata
 
-        final JsonElement owner = rootElement.path("user");
+        final JsonElement owner = rootElement.paths("user");
 
         final String channelName = owner.asString("username");
 
@@ -63,16 +63,16 @@ public SoundCloudTrackBuilder deserialize(final JsonParser p, final Deserializat
         final Long playbackCount = rootElement.asLong("playback_count");
         final Long streamAmount = playbackCount == null ? 0L : playbackCount; // Apparently can be 'null' in the JSON
 
-        final String thumbNailUrl = rootElement.path("artwork_url")
+        final String thumbNailUrl = rootElement.paths("artwork_url")
                 .orElse(rootElement)
-                .path("user", "avatar_url") // Fallback to channel thumbnail
+                .paths("user", "avatar_url") // Fallback to channel thumbnail
                 .asString();
 
         soundCloudTrackBuilder.trackMetadata(SoundCloudTrackMetadata.of(channelName, channelUrl, streamAmount, thumbNailUrl));
 
         // Formats
 
-        final List<SoundCloudTrackFormat> trackFormats = rootElement.path("media", "transcodings")
+        final List<SoundCloudTrackFormat> trackFormats = rootElement.paths("media", "transcodings")
                 .arrayElements()
                 .map(SoundCloudTrackDeserializer::transcodingToTrackFormat)
                 .collect(Collectors.toList());
@@ -87,7 +87,7 @@ private static SoundCloudTrackFormat transcodingToTrackFormat(JsonElement transc
         final String formatUrl = transcoding.asString("url");
         final String audioQuality = transcoding.asString("quality");
 
-        final JsonElement formatElement = transcoding.path("format");
+        final JsonElement formatElement = transcoding.paths("format");
         final String mimeType = formatElement.asString("mime_type");
         final String protocol = formatElement.asString("protocol");
 

diff --git a/...n/java/io/sfrei/tracksearch/tracks/deserializer/youtube/YouTubeListTrackDeserializer.java b/...n/java/io/sfrei/tracksearch/tracks/deserializer/youtube/YouTubeListTrackDeserializer.java
@@ -43,8 +43,8 @@ public YouTubeTrack.ListYouTubeTrackBuilder deserialize(final JsonParser p, fina
         // Track
 
         final String ref = rootElement.asString("videoId");
-        final String title = rootElement.path("title", "runs").firstElement().asString("text");
-        final String timeString = rootElement.path("lengthText").asString("simpleText");
+        final String title = rootElement.paths("title", "runs").firstElement().asString("text");
+        final String timeString = rootElement.paths("lengthText").asString("simpleText");
         final Duration duration = TimeUtility.getDurationForTimeString(timeString);
 
         if (title == null || duration == null || ref == null)
@@ -60,21 +60,21 @@ public YouTubeTrack.ListYouTubeTrackBuilder deserialize(final JsonParser p, fina
 
         // Metadata
 
-        final JsonElement owner = rootElement.path("ownerText", "runs").firstElement();
+        final JsonElement owner = rootElement.paths("ownerText", "runs").firstElement();
 
         final String channelName = owner.asString("text");
 
-        final String channelUrlSuffix = owner.path("navigationEndpoint", "commandMetadata", "webCommandMetadata")
+        final String channelUrlSuffix = owner.paths("navigationEndpoint", "commandMetadata", "webCommandMetadata")
                 .asString("url");
         final String channelUrl = YouTubeClient.URL.concat(channelUrlSuffix);
 
-        final String streamAmountText = rootElement.path("viewCountText").asString("simpleText");
+        final String streamAmountText = rootElement.paths("viewCountText").asString("simpleText");
         final String streamAmountDigits = streamAmountText == null || streamAmountText.isEmpty() ?
                 null : ReplaceUtility.replaceNonDigits(streamAmountText);
         final Long streamAmount = streamAmountDigits == null || streamAmountDigits.isEmpty() ?
                 0L : Long.parseLong(streamAmountDigits);
 
-        final Stream<JsonElement> thumbNailStream = rootElement.path("thumbnail", "thumbnails").elements();
+        final Stream<JsonElement> thumbNailStream = rootElement.paths("thumbnail", "thumbnails").elements();
         final Optional<JsonElement> lastThumbnail = thumbNailStream.findFirst();
         final String thumbNailUrl = lastThumbnail.map(thumbNail -> thumbNail.asString("url")).orElse(null);
 

diff --git a/...in/java/io/sfrei/tracksearch/tracks/deserializer/youtube/YouTubeURLTrackDeserializer.java b/...in/java/io/sfrei/tracksearch/tracks/deserializer/youtube/YouTubeURLTrackDeserializer.java
@@ -41,7 +41,7 @@ public YouTubeTrack.URLYouTubeTrackBuilder deserialize(final JsonParser p, final
 
         // Track
 
-        final JsonElement videoDetails = rootElement.path("videoDetails");
+        final JsonElement videoDetails = rootElement.paths("videoDetails");
 
         final String ref = videoDetails.asString("videoId");
         final String title = videoDetails.asString("title");
@@ -61,15 +61,15 @@ public YouTubeTrack.URLYouTubeTrackBuilder deserialize(final JsonParser p, final
 
         // Metadata
 
-        final JsonElement owner = rootElement.path("microformat", "playerMicroformatRenderer");
+        final JsonElement owner = rootElement.paths("microformat", "playerMicroformatRenderer");
 
         final String channelName = owner.asString("ownerChannelName");
 
         final String channelUrl = owner.asString("ownerProfileUrl").replaceFirst("^http", "https");
 
         final long streamAmount = Long.parseLong(owner.asString("viewCount"));
 
-        final Stream<JsonElement> thumbNailStream = owner.path("thumbnail", "thumbnails").elements();
+        final Stream<JsonElement> thumbNailStream = owner.paths("thumbnail", "thumbnails").elements();
         final Optional<JsonElement> firstThumbnail = thumbNailStream.findFirst();
         final String thumbNailUrl = firstThumbnail.map(thumbNail -> thumbNail.asString("url")).orElse(null);
 

diff --git a/src/main/java/io/sfrei/tracksearch/utils/json/JsonElement.java b/src/main/java/io/sfrei/tracksearch/utils/json/JsonElement.java
@@ -21,6 +21,7 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.List;
 import java.util.Optional;
 import java.util.Spliterator;
 import java.util.Spliterators;
@@ -76,18 +77,18 @@ public Stream<JsonElement> arrayElements() {
     }
 
     public String asString(final String... paths) {
-        return super.asString(path(paths).node());
+        return super.asString(paths(paths).node());
     }
 
     public Long asLong(final String... paths) {
-        return getAsLong(path(paths).node());
+        return getAsLong(paths(paths).node());
     }
 
-    public JsonElement path(final String... paths) {
-        return nextElement(e -> nodeForPath(paths));
+    public JsonElement paths(final String... paths) {
+        return nextElement(e -> nodeForPaths(paths));
     }
 
-    private JsonNode nodeForPath(String... paths) {
+    private JsonNode nodeForPaths(String... paths) {
         if (paths.length == 0)
             return node();
 
@@ -106,6 +107,14 @@ public JsonElement firstElement() {
         return nextElement(node -> atIndex(0));
     }
 
+    public JsonElement lastForPath(final String path) {
+        return nextElement(node -> {
+            final List<JsonNode> nodes = node.findParents(path);
+            if (node.isEmpty()) return null;
+            return nodes.get(nodes.size() - 1).path(path);
+        });
+    }
+
     public JsonElement elementAtIndex(final int index) {
         return nextElement(node -> atIndex(index));
     }
@@ -150,7 +159,7 @@ public boolean isPresent() {
     }
 
     public boolean nodePresent(String path) {
-        return JsonElement.of(nodeForPath(path)).isPresent();
+        return JsonElement.of(nodeForPaths(path)).isPresent();
     }
 
 }
diff --git a/src/main/java/io/sfrei/tracksearch/utils/json/JsonNodeResolver.java b/src/main/java/io/sfrei/tracksearch/utils/json/JsonNodeResolver.java
@@ -34,12 +34,12 @@ public JsonNode node() {
         return node;
     }
 
-    protected boolean nodeIsNull(JsonNode node) {
+    protected boolean isNodeNull(JsonNode node) {
         return node == null || node.isNull();
     }
 
     protected boolean nodeIsNull() {
-        return nodeIsNull(node);
+        return isNodeNull(node);
     }
 
     public boolean isArray() {
@@ -51,11 +51,11 @@ protected ArrayNode toArrayNode() {
     }
 
     public String asString(final JsonNode node) {
-        return nodeIsNull(node) ? null : node.asText();
+        return isNodeNull(node) ? null : node.asText();
     }
 
     protected Long getAsLong(final JsonNode node) {
-        return nodeIsNull(node) ? null : node.asLong();
+        return isNodeNull(node) ? null : node.asLong();
     }
 
     protected JsonNode atIndex(final int index) {