Skip to content

Commit

Permalink
fix: add TeamNewPipe#1050 fix to channel tab name extraction
Browse files Browse the repository at this point in the history 
use shared method for channel header extraction
  • Loading branch information
@Theta-Dev
Theta-Dev committed May 7, 2023
1 parent 2adc2ca commit e8fab3b
Show file tree
Hide file tree
Showing 3 changed files with 87 additions and 55 deletions.
44 changes: 39 additions & 5 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YouTubeChannelHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import javax.annotation.Nonnull;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Optional;

import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;

Expand All @@ -22,6 +23,7 @@ private YouTubeChannelHelper() {

/**
* Take a YouTube channel ID or URL path, resolve it if necessary and return a channel ID.
*
* @param idOrPath YouTube channel ID or URL path
* @return YouTube Channel ID
*/
Expand All @@ -38,7 +40,7 @@ public static String resolveChannelId(final String idOrPath)
// we couldn't get information about the channel associated with this URL, if there is one.
if (!channelId[0].equals("channel")) {
final byte[] body = JsonWriter.string(YoutubeParsingHelper.prepareDesktopJsonBuilder(
Localization.DEFAULT, ContentCountry.DEFAULT)
Localization.DEFAULT, ContentCountry.DEFAULT)
.value("url", "https://www.youtube.com/" + idOrPath)
.done())
.getBytes(StandardCharsets.UTF_8);
Expand Down Expand Up @@ -96,9 +98,9 @@ private ChannelResponseData(final JsonObject responseJson, final String channelI
* </ul>
*
* @param channelId YouTube channel ID
* @param params Parameters to specify the YouTube channel tab
* @param loc YouTube localization
* @param country YouTube content country
* @param params Parameters to specify the YouTube channel tab
* @param loc YouTube localization
* @param country YouTube content country
* @return Channel response data
*/
public static ChannelResponseData getChannelResponse(final String channelId,
Expand All @@ -112,7 +114,7 @@ public static ChannelResponseData getChannelResponse(final String channelId,
int level = 0;
while (level < 3) {
final byte[] body = JsonWriter.string(YoutubeParsingHelper.prepareDesktopJsonBuilder(
loc, country)
loc, country)
.value("browseId", id)
.value("params", params) // Equal to videos
.done())
Expand Down Expand Up @@ -161,6 +163,7 @@ public static ChannelResponseData getChannelResponse(final String channelId,

/**
* Assert that a channel JSON response does not contain a 404 error.
*
* @param jsonResponse channel JSON response
* @throws ContentNotAvailableException if the channel was not found
*/
Expand All @@ -178,4 +181,35 @@ private static void checkIfChannelResponseIsValid(@Nonnull final JsonObject json
}
}
}

public static final class ChannelHeader {
public final JsonObject json;
public final boolean isCarouselHeader;

private ChannelHeader(final JsonObject json, final boolean isCarouselHeader) {
this.json = json;
this.isCarouselHeader = isCarouselHeader;
}
}

public static Optional<ChannelHeader> getChannelHeader(final JsonObject initialData) {
final JsonObject h = initialData.getObject("header");

if (h.has("c4TabbedHeaderRenderer")) {
return Optional.of(h.getObject("c4TabbedHeaderRenderer"))
.map(json -> new ChannelHeader(json, false));
} else if (h.has("carouselHeaderRenderer")) {
return h.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
.findFirst()
.map(itm -> itm.getObject("topicChannelDetailsRenderer"))
.map(json -> new ChannelHeader(json, true));
} else {
return Optional.empty();
}
}
}
81 changes: 33 additions & 48 deletions ...ava/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;
Expand Down Expand Up @@ -55,8 +56,7 @@

public class YoutubeChannelExtractor extends ChannelExtractor {
private JsonObject initialData;
private Optional<JsonObject> channelHeader;
private boolean isCarouselHeader = false;
private Optional<YouTubeChannelHelper.ChannelHeader> channelHeader;
private JsonObject videoTab;

/**
Expand Down Expand Up @@ -90,25 +90,9 @@ public void onFetchPage(@Nonnull final Downloader downloader)
}

@Nonnull
private Optional<JsonObject> getChannelHeader() {
private Optional<YouTubeChannelHelper.ChannelHeader> getChannelHeader() {
if (channelHeader == null) {
final JsonObject h = initialData.getObject("header");

if (h.has("c4TabbedHeaderRenderer")) {
channelHeader = Optional.of(h.getObject("c4TabbedHeaderRenderer"));
} else if (h.has("carouselHeaderRenderer")) {
isCarouselHeader = true;
channelHeader = h.getObject("carouselHeaderRenderer")
.getArray("contents")
.stream()
.filter(JsonObject.class::isInstance)
.map(JsonObject.class::cast)
.filter(itm -> itm.has("topicChannelDetailsRenderer"))
.findFirst()
.map(itm -> itm.getObject("topicChannelDetailsRenderer"));
} else {
channelHeader = Optional.empty();
}
channelHeader = YouTubeChannelHelper.getChannelHeader(initialData);
}
return channelHeader;
}
Expand All @@ -127,8 +111,8 @@ public String getUrl() throws ParsingException {
@Override
public String getId() throws ParsingException {
return getChannelHeader()
.flatMap(header -> Optional.ofNullable(header.getString("channelId")).or(
() -> Optional.ofNullable(header.getObject("navigationEndpoint")
.flatMap(header -> Optional.ofNullable(header.json.getString("channelId")).or(
() -> Optional.ofNullable(header.json.getObject("navigationEndpoint")
.getObject("browseEndpoint")
.getString("browseId"))
))
Expand All @@ -146,26 +130,24 @@ public String getName() throws ParsingException {
return mdName;
}

final Optional<JsonObject> header = getChannelHeader();
if (header.isPresent()) {
final Object title = header.get().get("title");
return getChannelHeader().flatMap(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
return Optional.of((String) title);
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
return Optional.of(headerName);
}
}
}

throw new ParsingException("Could not get channel name");
return Optional.empty();
}).orElseThrow(() -> new ParsingException("Could not get channel name"));
}

@Override
public String getAvatarUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("avatar").getArray("thumbnails")
header.json.getObject("avatar").getArray("thumbnails")
.getObject(0).getString("url")
))
.map(YoutubeParsingHelper::fixThumbnailUrl)
Expand All @@ -175,7 +157,7 @@ public String getAvatarUrl() throws ParsingException {
@Override
public String getBannerUrl() throws ParsingException {
return getChannelHeader().flatMap(header -> Optional.ofNullable(
header.getObject("banner").getArray("thumbnails")
header.json.getObject("banner").getArray("thumbnails")
.getObject(0).getString("url")
))
.filter(url -> !url.contains("s.ytimg.com") && !url.contains("default_banner"))
Expand All @@ -194,14 +176,15 @@ public String getFeedUrl() throws ParsingException {

@Override
public long getSubscriberCount() throws ParsingException {
final Optional<JsonObject> header = getChannelHeader();
if (header.isPresent()) {
final Optional<YouTubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final JsonObject header = headerOpt.get().json;
JsonObject textObject = null;

if (header.get().has("subscriberCountText")) {
textObject = header.get().getObject("subscriberCountText");
} else if (header.get().has("subtitle")) {
textObject = header.get().getObject("subtitle");
if (header.has("subscriberCountText")) {
textObject = header.getObject("subscriberCountText");
} else if (header.has("subtitle")) {
textObject = header.getObject("subtitle");
}

if (textObject != null) {
Expand Down Expand Up @@ -242,17 +225,19 @@ public String getParentChannelAvatarUrl() {

@Override
public boolean isVerified() throws ParsingException {
// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (isCarouselHeader) {
return true;
final Optional<YouTubeChannelHelper.ChannelHeader> headerOpt = getChannelHeader();
if (headerOpt.isPresent()) {
final YouTubeChannelHelper.ChannelHeader header = headerOpt.get();

// The CarouselHeaderRenderer does not contain any verification badges.
// Since it is only shown on YT-internal channels or on channels of large organizations
// broadcasting live events, we can assume the channel to be verified.
if (header.isCarouselHeader) {
return true;
}
return YoutubeParsingHelper.isVerified(header.json.getArray("badges"));
}

return getChannelHeader()
.map(header -> header.getArray("badges"))
.map(YoutubeParsingHelper::isVerified)
.orElse(false);
return false;
}

@Nonnull
Expand Down
17 changes: 15 additions & 2 deletions .../org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelTabExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.schabi.newpipe.extractor.linkhandler.ChannelTabs;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.localization.Localization;
import org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelTabLinkHandlerFactory;

import javax.annotation.Nonnull;
Expand All @@ -31,6 +32,7 @@
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.getChannelResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.defaultAlertsCheck;
import static org.schabi.newpipe.extractor.services.youtube.YouTubeChannelHelper.resolveChannelId;
Expand Down Expand Up @@ -155,8 +157,19 @@ protected String getChannelName() {
return mdName;
}

return initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
.getString("title", "");
return YouTubeChannelHelper.getChannelHeader(initialData)
.map(header -> {
final Object title = header.json.get("title");
if (title instanceof String) {
return (String) title;
} else if (title instanceof JsonObject) {
final String headerName = getTextFromObject((JsonObject) title);
if (!isNullOrEmpty(headerName)) {
return headerName;
}
}
return "";
}).orElse("");
}

@Nonnull
Expand Down

0 comments on commit e8fab3b

Please sign in to comment.