Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[YouTube] Update clients versions, restore access to some streams and more #1168

Merged
merged 11 commits into from
Apr 20, 2024
Merged
  •  
  •  
  •  
Original file line number Diff line number Diff line change
Expand Up @@ -170,16 +170,34 @@ private static void getEmergencyOneboxRenderer(

// usually an encouragement like "We are with you"
final String title = getTextFromObjectOrThrow(r.getObject("title"), "title");

// usually a phone number
final String action = getTextFromObjectOrThrow(r.getObject("actionText"), "action");
final String action; // this variable is expected to start with "\n"
if (r.has("actionText")) {
action = "\n" + getTextFromObjectOrThrow(r.getObject("actionText"), "action");
} else if (r.has("contacts")) {
final JsonArray contacts = r.getArray("contacts");
final StringBuilder stringBuilder = new StringBuilder();
// Loop over contacts item from the first contact to the last one
for (int i = 0; i < contacts.size(); i++) {
stringBuilder.append("\n");
stringBuilder.append(getTextFromObjectOrThrow(contacts.getObject(i)
.getObject("actionText"), "contacts.actionText"));
}
action = stringBuilder.toString();
} else {
action = "";
}

// usually details about the phone number
final String details = getTextFromObjectOrThrow(r.getObject("detailsText"), "details");

// usually the name of an association
final String urlText = getTextFromObjectOrThrow(r.getObject("navigationText"),
"urlText");

metaInfo.setTitle(title);
metaInfo.setContent(new Description(details + "\n" + action, Description.PLAIN_TEXT));
metaInfo.setContent(new Description(details + action, Description.PLAIN_TEXT));
metaInfo.addUrlText(urlText);

// usually the webpage of the association
Expand Down
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You renamed a lot of things in this class, maybe it would be better to choose a naming that is independent of implementation details? E.g. isHardcodedClientValid instead of isHardcodedClientVersionValid. No need to make changes right now unless you have a clear idea, though.

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
Expand Down Expand Up @@ -422,8 +421,8 @@ private Page getNextPageFrom(final JsonObject continuations,
.done())
.getBytes(StandardCharsets.UTF_8);

return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, null, channelIds, null, body);
return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, null,
channelIds, null, body);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractCookieValue;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistId;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYouTubeHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
Expand Down Expand Up @@ -103,8 +102,8 @@ public void onFetchPage(@Nonnull final Downloader downloader)
final var headers = getYouTubeHeaders();

final Response response = getDownloader().postWithContentTypeJson(
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
headers, body, localization);
YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, headers, body,
localization);

initialData = JsonUtils.toJsonObject(getValidJsonResponseBody(response));
playlistData = initialData
Expand Down Expand Up @@ -225,7 +224,8 @@ private Page getNextPageFrom(@Nonnull final JsonObject playlistJson,
.done())
.getBytes(StandardCharsets.UTF_8);

return new Page(YOUTUBEI_V1_URL + "next?key=" + getKey(), null, null, cookies, body);
return new Page(YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER, null, null,
cookies, body);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getValidJsonResponseBody;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicClientVersion;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getYoutubeMusicHeaders;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ALBUMS;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.MUSIC_ARTISTS;
Expand All @@ -25,10 +26,8 @@
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandler;
import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;

import java.io.IOException;
Expand All @@ -52,10 +51,8 @@ public YoutubeMusicSearchExtractor(final StreamingService service,
@Override
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();

final String url = "https://music.youtube.com/youtubei/v1/search?key="
+ youtubeMusicKeys[0] + DISABLE_PRETTY_PRINT_PARAMETER;
final String url = "https://music.youtube.com/youtubei/v1/search?"
+ DISABLE_PRETTY_PRINT_PARAMETER;

final String params;

Expand Down Expand Up @@ -86,7 +83,7 @@ public void onFetchPage(@Nonnull final Downloader downloader)
.object("context")
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", youtubeMusicKeys[2])
.value("clientVersion", getYoutubeMusicClientVersion())
.value("hl", "en-GB")
.value("gl", getExtractorContentCountry().getCountryCode())
.value("platform", "DESKTOP")
Expand Down Expand Up @@ -206,15 +203,13 @@ public InfoItemsPage<InfoItem> getPage(final Page page)

final MultiInfoItemsCollector collector = new MultiInfoItemsCollector(getServiceId());

final String[] youtubeMusicKeys = YoutubeParsingHelper.getYoutubeMusicKey();

// @formatter:off
final byte[] json = JsonWriter.string()
.object()
.object("context")
.object("client")
.value("clientName", "WEB_REMIX")
.value("clientVersion", youtubeMusicKeys[2])
.value("clientVersion", getYoutubeMusicClientVersion())
.value("hl", "en-GB")
.value("gl", getExtractorContentCountry().getCountryCode())
.value("platform", "DESKTOP")
Expand Down Expand Up @@ -295,8 +290,7 @@ private void collectMusicStreamsFrom(final MultiInfoItemsCollector collector,
}

@Nullable
private Page getNextPageFrom(final JsonArray continuations)
throws IOException, ParsingException, ReCaptchaException {
private Page getNextPageFrom(final JsonArray continuations) {
if (isNullOrEmpty(continuations)) {
return null;
}
Expand All @@ -306,7 +300,6 @@ private Page getNextPageFrom(final JsonArray continuations)
final String continuation = nextContinuationData.getString("continuation");

return new Page("https://music.youtube.com/youtubei/v1/search?ctoken=" + continuation
+ "&continuation=" + continuation + "&key="
+ YoutubeParsingHelper.getYoutubeMusicKey()[0] + DISABLE_PRETTY_PRINT_PARAMETER);
+ "&continuation=" + continuation + "&" + DISABLE_PRETTY_PRINT_PARAMETER);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.extractPlaylistTypeFromPlaylistUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getImagesFromThumbnailsArray;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
Expand Down Expand Up @@ -387,8 +386,7 @@ private Page getNextPageFrom(final JsonArray contents)
.done())
.getBytes(StandardCharsets.UTF_8);

return new Page(YOUTUBEI_V1_URL + "browse?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER, body);
return new Page(YOUTUBEI_V1_URL + "browse?" + DISABLE_PRETTY_PRINT_PARAMETER, body);
} else {
return null;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonPostResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;
import static org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeSearchQueryHandlerFactory.ALL;
Expand Down Expand Up @@ -247,8 +246,7 @@ private void collectStreamsFrom(final MultiInfoItemsCollector collector,
}

@Nullable
private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws IOException,
ExtractionException {
private Page getNextPageFrom(final JsonObject continuationItemRenderer) {
if (isNullOrEmpty(continuationItemRenderer)) {
return null;
}
Expand All @@ -257,8 +255,7 @@ private Page getNextPageFrom(final JsonObject continuationItemRenderer) throws I
.getObject("continuationCommand")
.getString("token");

final String url = YOUTUBEI_V1_URL + "search?key=" + getKey()
+ DISABLE_PRETTY_PRINT_PARAMETER;
final String url = YOUTUBEI_V1_URL + "search?" + DISABLE_PRETTY_PRINT_PARAMETER;

return new Page(url, token);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -965,7 +965,7 @@ private void fetchAndroidMobileJsonPlayer(@Nonnull final ContentCountry contentC
.value(RACY_CHECK_OK, true)
// Workaround getting streaming URLs which return 403 HTTP response code by
// using some parameters for Android client requests
.value("params", "CgIQBg")
.value("params", "CgIIAQ%3D%3D")
Stypox marked this conversation as resolved.
Show resolved Hide resolved
.done())
.getBytes(StandardCharsets.UTF_8);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,8 +171,17 @@ public static void assertContains(
public static void assertTabsContain(@Nonnull final List<ListLinkHandler> tabs,
@Nonnull final String... expectedTabs) {
final Set<String> tabSet = tabs.stream()
.map(linkHandler -> linkHandler.getContentFilters().get(0))
.map(linkHandler -> {
assertEquals(1, linkHandler.getContentFilters().size(),
"Unexpected content filters for channel tab: "
+ linkHandler.getContentFilters());
return linkHandler.getContentFilters().get(0);
})
.collect(Collectors.toUnmodifiableSet());

assertEquals(expectedTabs.length, tabSet.size(),
"Different amount of tabs returned:\nExpected: "
+ Arrays.toString(expectedTabs) + "\nActual: " + tabSet);
Arrays.stream(expectedTabs)
.forEach(expectedTab -> assertTrue(tabSet.contains(expectedTab),
"Missing " + expectedTab + " tab (got " + tabSet + ")"));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.schabi.newpipe.extractor.exceptions.AccountTerminatedException;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.linkhandler.ReadyChannelTabListLinkHandler;
import org.schabi.newpipe.extractor.services.BaseChannelExtractorTest;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
Expand Down Expand Up @@ -540,7 +541,8 @@ public void testVerified() throws Exception {
@Test
@Override
public void testTabs() throws Exception {
assertTabsContain(extractor.getTabs(), ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS);
assertTabsContain(extractor.getTabs(),
ChannelTabs.VIDEOS, ChannelTabs.PLAYLISTS, ChannelTabs.SHORTS);
assertTrue(extractor.getTabs().stream()
.filter(it -> ChannelTabs.VIDEOS.equals(it.getContentFilters().get(0)))
.allMatch(ReadyChannelTabListLinkHandler.class::isInstance));
Expand Down Expand Up @@ -916,8 +918,10 @@ public void testVerified() throws Exception {
@Test
@Override
public void testTabs() throws Exception {
// Gaming topic channels tabs are not yet supported, so an empty list should be returned
assertTrue(extractor.getTabs().isEmpty());
// Gaming topic channels tabs are not yet supported
// However, a Shorts tab like on other channel types is returned, so it is supported
// Check that it is returned
assertTabsContain(extractor.getTabs(), ChannelTabs.SHORTS);
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeOtfDashManifestCreator;
import org.schabi.newpipe.extractor.services.youtube.dashmanifestcreators.YoutubeProgressiveDashManifestCreator;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
import org.schabi.newpipe.extractor.stream.AudioTrackType;
import org.schabi.newpipe.extractor.stream.DeliveryMethod;
import org.schabi.newpipe.extractor.stream.Stream;
import org.w3c.dom.Document;
Expand Down Expand Up @@ -89,7 +88,7 @@ class YoutubeDashManifestCreatorsTest {

@BeforeAll
public static void setUp() throws Exception {
YoutubeParsingHelper.resetClientVersionAndKey();
YoutubeParsingHelper.resetClientVersion();
YoutubeParsingHelper.setNumberGenerator(new Random(1));
NewPipe.init(DownloaderTestImpl.getInstance());

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import static org.schabi.newpipe.extractor.ServiceList.YouTube;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.DISABLE_PRETTY_PRINT_PARAMETER;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.YOUTUBEI_V1_URL;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getKey;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.prepareDesktopJsonBuilder;

import com.grack.nanojson.JsonWriter;
Expand Down Expand Up @@ -91,7 +90,7 @@ void getPage() throws Exception {
.getBytes(StandardCharsets.UTF_8);

final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
Expand Down Expand Up @@ -180,7 +179,7 @@ void getPage() throws Exception {
.getBytes(StandardCharsets.UTF_8);

final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
Expand Down Expand Up @@ -268,7 +267,7 @@ void getPage() throws Exception {
.getBytes(StandardCharsets.UTF_8);

final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
Expand Down Expand Up @@ -386,7 +385,7 @@ void getPage() throws Exception {
.getBytes(StandardCharsets.UTF_8);

final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
Expand Down Expand Up @@ -453,7 +452,7 @@ void getPage() throws Exception {
.getBytes(StandardCharsets.UTF_8);

final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
Expand Down Expand Up @@ -542,7 +541,7 @@ void getPage() throws Exception {
.getBytes(StandardCharsets.UTF_8);

final InfoItemsPage<StreamInfoItem> streams = extractor.getPage(new Page(
YOUTUBEI_V1_URL + "next?key=" + getKey() + DISABLE_PRETTY_PRINT_PARAMETER,
YOUTUBEI_V1_URL + "next?" + DISABLE_PRETTY_PRINT_PARAMETER,
null, null, dummyCookie, body));
assertFalse(streams.getItems().isEmpty());
assertTrue(streams.hasNextPage());
Expand Down
Loading
Loading