Add a static method to reset the YouTube deobfuscation code and use i…

…t in tests This method is needed for YouTube stream tests, because when all YouTube tests are ran, the signatureTimestamp is known (the sts string) so a different body than the body present in the mocks is send by the extractor instance. As a result, running all YouTube stream tests with the MockDownloader (like the CI does) will fail if this method is not called before fetching the page of a test.
TeamNewPipe · Jun 15, 2021 · e4b5115 · e4b5115
1 parent e4f3f02
commit e4b5115
Show file tree

Hide file tree

Showing 6 changed files with 39 additions and 1 deletion.
diff --git a/...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java b/...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
@@ -744,7 +744,8 @@ public void onFetchPage(@Nonnull final Downloader downloader)
     }
 
     private void checkPlayabilityStatus(final JsonObject youtubePlayerResponse,
-                                        JsonObject playabilityStatus) throws ParsingException {
+                                        @Nonnull JsonObject playabilityStatus)
+            throws ParsingException {
         String status = playabilityStatus.getString("status");
         // If status exist, and is not "OK", throw the specific exception based on error message
         // or a ContentNotAvailableException with the reason text if it's an unknown reason.
@@ -855,6 +856,7 @@ private void fetchVideoInfoPage() throws ParsingException, ReCaptchaException, I
         }
     }
 
+    @Nonnull
     private byte[] createPlayerBodyWithSts(final Localization localization,
                                            final ContentCountry contentCountry,
                                            final String videoId) throws ExtractionException,
@@ -972,6 +974,7 @@ private String getDeobfuscationFuncName(final String playerCode) throws Deobfusc
                 "Could not find deobfuscate function with any of the given patterns.", exception);
     }
 
+    @Nonnull
     private String loadDeobfuscationCode() throws DeobfuscateException {
         try {
             final String deobfuscationFunctionName = getDeobfuscationFuncName(playerCode);
@@ -1099,6 +1102,7 @@ private static String getVideoInfoUrl(final String id, final String sts) {
                 "&sts=" + sts + "&ps=default&gl=US&hl=en";
     }
 
+    @Nonnull
     private Map<String, ItagItem> getItags(final String streamingDataKey,
                                            final ItagItem.ItagType itagTypeWanted)
             throws ParsingException {
@@ -1346,4 +1350,21 @@ public List<MetaInfo> getMetaInfo() throws ParsingException {
                 nextResponse.getObject("contents").getObject("twoColumnWatchNextResults")
                         .getObject("results").getObject("results").getArray("contents"));
     }
+
+    /**
+     * Reset YouTube deobfuscation code.
+     * <p>
+     * This is needed for mocks in YouTube stream tests, because when they are ran, the
+     * {@code signatureTimestamp} is known (the {@code sts} string) so a different body than the
+     * body present in the mocks is send by the extractor instance. As a result, running all
+     * YouTube stream tests with the MockDownloader (like the CI does) will fail if this method is
+     * not called before fetching the page of a test.
+     * </p>
+     */
+    public static void resetDeobfuscationCode() {
+        cachedDeobfuscationCode = null;
+        playerCode = null;
+        playerJsUrl = null;
+        sts = null;
+    }
 }
diff --git a/...bi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java b/...bi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorAgeRestrictedTest.java
@@ -6,6 +6,7 @@
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
@@ -28,6 +29,7 @@ public class YoutubeStreamExtractorAgeRestrictedTest extends DefaultStreamExtrac
     public static void setUp() throws Exception {
         YoutubeParsingHelper.resetClientVersionAndKey();
         YoutubeParsingHelper.setNumberGenerator(new Random(1));
+        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ageRestricted"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

diff --git a/...bi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java b/...bi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorControversialTest.java
@@ -7,6 +7,7 @@
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeStreamLinkHandlerFactory;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
@@ -33,6 +34,7 @@ public class YoutubeStreamExtractorControversialTest extends DefaultStreamExtrac
     public static void setUp() throws Exception {
         YoutubeParsingHelper.resetClientVersionAndKey();
         YoutubeParsingHelper.setNumberGenerator(new Random(1));
+        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "controversial"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

diff --git a/...g/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java b/...g/schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorDefaultTest.java
@@ -63,6 +63,7 @@ public static class NotAvailable {
         public static void setUp() throws IOException {
             YoutubeParsingHelper.resetClientVersionAndKey();
             YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "notAvailable"));
         }
 
@@ -119,6 +120,7 @@ public static class DescriptionTestPewdiepie extends DefaultStreamExtractorTest
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "pewdiwpie"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -162,6 +164,7 @@ public static class DescriptionTestUnboxing extends DefaultStreamExtractorTest {
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unboxing"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -215,6 +218,7 @@ public static class RatingsDisabledTest extends DefaultStreamExtractorTest {
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "ratingsDisabled"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -252,6 +256,7 @@ public static class StreamSegmentsTestTagesschau extends DefaultStreamExtractorT
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsTagesschau"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -313,6 +318,7 @@ public static class StreamSegmentsTestMaiLab extends DefaultStreamExtractorTest
         public static void setUp() throws Exception {
             YoutubeParsingHelper.resetClientVersionAndKey();
             YoutubeParsingHelper.setNumberGenerator(new Random(1));
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "streamSegmentsMaiLab"));
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();
@@ -383,6 +389,7 @@ public static void setUp() throws Exception {
             YoutubeParsingHelper.setNumberGenerator(new Random(1));
             NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "publicBroadcast"));
             extractor = YouTube.getStreamExtractor(URL);
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             extractor.fetchPage();
         }
 
@@ -432,6 +439,7 @@ public static class UnlistedTest {
 
         @BeforeClass
         public static void setUp() throws Exception {
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = (YoutubeStreamExtractor) YouTube
                     .getStreamExtractor("https://www.youtube.com/watch?v=tjz2u2DiveM");
@@ -451,6 +459,7 @@ public static class CCLicensed {
 
         @BeforeClass
         public static void setUp() throws Exception {
+            YoutubeStreamExtractor.resetDeobfuscationCode();
             NewPipe.init(DownloaderTestImpl.getInstance());
             extractor = YouTube.getStreamExtractor(URL);
             extractor.fetchPage();

diff --git a/...chabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java b/...chabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorLivestreamTest.java
@@ -8,6 +8,7 @@
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
@@ -30,6 +31,7 @@ public class YoutubeStreamExtractorLivestreamTest extends DefaultStreamExtractor
     public static void setUp() throws Exception {
         YoutubeParsingHelper.resetClientVersionAndKey();
         YoutubeParsingHelper.setNumberGenerator(new Random(1));
+        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "live"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();

diff --git a/.../schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java b/.../schabi/newpipe/extractor/services/youtube/stream/YoutubeStreamExtractorUnlistedTest.java
@@ -6,6 +6,7 @@
 import org.schabi.newpipe.extractor.StreamingService;
 import org.schabi.newpipe.extractor.services.DefaultStreamExtractorTest;
 import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
+import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeStreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamExtractor;
 import org.schabi.newpipe.extractor.stream.StreamType;
 
@@ -28,6 +29,7 @@ public class YoutubeStreamExtractorUnlistedTest extends DefaultStreamExtractorTe
     public static void setUp() throws Exception {
         YoutubeParsingHelper.resetClientVersionAndKey();
         YoutubeParsingHelper.setNumberGenerator(new Random(1));
+        YoutubeStreamExtractor.resetDeobfuscationCode();
         NewPipe.init(new DownloaderFactory().getDownloader(RESOURCE_PATH + "unlisted"));
         extractor = YouTube.getStreamExtractor(URL);
         extractor.fetchPage();