Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[YouTube] Parse and execute deobfuscation code without JavaScript #529

Closed
wants to merge 1 commit into from
Closed

[YouTube] Parse and execute deobfuscation code without JavaScript #529

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion extractor/build.gradle
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ dependencies {

implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'org.jsoup:jsoup:1.13.1'
implementation 'org.mozilla:rhino:1.7.12'
implementation 'com.github.spotbugs:spotbugs-annotations:4.0.2'
implementation 'org.nibor.autolink:autolink:0.10.0'

Expand Down
161 changes: 123 additions & 38 deletions ...java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,11 @@
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.mozilla.javascript.Context;
import org.mozilla.javascript.Function;
import org.mozilla.javascript.ScriptableObject;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.MetaInfo;
import org.schabi.newpipe.extractor.NewPipe;
Expand All @@ -28,20 +26,41 @@
import org.schabi.newpipe.extractor.services.youtube.ItagItem;
import org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
import org.schabi.newpipe.extractor.stream.*;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Description;
import org.schabi.newpipe.extractor.stream.Frameset;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.StreamExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.stream.StreamSegment;
import org.schabi.newpipe.extractor.stream.StreamType;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.time.LocalDate;
import java.time.OffsetDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Pattern;

import javax.annotation.Nonnull;
import javax.annotation.Nullable;

import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.fixThumbnailUrl;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getJsonResponse;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getTextFromObject;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getUrlFromNavigationEndpoint;
import static org.schabi.newpipe.extractor.utils.Utils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;

Expand Down Expand Up @@ -78,10 +97,9 @@ public static class DeobfuscateException extends ParsingException {

/*//////////////////////////////////////////////////////////////////////////*/

@Nullable
private static String cachedDeobfuscationCode = null;
@Nullable
private String playerJsUrl = null;

@Nullable private static List<DeobfuscationFunction> cachedDeobfuscationFunctions = null;
@Nullable private String playerJsUrl = null;

private JsonArray initialAjaxJson;
private JsonObject initialData;
Expand Down Expand Up @@ -681,7 +699,6 @@ public String getErrorMessage() {
private static final String FORMATS = "formats";
private static final String ADAPTIVE_FORMATS = "adaptiveFormats";
private static final String HTTPS = "https:";
private static final String DEOBFUSCATION_FUNC_NAME = "deobfuscate";

private final static String[] REGEXES = {
"(?:\\b|[^a-zA-Z0-9$])([a-zA-Z0-9$]{2})\\s*=\\s*function\\(\\s*a\\s*\\)\\s*\\{\\s*a\\s*=\\s*a\\.split\\(\\s*\"\"\\s*\\)",
Expand Down Expand Up @@ -747,7 +764,7 @@ private String getEmbeddedInfoStsAndStorePlayerJsUrl() {
.replace("\\", "").replace("\"", "");
} catch (Parser.RegexException ex) {
// playerJsUrl is still available in the file, just somewhere else TODO
// it is ok not to find it, see how that's handled in getDeobfuscationCode()
// it is ok not to find it, see how that's handled in getDeobfuscationFunctions()
final Document doc = Jsoup.parse(embedPageContent);
final Elements elems = doc.select("script").attr("name", "player_ias/base");
for (Element elem : elems) {
Expand All @@ -767,6 +784,60 @@ private String getEmbeddedInfoStsAndStorePlayerJsUrl() {
}


///////////////////////////////////////////////////////////////
// Deobfuscation functions
///////////////////////////////////////////////////////////////

public static final Pattern SWAP_FIRST_AND_INDEX_REGEX = Pattern.compile("([a-zA-Z0-9$]+):function\\(a,b\\)\\{v");
public static final Pattern SLICE_AT_REGEX = Pattern.compile("([a-zA-Z0-9$]+):function\\(a,b\\)\\{a");
public static final Pattern REVERSE_REGEX = Pattern.compile("([a-zA-Z0-9$]+):function\\(a\\)");
public static final Pattern CONTAINER_REGEX = Pattern.compile("var ([a-zA-Z0-9$]+)=\\{");
public static final Pattern STEP_INDEX_REGEX = Pattern.compile("[a-zA-Z0-9$]+\\(a,([0-9]+)\\)");
public static final Pattern STEP_NAME_REGEX = Pattern.compile("([a-zA-Z0-9$]+)\\(");

private interface DeobfuscationFunction {
String transform(String cipher);
}

private static class SwapFirstAndIndexFunction implements DeobfuscationFunction {

final int index;
SwapFirstAndIndexFunction(final int index) {
this.index = index;
}

@Override
public String transform(final String cipher) {
return cipher.charAt(index) + cipher.substring(1, index) + cipher.charAt(0)
+ cipher.substring(index + 1);
}
}

private static class SliceAtFunction implements DeobfuscationFunction {

final int index;
SliceAtFunction(final int index) {
this.index = index;
}

@Override
public String transform(final String cipher) {
return cipher.substring(index);
}
}

private static class ReverseFunction implements DeobfuscationFunction {
@Override
public String transform(final String cipher) {
return new StringBuilder(cipher).reverse().toString();
}
}


///////////////////////////////////////////////////////////////
// Deobfuscation
///////////////////////////////////////////////////////////////

private String getDeobfuscationFuncName(final String playerCode) throws DeobfuscateException {
Parser.RegexException exception = null;
for (final String regex : REGEXES) {
Expand Down Expand Up @@ -800,10 +871,7 @@ private String loadDeobfuscationCode(@Nonnull final String playerJsUrl)
final String helperObject =
Parser.matchGroup1(helperPattern, playerCode.replace("\n", ""));

final String callerFunction =
"function " + DEOBFUSCATION_FUNC_NAME + "(a){return " + deobfuscationFunctionName + "(a);}";

return helperObject + deobfuscateFunction + callerFunction;
return helperObject + deobfuscateFunction;
} catch (IOException ioe) {
throw new DeobfuscateException("Could not load deobfuscate function", ioe);
} catch (Exception e) {
Expand All @@ -812,8 +880,9 @@ private String loadDeobfuscationCode(@Nonnull final String playerJsUrl)
}

@Nonnull
private String getDeobfuscationCode() throws ParsingException {
if (cachedDeobfuscationCode == null) {
private List<DeobfuscationFunction> getDeobfuscationFunctions() throws ParsingException {
if (cachedDeobfuscationFunctions == null) {
// extract deobfuscation code
if (playerJsUrl == null) {
// the currentPlayerJsUrl was not found in any page fetched so far and there is
// nothing cached, so try fetching embedded info
Expand All @@ -831,28 +900,44 @@ private String getDeobfuscationCode() throws ParsingException {
playerJsUrl = HTTPS + "//www.youtube.com" + playerJsUrl;
}

cachedDeobfuscationCode = loadDeobfuscationCode(playerJsUrl);
final String deobfuscationCode = loadDeobfuscationCode(playerJsUrl);
final String swapFirstAndIndexName = Parser.matchGroup1(SWAP_FIRST_AND_INDEX_REGEX, deobfuscationCode);
final String sliceAtName = Parser.matchGroup1(SLICE_AT_REGEX, deobfuscationCode);
final String reverseName = Parser.matchGroup1(REVERSE_REGEX, deobfuscationCode);
final String containerName = Parser.matchGroup1(CONTAINER_REGEX, deobfuscationCode);

final String[] stringSteps = deobfuscationCode.split(containerName + "\\.");
boolean first = true;
cachedDeobfuscationFunctions = new ArrayList<>();
for (final String step : stringSteps) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since this is an array you can just use a for i loop with i=1 instead of the weird first=true setup

if (first) {
first = false;
continue;
}

final int index = Integer.parseInt(Parser.matchGroup1(STEP_INDEX_REGEX, step));
final String name = Parser.matchGroup1(STEP_NAME_REGEX, step);

if (name.equals(swapFirstAndIndexName)) {
cachedDeobfuscationFunctions.add(new SwapFirstAndIndexFunction(index));
} else if (name.equals(sliceAtName)) {
cachedDeobfuscationFunctions.add(new SliceAtFunction(index));
} else if (name.equals(reverseName)) {
cachedDeobfuscationFunctions.add(new ReverseFunction());
} else {
throw new ParsingException("Unexpected function name: " + name);
}
}
}
return cachedDeobfuscationCode;
return cachedDeobfuscationFunctions;
}

private String deobfuscateSignature(final String obfuscatedSig) throws ParsingException {
final String deobfuscationCode = getDeobfuscationCode();

final Context context = Context.enter();
context.setOptimizationLevel(-1);
final Object result;
try {
final ScriptableObject scope = context.initSafeStandardObjects();
context.evaluateString(scope, deobfuscationCode, "deobfuscationCode", 1, null);
final Function deobfuscateFunc = (Function) scope.get(DEOBFUSCATION_FUNC_NAME, scope);
result = deobfuscateFunc.call(context, scope, scope, new Object[]{obfuscatedSig});
} catch (Exception e) {
throw new DeobfuscateException("Could not get deobfuscate signature", e);
} finally {
Context.exit();
private String deobfuscateSignature(String obfuscatedSig) throws ParsingException {
final List<DeobfuscationFunction> deobfuscationFunctions = getDeobfuscationFunctions();
for (final DeobfuscationFunction deobfuscationFunction : deobfuscationFunctions) {
obfuscatedSig = deobfuscationFunction.transform(obfuscatedSig);
}
return Objects.toString(result, "");
return obfuscatedSig;
}

/*//////////////////////////////////////////////////////////////////////////
Expand Down