Skip to content

Commit

Permalink
Retrieve MediaFormat for streams that could not be extracted by the e…
Browse files Browse the repository at this point in the history 
…xtractor
  • Loading branch information
@TobiGr
TobiGr committed Jul 18, 2023
1 parent 75891cb commit d0c1919
Show file tree
Hide file tree
Showing 3 changed files with 175 additions and 17 deletions.
2 changes: 1 addition & 1 deletion app/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,7 @@ dependencies {
// name and the commit hash with the commit hash of the (pushed) commit you want to test
// This works thanks to JitPack: https://jitpack.io/
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'com.github.TeamNewPipe:NewPipeExtractor:8495ad619e'
implementation 'com.github.TeamNewPipe:NewPipeExtractor:25718aeeda359c085945e6e7272a235ac2d03fb5'
implementation 'com.github.TeamNewPipe:NoNonsense-FilePicker:5.0.0'

/** Checkstyle **/
Expand Down
10 changes: 5 additions & 5 deletions app/src/main/java/org/schabi/newpipe/download/DownloadDialog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -766,7 +766,7 @@ private String getNameEditText() {
}

private void showFailedDialog(@StringRes final int msg) {
assureCorrectAppLanguage(getContext());
assureCorrectAppLanguage(requireContext());
new AlertDialog.Builder(context)
.setTitle(R.string.general_error)
.setMessage(msg)
Expand Down Expand Up @@ -799,7 +799,7 @@ private void prepareSelectedDownload() {
filenameTmp += "opus";
} else if (format != null) {
mimeTmp = format.mimeType;
filenameTmp += format.suffix;
filenameTmp += format.getSuffix();
}
break;
case R.id.video_button:
Expand All @@ -808,7 +808,7 @@ private void prepareSelectedDownload() {
format = videoStreamsAdapter.getItem(selectedVideoIndex).getFormat();
if (format != null) {
mimeTmp = format.mimeType;
filenameTmp += format.suffix;
filenameTmp += format.getSuffix();
}
break;
case R.id.subtitle_button:
Expand All @@ -820,9 +820,9 @@ private void prepareSelectedDownload() {
}

if (format == MediaFormat.TTML) {
filenameTmp += MediaFormat.SRT.suffix;
filenameTmp += MediaFormat.SRT.getSuffix();
} else if (format != null) {
filenameTmp += format.suffix;
filenameTmp += format.getSuffix();
}
break;
default:
Expand Down
180 changes: 169 additions & 11 deletions app/src/main/java/org/schabi/newpipe/util/StreamItemAdapter.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package org.schabi.newpipe.util;

import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;

import android.content.Context;
import android.view.LayoutInflater;
import android.view.View;
Expand All @@ -16,16 +18,23 @@
import org.schabi.newpipe.DownloaderImpl;
import org.schabi.newpipe.R;
import org.schabi.newpipe.extractor.MediaFormat;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.stream.AudioStream;
import org.schabi.newpipe.extractor.stream.Stream;
import org.schabi.newpipe.extractor.stream.SubtitlesStream;
import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Utils;

import java.io.Serializable;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.stream.Collectors;

import io.reactivex.rxjava3.android.schedulers.AndroidSchedulers;
import io.reactivex.rxjava3.core.Single;
Expand Down Expand Up @@ -228,6 +237,7 @@ public static class StreamInfoWrapper<T extends Stream> implements Serializable

private final List<T> streamsList;
private final long[] streamSizes;
private final MediaFormat[] streamFormats;
private final String unknownSize;

public StreamInfoWrapper(@NonNull final List<T> streamList,
Expand All @@ -236,31 +246,42 @@ public StreamInfoWrapper(@NonNull final List<T> streamList,
this.streamSizes = new long[streamsList.size()];
this.unknownSize = context == null
? "--.-" : context.getString(R.string.unknown_content);

resetSizes();
this.streamFormats = new MediaFormat[streamsList.size()];
resetInfo();
}

/**
* Helper method to fetch the sizes of all the streams in a wrapper.
* Helper method to fetch the sizes and missing media formats
* of all the streams in a wrapper.
*
* @param <X> the stream type's class extending {@link Stream}
* @param streamsWrapper the wrapper
* @return a {@link Single} that returns a boolean indicating if any elements were changed
*/
@NonNull
public static <X extends Stream> Single<Boolean> fetchSizeForWrapper(
public static <X extends Stream> Single<Boolean> fetchMoreInfoForWrapper(
final StreamInfoWrapper<X> streamsWrapper) {
final Callable<Boolean> fetchAndSet = () -> {
boolean hasChanged = false;
for (final X stream : streamsWrapper.getStreamsList()) {
if (streamsWrapper.getSizeInBytes(stream) > SIZE_UNSET) {
final boolean changeSize = streamsWrapper.getSizeInBytes(stream) <= SIZE_UNSET;
final boolean changeFormat = stream.getFormat() == null;
if (!changeSize && !changeFormat) {
continue;
}

final long contentLength = DownloaderImpl.getInstance().getContentLength(
stream.getContent());
streamsWrapper.setSize(stream, contentLength);
hasChanged = true;
final Response response = DownloaderImpl.getInstance()
.head(stream.getContent());
if (changeSize) {
final String contentLength = response.getHeader("Content-Length");
if (!isNullOrEmpty(contentLength)) {
streamsWrapper.setSize(stream, Long.parseLong(contentLength));
hasChanged = true;
}
}
if (changeFormat) {
hasChanged = retrieveMediaFormat(stream, streamsWrapper, response)
|| hasChanged;
}
}
return hasChanged;
};
Expand All @@ -271,8 +292,137 @@ public static <X extends Stream> Single<Boolean> fetchSizeForWrapper(
.onErrorReturnItem(true);
}

public void resetSizes() {
/**
* Try to retrieve the {@link MediaFormat} for a stream from the request headers.
*
* @param <X> the stream type to get the {@link MediaFormat} for
* @param stream the stream to find the {@link MediaFormat} for
* @param streamsWrapper the wrapper to store the found {@link MediaFormat} in
* @param response the response of the head request for the given stream
* @return {@code true} if the media format could be retrieved; {@code false} otherwise
*/
private static <X extends Stream> boolean retrieveMediaFormat(
@NonNull final X stream,
@NonNull final StreamInfoWrapper<X> streamsWrapper,
@NonNull final Response response) {
return retrieveMediaFormatFromFileTypeHeaders(stream, streamsWrapper, response)
|| retrieveMediaFormatFromContentDispositionHeader(
stream, streamsWrapper, response)
|| retrieveMediaFormatFromContentTypeHeader(stream, streamsWrapper, response);
}

private static <X extends Stream> boolean retrieveMediaFormatFromFileTypeHeaders(
@NonNull final X stream,
@NonNull final StreamInfoWrapper<X> streamsWrapper,
@NonNull final Response response) {
// try to use additional headers from CDNs or servers,
// e.g. x-amz-meta-file-type (e.g. for SoundCloud)
final List<String> keys = response.responseHeaders().keySet().stream()
.filter(k -> k.endsWith("file-type")).collect(Collectors.toList());
if (!keys.isEmpty()) {
for (final String key : keys) {
final String suffix = response.getHeader(key);
final MediaFormat format = MediaFormat.getFromSuffix(suffix);
if (format != null) {
streamsWrapper.setFormat(stream, format);
return true;
}
}
}
return false;
}

private static <X extends Stream> boolean retrieveMediaFormatFromContentDispositionHeader(
@NonNull final X stream,
@NonNull final StreamInfoWrapper<X> streamsWrapper,
@NonNull final Response response) {
// parse the Content-Disposition header,
// see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition
// there can be two filename directives
String contentDisposition = response.getHeader("Content-Disposition");
if (contentDisposition != null) {
try {
contentDisposition = Utils.decodeUrlUtf8(contentDisposition);
final String[] parts = contentDisposition.split(";");
for (String part : parts) {
final String fileName;
part = part.trim();
// extract the filename
if (part.startsWith("filename=")) {
// remove directive and decode
fileName = Utils.decodeUrlUtf8(part.substring(9));
} else if (part.startsWith("filename*=")) {
// The filename* directive passes the encoding of the filename, too.
// The encoding is prepended to the actual filename
// and the file name *should* be put into quotes.
// There needs to be a character (mostly one of the quotes)
// which separates the encoding from the filename
part = part.substring(10); // remove directive
final String encoding = Parser.matchGroup1(
"([0-9|A-Z|a-z|\\-|_]+)", part);
// remove encoding and separating character
part = part.substring(encoding.length() + 1);
final Charset charset = Charset.forName(encoding);
// decode info with the given encoding
if (charset == null) {
continue;
}
if (charset.equals(Charset.defaultCharset())) {
fileName = part;
} else {
fileName = new String(part.getBytes(charset),
StandardCharsets.UTF_8);
}
} else {
continue;
}

// extract the file extension / suffix
final String[] p = fileName.split("\\.");
String suffix = p[p.length - 1];
if (suffix.endsWith("\"") || suffix.endsWith("'")) {
// remove trailing quotes if present
suffix = suffix.substring(0, suffix.length() - 2);
}

final MediaFormat format = MediaFormat.getFromSuffix(suffix);
if (format != null) {
streamsWrapper.setFormat(stream, format);
return true;
}
}
} catch (final Exception ignored) { }
}
return false;
}

private static <X extends Stream> boolean retrieveMediaFormatFromContentTypeHeader(
@NonNull final X stream,
@NonNull final StreamInfoWrapper<X> streamsWrapper,
@NonNull final Response response) {
// try to get the format by content type
// some mime types are not unique for every format, those are omitted
final List<MediaFormat> formats = MediaFormat.getAllFromMimeType(
response.getHeader("Content-Type"));
final List<MediaFormat> uniqueFormats = new ArrayList<>(formats.size());
for (int i = 0; i < formats.size(); i++) {
final MediaFormat format = formats.get(i);
if (uniqueFormats.stream().filter(f -> f.id == format.id).count() == 0) {
uniqueFormats.add(format);
}
}
if (uniqueFormats.size() == 1) {
streamsWrapper.setFormat(stream, uniqueFormats.get(0));
return true;
}
return false;
}

public void resetInfo() {
Arrays.fill(streamSizes, SIZE_UNSET);
for (int i = 0; i < streamsList.size(); i++) {
streamFormats[i] = streamsList.get(i).getFormat();
}
}

public static <X extends Stream> StreamInfoWrapper<X> empty() {
Expand Down Expand Up @@ -306,5 +456,13 @@ private String formatSize(final long size) {
public void setSize(final T stream, final long sizeInBytes) {
streamSizes[streamsList.indexOf(stream)] = sizeInBytes;
}

public MediaFormat getFormat(final int streamIndex) {
return streamFormats[streamIndex];
}

public void setFormat(final T stream, final MediaFormat format) {
streamFormats[streamsList.indexOf(stream)] = format;
}
}
}

0 comments on commit d0c1919

Please sign in to comment.