Skip to content

Commit

Permalink
Align audio adaptive support checks with video
Browse files Browse the repository at this point in the history
In particular:
 - Add allowAudioNonSeamlessAdaptiveness parameter (default true, same
   as video and as already implemented by default)
 - Forward mixedMimeTypeAdaptation support to AudioTrackScore
   (as for VideoTrackScore) and adapt mixed MIME type adaptive
   support accordingly
 - Check adaptive support when deciding whether a track is allowed for
   adaptation (also same check as for video). This takes the new
   parameter into account.

#minor-release

PiperOrigin-RevId: 572191308
  • Loading branch information
tonihei authored and copybara-github committed Oct 10, 2023
1 parent d60596c commit f20d18e
Show file tree
Hide file tree
Showing 3 changed files with 166 additions and 13 deletions.
3 changes: 3 additions & 0 deletions RELEASENOTES.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,9 @@
* ExoPlayer:
* Transformer:
* Track Selection:
* Add `DefaultTrackSelector.Parameters.allowAudioNonSeamlessAdaptiveness`
to explicitly allow or disallow non-seamless adaptation. The default
stays at its current behavior of `true`.
* Extractors:
* Audio:
* Video:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -386,7 +386,9 @@ public ParametersBuilder setExceedAudioConstraintsIfNecessary(
/**
* Sets whether to allow adaptive audio selections containing mixed MIME types.
*
* <p>Adaptations between different MIME types may not be completely seamless.
* <p>Adaptations between different MIME types may not be completely seamless, in which case
* {@link Parameters.Builder#setAllowAudioNonSeamlessAdaptiveness(boolean)} also needs to be
* {@code true} for mixed MIME type selections to be made.
*
* @param allowAudioMixedMimeTypeAdaptiveness Whether to allow adaptive audio selections
* containing mixed MIME types.
Expand Down Expand Up @@ -769,6 +771,7 @@ public static final class Builder extends TrackSelectionParameters.Builder {
private boolean allowAudioMixedSampleRateAdaptiveness;
private boolean allowAudioMixedChannelCountAdaptiveness;
private boolean allowAudioMixedDecoderSupportAdaptiveness;
private boolean allowAudioNonSeamlessAdaptiveness;
private boolean constrainAudioChannelCountToDeviceCapabilities;
// General
private boolean exceedRendererCapabilitiesIfNecessary;
Expand Down Expand Up @@ -825,6 +828,7 @@ private Builder(Parameters initialValues) {
initialValues.allowAudioMixedChannelCountAdaptiveness;
allowAudioMixedDecoderSupportAdaptiveness =
initialValues.allowAudioMixedDecoderSupportAdaptiveness;
allowAudioNonSeamlessAdaptiveness = initialValues.allowAudioNonSeamlessAdaptiveness;
constrainAudioChannelCountToDeviceCapabilities =
initialValues.constrainAudioChannelCountToDeviceCapabilities;
// General
Expand Down Expand Up @@ -881,6 +885,10 @@ private Builder(Bundle bundle) {
bundle.getBoolean(
Parameters.FIELD_ALLOW_AUDIO_MIXED_DECODER_SUPPORT_ADAPTIVENESS,
defaultValue.allowAudioMixedDecoderSupportAdaptiveness));
setAllowAudioNonSeamlessAdaptiveness(
bundle.getBoolean(
Parameters.FIELD_ALLOW_AUDIO_NON_SEAMLESS_ADAPTIVENESS,
defaultValue.allowAudioNonSeamlessAdaptiveness));
setConstrainAudioChannelCountToDeviceCapabilities(
bundle.getBoolean(
Parameters.FIELD_CONSTRAIN_AUDIO_CHANNEL_COUNT_TO_DEVICE_CAPABILITIES,
Expand Down Expand Up @@ -1136,7 +1144,9 @@ public Builder setExceedAudioConstraintsIfNecessary(
/**
* Sets whether to allow adaptive audio selections containing mixed MIME types.
*
* <p>Adaptations between different MIME types may not be completely seamless.
* <p>Adaptations between different MIME types may not be completely seamless, in which case
* {@link #setAllowAudioNonSeamlessAdaptiveness(boolean)} also needs to be {@code true} for
* mixed MIME type selections to be made.
*
* @param allowAudioMixedMimeTypeAdaptiveness Whether to allow adaptive audio selections
* containing mixed MIME types.
Expand Down Expand Up @@ -1211,6 +1221,21 @@ public Builder setPreferredAudioMimeTypes(String... mimeTypes) {
return this;
}

/**
* Sets whether to allow adaptive audio selections where adaptation may not be completely
* seamless.
*
* @param allowAudioNonSeamlessAdaptiveness Whether to allow adaptive audio selections where
* adaptation may not be completely seamless.
* @return This builder.
*/
@CanIgnoreReturnValue
public Builder setAllowAudioNonSeamlessAdaptiveness(
boolean allowAudioNonSeamlessAdaptiveness) {
this.allowAudioNonSeamlessAdaptiveness = allowAudioNonSeamlessAdaptiveness;
return this;
}

/**
* Whether to only select audio tracks with channel counts that don't exceed the device's
* output capabilities. The default value is {@code true}.
Expand Down Expand Up @@ -1580,6 +1605,7 @@ private void init(Builder this) {
allowAudioMixedSampleRateAdaptiveness = false;
allowAudioMixedChannelCountAdaptiveness = false;
allowAudioMixedDecoderSupportAdaptiveness = false;
allowAudioNonSeamlessAdaptiveness = true;
constrainAudioChannelCountToDeviceCapabilities = true;
// General
exceedRendererCapabilitiesIfNecessary = true;
Expand Down Expand Up @@ -1713,7 +1739,9 @@ public static Parameters getDefaults(Context context) {

/**
* Whether to allow adaptive audio selections containing mixed MIME types. Adaptations between
* different MIME types may not be completely seamless. The default value is {@code false}.
* different MIME types may not be completely seamless, in which case {@link
* #allowAudioNonSeamlessAdaptiveness} also needs to be {@code true} for mixed MIME type
* selections to be made. The default value is {@code false}.
*/
public final boolean allowAudioMixedMimeTypeAdaptiveness;

Expand All @@ -1737,6 +1765,12 @@ public static Parameters getDefaults(Context context) {
*/
public final boolean allowAudioMixedDecoderSupportAdaptiveness;

/**
* Whether to allow adaptive audio selections where adaptation may not be completely seamless.
* The default value is {@code true}.
*/
public final boolean allowAudioNonSeamlessAdaptiveness;

/**
* Whether to constrain audio track selection so that the selected track's channel count does
* not exceed the device's output capabilities. The default value is {@code true}.
Expand Down Expand Up @@ -1793,6 +1827,7 @@ private Parameters(Builder builder) {
allowAudioMixedSampleRateAdaptiveness = builder.allowAudioMixedSampleRateAdaptiveness;
allowAudioMixedChannelCountAdaptiveness = builder.allowAudioMixedChannelCountAdaptiveness;
allowAudioMixedDecoderSupportAdaptiveness = builder.allowAudioMixedDecoderSupportAdaptiveness;
allowAudioNonSeamlessAdaptiveness = builder.allowAudioNonSeamlessAdaptiveness;
constrainAudioChannelCountToDeviceCapabilities =
builder.constrainAudioChannelCountToDeviceCapabilities;
// General
Expand Down Expand Up @@ -1885,6 +1920,7 @@ public boolean equals(@Nullable Object obj) {
== other.allowAudioMixedChannelCountAdaptiveness
&& allowAudioMixedDecoderSupportAdaptiveness
== other.allowAudioMixedDecoderSupportAdaptiveness
&& allowAudioNonSeamlessAdaptiveness == other.allowAudioNonSeamlessAdaptiveness
&& constrainAudioChannelCountToDeviceCapabilities
== other.constrainAudioChannelCountToDeviceCapabilities
// General
Expand Down Expand Up @@ -1913,6 +1949,7 @@ public int hashCode() {
result = 31 * result + (allowAudioMixedSampleRateAdaptiveness ? 1 : 0);
result = 31 * result + (allowAudioMixedChannelCountAdaptiveness ? 1 : 0);
result = 31 * result + (allowAudioMixedDecoderSupportAdaptiveness ? 1 : 0);
result = 31 * result + (allowAudioNonSeamlessAdaptiveness ? 1 : 0);
result = 31 * result + (constrainAudioChannelCountToDeviceCapabilities ? 1 : 0);
// General
result = 31 * result + (exceedRendererCapabilitiesIfNecessary ? 1 : 0);
Expand Down Expand Up @@ -1961,6 +1998,8 @@ public int hashCode() {
Util.intToStringMaxRadix(FIELD_CUSTOM_ID_BASE + 16);
private static final String FIELD_ALLOW_INVALIDATE_SELECTIONS_ON_RENDERER_CAPABILITIES_CHANGE =
Util.intToStringMaxRadix(FIELD_CUSTOM_ID_BASE + 17);
private static final String FIELD_ALLOW_AUDIO_NON_SEAMLESS_ADAPTIVENESS =
Util.intToStringMaxRadix(FIELD_CUSTOM_ID_BASE + 18);

@Override
public Bundle toBundle() {
Expand Down Expand Up @@ -1989,6 +2028,8 @@ public Bundle toBundle() {
bundle.putBoolean(
FIELD_ALLOW_AUDIO_MIXED_DECODER_SUPPORT_ADAPTIVENESS,
allowAudioMixedDecoderSupportAdaptiveness);
bundle.putBoolean(
FIELD_ALLOW_AUDIO_NON_SEAMLESS_ADAPTIVENESS, allowAudioNonSeamlessAdaptiveness);
bundle.putBoolean(
FIELD_CONSTRAIN_AUDIO_CHANNEL_COUNT_TO_DEVICE_CAPABILITIES,
constrainAudioChannelCountToDeviceCapabilities);
Expand Down Expand Up @@ -2678,7 +2719,8 @@ protected Pair<ExoTrackSelection.Definition, Integer> selectAudioTrack(
params,
support,
hasVideoRendererWithMappedTracksFinal,
this::isAudioFormatWithinAudioChannelCountConstraints),
this::isAudioFormatWithinAudioChannelCountConstraints,
rendererMixedMimeTypeAdaptationSupports[rendererIndex]),
AudioTrackInfo::compareSelections);
}

Expand Down Expand Up @@ -3348,7 +3390,7 @@ public static ImmutableList<VideoTrackInfo> createForTrackGroup(
TrackGroup trackGroup,
Parameters params,
@Capabilities int[] formatSupport,
@AdaptiveSupport int mixedMimeTypeAdaptionSupport) {
@AdaptiveSupport int mixedMimeTypeAdaptationSupport) {
int maxPixelsToRetainForViewport =
getMaxVideoPixelsToRetainForViewport(
trackGroup,
Expand All @@ -3368,7 +3410,7 @@ public static ImmutableList<VideoTrackInfo> createForTrackGroup(
/* trackIndex= */ i,
params,
formatSupport[i],
mixedMimeTypeAdaptionSupport,
mixedMimeTypeAdaptationSupport,
isSuitableForViewport));
}
return listBuilder.build();
Expand Down Expand Up @@ -3482,7 +3524,7 @@ public boolean isCompatibleForAdaptationWith(VideoTrackInfo otherTrack) {
&& format.bitrate != Format.NO_VALUE
&& !parameters.forceHighestSupportedBitrate
&& !parameters.forceLowestBitrate
&& ((rendererSupport & requiredAdaptiveSupport) != 0)
&& (rendererSupport & requiredAdaptiveSupport) != 0
? SELECTION_ELIGIBILITY_ADAPTIVE
: SELECTION_ELIGIBILITY_FIXED;
}
Expand Down Expand Up @@ -3562,7 +3604,8 @@ public static ImmutableList<AudioTrackInfo> createForTrackGroup(
Parameters params,
@Capabilities int[] formatSupport,
boolean hasMappedVideoTracks,
Predicate<Format> withinAudioChannelCountConstraints) {
Predicate<Format> withinAudioChannelCountConstraints,
@AdaptiveSupport int mixedMimeTypeAdaptationSupport) {
ImmutableList.Builder<AudioTrackInfo> listBuilder = ImmutableList.builder();
for (int i = 0; i < trackGroup.length; i++) {
listBuilder.add(
Expand All @@ -3573,7 +3616,8 @@ public static ImmutableList<AudioTrackInfo> createForTrackGroup(
params,
formatSupport[i],
hasMappedVideoTracks,
withinAudioChannelCountConstraints));
withinAudioChannelCountConstraints,
mixedMimeTypeAdaptationSupport));
}
return listBuilder.build();
}
Expand All @@ -3586,6 +3630,7 @@ public static ImmutableList<AudioTrackInfo> createForTrackGroup(
private final int preferredLanguageScore;
private final int preferredLanguageIndex;
private final int preferredRoleFlagsScore;
private final boolean allowMixedMimeTypes;
private final boolean hasMainOrNoRoleFlag;
private final int localeLanguageMatchIndex;
private final int localeLanguageScore;
Expand All @@ -3604,9 +3649,19 @@ public AudioTrackInfo(
Parameters parameters,
@Capabilities int formatSupport,
boolean hasMappedVideoTracks,
Predicate<Format> withinAudioChannelCountConstraints) {
Predicate<Format> withinAudioChannelCountConstraints,
@AdaptiveSupport int mixedMimeTypeAdaptationSupport) {
super(rendererIndex, trackGroup, trackIndex);
this.parameters = parameters;
@SuppressLint("WrongConstant")
int requiredAdaptiveSupport =
parameters.allowAudioNonSeamlessAdaptiveness
? (RendererCapabilities.ADAPTIVE_NOT_SEAMLESS
| RendererCapabilities.ADAPTIVE_SEAMLESS)
: RendererCapabilities.ADAPTIVE_SEAMLESS;
allowMixedMimeTypes =
parameters.allowAudioMixedMimeTypeAdaptiveness
&& (mixedMimeTypeAdaptationSupport & requiredAdaptiveSupport) != 0;
this.language = normalizeUndeterminedLanguageToNull(format.language);
isWithinRendererCapabilities =
isSupported(formatSupport, /* allowExceedsCapabilities= */ false);
Expand Down Expand Up @@ -3669,7 +3724,8 @@ public AudioTrackInfo(
RendererCapabilities.getHardwareAccelerationSupport(formatSupport)
== RendererCapabilities.HARDWARE_ACCELERATION_SUPPORTED;
selectionEligibility =
evaluateSelectionEligibility(parameters, formatSupport, hasMappedVideoTracks);
evaluateSelectionEligibility(
parameters, formatSupport, hasMappedVideoTracks, requiredAdaptiveSupport);
}

@Override
Expand All @@ -3682,7 +3738,7 @@ public boolean isCompatibleForAdaptationWith(AudioTrackInfo otherTrack) {
return (parameters.allowAudioMixedChannelCountAdaptiveness
|| (format.channelCount != Format.NO_VALUE
&& format.channelCount == otherTrack.format.channelCount))
&& (parameters.allowAudioMixedMimeTypeAdaptiveness
&& (allowMixedMimeTypes
|| (format.sampleMimeType != null
&& TextUtils.equals(format.sampleMimeType, otherTrack.format.sampleMimeType)))
&& (parameters.allowAudioMixedSampleRateAdaptiveness
Expand Down Expand Up @@ -3743,7 +3799,10 @@ public int compareTo(AudioTrackInfo other) {
}

private @SelectionEligibility int evaluateSelectionEligibility(
Parameters params, @Capabilities int rendererSupport, boolean hasMappedVideoTracks) {
Parameters params,
@Capabilities int rendererSupport,
boolean hasMappedVideoTracks,
@AdaptiveSupport int requiredAdaptiveSupport) {
if (!isSupported(rendererSupport, parameters.exceedRendererCapabilitiesIfNecessary)) {
return SELECTION_ELIGIBILITY_NO;
}
Expand All @@ -3761,6 +3820,7 @@ public int compareTo(AudioTrackInfo other) {
&& !parameters.forceLowestBitrate
&& (parameters.allowMultipleAdaptiveSelections || !hasMappedVideoTracks)
&& params.audioOffloadPreferences.audioOffloadMode != AUDIO_OFFLOAD_MODE_REQUIRED
&& (rendererSupport & requiredAdaptiveSupport) != 0
? SELECTION_ELIGIBILITY_ADAPTIVE
: SELECTION_ELIGIBILITY_FIXED;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
import static androidx.media3.common.TrackSelectionParameters.AudioOffloadPreferences.AUDIO_OFFLOAD_MODE_ENABLED;
import static androidx.media3.common.TrackSelectionParameters.AudioOffloadPreferences.AUDIO_OFFLOAD_MODE_REQUIRED;
import static androidx.media3.exoplayer.RendererCapabilities.ADAPTIVE_NOT_SEAMLESS;
import static androidx.media3.exoplayer.RendererCapabilities.ADAPTIVE_NOT_SUPPORTED;
import static androidx.media3.exoplayer.RendererCapabilities.ADAPTIVE_SEAMLESS;
import static androidx.media3.exoplayer.RendererCapabilities.AUDIO_OFFLOAD_GAPLESS_SUPPORTED;
import static androidx.media3.exoplayer.RendererCapabilities.AUDIO_OFFLOAD_SUPPORTED;
import static androidx.media3.exoplayer.RendererCapabilities.DECODER_SUPPORT_FALLBACK;
Expand Down Expand Up @@ -1860,6 +1862,94 @@ public void selectTracksWithMultipleAudioTracksWithMixedDecoderSupportLevels() t
assertAdaptiveSelection(result.selections[0], trackGroups.get(0), 3, 2, 0);
}

@Test
public void selectTracks_withMultipleAudioTracksWithoutAdaptationSupport_selectsFixed()
throws Exception {
FakeRendererCapabilities seamlessAudioCapabilities =
new FakeRendererCapabilities(
C.TRACK_TYPE_AUDIO,
RendererCapabilities.create(
FORMAT_HANDLED, ADAPTIVE_NOT_SUPPORTED, TUNNELING_NOT_SUPPORTED));

Format.Builder formatBuilder = AUDIO_FORMAT.buildUpon();
TrackGroupArray trackGroups =
singleTrackGroup(formatBuilder.setId("0").build(), formatBuilder.setId("1").build());
// Explicitly allow non-seamless adaptation to ensure it's not used.
trackSelector.setParameters(
defaultParameters.buildUpon().setAllowAudioNonSeamlessAdaptiveness(true));
TrackSelectorResult result =
trackSelector.selectTracks(
new RendererCapabilities[] {seamlessAudioCapabilities},
trackGroups,
periodId,
TIMELINE);
assertThat(result.length).isEqualTo(1);
assertFixedSelection(result.selections[0], trackGroups.get(0), /* expectedTrack= */ 0);
}

@Test
public void
selectTracks_withMultipleAudioTracksWithNonSeamlessAdaptiveness_selectsAdaptiveOnlyIfAllowed()
throws Exception {
FakeRendererCapabilities nonSeamlessAudioCapabilities =
new FakeRendererCapabilities(
C.TRACK_TYPE_AUDIO,
RendererCapabilities.create(
FORMAT_HANDLED, ADAPTIVE_NOT_SEAMLESS, TUNNELING_NOT_SUPPORTED));

Format.Builder formatBuilder = AUDIO_FORMAT.buildUpon();
TrackGroupArray trackGroups =
singleTrackGroup(formatBuilder.setId("0").build(), formatBuilder.setId("1").build());
trackSelector.setParameters(
defaultParameters.buildUpon().setAllowAudioNonSeamlessAdaptiveness(true));
TrackSelectorResult result =
trackSelector.selectTracks(
new RendererCapabilities[] {nonSeamlessAudioCapabilities},
trackGroups,
periodId,
TIMELINE);
assertThat(result.length).isEqualTo(1);
assertAdaptiveSelection(
result.selections[0], trackGroups.get(0), /* expectedTracks...= */ 0, 1);

trackSelector.setParameters(
defaultParameters.buildUpon().setAllowAudioNonSeamlessAdaptiveness(false));
result =
trackSelector.selectTracks(
new RendererCapabilities[] {nonSeamlessAudioCapabilities},
trackGroups,
periodId,
TIMELINE);
assertThat(result.length).isEqualTo(1);
assertFixedSelection(result.selections[0], trackGroups.get(0), /* expectedTrack= */ 0);
}

@Test
public void selectTracks_withMultipleAudioTracksWithSeamlessAdaptiveness_selectsAdaptive()
throws Exception {
FakeRendererCapabilities seamlessAudioCapabilities =
new FakeRendererCapabilities(
C.TRACK_TYPE_AUDIO,
RendererCapabilities.create(
FORMAT_HANDLED, ADAPTIVE_SEAMLESS, TUNNELING_NOT_SUPPORTED));

Format.Builder formatBuilder = AUDIO_FORMAT.buildUpon();
TrackGroupArray trackGroups =
singleTrackGroup(formatBuilder.setId("0").build(), formatBuilder.setId("1").build());
// Explicitly disallow non-seamless adaptation to ensure it's not used.
trackSelector.setParameters(
defaultParameters.buildUpon().setAllowAudioNonSeamlessAdaptiveness(false));
TrackSelectorResult result =
trackSelector.selectTracks(
new RendererCapabilities[] {seamlessAudioCapabilities},
trackGroups,
periodId,
TIMELINE);
assertThat(result.length).isEqualTo(1);
assertAdaptiveSelection(
result.selections[0], trackGroups.get(0), /* expectedTracks...= */ 0, 1);
}

@Test
public void selectTracksWithMultipleAudioTracksOverrideReturnsAdaptiveTrackSelection()
throws Exception {
Expand Down

0 comments on commit f20d18e

Please sign in to comment.