Skip to content

Commit

Permalink
Move video frame release logic to VideoFrameReleaseControl
Browse files Browse the repository at this point in the history 
This change moves the release timestamp adjustment logic out of
MediaCodecVideoRenderer and into a standalone component, the
VideoFrameReleaseControl. The plan for VideoFrameReleaseControl is to use
it:
- from MediaCodecVideoRenderer, when ExoPlayer plays video in standalone
  mode.
- from the CompositionPlayer's DefaultVideoSink, when CompositionPlayer
  supports multiple sequences.
- (temporarily) from the CompositionPlayer's custom ImageRenderer while
  we are implementing single-sequence preview, which is an intermediate
  milestone for composition preview.
PiperOrigin-RevId: 574420427
  • Loading branch information
@christosts @copybara-github
christosts authored and copybara-github committed Oct 18, 2023
1 parent ff330bd commit eafe2e3
Show file tree
Hide file tree
Showing 7 changed files with 1,247 additions and 410 deletions.
6 changes: 6 additions & 0 deletions libraries/exoplayer/src/main/java/androidx/media3/exoplayer/BaseRenderer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public final void init(int index, PlayerId playerId, Clock clock) {
this.index = index;
this.playerId = playerId;
this.clock = clock;
onInit();
}

@Override
Expand Down Expand Up @@ -263,6 +264,11 @@ public void handleMessage(@MessageType int messageType, @Nullable Object message

// Methods to be overridden by subclasses.

/** Called when the renderer is initialized. */
protected void onInit() {
// Do nothing
}

/**
* Called when the renderer is enabled.
*
Expand Down
151 changes: 93 additions & 58 deletions ...exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
import androidx.media3.common.VideoSize;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.LongArrayQueue;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.TimedValueQueue;
Expand All @@ -62,7 +63,7 @@

private final Context context;
private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
private final VideoSink.RenderControl renderControl;
private final VideoFrameReleaseControl videoFrameReleaseControl;

@Nullable private VideoSinkImpl videoSinkImpl;
@Nullable private List<Effect> videoEffects;
Expand All @@ -73,7 +74,7 @@
public CompositingVideoSinkProvider(
Context context,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
VideoSink.RenderControl renderControl) {
VideoFrameReleaseControl renderControl) {
this(
context,
new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory),
Expand All @@ -84,10 +85,10 @@ public CompositingVideoSinkProvider(
/* package */ CompositingVideoSinkProvider(
Context context,
PreviewingVideoGraph.Factory previewingVideoGraphFactory,
VideoSink.RenderControl renderControl) {
VideoFrameReleaseControl releaseControl) {
this.context = context;
this.previewingVideoGraphFactory = previewingVideoGraphFactory;
this.renderControl = renderControl;
this.videoFrameReleaseControl = releaseControl;
}

@Override
Expand All @@ -97,7 +98,8 @@ public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException

try {
videoSinkImpl =
new VideoSinkImpl(context, previewingVideoGraphFactory, renderControl, sourceFormat);
new VideoSinkImpl(
context, previewingVideoGraphFactory, videoFrameReleaseControl, sourceFormat);
} catch (VideoFrameProcessingException e) {
throw new VideoSink.VideoSinkException(e, sourceFormat);
}
Expand Down Expand Up @@ -173,7 +175,8 @@ public void setVideoFrameMetadataListener(VideoFrameMetadataListener videoFrameM
private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listener {

private final Context context;
private final VideoSink.RenderControl renderControl;
private final VideoFrameReleaseControl videoFrameReleaseControl;
private final VideoFrameReleaseControl.FrameReleaseInfo videoFrameReleaseInfo;
private final VideoFrameProcessor videoFrameProcessor;
private final LongArrayQueue processedFramesBufferTimestampsUs;
private final TimedValueQueue<Long> streamOffsets;
Expand Down Expand Up @@ -207,11 +210,9 @@ private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listen
private VideoSize processedFrameSize;
private VideoSize reportedVideoSize;
private boolean pendingVideoSizeChange;
private boolean renderedFirstFrame;
private long inputStreamOffsetUs;
private boolean pendingInputStreamOffsetChange;
private long outputStreamOffsetUs;
private float playbackSpeed;

// TODO b/292111083 - Remove the field and trigger the callback on every video size change.
private boolean onVideoSizeChangedCalled;
Expand All @@ -220,11 +221,12 @@ private static final class VideoSinkImpl implements VideoSink, VideoGraph.Listen
public VideoSinkImpl(
Context context,
PreviewingVideoGraph.Factory previewingVideoGraphFactory,
RenderControl renderControl,
VideoFrameReleaseControl videoFrameReleaseControl,
Format sourceFormat)
throws VideoFrameProcessingException {
this.context = context;
this.renderControl = renderControl;
this.videoFrameReleaseControl = videoFrameReleaseControl;
videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
processedFramesBufferTimestampsUs = new LongArrayQueue();
streamOffsets = new TimedValueQueue<>();
videoSizeChanges = new TimedValueQueue<>();
Expand All @@ -237,7 +239,6 @@ public VideoSinkImpl(
lastCodecBufferPresentationTimestampUs = C.TIME_UNSET;
processedFrameSize = VideoSize.UNKNOWN;
reportedVideoSize = VideoSize.UNKNOWN;
playbackSpeed = 1f;

// Playback thread handler.
handler = Util.createHandlerForCurrentLooper();
Expand Down Expand Up @@ -293,7 +294,7 @@ public void flush() {
processedFramesBufferTimestampsUs.clear();
streamOffsets.clear();
handler.removeCallbacksAndMessages(/* token= */ null);
renderedFirstFrame = false;
videoFrameReleaseControl.reset();
if (registeredLastFrame) {
registeredLastFrame = false;
processedLastFrame = false;
Expand All @@ -303,7 +304,7 @@ public void flush() {

@Override
public boolean isReady() {
return renderedFirstFrame;
return videoFrameReleaseControl.isReady(/* rendererReady= */ true);
}

@Override
Expand Down Expand Up @@ -385,47 +386,50 @@ public void render(long positionUs, long elapsedRealtimeUs) {
long bufferPresentationTimeUs = processedFramesBufferTimestampsUs.element();
// check whether this buffer comes with a new stream offset.
if (maybeUpdateOutputStreamOffset(bufferPresentationTimeUs)) {
renderedFirstFrame = false;
videoFrameReleaseControl.onProcessedStreamChange();
}
long framePresentationTimeUs = bufferPresentationTimeUs - outputStreamOffsetUs;
boolean isLastFrame = processedLastFrame && processedFramesBufferTimestampsUs.size() == 1;
long frameRenderTimeNs =
renderControl.getFrameRenderTimeNs(
bufferPresentationTimeUs, positionUs, elapsedRealtimeUs, playbackSpeed);
if (frameRenderTimeNs == RenderControl.RENDER_TIME_TRY_AGAIN_LATER) {
return;
} else if (framePresentationTimeUs == RenderControl.RENDER_TIME_DROP) {
// TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
// VideoFrameProcessor input frames in this case.
releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame);
continue;
@VideoFrameReleaseControl.FrameReleaseAction
int frameReleaseAction =
videoFrameReleaseControl.getFrameReleaseAction(
bufferPresentationTimeUs,
positionUs,
elapsedRealtimeUs,
outputStreamOffsetUs,
isLastFrame,
videoFrameReleaseInfo);
switch (frameReleaseAction) {
case VideoFrameReleaseControl.FRAME_RELEASE_TRY_AGAIN_LATER:
return;
case VideoFrameReleaseControl.FRAME_RELEASE_SKIP:
case VideoFrameReleaseControl.FRAME_RELEASE_SKIP_TO_KEYFRAME:
// Skipped frames are dropped.
case VideoFrameReleaseControl.FRAME_RELEASE_DROP:
case VideoFrameReleaseControl.FRAME_RELEASE_DROP_TO_KEYFRAME:
// TODO b/293873191 - Handle very late buffers and drop to key frame. Need to flush
// VideoFrameProcessor input frames in this case.
dropFrame(isLastFrame);
break;
case VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY:
case VideoFrameReleaseControl.FRAME_RELEASE_SCHEDULED:
renderFrame(
framePresentationTimeUs, bufferPresentationTimeUs, frameReleaseAction, isLastFrame);
break;
default:
throw new IllegalStateException(String.valueOf(frameReleaseAction));
}
renderControl.onNextFrame(bufferPresentationTimeUs);
if (videoFrameMetadataListener != null) {
videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
framePresentationTimeUs,
frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY
? System.nanoTime()
: frameRenderTimeNs,
checkNotNull(inputFormat),
/* mediaFormat= */ null);
}
releaseProcessedFrameInternal(
frameRenderTimeNs == RenderControl.RENDER_TIME_IMMEDIATELY
? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
: frameRenderTimeNs,
isLastFrame);

maybeNotifyVideoSizeChanged(bufferPresentationTimeUs);
}
}

@Override
public void setPlaybackSpeed(float speed) {
checkArgument(speed >= 0.0);
this.playbackSpeed = speed;
videoFrameReleaseControl.setPlaybackSpeed(speed);
}

// VideoGraph.Listener methods

@Override
public void onOutputSizeChanged(int width, int height) {
VideoSize newVideoSize = new VideoSize(width, height);
Expand Down Expand Up @@ -477,12 +481,13 @@ public void onEnded(long finalFramePresentationTimeUs) {
throw new IllegalStateException();
}

// Other methods

public void release() {
videoFrameProcessor.release();
handler.removeCallbacksAndMessages(/* token= */ null);
streamOffsets.clear();
processedFramesBufferTimestampsUs.clear();
renderedFirstFrame = false;
}

/** Sets the {@linkplain Effect video effects} to apply immediately. */
Expand Down Expand Up @@ -541,18 +546,17 @@ public void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution) {
&& currentSurfaceAndSize.second.equals(outputResolution)) {
return;
}
renderedFirstFrame =
currentSurfaceAndSize == null || currentSurfaceAndSize.first.equals(outputSurface);
videoFrameReleaseControl.setOutputSurface(outputSurface);
currentSurfaceAndSize = Pair.create(outputSurface, outputResolution);
videoFrameProcessor.setOutputSurfaceInfo(
new SurfaceInfo(
outputSurface, outputResolution.getWidth(), outputResolution.getHeight()));
}

/** Clears the output surface info. */
public void clearOutputSurfaceInfo() {
videoFrameProcessor.setOutputSurfaceInfo(null);
currentSurfaceAndSize = null;
renderedFirstFrame = false;
}

private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) {
Expand All @@ -565,21 +569,52 @@ private boolean maybeUpdateOutputStreamOffset(long bufferPresentationTimeUs) {
return updatedOffset;
}

private void dropFrame(boolean isLastFrame) {
if (listenerExecutor != null) {
listenerExecutor.execute(
() -> {
if (listener != null) {
listener.onFrameDropped(this);
}
});
}
releaseProcessedFrameInternal(VideoFrameProcessor.DROP_OUTPUT_FRAME, isLastFrame);
}

private void renderFrame(
long framePresentationTimeUs,
long bufferPresentationTimeUs,
@VideoFrameReleaseControl.FrameReleaseAction int frameReleaseAction,
boolean isLastFrame) {
if (videoFrameMetadataListener != null) {
videoFrameMetadataListener.onVideoFrameAboutToBeRendered(
framePresentationTimeUs,
frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY
? Clock.DEFAULT.nanoTime()
: videoFrameReleaseInfo.getReleaseTimeNs(),
checkNotNull(inputFormat),
/* mediaFormat= */ null);
}
if (videoFrameReleaseControl.onFrameReleasedIsFirstFrame() && listenerExecutor != null) {
listenerExecutor.execute(
() -> {
if (listener != null) {
listener.onFirstFrameRendered(/* videoSink= */ this);
}
});
}
releaseProcessedFrameInternal(
frameReleaseAction == VideoFrameReleaseControl.FRAME_RELEASE_IMMEDIATELY
? VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
: videoFrameReleaseInfo.getReleaseTimeNs(),
isLastFrame);

maybeNotifyVideoSizeChanged(bufferPresentationTimeUs);
}

private void releaseProcessedFrameInternal(long releaseTimeNs, boolean isLastFrame) {
videoFrameProcessor.renderOutputFrame(releaseTimeNs);
processedFramesBufferTimestampsUs.remove();
if (releaseTimeNs == VideoFrameProcessor.DROP_OUTPUT_FRAME) {
renderControl.onFrameDropped();
} else {
renderControl.onFrameRendered();
if (!renderedFirstFrame) {
if (listener != null) {
checkNotNull(listenerExecutor)
.execute(() -> checkNotNull(listener).onFirstFrameRendered(this));
}
renderedFirstFrame = true;
}
}
if (isLastFrame) {
releasedLastFrame = true;
}
Expand Down
Loading

0 comments on commit eafe2e3

Please sign in to comment.