loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Previewing: set VideoFrameReleaseControl after CVSP is created.

Browse Source 
This allows us to inject a videoFrameProcessorFactory into
MediaCodecVideoRenderer, without issues about creating the
VideoFrameReleaseControl in the MediaCodecVideoRenderer.

Unfortunately, this does result in more complex CVSP state, where
VideoFrameReleaseControl is no longer final, may be null, and may potentially
change. However, this tries to be careful with assertions to guarantee good
state, and is cleaner than modifying the long-standing MediaCodecVideoRenderer
interface.

Tested that this works on the ExoPlayer demo with setVideoEffects applied, and
using a playlist with SDR->HDR and HDR->SDR items.

PiperOrigin-RevId: 599823412
This commit is contained in:
huangdarwin 2024-01-19 07:06:02 -08:00 committed by Copybara-Service
parent e364510937
commit cb0f5a7fff
4 changed files with 55 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
View File

@ -50,7 +50,6 @@ import androidx.media3.common.util.TimestampIterator;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.ExoPlaybackException;
import androidx.media3.exoplayer.video.VideoFrameReleaseControl.FrameTimingEvaluator;
import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
@ -80,7 +79,6 @@ public final class CompositingVideoSinkProvider
private VideoFrameProcessor.@MonotonicNonNull Factory videoFrameProcessorFactory;
private PreviewingVideoGraph.@MonotonicNonNull Factory previewingVideoGraphFactory;
private @MonotonicNonNull VideoFrameReleaseControl videoFrameReleaseControl;
private boolean built;
/** Creates a builder with the supplied {@linkplain Context application context}. */
@ -119,32 +117,6 @@ public final class CompositingVideoSinkProvider
return this;
}
/**
* Sets the {@link VideoFrameReleaseControl} that will be used.
*
* <p>By default, a {@link VideoFrameReleaseControl} will be used with a {@link
* FrameTimingEvaluator} implementation which:
*
* <ul>
* <li>Signals to {@linkplain FrameTimingEvaluator#shouldForceReleaseFrame(long, long) force
* release} a frame if the frame is late by more than {@link #FRAME_LATE_THRESHOLD_US} and
* the elapsed time since the previous frame release is greater than {@link
* #FRAME_RELEASE_THRESHOLD_US}.
* <li>Signals to {@linkplain FrameTimingEvaluator#shouldDropFrame(long, long, boolean) drop a
* frame} if the frame is late by more than {@link #FRAME_LATE_THRESHOLD_US} and the frame
* is not marked as the last one.
* <li>Signals to never {@linkplain FrameTimingEvaluator#shouldIgnoreFrame(long, long, long,
* boolean, boolean) ignore} a frame.
* </ul>
*
* @param videoFrameReleaseControl The {@link VideoFrameReleaseControl}.
* @return This builder, for convenience.
*/
public Builder setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl) {
this.videoFrameReleaseControl = videoFrameReleaseControl;
return this;
}
/**
* Builds the {@link CompositingVideoSinkProvider}.
*
@ -161,11 +133,6 @@ public final class CompositingVideoSinkProvider
previewingVideoGraphFactory =
new ReflectivePreviewingSingleInputVideoGraphFactory(videoFrameProcessorFactory);
}
if (videoFrameReleaseControl == null) {
videoFrameReleaseControl =
new VideoFrameReleaseControl(
context, new CompositionFrameTimingEvaluator(), /* allowedJoiningTimeMs= */ 0);
}
CompositingVideoSinkProvider compositingVideoSinkProvider =
new CompositingVideoSinkProvider(this);
built = true;
@ -173,41 +140,6 @@ public final class CompositingVideoSinkProvider
}
}
/** The time threshold, in microseconds, after which a frame is considered late. */
public static final long FRAME_LATE_THRESHOLD_US = -30_000;
/**
* The maximum elapsed time threshold, in microseconds, since last releasing a frame after which a
* frame can be force released.
*/
public static final long FRAME_RELEASE_THRESHOLD_US = 100_000;
/** A {@link FrameTimingEvaluator} for composition frames. */
private static final class CompositionFrameTimingEvaluator implements FrameTimingEvaluator {
@Override
public boolean shouldForceReleaseFrame(long earlyUs, long elapsedSinceLastReleaseUs) {
return earlyUs < FRAME_LATE_THRESHOLD_US
&& elapsedSinceLastReleaseUs > FRAME_RELEASE_THRESHOLD_US;
}
@Override
public boolean shouldDropFrame(long earlyUs, long elapsedRealtimeUs, boolean isLastFrame) {
return earlyUs < FRAME_LATE_THRESHOLD_US && !isLastFrame;
}
@Override
public boolean shouldIgnoreFrame(
long earlyUs,
long positionUs,
long elapsedRealtimeUs,
boolean isLastFrame,
boolean treatDroppedBuffersAsSkipped) {
// TODO b/293873191 - Handle very late buffers and drop to key frame.
return false;
}
}
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@ -222,10 +154,10 @@ public final class CompositingVideoSinkProvider
private final Context context;
private final PreviewingVideoGraph.Factory previewingVideoGraphFactory;
private final VideoFrameReleaseControl videoFrameReleaseControl;
private final VideoFrameRenderControl videoFrameRenderControl;
private Clock clock;
private @MonotonicNonNull VideoFrameReleaseControl videoFrameReleaseControl;
private @MonotonicNonNull VideoFrameRenderControl videoFrameRenderControl;
private @MonotonicNonNull Format outputFormat;
private @MonotonicNonNull VideoFrameMetadataListener videoFrameMetadataListener;
private @MonotonicNonNull HandlerWrapper handler;
@ -241,11 +173,6 @@ public final class CompositingVideoSinkProvider
private CompositingVideoSinkProvider(Builder builder) {
this.context = builder.context;
this.previewingVideoGraphFactory = checkStateNotNull(builder.previewingVideoGraphFactory);
videoFrameReleaseControl = checkStateNotNull(builder.videoFrameReleaseControl);
@SuppressWarnings("nullness:assignment")
VideoFrameRenderControl.@Initialized FrameRenderer thisRef = this;
videoFrameRenderControl =
new VideoFrameRenderControl(/* frameRenderer= */ thisRef, videoFrameReleaseControl);
clock = Clock.DEFAULT;
listener = VideoSink.Listener.NO_OP;
listenerExecutor = NO_OP_EXECUTOR;
@ -258,6 +185,7 @@ public final class CompositingVideoSinkProvider
public void initialize(Format sourceFormat) throws VideoSink.VideoSinkException {
checkState(state == STATE_CREATED);
checkStateNotNull(videoEffects);
checkState(videoFrameRenderControl != null && videoFrameReleaseControl != null);
// Lazily initialize the handler here so it's initialized on the playback looper.
handler = clock.createHandler(checkStateNotNull(Looper.myLooper()), /* callback= */ null);
@ -356,6 +284,14 @@ public final class CompositingVideoSinkProvider
outputSurface, outputResolution.getWidth(), outputResolution.getHeight());
}
@Override
public void setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl) {
checkState(!isInitialized());
this.videoFrameReleaseControl = videoFrameReleaseControl;
videoFrameRenderControl =
new VideoFrameRenderControl(/* frameRenderer= */ this, videoFrameReleaseControl);
}
@Override
public void clearOutputSurfaceInfo() {
maybeSetOutputSurfaceInfo(
@ -371,6 +307,7 @@ public final class CompositingVideoSinkProvider
}
@Override
@Nullable
public VideoFrameReleaseControl getVideoFrameReleaseControl() {
return videoFrameReleaseControl;
}
@ -386,7 +323,7 @@ public final class CompositingVideoSinkProvider
@Override
public void onOutputSizeChanged(int width, int height) {
// We forward output size changes to render control even if we are still flushing.
videoFrameRenderControl.onOutputSizeChanged(width, height);
checkStateNotNull(videoFrameRenderControl).onOutputSizeChanged(width, height);
}
@Override
@ -395,7 +332,8 @@ public final class CompositingVideoSinkProvider
// Ignore available frames while the sink provider is flushing
return;
}
videoFrameRenderControl.onOutputFrameAvailableForRendering(presentationTimeUs);
checkStateNotNull(videoFrameRenderControl)
.onOutputFrameAvailableForRendering(presentationTimeUs);
}
@Override
@ -471,7 +409,7 @@ public final class CompositingVideoSinkProvider
*/
public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
if (pendingFlushCount == 0) {
videoFrameRenderControl.render(positionUs, elapsedRealtimeUs);
checkStateNotNull(videoFrameRenderControl).render(positionUs, elapsedRealtimeUs);
}
}
@ -502,23 +440,24 @@ public final class CompositingVideoSinkProvider
// Update the surface on the video graph and the video frame release control together.
SurfaceInfo surfaceInfo = surface != null ? new SurfaceInfo(surface, width, height) : null;
videoGraph.setOutputSurfaceInfo(surfaceInfo);
videoFrameReleaseControl.setOutputSurface(surface);
checkNotNull(videoFrameReleaseControl).setOutputSurface(surface);
}
}
private boolean isReady() {
return pendingFlushCount == 0 && videoFrameRenderControl.isReady();
return pendingFlushCount == 0 && checkStateNotNull(videoFrameRenderControl).isReady();
}
private boolean hasReleasedFrame(long presentationTimeUs) {
return pendingFlushCount == 0 && videoFrameRenderControl.hasReleasedFrame(presentationTimeUs);
return pendingFlushCount == 0
&& checkStateNotNull(videoFrameRenderControl).hasReleasedFrame(presentationTimeUs);
}
private void flush() {
pendingFlushCount++;
// Flush the render control now to ensure it has no data, eg calling isReady() must return false
// and render() should not render any frames.
videoFrameRenderControl.flush();
checkStateNotNull(videoFrameRenderControl).flush();
// Finish flushing after handling pending video graph callbacks to ensure video size changes
// reach the video render control.
checkStateNotNull(handler).post(this::flushInternal);
@ -533,15 +472,16 @@ public final class CompositingVideoSinkProvider
throw new IllegalStateException(String.valueOf(pendingFlushCount));
}
// Flush the render control again.
videoFrameRenderControl.flush();
checkStateNotNull(videoFrameRenderControl).flush();
}
private void setPlaybackSpeed(float speed) {
videoFrameRenderControl.setPlaybackSpeed(speed);
checkStateNotNull(videoFrameRenderControl).setPlaybackSpeed(speed);
}
private void onStreamOffsetChange(long bufferPresentationTimeUs, long streamOffsetUs) {
videoFrameRenderControl.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs);
checkStateNotNull(videoFrameRenderControl)
.onStreamOffsetChange(bufferPresentationTimeUs, streamOffsetUs);
}
private static ColorInfo getAdjustedInputColorInfo(@Nullable ColorInfo inputColorInfo) {

15
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/MediaCodecVideoRenderer.java
View File

@ -393,17 +393,18 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer
this.context = context.getApplicationContext();
eventDispatcher = new EventDispatcher(eventHandler, eventListener);
if (videoSinkProvider == null) {
videoSinkProvider = new CompositingVideoSinkProvider.Builder(this.context).build();
}
if (videoSinkProvider.getVideoFrameReleaseControl() == null) {
@SuppressWarnings("nullness:assignment")
VideoFrameReleaseControl.@Initialized FrameTimingEvaluator thisRef = this;
videoSinkProvider =
new CompositingVideoSinkProvider.Builder(this.context)
.setVideoFrameReleaseControl(
new VideoFrameReleaseControl(
this.context, /* frameTimingEvaluator= */ thisRef, allowedJoiningTimeMs))
.build();
videoSinkProvider.setVideoFrameReleaseControl(
new VideoFrameReleaseControl(
this.context, /* frameTimingEvaluator= */ thisRef, allowedJoiningTimeMs));
}
this.videoSinkProvider = videoSinkProvider;
this.videoFrameReleaseControl = this.videoSinkProvider.getVideoFrameReleaseControl();
this.videoFrameReleaseControl =
checkStateNotNull(this.videoSinkProvider.getVideoFrameReleaseControl());
videoFrameReleaseInfo = new VideoFrameReleaseControl.FrameReleaseInfo();
deviceNeedsNoPostProcessWorkaround = deviceNeedsNoPostProcessWorkaround();
scalingMode = C.VIDEO_SCALING_MODE_DEFAULT;

15
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/VideoSinkProvider.java
View File

@ -23,6 +23,7 @@ import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.UnstableApi;
import java.util.List;
import org.checkerframework.checker.nullness.qual.Nullable;
/** A provider of {@link VideoSink VideoSinks}. */
@UnstableApi
@ -70,6 +71,15 @@ public interface VideoSinkProvider {
*/
void setOutputSurfaceInfo(Surface outputSurface, Size outputResolution);
/**
* Sets the {@link VideoFrameReleaseControl} that will be used for releasing of video frames
* during rendering.
*
* <p>Must be called before, not after, the sink provider is {@linkplain #initialize(Format)
* initialized}.
*/
void setVideoFrameReleaseControl(VideoFrameReleaseControl videoFrameReleaseControl);
/**
* Clears the set output surface info.
*
@ -83,8 +93,11 @@ public interface VideoSinkProvider {
/**
* Returns the {@link VideoFrameReleaseControl} that will be used for releasing of video frames
* during rendering.
*
* <p>If this value is {@code null}, it must be {@linkplain #setVideoFrameReleaseControl set} to a
* non-null value before rendering begins.
*/
VideoFrameReleaseControl getVideoFrameReleaseControl();
@Nullable VideoFrameReleaseControl getVideoFrameReleaseControl();
/**
* Sets the {@link Clock} that the provider should use internally.

21
libraries/exoplayer/src/test/java/androidx/media3/exoplayer/video/CompositingVideoSinkProviderTest.java
View File

@ -18,7 +18,6 @@ package androidx.media3.exoplayer.video;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import android.content.Context;
@ -45,12 +44,7 @@ public final class CompositingVideoSinkProviderTest {
@Test
public void builder_calledMultipleTimes_throws() {
CompositingVideoSinkProvider.Builder builder =
new CompositingVideoSinkProvider.Builder(ApplicationProvider.getApplicationContext())
.setVideoFrameReleaseControl(
new VideoFrameReleaseControl(
ApplicationProvider.getApplicationContext(),
mock(VideoFrameReleaseControl.FrameTimingEvaluator.class),
/* allowedJoiningTimeMs= */ 0));
new CompositingVideoSinkProvider.Builder(ApplicationProvider.getApplicationContext());
builder.build();
@ -169,12 +163,13 @@ public final class CompositingVideoSinkProviderTest {
return false;
}
};
VideoFrameReleaseControl releaseControl =
new VideoFrameReleaseControl(context, frameTimingEvaluator, /* allowedJoiningTimeMs= */ 0);
return new CompositingVideoSinkProvider.Builder(context)
.setPreviewingVideoGraphFactory(new TestPreviewingVideoGraphFactory())
.setVideoFrameReleaseControl(releaseControl)
.build();
CompositingVideoSinkProvider compositingVideoSinkProvider =
new CompositingVideoSinkProvider.Builder(context)
.setPreviewingVideoGraphFactory(new TestPreviewingVideoGraphFactory())
.build();
compositingVideoSinkProvider.setVideoFrameReleaseControl(
new VideoFrameReleaseControl(context, frameTimingEvaluator, /* allowedJoiningTimeMs= */ 0));
return compositingVideoSinkProvider;
}
private static class TestPreviewingVideoGraphFactory implements PreviewingVideoGraph.Factory {