Use Clock in video renderers

This makes tests more realistic because the returned value matches
the rest of the simulated test time.

It also prevents test flakiness in (yet to be written) tests that
may accidentally drop output buffers or calculate the wrong values.

PiperOrigin-RevId: 545690008
This commit is contained in:
tonihei 2023-07-05 16:56:48 +01:00 committed by Rohit Singh
parent 07d4e5986b
commit 63331dbd1b
10 changed files with 64 additions and 37 deletions

View File

@ -24,6 +24,7 @@ import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.util.Assertions;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.decoder.DecoderInputBuffer;
import androidx.media3.decoder.DecoderInputBuffer.InsufficientCapacityException;
@ -45,6 +46,7 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities {
@Nullable private RendererConfiguration configuration;
private int index;
private @MonotonicNonNull PlayerId playerId;
private @MonotonicNonNull Clock clock;
private int state;
@Nullable private SampleStream stream;
@Nullable private Format[] streamFormats;
@ -80,9 +82,10 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities {
}
@Override
public final void init(int index, PlayerId playerId) {
public final void init(int index, PlayerId playerId, Clock clock) {
this.index = index;
this.playerId = playerId;
this.clock = clock;
}
@Override
@ -393,6 +396,15 @@ public abstract class BaseRenderer implements Renderer, RendererCapabilities {
return checkNotNull(playerId);
}
/**
* Returns the {@link Clock}.
*
* <p>Must only be used after the renderer has been initialized by the player.
*/
protected final Clock getClock() {
return checkNotNull(clock);
}
/**
* Creates an {@link ExoPlaybackException} of type {@link ExoPlaybackException#TYPE_RENDERER} for
* this renderer.

View File

@ -17,6 +17,7 @@ package androidx.media3.exoplayer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Util.castNonNull;
import static androidx.media3.common.util.Util.msToUs;
import static java.lang.Math.max;
import static java.lang.Math.min;
@ -25,7 +26,6 @@ import android.os.HandlerThread;
import android.os.Looper;
import android.os.Message;
import android.os.Process;
import android.os.SystemClock;
import android.util.Pair;
import androidx.annotation.CheckResult;
import androidx.annotation.Nullable;
@ -277,7 +277,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
RendererCapabilities.Listener rendererCapabilitiesListener =
trackSelector.getRendererCapabilitiesListener();
for (int i = 0; i < renderers.length; i++) {
renderers[i].init(/* index= */ i, playerId);
renderers[i].init(/* index= */ i, playerId, clock);
rendererCapabilities[i] = renderers[i].getCapabilities();
if (rendererCapabilitiesListener != null) {
rendererCapabilities[i].setListener(rendererCapabilitiesListener);
@ -1056,7 +1056,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
boolean renderersEnded = true;
boolean renderersAllowPlayback = true;
if (playingPeriodHolder.prepared) {
long rendererPositionElapsedRealtimeUs = SystemClock.elapsedRealtime() * 1000;
long rendererPositionElapsedRealtimeUs = msToUs(clock.elapsedRealtime());
playingPeriodHolder.mediaPeriod.discardBuffer(
playbackInfo.positionUs - backBufferDurationUs, retainBackBufferFromKeyframe);
for (int i = 0; i < renderers.length; i++) {

View File

@ -19,6 +19,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.Assertions;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.exoplayer.analytics.PlayerId;
import androidx.media3.exoplayer.source.SampleStream;
@ -49,7 +50,7 @@ public abstract class NoSampleRenderer implements Renderer, RendererCapabilities
}
@Override
public final void init(int index, PlayerId playerId) {
public final void init(int index, PlayerId playerId, Clock clock) {
this.index = index;
}

View File

@ -27,6 +27,7 @@ import androidx.media3.common.C;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
import androidx.media3.common.Player;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
@ -305,8 +306,9 @@ public interface Renderer extends PlayerMessage.Target {
*
* @param index The renderer index within the player.
* @param playerId The {@link PlayerId} of the player.
* @param clock The {@link Clock}.
*/
void init(int index, PlayerId playerId);
void init(int index, PlayerId playerId, Clock clock);
/**
* If the renderer advances its own playback position then this method returns a corresponding

View File

@ -573,7 +573,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer {
throws DecoderException {
if (frameMetadataListener != null) {
frameMetadataListener.onVideoFrameAboutToBeRendered(
presentationTimeUs, System.nanoTime(), outputFormat, /* mediaFormat= */ null);
presentationTimeUs, getClock().nanoTime(), outputFormat, /* mediaFormat= */ null);
}
lastRenderTimeUs = Util.msToUs(SystemClock.elapsedRealtime() * 1000);
int bufferMode = outputBuffer.mode;

View File

@ -19,6 +19,7 @@ import static android.view.Display.DEFAULT_DISPLAY;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.common.util.Util.msToUs;
import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_MAX_INPUT_SIZE_EXCEEDED;
import static androidx.media3.exoplayer.DecoderReuseEvaluation.DISCARD_REASON_VIDEO_MAX_RESOLUTION_EXCEEDED;
import static androidx.media3.exoplayer.DecoderReuseEvaluation.REUSE_RESULT_NO;
@ -59,6 +60,7 @@ import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoSize;
import androidx.media3.common.util.Clock;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.MediaFormatUtil;
import androidx.media3.common.util.Size;
@ -644,7 +646,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
} else if (joiningDeadlineMs == C.TIME_UNSET) {
// Not joining.
return false;
} else if (SystemClock.elapsedRealtime() < joiningDeadlineMs) {
} else if (getClock().elapsedRealtime() < joiningDeadlineMs) {
// Joining and still within the joining deadline.
return true;
} else {
@ -658,8 +660,9 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
protected void onStarted() {
super.onStarted();
droppedFrames = 0;
droppedFrameAccumulationStartTimeMs = SystemClock.elapsedRealtime();
lastRenderRealtimeUs = SystemClock.elapsedRealtime() * 1000;
long elapsedRealtimeMs = getClock().elapsedRealtime();
droppedFrameAccumulationStartTimeMs = elapsedRealtimeMs;
lastRenderRealtimeUs = msToUs(elapsedRealtimeMs);
totalVideoFrameProcessingOffsetUs = 0;
videoFrameProcessingOffsetCount = 0;
frameReleaseHelper.onStarted();
@ -1009,7 +1012,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
@Override
protected void onReadyToInitializeCodec(Format format) throws ExoPlaybackException {
if (!videoFrameProcessorManager.isEnabled()) {
videoFrameProcessorManager.maybeEnable(format, getOutputStreamOffsetUs());
videoFrameProcessorManager.maybeEnable(format, getOutputStreamOffsetUs(), getClock());
}
}
@ -1202,7 +1205,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
// Note: Use of double rather than float is intentional for accuracy in the calculations below.
boolean isStarted = getState() == STATE_STARTED;
long elapsedRealtimeNowUs = SystemClock.elapsedRealtime() * 1000;
long elapsedRealtimeNowUs = msToUs(getClock().elapsedRealtime());
long earlyUs =
calculateEarlyTimeUs(
positionUs,
@ -1244,7 +1247,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
// Compute the buffer's desired release time in nanoseconds.
long systemTimeNs = System.nanoTime();
long systemTimeNs = getClock().nanoTime();
long unadjustedFrameReleaseTimeNs = systemTimeNs + (earlyUs * 1000);
// Apply a timestamp adjustment, if there is one.
@ -1330,7 +1333,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
!renderedFirstFrameAfterEnable
? (isStarted || mayRenderFirstFrameAfterEnableIfNotStarted)
: !renderedFirstFrameAfterReset;
long elapsedSinceLastRenderUs = SystemClock.elapsedRealtime() * 1000 - lastRenderRealtimeUs;
long elapsedSinceLastRenderUs = msToUs(getClock().elapsedRealtime()) - lastRenderRealtimeUs;
// Don't force output until we joined and the position reached the current stream.
return joiningDeadlineMs == C.TIME_UNSET
&& positionUs >= getOutputStreamOffsetUs()
@ -1597,7 +1600,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
? videoFrameProcessorManager.getCorrectedFramePresentationTimeUs(
presentationTimeUs, getOutputStreamOffsetUs())
* 1000
: System.nanoTime();
: getClock().nanoTime();
if (notifyFrameMetadataListener) {
notifyFrameMetadataListener(presentationTimeUs, releaseTimeNs, format);
}
@ -1627,7 +1630,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
decoderCounters.renderedOutputBufferCount++;
consecutiveDroppedFrameCount = 0;
if (!videoFrameProcessorManager.isEnabled()) {
lastRenderRealtimeUs = SystemClock.elapsedRealtime() * 1000;
lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime());
maybeNotifyVideoSizeChanged(decodedVideoSize);
maybeNotifyRenderedFirstFrame();
}
@ -1655,7 +1658,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
decoderCounters.renderedOutputBufferCount++;
consecutiveDroppedFrameCount = 0;
if (!videoFrameProcessorManager.isEnabled()) {
lastRenderRealtimeUs = SystemClock.elapsedRealtime() * 1000;
lastRenderRealtimeUs = msToUs(getClock().elapsedRealtime());
maybeNotifyVideoSizeChanged(decodedVideoSize);
maybeNotifyRenderedFirstFrame();
}
@ -1680,7 +1683,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private void setJoiningDeadlineMs() {
joiningDeadlineMs =
allowedJoiningTimeMs > 0
? (SystemClock.elapsedRealtime() + allowedJoiningTimeMs)
? (getClock().elapsedRealtime() + allowedJoiningTimeMs)
: C.TIME_UNSET;
}
@ -1734,7 +1737,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
private void maybeNotifyDroppedFrames() {
if (droppedFrames > 0) {
long now = SystemClock.elapsedRealtime();
long now = getClock().elapsedRealtime();
long elapsedMs = now - droppedFrameAccumulationStartTimeMs;
eventDispatcher.droppedFrames(droppedFrames, elapsedMs);
droppedFrames = 0;
@ -1936,6 +1939,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
*/
private @MonotonicNonNull Pair<Long, Format> currentFrameFormat;
private @MonotonicNonNull Clock clock;
@Nullable private Pair<Surface, Size> currentSurfaceAndSize;
private int videoFrameProcessorMaxPendingFrameCount;
@ -2039,7 +2043,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
* @throws ExoPlaybackException When enabling the {@link VideoFrameProcessor} failed.
*/
@CanIgnoreReturnValue
public boolean maybeEnable(Format inputFormat, long initialStreamOffsetUs)
public boolean maybeEnable(Format inputFormat, long initialStreamOffsetUs, Clock clock)
throws ExoPlaybackException {
checkState(!isEnabled());
if (!canEnableFrameProcessing) {
@ -2052,6 +2056,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
// Playback thread handler.
handler = Util.createHandlerForCurrentLooper();
this.clock = clock;
Pair<ColorInfo, ColorInfo> inputAndOutputColorInfos =
renderer.experimentalGetVideoFrameProcessorColorConfiguration(inputFormat.colorInfo);
@ -2292,7 +2297,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
renderer.calculateEarlyTimeUs(
positionUs,
elapsedRealtimeUs,
SystemClock.elapsedRealtime() * 1000,
msToUs(clock.elapsedRealtime()),
bufferPresentationTimeUs,
isStarted);
@ -2313,10 +2318,11 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
}
frameReleaseHelper.onNextFrame(bufferPresentationTimeUs);
long unadjustedFrameReleaseTimeNs = System.nanoTime() + earlyUs * 1000;
long systemNanoTime = checkNotNull(clock).nanoTime();
long unadjustedFrameReleaseTimeNs = systemNanoTime + earlyUs * 1000;
long adjustedFrameReleaseTimeNs =
frameReleaseHelper.adjustReleaseTime(unadjustedFrameReleaseTimeNs);
earlyUs = (adjustedFrameReleaseTimeNs - System.nanoTime()) / 1000;
earlyUs = (adjustedFrameReleaseTimeNs - systemNanoTime) / 1000;
// TODO(b/238302341) Handle very late buffers and drop to key frame. Need to flush
// VideoFrameProcessor input frames in this case.
@ -2364,7 +2370,7 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer {
checkStateNotNull(videoFrameProcessor);
videoFrameProcessor.renderOutputFrame(releaseTimeNs);
processedFramesTimestampsUs.remove();
renderer.lastRenderRealtimeUs = SystemClock.elapsedRealtime() * 1000;
renderer.lastRenderRealtimeUs = msToUs(clock.elapsedRealtime());
if (releaseTimeNs != VideoFrameProcessor.DROP_OUTPUT_FRAME) {
renderer.maybeNotifyRenderedFirstFrame();
}

View File

@ -35,6 +35,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Clock;
import androidx.media3.decoder.CryptoConfig;
import androidx.media3.decoder.DecoderException;
import androidx.media3.decoder.DecoderInputBuffer;
@ -91,7 +92,7 @@ public class DecoderAudioRendererTest {
return FORMAT;
}
};
audioRenderer.init(/* index= */ 0, PlayerId.UNSET);
audioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
}
@Config(sdk = 19)

View File

@ -38,6 +38,7 @@ import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.PlaybackException;
import androidx.media3.common.util.Clock;
import androidx.media3.exoplayer.ExoPlaybackException;
import androidx.media3.exoplayer.RendererCapabilities;
import androidx.media3.exoplayer.RendererCapabilities.Capabilities;
@ -134,7 +135,7 @@ public class MediaCodecAudioRendererTest {
eventHandler,
audioRendererEventListener,
audioSink);
mediaCodecAudioRenderer.init(/* index= */ 0, PlayerId.UNSET);
mediaCodecAudioRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
}
@Test
@ -289,7 +290,7 @@ public class MediaCodecAudioRendererTest {
oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM));
fakeSampleStream.writeData(/* startPositionUs= */ 0);
exceptionThrowingRenderer.init(/* index= */ 0, PlayerId.UNSET);
exceptionThrowingRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
exceptionThrowingRenderer.enable(
RendererConfiguration.DEFAULT,
new Format[] {AUDIO_AAC, changedFormat},
@ -438,7 +439,7 @@ public class MediaCodecAudioRendererTest {
/* eventHandler= */ new Handler(Looper.getMainLooper()),
audioRendererEventListener,
audioSink);
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
@Capabilities int capabilities = renderer.supportsFormat(mediaFormat);

View File

@ -30,6 +30,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Clock;
import androidx.media3.exoplayer.DecoderReuseEvaluation;
import androidx.media3.exoplayer.ExoPlaybackException;
import androidx.media3.exoplayer.RendererCapabilities;
@ -63,7 +64,7 @@ public class MediaCodecRendererTest {
FakeSampleStream fakeSampleStream2 =
createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200);
MediaCodecRenderer renderer = spy(new TestRenderer());
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
renderer.enable(
RendererConfiguration.DEFAULT,
@ -116,7 +117,7 @@ public class MediaCodecRendererTest {
FakeSampleStream fakeSampleStream2 =
createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200);
MediaCodecRenderer renderer = spy(new TestRenderer());
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
renderer.enable(
RendererConfiguration.DEFAULT,
@ -172,7 +173,7 @@ public class MediaCodecRendererTest {
FakeSampleStream fakeSampleStream2 =
createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200, 300, 400);
MediaCodecRenderer renderer = spy(new TestRenderer());
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
renderer.enable(
RendererConfiguration.DEFAULT,
@ -226,7 +227,7 @@ public class MediaCodecRendererTest {
FakeSampleStream fakeSampleStream2 =
createFakeSampleStream(format2, /* sampleTimesUs...= */ 0, 100, 200);
MediaCodecRenderer renderer = spy(new TestRenderer());
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
renderer.enable(
RendererConfiguration.DEFAULT,
@ -277,7 +278,7 @@ public class MediaCodecRendererTest {
FakeSampleStream fakeSampleStream3 =
createFakeSampleStream(format3, /* sampleTimesUs...= */ 0, 100, 200);
MediaCodecRenderer renderer = spy(new TestRenderer());
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
renderer.enable(
RendererConfiguration.DEFAULT,

View File

@ -47,6 +47,7 @@ import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.VideoSize;
import androidx.media3.common.util.Clock;
import androidx.media3.decoder.CryptoInfo;
import androidx.media3.exoplayer.DecoderCounters;
import androidx.media3.exoplayer.Renderer;
@ -169,6 +170,7 @@ public class MediaCodecVideoRendererTest {
}
};
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
surface = new Surface(new SurfaceTexture(/* texName= */ 0));
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
}
@ -247,6 +249,7 @@ public class MediaCodecVideoRendererTest {
/* eventHandler= */ new Handler(testMainLooper),
/* eventListener= */ eventListener,
/* maxDroppedFramesToNotify= */ 1);
mediaCodecVideoRenderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
mediaCodecVideoRenderer.handleMessage(Renderer.MSG_SET_VIDEO_OUTPUT, surface);
mediaCodecVideoRenderer.enable(
RendererConfiguration.DEFAULT,
@ -697,7 +700,7 @@ public class MediaCodecVideoRendererTest {
/* eventHandler= */ new Handler(testMainLooper),
/* eventListener= */ eventListener,
/* maxDroppedFramesToNotify= */ 1);
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
@Capabilities
int capabilitiesDvheDtrFallbackToH265 = renderer.supportsFormat(formatDvheDtrFallbackToH265);
@ -782,7 +785,7 @@ public class MediaCodecVideoRendererTest {
/* eventHandler= */ new Handler(testMainLooper),
/* eventListener= */ eventListener,
/* maxDroppedFramesToNotify= */ 1);
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
@Capabilities int capabilitiesDvheDtr = renderer.supportsFormat(formatDvheDtr);
@ -842,7 +845,7 @@ public class MediaCodecVideoRendererTest {
/* eventHandler= */ new Handler(testMainLooper),
/* eventListener= */ eventListener,
/* maxDroppedFramesToNotify= */ 1);
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
List<MediaCodecInfo> mediaCodecInfoList =
renderer.getDecoderInfos(mediaCodecSelector, avcFormat, false);
@ -885,7 +888,7 @@ public class MediaCodecVideoRendererTest {
/* eventHandler= */ new Handler(testMainLooper),
/* eventListener= */ eventListener,
/* maxDroppedFramesToNotify= */ 1);
renderer.init(/* index= */ 0, PlayerId.UNSET);
renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT);
List<MediaCodecInfo> mediaCodecInfoList =
renderer.getDecoderInfos(mediaCodecSelector, avcFormat, false);