Crop and pass thumbnails to ImageOutput
Image grids are now cropped into tiles. The tiles are provided to ImageOutput at their correct timestamps. PiperOrigin-RevId: 597553029
This commit is contained in:
lpribanic
Copybara-Service
parent
cd2d7f5da5
commit
e93188fe7f
@ -162,6 +162,10 @@ This release includes the following changes since the
|
||||
* Catch `OutOfMemoryError` when parsing very large ID3 frames, meaning
|
||||
playback can continue without the tag info instead of playback failing
|
||||
completely.
|
||||
* Image:
|
||||
* Add support for DASH thumbnails. Grid images are cropped and individual
|
||||
thumbnails are provided to `ImageOutput` close to their presentation
|
||||
times.
|
||||
* DRM:
|
||||
* Extend workaround for spurious ClearKey `https://default.url` license
|
||||
URL to API 33+ (previously the workaround only applied on API 33
|
||||
|
||||
@ -46,6 +46,7 @@ import java.lang.annotation.Retention;
|
||||
import java.lang.annotation.RetentionPolicy;
|
||||
import java.lang.annotation.Target;
|
||||
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
import org.checkerframework.checker.nullness.qual.Nullable;
|
||||
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
|
||||
@ -83,6 +84,11 @@ public class ImageRenderer extends BaseRenderer {
|
||||
*/
|
||||
private static final int REINITIALIZATION_STATE_WAIT_END_OF_STREAM = 3;
|
||||
|
||||
/**
|
||||
* A time threshold, in microseconds, for the window during which an image should be presented.
|
||||
*/
|
||||
private static final long IMAGE_PRESENTATION_WINDOW_THRESHOLD_US = 30_000;
|
||||
|
||||
private final ImageDecoder.Factory decoderFactory;
|
||||
private final DecoderInputBuffer flagsOnlyBuffer;
|
||||
private final LongArrayQueue offsetQueue;
|
||||
@ -94,8 +100,12 @@ public class ImageRenderer extends BaseRenderer {
|
||||
private @Nullable Format inputFormat;
|
||||
private @Nullable ImageDecoder decoder;
|
||||
private @Nullable DecoderInputBuffer inputBuffer;
|
||||
private @Nullable ImageOutputBuffer outputBuffer;
|
||||
private ImageOutput imageOutput;
|
||||
private @Nullable Bitmap outputBitmap;
|
||||
private boolean readyToOutputTiles;
|
||||
private @Nullable TileInfo tileInfo;
|
||||
private @Nullable TileInfo nextTileInfo;
|
||||
private int currentTileIndex;
|
||||
|
||||
/**
|
||||
* Creates an instance.
|
||||
@ -156,8 +166,8 @@ public class ImageRenderer extends BaseRenderer {
|
||||
try {
|
||||
// Rendering loop.
|
||||
TraceUtil.beginSection("drainAndFeedDecoder");
|
||||
while (drainOutputBuffer(positionUs, elapsedRealtimeUs)) {}
|
||||
while (feedInputBuffer()) {}
|
||||
while (drainOutput(positionUs, elapsedRealtimeUs)) {}
|
||||
while (feedInputBuffer(positionUs)) {}
|
||||
TraceUtil.endSection();
|
||||
} catch (ImageDecoderException e) {
|
||||
throw createRendererException(e, null, PlaybackException.ERROR_CODE_DECODING_FAILED);
|
||||
@ -168,7 +178,7 @@ public class ImageRenderer extends BaseRenderer {
|
||||
public boolean isReady() {
|
||||
return firstFrameState == FIRST_FRAME_RENDERED
|
||||
|| (firstFrameState == FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED
|
||||
&& outputBuffer != null);
|
||||
&& readyToOutputTiles);
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -198,8 +208,18 @@ public class ImageRenderer extends BaseRenderer {
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void onPositionReset(long positionUs, boolean joining) {
|
||||
protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException {
|
||||
lowerFirstFrameState(FIRST_FRAME_NOT_RENDERED);
|
||||
outputStreamEnded = false;
|
||||
inputStreamEnded = false;
|
||||
outputBitmap = null;
|
||||
tileInfo = null;
|
||||
nextTileInfo = null;
|
||||
readyToOutputTiles = false;
|
||||
inputBuffer = null;
|
||||
if (decoder != null) {
|
||||
decoder.flush();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -238,63 +258,108 @@ public class ImageRenderer extends BaseRenderer {
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to dequeue an output buffer from the decoder and, if successful and permitted to,
|
||||
* renders it.
|
||||
* Checks if there is data to output. If there is no data to output, it attempts dequeuing the
|
||||
* output buffer from the decoder. If there is data to output, it attempts to render it.
|
||||
*
|
||||
* @param positionUs The player's current position.
|
||||
* @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds,
|
||||
* measured at the start of the current iteration of the rendering loop.
|
||||
* @return Whether it may be possible to drain more output data.
|
||||
* @return Whether it may be possible to output more data.
|
||||
* @throws ImageDecoderException If an error occurs draining the output buffer.
|
||||
*/
|
||||
private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs)
|
||||
private boolean drainOutput(long positionUs, long elapsedRealtimeUs)
|
||||
throws ImageDecoderException, ExoPlaybackException {
|
||||
if (outputBuffer == null) {
|
||||
checkStateNotNull(decoder);
|
||||
outputBuffer = decoder.dequeueOutputBuffer();
|
||||
if (outputBuffer == null) {
|
||||
return false;
|
||||
}
|
||||
// If tileInfo and outputBitmap are both null, we must not return early. The EOS may have been
|
||||
// queued to the decoder, and we must stay in this method to deque it further down.
|
||||
if (outputBitmap != null && tileInfo == null) {
|
||||
return false;
|
||||
}
|
||||
if (firstFrameState == FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED
|
||||
&& getState() != STATE_STARTED) {
|
||||
return false;
|
||||
}
|
||||
if (checkStateNotNull(outputBuffer).isEndOfStream()) {
|
||||
offsetQueue.remove();
|
||||
if (decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM) {
|
||||
// We're waiting to re-initialize the decoder, and have now processed all final buffers.
|
||||
releaseDecoderResources();
|
||||
checkStateNotNull(inputFormat);
|
||||
initDecoder();
|
||||
} else {
|
||||
checkStateNotNull(outputBuffer).release();
|
||||
outputBuffer = null;
|
||||
if (offsetQueue.isEmpty()) {
|
||||
outputStreamEnded = true;
|
||||
}
|
||||
if (outputBitmap == null) {
|
||||
checkStateNotNull(decoder);
|
||||
ImageOutputBuffer outputBuffer = decoder.dequeueOutputBuffer();
|
||||
if (outputBuffer == null) {
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
if (checkStateNotNull(outputBuffer).isEndOfStream()) {
|
||||
offsetQueue.remove();
|
||||
if (decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM) {
|
||||
// We're waiting to re-initialize the decoder, and have now processed all final buffers.
|
||||
releaseDecoderResources();
|
||||
checkStateNotNull(inputFormat);
|
||||
initDecoder();
|
||||
} else {
|
||||
checkStateNotNull(outputBuffer).release();
|
||||
if (offsetQueue.isEmpty()) {
|
||||
outputStreamEnded = true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
checkStateNotNull(
|
||||
outputBuffer.bitmap, "Non-EOS buffer came back from the decoder without bitmap.");
|
||||
outputBitmap = outputBuffer.bitmap;
|
||||
checkStateNotNull(outputBuffer).release();
|
||||
}
|
||||
|
||||
ImageOutputBuffer imageOutputBuffer = checkStateNotNull(outputBuffer);
|
||||
checkStateNotNull(
|
||||
imageOutputBuffer.bitmap, "Non-EOS buffer came back from the decoder without bitmap.");
|
||||
if (!processOutputBuffer(
|
||||
positionUs, elapsedRealtimeUs, imageOutputBuffer.bitmap, imageOutputBuffer.timeUs)) {
|
||||
return false;
|
||||
if (readyToOutputTiles && outputBitmap != null && tileInfo != null) {
|
||||
checkStateNotNull(inputFormat);
|
||||
boolean isThumbnailGrid =
|
||||
(inputFormat.tileCountHorizontal != 1 || inputFormat.tileCountVertical != 1)
|
||||
&& inputFormat.tileCountHorizontal != Format.NO_VALUE
|
||||
&& inputFormat.tileCountVertical != Format.NO_VALUE;
|
||||
// Lazily crop and store the bitmap to ensure we only have one tile in memory rather than
|
||||
// proactively storing a tile whenever creating TileInfos.
|
||||
if (!tileInfo.hasTileBitmap()) {
|
||||
tileInfo.setTileBitmap(
|
||||
isThumbnailGrid
|
||||
? cropTileFromImageGrid(tileInfo.getTileIndex())
|
||||
: checkStateNotNull(outputBitmap));
|
||||
}
|
||||
if (!processOutputBuffer(
|
||||
positionUs,
|
||||
elapsedRealtimeUs,
|
||||
checkStateNotNull(tileInfo.getTileBitmap()),
|
||||
tileInfo.getPresentationTimeUs())) {
|
||||
return false;
|
||||
}
|
||||
firstFrameState = FIRST_FRAME_RENDERED;
|
||||
if (!isThumbnailGrid
|
||||
|| checkStateNotNull(tileInfo).getTileIndex()
|
||||
== checkStateNotNull(inputFormat).tileCountVertical
|
||||
* checkStateNotNull(inputFormat).tileCountHorizontal
|
||||
- 1) {
|
||||
outputBitmap = null;
|
||||
}
|
||||
tileInfo = nextTileInfo;
|
||||
nextTileInfo = null;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private boolean shouldForceRender() {
|
||||
boolean isStarted = getState() == STATE_STARTED;
|
||||
switch (firstFrameState) {
|
||||
case C.FIRST_FRAME_NOT_RENDERED_ONLY_ALLOWED_IF_STARTED:
|
||||
return isStarted;
|
||||
case C.FIRST_FRAME_NOT_RENDERED:
|
||||
return true;
|
||||
case C.FIRST_FRAME_RENDERED:
|
||||
return false;
|
||||
default:
|
||||
throw new IllegalStateException();
|
||||
}
|
||||
checkStateNotNull(outputBuffer).release();
|
||||
outputBuffer = null;
|
||||
firstFrameState = FIRST_FRAME_RENDERED;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Processes an output image.
|
||||
*
|
||||
* @param positionUs The current media time in microseconds, measured at the start of the current
|
||||
* iteration of the rendering loop.
|
||||
* @param positionUs The current playback position in microseconds, measured at the start of the
|
||||
* current iteration of the rendering loop.
|
||||
* @param elapsedRealtimeUs {@link SystemClock#elapsedRealtime()} in microseconds, measured at the
|
||||
* start of the current iteration of the rendering loop.
|
||||
* @param outputBitmap The {@link Bitmap}.
|
||||
@ -305,18 +370,25 @@ public class ImageRenderer extends BaseRenderer {
|
||||
protected boolean processOutputBuffer(
|
||||
long positionUs, long elapsedRealtimeUs, Bitmap outputBitmap, long bufferPresentationTimeUs)
|
||||
throws ExoPlaybackException {
|
||||
if (positionUs < bufferPresentationTimeUs) {
|
||||
// It's too early to render the buffer.
|
||||
return false;
|
||||
// TODO: b/319484746 - ImageRenderer should consider startPositionUs when choosing to output an
|
||||
// image.
|
||||
long earlyUs = bufferPresentationTimeUs - positionUs;
|
||||
if (shouldForceRender() || earlyUs < IMAGE_PRESENTATION_WINDOW_THRESHOLD_US) {
|
||||
imageOutput.onImageAvailable(bufferPresentationTimeUs - offsetQueue.element(), outputBitmap);
|
||||
return true;
|
||||
}
|
||||
imageOutput.onImageAvailable(bufferPresentationTimeUs - offsetQueue.element(), outputBitmap);
|
||||
return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param positionUs The current playback position in microseconds, measured at the start of the
|
||||
* current iteration of the rendering loop.
|
||||
* @return Whether we can feed more input data to the decoder.
|
||||
*/
|
||||
private boolean feedInputBuffer() throws ImageDecoderException {
|
||||
private boolean feedInputBuffer(long positionUs) throws ImageDecoderException {
|
||||
if (readyToOutputTiles && tileInfo != null) {
|
||||
return false;
|
||||
}
|
||||
FormatHolder formatHolder = getFormatHolder();
|
||||
if (decoder == null
|
||||
|| decoderReinitializationState == REINITIALIZATION_STATE_WAIT_END_OF_STREAM
|
||||
@ -349,8 +421,12 @@ public class ImageRenderer extends BaseRenderer {
|
||||
checkStateNotNull(inputBuffer.data).remaining() > 0
|
||||
|| checkStateNotNull(inputBuffer).isEndOfStream();
|
||||
if (shouldQueueBuffer) {
|
||||
// TODO: b/318696449 - Don't use the deprecated BUFFER_FLAG_DECODE_ONLY with image chunks.
|
||||
checkStateNotNull(inputBuffer).clearFlag(C.BUFFER_FLAG_DECODE_ONLY);
|
||||
checkStateNotNull(decoder).queueInputBuffer(checkStateNotNull(inputBuffer));
|
||||
currentTileIndex = 0;
|
||||
}
|
||||
maybeAdvanceTileInfo(positionUs, checkStateNotNull(inputBuffer));
|
||||
if (checkStateNotNull(inputBuffer).isEndOfStream()) {
|
||||
inputStreamEnded = true;
|
||||
inputBuffer = null;
|
||||
@ -363,7 +439,7 @@ public class ImageRenderer extends BaseRenderer {
|
||||
} else {
|
||||
checkStateNotNull(inputBuffer).clear();
|
||||
}
|
||||
return true;
|
||||
return !readyToOutputTiles;
|
||||
case C.RESULT_FORMAT_READ:
|
||||
inputFormat = checkStateNotNull(formatHolder.format);
|
||||
decoderReinitializationState = REINITIALIZATION_STATE_SIGNAL_END_OF_STREAM_THEN_WAIT;
|
||||
@ -401,10 +477,6 @@ public class ImageRenderer extends BaseRenderer {
|
||||
|
||||
private void releaseDecoderResources() {
|
||||
inputBuffer = null;
|
||||
if (outputBuffer != null) {
|
||||
outputBuffer.release();
|
||||
}
|
||||
outputBuffer = null;
|
||||
decoderReinitializationState = REINITIALIZATION_STATE_NONE;
|
||||
if (decoder != null) {
|
||||
decoder.release();
|
||||
@ -416,7 +488,72 @@ public class ImageRenderer extends BaseRenderer {
|
||||
this.imageOutput = getImageOutput(imageOutput);
|
||||
}
|
||||
|
||||
private void maybeAdvanceTileInfo(long positionUs, DecoderInputBuffer inputBuffer) {
|
||||
if (inputBuffer.isEndOfStream()) {
|
||||
readyToOutputTiles = true;
|
||||
return;
|
||||
}
|
||||
nextTileInfo = new TileInfo(currentTileIndex, inputBuffer.timeUs);
|
||||
currentTileIndex++;
|
||||
// TODO: b/319484746 - ImageRenderer should consider startPositionUs when choosing to output an
|
||||
// image.
|
||||
if (nextTileInfo.getPresentationTimeUs() - IMAGE_PRESENTATION_WINDOW_THRESHOLD_US <= positionUs
|
||||
&& positionUs
|
||||
<= nextTileInfo.getPresentationTimeUs() + IMAGE_PRESENTATION_WINDOW_THRESHOLD_US) {
|
||||
readyToOutputTiles = true;
|
||||
} else if (tileInfo != null
|
||||
&& nextTileInfo != null
|
||||
&& tileInfo.getPresentationTimeUs() <= positionUs
|
||||
&& positionUs < checkStateNotNull(nextTileInfo).getPresentationTimeUs()) {
|
||||
readyToOutputTiles = true;
|
||||
return;
|
||||
}
|
||||
tileInfo = nextTileInfo;
|
||||
nextTileInfo = null;
|
||||
}
|
||||
|
||||
private Bitmap cropTileFromImageGrid(int tileIndex) {
|
||||
checkStateNotNull(outputBitmap);
|
||||
int tileWidth = outputBitmap.getWidth() / checkStateNotNull(inputFormat).tileCountHorizontal;
|
||||
int tileHeight = outputBitmap.getHeight() / checkStateNotNull(inputFormat).tileCountVertical;
|
||||
int tileStartXCoordinate = tileWidth * (tileIndex % inputFormat.tileCountVertical);
|
||||
int tileStartYCoordinate = tileHeight * (tileIndex / inputFormat.tileCountHorizontal);
|
||||
return Bitmap.createBitmap(
|
||||
outputBitmap, tileStartXCoordinate, tileStartYCoordinate, tileWidth, tileHeight);
|
||||
}
|
||||
|
||||
private static ImageOutput getImageOutput(@Nullable ImageOutput imageOutput) {
|
||||
return imageOutput == null ? ImageOutput.NO_OP : imageOutput;
|
||||
}
|
||||
|
||||
private static class TileInfo {
|
||||
private final int tileIndex;
|
||||
private final long presentationTimeUs;
|
||||
private @MonotonicNonNull Bitmap tileBitmap;
|
||||
|
||||
public TileInfo(int tileIndex, long presentationTimeUs) {
|
||||
this.tileIndex = tileIndex;
|
||||
this.presentationTimeUs = presentationTimeUs;
|
||||
}
|
||||
|
||||
public int getTileIndex() {
|
||||
return this.tileIndex;
|
||||
}
|
||||
|
||||
public long getPresentationTimeUs() {
|
||||
return presentationTimeUs;
|
||||
}
|
||||
|
||||
public @Nullable Bitmap getTileBitmap() {
|
||||
return tileBitmap;
|
||||
}
|
||||
|
||||
public void setTileBitmap(Bitmap tileBitmap) {
|
||||
this.tileBitmap = tileBitmap;
|
||||
}
|
||||
|
||||
public boolean hasTileBitmap() {
|
||||
return tileBitmap != null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -15,8 +15,6 @@
|
||||
*/
|
||||
package androidx.media3.exoplayer.source.chunk;
|
||||
|
||||
import static androidx.media3.common.C.BUFFER_FLAG_KEY_FRAME;
|
||||
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.C;
|
||||
import androidx.media3.common.Format;
|
||||
@ -171,11 +169,7 @@ public class ContainerMediaChunk extends BaseMediaChunk {
|
||||
long tileStartTimeUs = i * tileDurationUs;
|
||||
trackOutput.sampleData(new ParsableByteArray(), /* length= */ 0);
|
||||
trackOutput.sampleMetadata(
|
||||
tileStartTimeUs,
|
||||
/* flags= */ BUFFER_FLAG_KEY_FRAME,
|
||||
/* size= */ 0,
|
||||
/* offset= */ 0,
|
||||
/* cryptoData= */ null);
|
||||
tileStartTimeUs, /* flags= */ 0, /* size= */ 0, /* offset= */ 0, /* cryptoData= */ null);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -17,6 +17,7 @@ package androidx.media3.exoplayer.image;
|
||||
|
||||
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM;
|
||||
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample;
|
||||
import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.sample;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import android.graphics.Bitmap;
|
||||
@ -65,12 +66,18 @@ public class ImageRendererTest {
|
||||
.setTileCountVertical(1)
|
||||
.setTileCountHorizontal(1)
|
||||
.build();
|
||||
private static final Format JPEG_FORMAT_WITH_FOUR_TILES =
|
||||
new Format.Builder()
|
||||
.setContainerMimeType(MimeTypes.IMAGE_JPEG)
|
||||
.setTileCountVertical(2)
|
||||
.setTileCountHorizontal(2)
|
||||
.build();
|
||||
|
||||
private final List<Pair<Long, Bitmap>> renderedBitmaps = new ArrayList<>();
|
||||
private final Bitmap fakeDecodedBitmap1 =
|
||||
Bitmap.createBitmap(/* width= */ 1, /* height= */ 1, Bitmap.Config.ARGB_8888);
|
||||
private final Bitmap fakeDecodedBitmap2 =
|
||||
Bitmap.createBitmap(/* width= */ 2, /* height= */ 2, Bitmap.Config.ARGB_8888);
|
||||
private final Bitmap fakeDecodedBitmap2 =
|
||||
Bitmap.createBitmap(/* width= */ 4, /* height= */ 4, Bitmap.Config.ARGB_8888);
|
||||
|
||||
private ImageRenderer renderer;
|
||||
private int decodeCallCount;
|
||||
@ -256,6 +263,285 @@ public class ImageRendererTest {
|
||||
assertThat(renderedBitmaps.get(1).second).isSameInstanceAs(fakeDecodedBitmap2);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void render_tiledImage_cropsAndRendersToImageOutput() throws Exception {
|
||||
FakeSampleStream fakeSampleStream =
|
||||
createSampleStream(
|
||||
JPEG_FORMAT_WITH_FOUR_TILES,
|
||||
ImmutableList.of(
|
||||
oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME),
|
||||
emptySample(/* timeUs= */ 100_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 200_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 300_000L, /* flags= */ 0),
|
||||
END_OF_STREAM_ITEM));
|
||||
fakeSampleStream.writeData(/* startPositionUs= */ 0);
|
||||
renderer.enable(
|
||||
RendererConfiguration.DEFAULT,
|
||||
new Format[] {JPEG_FORMAT_WITH_FOUR_TILES},
|
||||
fakeSampleStream,
|
||||
/* positionUs= */ 0,
|
||||
/* joining= */ false,
|
||||
/* mayRenderStartOfStream= */ true,
|
||||
/* startPositionUs= */ 0,
|
||||
/* offsetUs= */ 0,
|
||||
new MediaSource.MediaPeriodId(new Object()));
|
||||
renderer.setCurrentStreamFinal();
|
||||
|
||||
StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE);
|
||||
while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
/* positionUs= */ 0,
|
||||
/* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
}
|
||||
StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE);
|
||||
long positionUs = 0;
|
||||
while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
positionUs += 100_000;
|
||||
}
|
||||
|
||||
assertThat(renderedBitmaps).hasSize(4);
|
||||
assertThat(renderedBitmaps.get(0).first).isEqualTo(0L);
|
||||
assertThat(renderedBitmaps.get(0).second.getHeight()).isEqualTo(1);
|
||||
assertThat(renderedBitmaps.get(0).second.getWidth()).isEqualTo(1);
|
||||
assertThat(renderedBitmaps.get(1).first).isEqualTo(100_000L);
|
||||
assertThat(renderedBitmaps.get(1).second.getHeight()).isEqualTo(1);
|
||||
assertThat(renderedBitmaps.get(1).second.getWidth()).isEqualTo(1);
|
||||
assertThat(renderedBitmaps.get(2).first).isEqualTo(200_000L);
|
||||
assertThat(renderedBitmaps.get(2).second.getHeight()).isEqualTo(1);
|
||||
assertThat(renderedBitmaps.get(2).second.getWidth()).isEqualTo(1);
|
||||
assertThat(renderedBitmaps.get(3).first).isEqualTo(300_000L);
|
||||
assertThat(renderedBitmaps.get(3).second.getHeight()).isEqualTo(1);
|
||||
assertThat(renderedBitmaps.get(3).second.getWidth()).isEqualTo(1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void render_tiledImageWithNonZeroStartPosition_rendersToImageOutput() throws Exception {
|
||||
FakeSampleStream fakeSampleStream =
|
||||
createSampleStream(
|
||||
JPEG_FORMAT_WITH_FOUR_TILES,
|
||||
ImmutableList.of(
|
||||
oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME),
|
||||
emptySample(/* timeUs= */ 100_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 200_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 300_000L, /* flags= */ 0),
|
||||
END_OF_STREAM_ITEM));
|
||||
fakeSampleStream.writeData(/* startPositionUs= */ 0);
|
||||
renderer.enable(
|
||||
RendererConfiguration.DEFAULT,
|
||||
new Format[] {JPEG_FORMAT_WITH_FOUR_TILES},
|
||||
fakeSampleStream,
|
||||
/* positionUs= */ 200_000,
|
||||
/* joining= */ false,
|
||||
/* mayRenderStartOfStream= */ true,
|
||||
/* startPositionUs= */ 0,
|
||||
/* offsetUs= */ 0,
|
||||
new MediaSource.MediaPeriodId(new Object()));
|
||||
renderer.setCurrentStreamFinal();
|
||||
|
||||
StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE);
|
||||
while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
/* positionUs= */ 200_000,
|
||||
/* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
}
|
||||
StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE);
|
||||
long positionUs = 200_000;
|
||||
while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
positionUs += 100_000;
|
||||
}
|
||||
|
||||
assertThat(renderedBitmaps).hasSize(2);
|
||||
assertThat(renderedBitmaps.get(0).first).isEqualTo(200_000L);
|
||||
assertThat(renderedBitmaps.get(1).first).isEqualTo(300_000L);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void render_tiledImageStartPositionIsAfterLastTile_rendersToImageOutput()
|
||||
throws Exception {
|
||||
FakeSampleStream fakeSampleStream =
|
||||
createSampleStream(
|
||||
JPEG_FORMAT_WITH_FOUR_TILES,
|
||||
ImmutableList.of(
|
||||
oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME),
|
||||
emptySample(/* timeUs= */ 100_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 200_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 300_000L, /* flags= */ 0),
|
||||
END_OF_STREAM_ITEM));
|
||||
fakeSampleStream.writeData(/* startPositionUs= */ 0);
|
||||
renderer.enable(
|
||||
RendererConfiguration.DEFAULT,
|
||||
new Format[] {JPEG_FORMAT_WITH_FOUR_TILES},
|
||||
fakeSampleStream,
|
||||
/* positionUs= */ 0,
|
||||
/* joining= */ false,
|
||||
/* mayRenderStartOfStream= */ true,
|
||||
/* startPositionUs= */ 0,
|
||||
/* offsetUs= */ 0,
|
||||
new MediaSource.MediaPeriodId(new Object()));
|
||||
renderer.setCurrentStreamFinal();
|
||||
|
||||
StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE);
|
||||
while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
/* positionUs= */ 350_000,
|
||||
/* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
}
|
||||
StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE);
|
||||
long positionUs = 350_000;
|
||||
while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
positionUs += 100_000;
|
||||
}
|
||||
|
||||
assertThat(renderedBitmaps).hasSize(1);
|
||||
assertThat(renderedBitmaps.get(0).first).isEqualTo(300_000L);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void
|
||||
render_tiledImageStartPositionBeforePresentationTimeAndWithinThreshold_rendersIncomingTile()
|
||||
throws Exception {
|
||||
FakeSampleStream fakeSampleStream =
|
||||
createSampleStream(
|
||||
JPEG_FORMAT_WITH_FOUR_TILES,
|
||||
ImmutableList.of(
|
||||
oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME),
|
||||
emptySample(/* timeUs= */ 100_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 200_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 300_000L, /* flags= */ 0),
|
||||
END_OF_STREAM_ITEM));
|
||||
fakeSampleStream.writeData(/* startPositionUs= */ 0);
|
||||
renderer.enable(
|
||||
RendererConfiguration.DEFAULT,
|
||||
new Format[] {JPEG_FORMAT_WITH_FOUR_TILES},
|
||||
fakeSampleStream,
|
||||
/* positionUs= */ 0,
|
||||
/* joining= */ false,
|
||||
/* mayRenderStartOfStream= */ true,
|
||||
/* startPositionUs= */ 0,
|
||||
/* offsetUs= */ 0,
|
||||
new MediaSource.MediaPeriodId(new Object()));
|
||||
renderer.setCurrentStreamFinal();
|
||||
|
||||
StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE);
|
||||
while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
/* positionUs= */ 70_000,
|
||||
/* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
}
|
||||
StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE);
|
||||
long positionUs = 70_000;
|
||||
while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
positionUs += 100_000;
|
||||
}
|
||||
|
||||
assertThat(renderedBitmaps).hasSize(3);
|
||||
assertThat(renderedBitmaps.get(0).first).isEqualTo(100_000L);
|
||||
assertThat(renderedBitmaps.get(1).first).isEqualTo(200_000L);
|
||||
assertThat(renderedBitmaps.get(2).first).isEqualTo(300_000L);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void
|
||||
render_tiledImageStartPositionAfterPresentationTimeAndWithinThreshold_rendersLastReadTile()
|
||||
throws Exception {
|
||||
FakeSampleStream fakeSampleStream =
|
||||
createSampleStream(
|
||||
JPEG_FORMAT_WITH_FOUR_TILES,
|
||||
ImmutableList.of(
|
||||
oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME),
|
||||
emptySample(/* timeUs= */ 100_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 200_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 300_000L, /* flags= */ 0),
|
||||
END_OF_STREAM_ITEM));
|
||||
fakeSampleStream.writeData(/* startPositionUs= */ 0);
|
||||
renderer.enable(
|
||||
RendererConfiguration.DEFAULT,
|
||||
new Format[] {JPEG_FORMAT_WITH_FOUR_TILES},
|
||||
fakeSampleStream,
|
||||
/* positionUs= */ 0,
|
||||
/* joining= */ false,
|
||||
/* mayRenderStartOfStream= */ true,
|
||||
/* startPositionUs= */ 0,
|
||||
/* offsetUs= */ 0,
|
||||
new MediaSource.MediaPeriodId(new Object()));
|
||||
renderer.setCurrentStreamFinal();
|
||||
|
||||
StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE);
|
||||
while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
/* positionUs= */ 130_000,
|
||||
/* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
}
|
||||
StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE);
|
||||
long positionUs = 130_000;
|
||||
while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
positionUs += 100_000;
|
||||
}
|
||||
|
||||
assertThat(renderedBitmaps).hasSize(3);
|
||||
assertThat(renderedBitmaps.get(0).first).isEqualTo(100_000L);
|
||||
assertThat(renderedBitmaps.get(1).first).isEqualTo(200_000L);
|
||||
assertThat(renderedBitmaps.get(2).first).isEqualTo(300_000L);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void render_tiledImageStartPositionRightBeforeEOSAndWithinThreshold_rendersLastTileInGrid()
|
||||
throws Exception {
|
||||
FakeSampleStream fakeSampleStream =
|
||||
createSampleStream(
|
||||
JPEG_FORMAT_WITH_FOUR_TILES,
|
||||
ImmutableList.of(
|
||||
oneByteSample(/* timeUs= */ 0L, /* flags= */ C.BUFFER_FLAG_KEY_FRAME),
|
||||
emptySample(/* timeUs= */ 100_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 200_000L, /* flags= */ 0),
|
||||
emptySample(/* timeUs= */ 300_000L, /* flags= */ 0),
|
||||
END_OF_STREAM_ITEM));
|
||||
fakeSampleStream.writeData(/* startPositionUs= */ 0);
|
||||
renderer.enable(
|
||||
RendererConfiguration.DEFAULT,
|
||||
new Format[] {JPEG_FORMAT_WITH_FOUR_TILES},
|
||||
fakeSampleStream,
|
||||
/* positionUs= */ 0,
|
||||
/* joining= */ false,
|
||||
/* mayRenderStartOfStream= */ true,
|
||||
/* startPositionUs= */ 0,
|
||||
/* offsetUs= */ 0,
|
||||
new MediaSource.MediaPeriodId(new Object()));
|
||||
renderer.setCurrentStreamFinal();
|
||||
|
||||
StopWatch isReadyStopWatch = new StopWatch(IS_READY_TIMEOUT_MESSAGE);
|
||||
while (!renderer.isReady() && isReadyStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
/* positionUs= */ 330_000,
|
||||
/* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
}
|
||||
StopWatch isEndedStopWatch = new StopWatch(IS_ENDED_TIMEOUT_MESSAGE);
|
||||
long positionUs = 330_000;
|
||||
while (!renderer.isEnded() && isEndedStopWatch.ensureNotExpired()) {
|
||||
renderer.render(
|
||||
positionUs, /* elapsedRealtimeUs= */ SystemClock.DEFAULT.elapsedRealtime() * 1000);
|
||||
positionUs += 100_000;
|
||||
}
|
||||
|
||||
assertThat(renderedBitmaps).hasSize(1);
|
||||
assertThat(renderedBitmaps.get(0).first).isEqualTo(300_000L);
|
||||
}
|
||||
|
||||
private static FakeSampleStream.FakeSampleStreamItem emptySample(
|
||||
long timeUs, @C.BufferFlags int flags) {
|
||||
return sample(timeUs, flags, new byte[] {});
|
||||
}
|
||||
|
||||
private static FakeSampleStream createSampleStream(long timeUs) {
|
||||
return new FakeSampleStream(
|
||||
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
|
||||
@ -266,6 +552,17 @@ public class ImageRendererTest {
|
||||
ImmutableList.of(oneByteSample(timeUs, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM));
|
||||
}
|
||||
|
||||
private static FakeSampleStream createSampleStream(
|
||||
Format format, List<FakeSampleStream.FakeSampleStreamItem> fakeSampleStreamItems) {
|
||||
return new FakeSampleStream(
|
||||
new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024),
|
||||
/* mediaSourceEventDispatcher= */ null,
|
||||
DrmSessionManager.DRM_UNSUPPORTED,
|
||||
new DrmSessionEventListener.EventDispatcher(),
|
||||
format,
|
||||
fakeSampleStreamItems);
|
||||
}
|
||||
|
||||
private static final class StopWatch {
|
||||
private final long startTimeMs;
|
||||
private final long timeOutMs;
|
||||
|
||||
@ -322,6 +322,11 @@ public final class DashPlaybackTest {
|
||||
applicationContext, playbackOutput, "playbackdumps/dash/metadata_from_early_output.dump");
|
||||
}
|
||||
|
||||
/**
|
||||
* This test might be flaky. The {@link ExoPlayer} instantiated in this test uses a {@link
|
||||
* FakeClock} that runs much faster than real time. This might cause the {@link ExoPlayer} to skip
|
||||
* and not present some images. That will cause the test to fail.
|
||||
*/
|
||||
@Test
|
||||
public void playThumbnailGrid() throws Exception {
|
||||
Context applicationContext = ApplicationProvider.getApplicationContext();
|
||||
|
||||
@ -1,5 +1,194 @@
|
||||
ImageOutput:
|
||||
rendered image count = 1
|
||||
rendered image count = 64
|
||||
image output #1:
|
||||
presentationTimeUs = 0
|
||||
bitmap hash = 90169190
|
||||
bitmap hash = 1662699756
|
||||
image output #2:
|
||||
presentationTimeUs = 937500
|
||||
bitmap hash = -1574796305
|
||||
image output #3:
|
||||
presentationTimeUs = 1875000
|
||||
bitmap hash = 113939771
|
||||
image output #4:
|
||||
presentationTimeUs = 2812500
|
||||
bitmap hash = -1947261277
|
||||
image output #5:
|
||||
presentationTimeUs = 3750000
|
||||
bitmap hash = 516207300
|
||||
image output #6:
|
||||
presentationTimeUs = 4687500
|
||||
bitmap hash = -1362626860
|
||||
image output #7:
|
||||
presentationTimeUs = 5625000
|
||||
bitmap hash = 1681263366
|
||||
image output #8:
|
||||
presentationTimeUs = 6562500
|
||||
bitmap hash = -2107026381
|
||||
image output #9:
|
||||
presentationTimeUs = 7500000
|
||||
bitmap hash = -1582997950
|
||||
image output #10:
|
||||
presentationTimeUs = 8437500
|
||||
bitmap hash = -1077263789
|
||||
image output #11:
|
||||
presentationTimeUs = 9375000
|
||||
bitmap hash = 1900793242
|
||||
image output #12:
|
||||
presentationTimeUs = 10312500
|
||||
bitmap hash = -310799297
|
||||
image output #13:
|
||||
presentationTimeUs = 11250000
|
||||
bitmap hash = 1499050092
|
||||
image output #14:
|
||||
presentationTimeUs = 12187500
|
||||
bitmap hash = -889397582
|
||||
image output #15:
|
||||
presentationTimeUs = 13125000
|
||||
bitmap hash = -522250446
|
||||
image output #16:
|
||||
presentationTimeUs = 14062500
|
||||
bitmap hash = -581506812
|
||||
image output #17:
|
||||
presentationTimeUs = 15000000
|
||||
bitmap hash = -767045995
|
||||
image output #18:
|
||||
presentationTimeUs = 15937500
|
||||
bitmap hash = -1205030348
|
||||
image output #19:
|
||||
presentationTimeUs = 16875000
|
||||
bitmap hash = 1338041318
|
||||
image output #20:
|
||||
presentationTimeUs = 17812500
|
||||
bitmap hash = 1352738203
|
||||
image output #21:
|
||||
presentationTimeUs = 18750000
|
||||
bitmap hash = 883432223
|
||||
image output #22:
|
||||
presentationTimeUs = 19687500
|
||||
bitmap hash = -535832768
|
||||
image output #23:
|
||||
presentationTimeUs = 20625000
|
||||
bitmap hash = -1338903762
|
||||
image output #24:
|
||||
presentationTimeUs = 21562500
|
||||
bitmap hash = -125811797
|
||||
image output #25:
|
||||
presentationTimeUs = 22500000
|
||||
bitmap hash = 549453620
|
||||
image output #26:
|
||||
presentationTimeUs = 23437500
|
||||
bitmap hash = 1677558048
|
||||
image output #27:
|
||||
presentationTimeUs = 24375000
|
||||
bitmap hash = -256549269
|
||||
image output #28:
|
||||
presentationTimeUs = 25312500
|
||||
bitmap hash = -630960808
|
||||
image output #29:
|
||||
presentationTimeUs = 26250000
|
||||
bitmap hash = 1015145670
|
||||
image output #30:
|
||||
presentationTimeUs = 27187500
|
||||
bitmap hash = -1795307136
|
||||
image output #31:
|
||||
presentationTimeUs = 28125000
|
||||
bitmap hash = 1272159394
|
||||
image output #32:
|
||||
presentationTimeUs = 29062500
|
||||
bitmap hash = -93678600
|
||||
image output #33:
|
||||
presentationTimeUs = 30000000
|
||||
bitmap hash = -600076145
|
||||
image output #34:
|
||||
presentationTimeUs = 30937500
|
||||
bitmap hash = -97251290
|
||||
image output #35:
|
||||
presentationTimeUs = 31875000
|
||||
bitmap hash = 1281484249
|
||||
image output #36:
|
||||
presentationTimeUs = 32812500
|
||||
bitmap hash = -1728867849
|
||||
image output #37:
|
||||
presentationTimeUs = 33750000
|
||||
bitmap hash = 380034424
|
||||
image output #38:
|
||||
presentationTimeUs = 34687500
|
||||
bitmap hash = 1913328953
|
||||
image output #39:
|
||||
presentationTimeUs = 35625000
|
||||
bitmap hash = 1616828465
|
||||
image output #40:
|
||||
presentationTimeUs = 36562500
|
||||
bitmap hash = 1579225474
|
||||
image output #41:
|
||||
presentationTimeUs = 37500000
|
||||
bitmap hash = -1263537508
|
||||
image output #42:
|
||||
presentationTimeUs = 38437500
|
||||
bitmap hash = 1469560805
|
||||
image output #43:
|
||||
presentationTimeUs = 39375000
|
||||
bitmap hash = -1949117971
|
||||
image output #44:
|
||||
presentationTimeUs = 40312500
|
||||
bitmap hash = 1890332461
|
||||
image output #45:
|
||||
presentationTimeUs = 41250000
|
||||
bitmap hash = 381486112
|
||||
image output #46:
|
||||
presentationTimeUs = 42187500
|
||||
bitmap hash = 943544370
|
||||
image output #47:
|
||||
presentationTimeUs = 43125000
|
||||
bitmap hash = -449507486
|
||||
image output #48:
|
||||
presentationTimeUs = 44062500
|
||||
bitmap hash = -1456959112
|
||||
image output #49:
|
||||
presentationTimeUs = 45000000
|
||||
bitmap hash = -919717716
|
||||
image output #50:
|
||||
presentationTimeUs = 45937500
|
||||
bitmap hash = -1852787702
|
||||
image output #51:
|
||||
presentationTimeUs = 46875000
|
||||
bitmap hash = 2000481270
|
||||
image output #52:
|
||||
presentationTimeUs = 47812500
|
||||
bitmap hash = 1399518428
|
||||
image output #53:
|
||||
presentationTimeUs = 48750000
|
||||
bitmap hash = -158658633
|
||||
image output #54:
|
||||
presentationTimeUs = 49687500
|
||||
bitmap hash = 587265344
|
||||
image output #55:
|
||||
presentationTimeUs = 50625000
|
||||
bitmap hash = -1857190760
|
||||
image output #56:
|
||||
presentationTimeUs = 51562500
|
||||
bitmap hash = -392855012
|
||||
image output #57:
|
||||
presentationTimeUs = 52500000
|
||||
bitmap hash = -1222466861
|
||||
image output #58:
|
||||
presentationTimeUs = 53437500
|
||||
bitmap hash = 2060648653
|
||||
image output #59:
|
||||
presentationTimeUs = 54375000
|
||||
bitmap hash = 1407821609
|
||||
image output #60:
|
||||
presentationTimeUs = 55312500
|
||||
bitmap hash = -1744072926
|
||||
image output #61:
|
||||
presentationTimeUs = 56250000
|
||||
bitmap hash = -1355216794
|
||||
image output #62:
|
||||
presentationTimeUs = 57187500
|
||||
bitmap hash = -7610058
|
||||
image output #63:
|
||||
presentationTimeUs = 58125000
|
||||
bitmap hash = 1362483058
|
||||
image output #64:
|
||||
presentationTimeUs = 59062500
|
||||
bitmap hash = 442567684
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user