From 1096ae9145757c557d1701f8c215a658c6282a5c Mon Sep 17 00:00:00 2001 From: lpribanic Date: Fri, 15 Dec 2023 06:15:52 -0800 Subject: [PATCH] Write empty image byte arrays to sample queue The number of empty image byte arrays written is one less than the total number of tiles in the image. The empty byte arrays act as placeholders for individual tiles inside ImageRenderer. PiperOrigin-RevId: 591231432 --- .../media3/exoplayer/image/ImageRenderer.java | 19 +++++++++-- .../source/chunk/ContainerMediaChunk.java | 34 ++++++++++++++++++- .../dash/DefaultDashChunkSource.java | 4 +++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java index 52d6dece14..a0c4c60545 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java @@ -344,13 +344,26 @@ public class ImageRenderer extends BaseRenderer { return false; case C.RESULT_BUFFER_READ: inputBuffer.flip(); - checkNotNull(decoder).queueInputBuffer(inputBuffer); - if (inputBuffer.isEndOfStream()) { + // Input buffers with no data that are also non-EOS, only carry the timestamp for a grid + // tile. These buffers are not queued. + boolean shouldQueueBuffer = + checkNotNull(inputBuffer.data).remaining() > 0 + || checkNotNull(inputBuffer).isEndOfStream(); + if (shouldQueueBuffer) { + checkNotNull(decoder).queueInputBuffer(checkNotNull(inputBuffer)); + } + if (checkNotNull(inputBuffer).isEndOfStream()) { inputStreamEnded = true; inputBuffer = null; return false; } - inputBuffer = null; + // If inputBuffer was queued, the decoder already cleared it. Otherwise, inputBuffer is + // cleared here. + if (shouldQueueBuffer) { + inputBuffer = null; + } else { + checkNotNull(inputBuffer).clear(); + } return true; case C.RESULT_FORMAT_READ: inputFormat = checkNotNull(formatHolder.format); diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ContainerMediaChunk.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ContainerMediaChunk.java index 3cd7bff93f..11d358e852 100644 --- a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ContainerMediaChunk.java +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/source/chunk/ContainerMediaChunk.java @@ -15,9 +15,13 @@ */ package androidx.media3.exoplayer.source.chunk; +import static androidx.media3.common.C.BUFFER_FLAG_KEY_FRAME; + import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.ParsableByteArray; import androidx.media3.common.util.UnstableApi; import androidx.media3.datasource.DataSource; import androidx.media3.datasource.DataSourceUtil; @@ -26,6 +30,7 @@ import androidx.media3.exoplayer.source.chunk.ChunkExtractor.TrackOutputProvider import androidx.media3.extractor.DefaultExtractorInput; import androidx.media3.extractor.Extractor; import androidx.media3.extractor.ExtractorInput; +import androidx.media3.extractor.TrackOutput; import java.io.IOException; /** A {@link BaseMediaChunk} that uses an {@link Extractor} to decode sample data. */ @@ -109,9 +114,9 @@ public class ContainerMediaChunk extends BaseMediaChunk { @SuppressWarnings("NonAtomicVolatileUpdate") @Override public final void load() throws IOException { + BaseMediaChunkOutput output = getOutput(); if (nextLoadPosition == 0) { // Configure the output and set it as the target for the extractor wrapper. - BaseMediaChunkOutput output = getOutput(); output.setSampleOffsetUs(sampleOffsetUs); chunkExtractor.init( getTrackOutputProvider(output), @@ -127,6 +132,7 @@ public class ContainerMediaChunk extends BaseMediaChunk { // Load and decode the sample data. try { while (!loadCanceled && chunkExtractor.read(input)) {} + maybeWriteEmptySamples(output); } finally { nextLoadPosition = input.getPosition() - dataSpec.position; } @@ -146,4 +152,30 @@ public class ContainerMediaChunk extends BaseMediaChunk { protected TrackOutputProvider getTrackOutputProvider(BaseMediaChunkOutput baseMediaChunkOutput) { return baseMediaChunkOutput; } + + private void maybeWriteEmptySamples(BaseMediaChunkOutput output) { + if (!MimeTypes.isImage(trackFormat.containerMimeType)) { + return; + } + if ((trackFormat.tileCountHorizontal <= 1 && trackFormat.tileCountVertical <= 1) + || trackFormat.tileCountHorizontal == Format.NO_VALUE + || trackFormat.tileCountVertical == Format.NO_VALUE) { + return; + } + + TrackOutput trackOutput = output.track(/* id= */ 0, C.TRACK_TYPE_IMAGE); + int tileCount = trackFormat.tileCountHorizontal * trackFormat.tileCountVertical; + long tileDurationUs = (endTimeUs - startTimeUs) / tileCount; + + for (int i = 1; i < tileCount; ++i) { + long tileStartTimeUs = i * tileDurationUs; + trackOutput.sampleData(new ParsableByteArray(), /* length= */ 0); + trackOutput.sampleMetadata( + tileStartTimeUs, + /* flags= */ BUFFER_FLAG_KEY_FRAME, + /* size= */ 0, + /* offset= */ 0, + /* cryptoData= */ null); + } + } } diff --git a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java index 429e8fc5e2..5b756401dd 100644 --- a/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java +++ b/libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DefaultDashChunkSource.java @@ -27,6 +27,7 @@ import androidx.annotation.CheckResult; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.UriUtil; import androidx.media3.common.util.Util; @@ -852,6 +853,9 @@ public class DefaultDashChunkSource implements DashChunkSource { dataSpec = cmcdData.addToDataSpec(dataSpec); } long sampleOffsetUs = -representation.presentationTimeOffsetUs; + if (MimeTypes.isImage(trackFormat.sampleMimeType)) { + sampleOffsetUs += startTimeUs; + } return new ContainerMediaChunk( dataSource, dataSpec,