diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageOutput.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageOutput.java new file mode 100644 index 0000000000..e7fec96f60 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageOutput.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import android.graphics.Bitmap; +import androidx.media3.common.Timeline; +import androidx.media3.common.util.UnstableApi; + +/** A listener for image output. */ +@UnstableApi +public interface ImageOutput { + + /** + * Called when an there is a new image available. + * + * @param presentationTimeUs The presentation time of the image, in microseconds. This time is an + * offset from the start of the current {@link Timeline.Period}. + * @param bitmap The new image available. + */ + void onImageAvailable(long presentationTimeUs, Bitmap bitmap); +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java new file mode 100644 index 0000000000..ba83ed7e25 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageRenderer.java @@ -0,0 +1,291 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import static androidx.media3.common.PlaybackException.ERROR_CODE_FAILED_RUNTIME_CHECK; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Assertions.checkStateNotNull; +import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT; + +import android.graphics.Bitmap; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.PlaybackException; +import androidx.media3.common.util.TraceUtil; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.exoplayer.BaseRenderer; +import androidx.media3.exoplayer.ExoPlaybackException; +import androidx.media3.exoplayer.FormatHolder; +import androidx.media3.exoplayer.Renderer; +import androidx.media3.exoplayer.RendererCapabilities; +import androidx.media3.exoplayer.source.SampleStream; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; + +// TODO(b/289989736): Currently works for one stream only. Refactor so that it works for multiple +// inputs streams. +/** A {@link Renderer} implementation for images. */ +@UnstableApi +public final class ImageRenderer extends BaseRenderer { + private static final String TAG = "ImageRenderer"; + + private final DecoderInputBuffer flagsOnlyBuffer; + private final ImageDecoder.Factory decoderFactory; + private final ImageOutput imageOutput; + + private @C.FirstFrameState int firstFrameState; + private boolean inputStreamEnded; + private boolean outputStreamEnded; + private long durationUs; + private long offsetUs; + private @Nullable ImageDecoder decoder; + private @Nullable DecoderInputBuffer inputBuffer; + private @Nullable ImageOutputBuffer outputBuffer; + private @MonotonicNonNull Format inputFormat; + + /** + * Creates an instance. + * + * @param decoderFactory A {@link ImageDecoder.Factory} that supplies a decoder depending on the + * format provided. + * @param imageOutput The rendering component to send the {@link Bitmap} and rendering commands + * to. + */ + public ImageRenderer(ImageDecoder.Factory decoderFactory, ImageOutput imageOutput) { + super(C.TRACK_TYPE_IMAGE); + flagsOnlyBuffer = DecoderInputBuffer.newNoDataInstance(); + this.decoderFactory = decoderFactory; + this.imageOutput = imageOutput; + durationUs = C.TIME_UNSET; + firstFrameState = C.FIRST_FRAME_NOT_RENDERED; + } + + @Override + public String getName() { + return TAG; + } + + @Override + public @Capabilities int supportsFormat(Format format) { + return decoderFactory.supportsFormat(format); + } + + @Override + public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + checkState(durationUs != C.TIME_UNSET); + if (outputStreamEnded) { + return; + } + + if (inputFormat == null) { + // We don't have a format yet, so try and read one. + FormatHolder formatHolder = getFormatHolder(); + flagsOnlyBuffer.clear(); + @SampleStream.ReadDataResult + int result = readSource(formatHolder, flagsOnlyBuffer, FLAG_REQUIRE_FORMAT); + if (result == C.RESULT_FORMAT_READ) { + // Note that this works because we only expect to enter this if-condition once per playback + // for now. + maybeInitDecoder(checkNotNull(formatHolder.format)); + } else if (result == C.RESULT_BUFFER_READ) { + // End of stream read having not read a format. + checkState(flagsOnlyBuffer.isEndOfStream()); + inputStreamEnded = true; + outputStreamEnded = true; + return; + } else { + // We still don't have a format and can't make progress without one. + return; + } + } + + try { + // Rendering loop. + TraceUtil.beginSection("drainAndFeedDecoder"); + while (drainOutputBuffer(positionUs, elapsedRealtimeUs)) {} + while (feedInputBuffer()) {} + TraceUtil.endSection(); + } catch (ImageDecoderException e) { + throw createRendererException(e, null, PlaybackException.ERROR_CODE_DECODING_FAILED); + } + } + + @Override + public boolean isReady() { + return firstFrameState == C.FIRST_FRAME_RENDERED; + } + + @Override + public boolean isEnded() { + return outputStreamEnded; + } + + @Override + protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) + throws ExoPlaybackException { + // TODO(b/289989736): when the mediaPeriodId is signalled to the renders, collect and set + // durationUs here. + durationUs = 2 * C.MICROS_PER_SECOND; + this.offsetUs = offsetUs; + super.onStreamChanged(formats, startPositionUs, offsetUs); + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) { + // Since the renderer only supports playing one image from, this is currently a no-op (don't + // need to consider a new stream because it will be the same as the last one). + } + + @Override + protected void onDisabled() { + releaseResources(); + } + + @Override + protected void onReset() { + releaseResources(); + } + + @Override + protected void onRelease() { + releaseResources(); + } + + /** + * Attempts to dequeue an output buffer from the decoder and, if successful, renders it. + * + * @param positionUs The player's current position. + * @param elapsedRealtimeUs {@link android.os.SystemClock#elapsedRealtime()} in microseconds, + * measured at the start of the current iteration of the rendering loop. + * @return Whether it may be possible to drain more output data. + * @throws ImageDecoderException If an error occurs draining the output buffer. + */ + private boolean drainOutputBuffer(long positionUs, long elapsedRealtimeUs) + throws ImageDecoderException { + if (outputBuffer == null) { + checkStateNotNull(decoder); + outputBuffer = decoder.dequeueOutputBuffer(); + if (outputBuffer == null) { + return false; + } + } + if (outputBuffer.isEndOfStream()) { + outputBuffer.release(); + outputBuffer = null; + outputStreamEnded = true; + return false; + } + + if (!processOutputBuffer(positionUs, elapsedRealtimeUs)) { + return false; + } + + firstFrameState = C.FIRST_FRAME_RENDERED; + return true; + } + + @RequiresNonNull("outputBuffer") + @SuppressWarnings("unused") // Will be used or removed when the integrated with the videoSink. + private boolean processOutputBuffer(long positionUs, long elapsedRealtimeUs) { + checkStateNotNull( + outputBuffer.bitmap, "Non-EOS buffer came back from the decoder without bitmap."); + imageOutput.onImageAvailable(positionUs - offsetUs, outputBuffer.bitmap); + checkNotNull(outputBuffer).release(); + outputBuffer = null; + return true; + } + + /** + * @return Whether we can feed more input data to the decoder. + */ + private boolean feedInputBuffer() throws ExoPlaybackException, ImageDecoderException { + FormatHolder formatHolder = getFormatHolder(); + if (decoder == null || inputStreamEnded) { + return false; + } + if (inputBuffer == null) { + inputBuffer = decoder.dequeueInputBuffer(); + if (inputBuffer == null) { + return false; + } + } + switch (readSource(formatHolder, inputBuffer, /* readFlags= */ 0)) { + case C.RESULT_NOTHING_READ: + return false; + case C.RESULT_BUFFER_READ: + checkNotNull(decoder).queueInputBuffer(inputBuffer); + if (inputBuffer.isEndOfStream()) { + inputStreamEnded = true; + inputBuffer = null; + return false; + } + inputBuffer = null; + return true; + case C.RESULT_FORMAT_READ: + if (checkNotNull(formatHolder.format).equals(inputFormat)) { + return true; + } + throw createRendererException( + new UnsupportedOperationException( + "Changing format is not supported in the ImageRenderer."), + formatHolder.format, + ERROR_CODE_FAILED_RUNTIME_CHECK); + default: + throw new IllegalStateException(); + } + } + + @EnsuresNonNull("decoder") + private void maybeInitDecoder(Format format) throws ExoPlaybackException { + if (inputFormat != null && inputFormat.equals(format) && decoder != null) { + return; + } + inputFormat = format; + if (canCreateDecoderForFormat(format)) { + if (decoder != null) { + decoder.release(); + } + decoder = decoderFactory.createImageDecoder(); + } else { + throw createRendererException( + new ImageDecoderException("Provided decoder factory can't create decoder for format."), + format, + PlaybackException.ERROR_CODE_DECODING_FORMAT_UNSUPPORTED); + } + } + + private boolean canCreateDecoderForFormat(Format format) { + @Capabilities int supportsFormat = decoderFactory.supportsFormat(format); + return supportsFormat == RendererCapabilities.create(C.FORMAT_HANDLED) + || supportsFormat == RendererCapabilities.create(C.FORMAT_EXCEEDS_CAPABILITIES); + } + + private void releaseResources() { + inputBuffer = null; + if (outputBuffer != null) { + outputBuffer.release(); + } + outputBuffer = null; + if (decoder != null) { + decoder.release(); + decoder = null; + } + } +} diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java new file mode 100644 index 0000000000..6e46b84812 --- /dev/null +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/ImageRendererTest.java @@ -0,0 +1,110 @@ +/* + * Copyright (C) 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.END_OF_STREAM_ITEM; +import static androidx.media3.test.utils.FakeSampleStream.FakeSampleStreamItem.oneByteSample; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Bitmap; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.Clock; +import androidx.media3.common.util.TimedValueQueue; +import androidx.media3.exoplayer.RendererConfiguration; +import androidx.media3.exoplayer.analytics.PlayerId; +import androidx.media3.exoplayer.drm.DrmSessionEventListener; +import androidx.media3.exoplayer.drm.DrmSessionManager; +import androidx.media3.exoplayer.upstream.DefaultAllocator; +import androidx.media3.test.utils.FakeSampleStream; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableList; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit test for {@link ImageRenderer}. */ +@RunWith(AndroidJUnit4.class) +public class ImageRendererTest { + + private static final Format FORMAT = + new Format.Builder() + .setContainerMimeType(MimeTypes.IMAGE_PNG) + .setTileCountVertical(1) + .setTileCountHorizontal(1) + .build(); + + private final TimedValueQueue renderedBitmaps = new TimedValueQueue<>(); + private final Bitmap fakeDecodedBitmap = + Bitmap.createBitmap(/* width= */ 1, /* height= */ 1, Bitmap.Config.ARGB_8888); + + private ImageRenderer renderer; + + @Before + public void setUp() throws Exception { + ImageDecoder.Factory fakeDecoderFactory = + new DefaultImageDecoder.Factory((data, length) -> fakeDecodedBitmap); + ImageOutput capturingImageOutput = renderedBitmaps::add; + renderer = new ImageRenderer(fakeDecoderFactory, capturingImageOutput); + renderer.init(/* index= */ 0, PlayerId.UNSET, Clock.DEFAULT); + } + + @After + public void tearDown() throws Exception { + renderedBitmaps.clear(); + renderer.disable(); + renderer.release(); + } + + @Test + public void renderOneStream_rendersToImageOutput() throws Exception { + FakeSampleStream fakeSampleStream = + new FakeSampleStream( + new DefaultAllocator(/* trimOnReset= */ true, /* individualAllocationSize= */ 1024), + /* mediaSourceEventDispatcher= */ null, + DrmSessionManager.DRM_UNSUPPORTED, + new DrmSessionEventListener.EventDispatcher(), + FORMAT, + ImmutableList.of( + oneByteSample(/* timeUs= */ 0, C.BUFFER_FLAG_KEY_FRAME), END_OF_STREAM_ITEM)); + fakeSampleStream.writeData(/* startPositionUs= */ 0); + // TODO(b/289989736): When the mediaPeriodId is signalled to the renders set durationUs here and + // assert on it. + renderer.enable( + RendererConfiguration.DEFAULT, + new Format[] {FORMAT}, + fakeSampleStream, + /* positionUs= */ 0, + /* joining= */ false, + /* mayRenderStartOfStream= */ true, + /* startPositionUs= */ 0, + /* offsetUs= */ 0); + renderer.setCurrentStreamFinal(); + + while (!renderer.isReady()) { + renderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); + } + + assertThat(renderedBitmaps.size()).isEqualTo(1); + assertThat(renderedBitmaps.poll(0)).isSameInstanceAs(fakeDecodedBitmap); + + renderer.render( + /* positionUs= */ C.MICROS_PER_SECOND, /* elapsedRealtimeUs= */ C.MICROS_PER_SECOND); + assertThat(renderer.isEnded()).isTrue(); + } +}