From cacf9b33b6282cd1fc917fb8d3f38fc836452d70 Mon Sep 17 00:00:00 2001 From: tofunmi Date: Tue, 15 Aug 2023 14:03:18 +0100 Subject: [PATCH] Add image decoding interfaces and capabilities PiperOrigin-RevId: 557108362 --- libraries/exoplayer/build.gradle | 1 + .../image/DefaultImageDecoderTest.java | 92 ++++++++++++ .../exoplayer/image/DefaultImageDecoder.java | 131 ++++++++++++++++++ .../media3/exoplayer/image/ImageDecoder.java | 47 +++++++ .../image/ImageDecoderException.java | 53 +++++++ .../exoplayer/image/ImageOutputBuffer.java | 28 ++++ .../DefaultImageDecoderBufferQueueTest.java | 115 +++++++++++++++ 7 files changed, 467 insertions(+) create mode 100644 libraries/exoplayer/src/androidTest/java/androidx/media3/exoplayer/image/DefaultImageDecoderTest.java create mode 100644 libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/DefaultImageDecoder.java create mode 100644 libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoder.java create mode 100644 libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoderException.java create mode 100644 libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageOutputBuffer.java create mode 100644 libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/DefaultImageDecoderBufferQueueTest.java diff --git a/libraries/exoplayer/build.gradle b/libraries/exoplayer/build.gradle index 8177f3d0af..730cd5bf8b 100644 --- a/libraries/exoplayer/build.gradle +++ b/libraries/exoplayer/build.gradle @@ -51,6 +51,7 @@ dependencies { api project(modulePrefix + 'lib-database') implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion implementation 'androidx.core:core:' + androidxCoreVersion + implementation 'androidx.exifinterface:exifinterface:1.3.6' compileOnly 'com.google.code.findbugs:jsr305:' + jsr305Version compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion diff --git a/libraries/exoplayer/src/androidTest/java/androidx/media3/exoplayer/image/DefaultImageDecoderTest.java b/libraries/exoplayer/src/androidTest/java/androidx/media3/exoplayer/image/DefaultImageDecoderTest.java new file mode 100644 index 0000000000..13db2f195a --- /dev/null +++ b/libraries/exoplayer/src/androidTest/java/androidx/media3/exoplayer/image/DefaultImageDecoderTest.java @@ -0,0 +1,92 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Matrix; +import androidx.media3.test.utils.TestUtil; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Unit tests for {@link DefaultImageDecoder}. */ +@RunWith(AndroidJUnit4.class) +public class DefaultImageDecoderTest { + + private static final String PNG_TEST_IMAGE_PATH = "media/png/non-motion-photo-shortened.png"; + private static final String JPEG_TEST_IMAGE_PATH = "media/jpeg/non-motion-photo-shortened.jpg"; + + private DefaultImageDecoder imageDecoder; + + @Before + public void setUp() { + imageDecoder = new DefaultImageDecoder(); + } + + @After + public void tearDown() { + imageDecoder.release(); + } + + @Test + public void decode_png_loadsCorrectData() throws Exception { + byte[] imageData = + TestUtil.getByteArray(ApplicationProvider.getApplicationContext(), PNG_TEST_IMAGE_PATH); + + Bitmap bitmap = imageDecoder.decode(imageData, imageData.length); + + assertThat( + bitmap.sameAs( + BitmapFactory.decodeByteArray(imageData, /* offset= */ 0, imageData.length))) + .isTrue(); + } + + @Test + public void decode_jpegWithExifRotation_loadsCorrectData() throws Exception { + byte[] imageData = + TestUtil.getByteArray(ApplicationProvider.getApplicationContext(), JPEG_TEST_IMAGE_PATH); + Bitmap bitmapWithoutRotation = + BitmapFactory.decodeByteArray(imageData, /* offset= */ 0, imageData.length); + Matrix rotationMatrix = new Matrix(); + rotationMatrix.postRotate(/* degrees= */ 90); + Bitmap expectedBitmap = + Bitmap.createBitmap( + bitmapWithoutRotation, + /* x= */ 0, + /* y= */ 0, + bitmapWithoutRotation.getWidth(), + bitmapWithoutRotation.getHeight(), + rotationMatrix, + /* filter= */ false); + + Bitmap actualBitmap = imageDecoder.decode(imageData, imageData.length); + + assertThat(actualBitmap.sameAs(expectedBitmap)).isTrue(); + } + + @Test + public void decodeBitmap_withInvalidData_throws() throws ImageDecoderException { + assertThrows( + ImageDecoderException.class, () -> imageDecoder.decode(new byte[1], /* length= */ 1)); + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/DefaultImageDecoder.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/DefaultImageDecoder.java new file mode 100644 index 0000000000..2df31081f5 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/DefaultImageDecoder.java @@ -0,0 +1,131 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL; + +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.Matrix; +import androidx.annotation.Nullable; +import androidx.exifinterface.media.ExifInterface; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.decoder.SimpleDecoder; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +/** + * An image decoder that uses {@link BitmapFactory} to decode images. + * + *

Only supports decoding one input buffer into one output buffer (i.e. one {@link Bitmap} + * alongside one timestamp)). + */ +@UnstableApi +public class DefaultImageDecoder + extends SimpleDecoder + implements ImageDecoder { + + /** Creates an instance. */ + public DefaultImageDecoder() { + super(new DecoderInputBuffer[1], new ImageOutputBuffer[1]); + } + + @Override + public final String getName() { + return "DefaultImageDecoder"; + } + + @Override + protected DecoderInputBuffer createInputBuffer() { + return new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_NORMAL); + } + + @Override + protected ImageOutputBuffer createOutputBuffer() { + return new ImageOutputBuffer() { + @Override + public void release() { + DefaultImageDecoder.this.releaseOutputBuffer(this); + } + }; + } + + @Override + protected ImageDecoderException createUnexpectedDecodeException(Throwable error) { + return new ImageDecoderException("Unexpected decode error", error); + } + + @Nullable + @Override + protected ImageDecoderException decode( + DecoderInputBuffer inputBuffer, ImageOutputBuffer outputBuffer, boolean reset) { + try { + ByteBuffer inputData = checkNotNull(inputBuffer.data); + checkState(inputData.hasArray()); + checkArgument(inputData.arrayOffset() == 0); + outputBuffer.bitmap = decode(inputData.array(), inputData.remaining()); + outputBuffer.timeUs = inputBuffer.timeUs; + return null; + } catch (ImageDecoderException e) { + return e; + } + } + + /** + * Decodes data into a {@link Bitmap}. + * + * @param data An array holding the data to be decoded, starting at position 0. + * @param length The length of the input to be decoded. + * @return The decoded {@link Bitmap}. + * @throws ImageDecoderException If a decoding error occurs. + */ + protected Bitmap decode(byte[] data, int length) throws ImageDecoderException { + @Nullable Bitmap bitmap = BitmapFactory.decodeByteArray(data, /* offset= */ 0, length); + if (bitmap == null) { + throw new ImageDecoderException( + "Could not decode image data with BitmapFactory. (data length = " + data.length + ")"); + } + // BitmapFactory doesn't read the exif header, so we use the ExifInterface to this do ensure the + // bitmap is correctly orientated. + ExifInterface exifInterface; + try (InputStream inputStream = new ByteArrayInputStream(data)) { + exifInterface = new ExifInterface(inputStream); + } catch (IOException e) { + throw new ImageDecoderException(e); + } + int rotationDegrees = exifInterface.getRotationDegrees(); + if (rotationDegrees != 0) { + Matrix matrix = new Matrix(); + matrix.postRotate(rotationDegrees); + bitmap = + Bitmap.createBitmap( + bitmap, + /* x= */ 0, + /* y= */ 0, + bitmap.getWidth(), + bitmap.getHeight(), + matrix, + /* filter= */ false); + } + return bitmap; + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoder.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoder.java new file mode 100644 index 0000000000..4b0feffec0 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoder.java @@ -0,0 +1,47 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import android.graphics.Bitmap; +import androidx.annotation.Nullable; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.decoder.Decoder; +import androidx.media3.decoder.DecoderInputBuffer; + +/** A {@link Decoder} implementation for images. */ +@UnstableApi +public interface ImageDecoder + extends Decoder { + + /** + * Queues an {@link DecoderInputBuffer} to the decoder. + * + * @param inputBuffer The input buffer containing the byte data corresponding to the image(s). + * @throws ImageDecoderException If a decoder error has occurred. + */ + @Override + void queueInputBuffer(DecoderInputBuffer inputBuffer) throws ImageDecoderException; + + /** + * Returns the next decoded {@link Bitmap} in an {@link ImageOutputBuffer}. + * + * @return The output buffer, or {@code null} if an output buffer isn't available. + * @throws ImageDecoderException If a decoder error has occurred. + */ + @Nullable + @Override + ImageOutputBuffer dequeueOutputBuffer() throws ImageDecoderException; +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoderException.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoderException.java new file mode 100644 index 0000000000..7f5f740d75 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoderException.java @@ -0,0 +1,53 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import androidx.annotation.Nullable; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.decoder.DecoderException; + +/** Thrown when an error occurs decoding image data. */ +@UnstableApi +public final class ImageDecoderException extends DecoderException { + + /** + * Creates an instance. + * + * @param message The detail message for this exception. + */ + public ImageDecoderException(String message) { + super(message); + } + + /** + * Creates an instance. + * + * @param cause The cause of this exception, or {@code null}. + */ + public ImageDecoderException(@Nullable Throwable cause) { + super(cause); + } + + /** + * Creates an instance. + * + * @param message The detail message for this exception. + * @param cause The cause of this exception, or {@code null}. + */ + public ImageDecoderException(String message, @Nullable Throwable cause) { + super(message, cause); + } +} diff --git a/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageOutputBuffer.java b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageOutputBuffer.java new file mode 100644 index 0000000000..52931bf0b6 --- /dev/null +++ b/libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageOutputBuffer.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import android.graphics.Bitmap; +import androidx.annotation.Nullable; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.decoder.DecoderOutputBuffer; + +/** Output buffer for {@link ImageDecoder}s. */ +@UnstableApi +public abstract class ImageOutputBuffer extends DecoderOutputBuffer { + + @Nullable public Bitmap bitmap; +} diff --git a/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/DefaultImageDecoderBufferQueueTest.java b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/DefaultImageDecoderBufferQueueTest.java new file mode 100644 index 0000000000..a72a5765c9 --- /dev/null +++ b/libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/DefaultImageDecoderBufferQueueTest.java @@ -0,0 +1,115 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.image; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static com.google.common.truth.Truth.assertThat; + +import android.graphics.Bitmap; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import java.nio.ByteBuffer; +import java.util.concurrent.TimeoutException; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** + * Unit tests for {@link DefaultImageDecoder} ensuring the buffer queue system operates correctly. + */ +@RunWith(AndroidJUnit4.class) +public class DefaultImageDecoderBufferQueueTest { + + private static final long TIMEOUT_MS = 5 * C.MICROS_PER_SECOND; + + private DefaultImageDecoder fakeImageDecoder; + private Bitmap decodedBitmap1; + private Bitmap decodedBitmap2; + + @Before + public void setUp() throws Exception { + decodedBitmap1 = Bitmap.createBitmap(/* width= */ 1, /* height= */ 1, Bitmap.Config.ARGB_8888); + decodedBitmap2 = Bitmap.createBitmap(/* width= */ 2, /* height= */ 2, Bitmap.Config.ARGB_8888); + fakeImageDecoder = + new DefaultImageDecoder() { + + public int decodeCallCount; + + /** Overrides the decode method to fake it. */ + @Override + protected Bitmap decode(byte[] data, int length) { + decodeCallCount++; + return decodeCallCount == 1 ? decodedBitmap1 : decodedBitmap2; + } + }; + } + + @After + public void tearDown() throws Exception { + fakeImageDecoder.release(); + } + + @Test + public void decodeIndirectly_returnBitmapAtTheCorrectTimestamp() throws Exception { + DecoderInputBuffer inputBuffer = checkNotNull(fakeImageDecoder.dequeueInputBuffer()); + inputBuffer.timeUs = 2 * C.MILLIS_PER_SECOND; + inputBuffer.data = ByteBuffer.wrap(new byte[1]); + fakeImageDecoder.queueInputBuffer(inputBuffer); + ImageOutputBuffer outputBuffer = getDecodedOutput(); + + assertThat(outputBuffer.timeUs).isEqualTo(inputBuffer.timeUs); + assertThat(outputBuffer.bitmap).isEqualTo(decodedBitmap1); + } + + @Test + public void decodeIndirectlyTwice_returnsSecondBitmapAtTheCorrectTimestamp() throws Exception { + DecoderInputBuffer inputBuffer1 = checkNotNull(fakeImageDecoder.dequeueInputBuffer()); + inputBuffer1.timeUs = 0; + inputBuffer1.data = ByteBuffer.wrap(new byte[1]); + fakeImageDecoder.queueInputBuffer(inputBuffer1); + checkNotNull(getDecodedOutput()).release(); + DecoderInputBuffer inputBuffer2 = checkNotNull(fakeImageDecoder.dequeueInputBuffer()); + inputBuffer2.timeUs = C.MICROS_PER_SECOND; + inputBuffer2.data = ByteBuffer.wrap(new byte[1]); + fakeImageDecoder.queueInputBuffer(inputBuffer2); + + ImageOutputBuffer outputBuffer2 = checkNotNull(getDecodedOutput()); + + assertThat(outputBuffer2.timeUs).isEqualTo(inputBuffer2.timeUs); + assertThat(outputBuffer2.bitmap).isEqualTo(decodedBitmap2); + } + + // Polling to see whether the output is available yet since the decode thread doesn't finish + // decoding immediately. + private ImageOutputBuffer getDecodedOutput() throws Exception { + @Nullable ImageOutputBuffer outputBuffer; + // Use System.currentTimeMillis() to calculate the wait duration more accurately. + long deadlineMs = System.currentTimeMillis() + TIMEOUT_MS; + long remainingMs = TIMEOUT_MS; + while (remainingMs > 0) { + outputBuffer = fakeImageDecoder.dequeueOutputBuffer(); + if (outputBuffer != null) { + return outputBuffer; + } + Thread.sleep(/* millis= */ 5); + remainingMs = deadlineMs - System.currentTimeMillis(); + } + throw new TimeoutException(); + } +}