Add image decoding interfaces and capabilities

PiperOrigin-RevId: 557108362

libraries/exoplayer/build.gradle

@@ -51,6 +51,7 @@ dependencies {
     api project(modulePrefix + 'lib-database')
     implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
     implementation 'androidx.core:core:' + androidxCoreVersion
+    implementation 'androidx.exifinterface:exifinterface:1.3.6'
     compileOnly 'com.google.code.findbugs:jsr305:' + jsr305Version
     compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion
     compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion

libraries/exoplayer/src/androidTest/java/androidx/media3/exoplayer/image/DefaultImageDecoderTest.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.image;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.assertThrows;
+
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.Matrix;
+import androidx.media3.test.utils.TestUtil;
+import androidx.test.core.app.ApplicationProvider;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/** Unit tests for {@link DefaultImageDecoder}. */
+@RunWith(AndroidJUnit4.class)
+public class DefaultImageDecoderTest {
+
+  private static final String PNG_TEST_IMAGE_PATH = "media/png/non-motion-photo-shortened.png";
+  private static final String JPEG_TEST_IMAGE_PATH = "media/jpeg/non-motion-photo-shortened.jpg";
+
+  private DefaultImageDecoder imageDecoder;
+
+  @Before
+  public void setUp() {
+    imageDecoder = new DefaultImageDecoder();
+  }
+
+  @After
+  public void tearDown() {
+    imageDecoder.release();
+  }
+
+  @Test
+  public void decode_png_loadsCorrectData() throws Exception {
+    byte[] imageData =
+        TestUtil.getByteArray(ApplicationProvider.getApplicationContext(), PNG_TEST_IMAGE_PATH);
+
+    Bitmap bitmap = imageDecoder.decode(imageData, imageData.length);
+
+    assertThat(
+            bitmap.sameAs(
+                BitmapFactory.decodeByteArray(imageData, /* offset= */ 0, imageData.length)))
+        .isTrue();
+  }
+
+  @Test
+  public void decode_jpegWithExifRotation_loadsCorrectData() throws Exception {
+    byte[] imageData =
+        TestUtil.getByteArray(ApplicationProvider.getApplicationContext(), JPEG_TEST_IMAGE_PATH);
+    Bitmap bitmapWithoutRotation =
+        BitmapFactory.decodeByteArray(imageData, /* offset= */ 0, imageData.length);
+    Matrix rotationMatrix = new Matrix();
+    rotationMatrix.postRotate(/* degrees= */ 90);
+    Bitmap expectedBitmap =
+        Bitmap.createBitmap(
+            bitmapWithoutRotation,
+            /* x= */ 0,
+            /* y= */ 0,
+            bitmapWithoutRotation.getWidth(),
+            bitmapWithoutRotation.getHeight(),
+            rotationMatrix,
+            /* filter= */ false);
+
+    Bitmap actualBitmap = imageDecoder.decode(imageData, imageData.length);
+
+    assertThat(actualBitmap.sameAs(expectedBitmap)).isTrue();
+  }
+
+  @Test
+  public void decodeBitmap_withInvalidData_throws() throws ImageDecoderException {
+    assertThrows(
+        ImageDecoderException.class, () -> imageDecoder.decode(new byte[1], /* length= */ 1));
+  }
+}

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/DefaultImageDecoder.java

@@ -0,0 +1,131 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.image;
+
+import static androidx.media3.common.util.Assertions.checkArgument;
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static androidx.media3.common.util.Assertions.checkState;
+import static androidx.media3.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL;
+
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
+import android.graphics.Matrix;
+import androidx.annotation.Nullable;
+import androidx.exifinterface.media.ExifInterface;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.DecoderInputBuffer;
+import androidx.media3.decoder.SimpleDecoder;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+
+/**
+ * An image decoder that uses {@link BitmapFactory} to decode images.
+ *
+ * <p>Only supports decoding one input buffer into one output buffer (i.e. one {@link Bitmap}
+ * alongside one timestamp)).
+ */
+@UnstableApi
+public class DefaultImageDecoder
+    extends SimpleDecoder<DecoderInputBuffer, ImageOutputBuffer, ImageDecoderException>
+    implements ImageDecoder {
+
+  /** Creates an instance. */
+  public DefaultImageDecoder() {
+    super(new DecoderInputBuffer[1], new ImageOutputBuffer[1]);
+  }
+
+  @Override
+  public final String getName() {
+    return "DefaultImageDecoder";
+  }
+
+  @Override
+  protected DecoderInputBuffer createInputBuffer() {
+    return new DecoderInputBuffer(BUFFER_REPLACEMENT_MODE_NORMAL);
+  }
+
+  @Override
+  protected ImageOutputBuffer createOutputBuffer() {
+    return new ImageOutputBuffer() {
+      @Override
+      public void release() {
+        DefaultImageDecoder.this.releaseOutputBuffer(this);
+      }
+    };
+  }
+
+  @Override
+  protected ImageDecoderException createUnexpectedDecodeException(Throwable error) {
+    return new ImageDecoderException("Unexpected decode error", error);
+  }
+
+  @Nullable
+  @Override
+  protected ImageDecoderException decode(
+      DecoderInputBuffer inputBuffer, ImageOutputBuffer outputBuffer, boolean reset) {
+    try {
+      ByteBuffer inputData = checkNotNull(inputBuffer.data);
+      checkState(inputData.hasArray());
+      checkArgument(inputData.arrayOffset() == 0);
+      outputBuffer.bitmap = decode(inputData.array(), inputData.remaining());
+      outputBuffer.timeUs = inputBuffer.timeUs;
+      return null;
+    } catch (ImageDecoderException e) {
+      return e;
+    }
+  }
+
+  /**
+   * Decodes data into a {@link Bitmap}.
+   *
+   * @param data An array holding the data to be decoded, starting at position 0.
+   * @param length The length of the input to be decoded.
+   * @return The decoded {@link Bitmap}.
+   * @throws ImageDecoderException If a decoding error occurs.
+   */
+  protected Bitmap decode(byte[] data, int length) throws ImageDecoderException {
+    @Nullable Bitmap bitmap = BitmapFactory.decodeByteArray(data, /* offset= */ 0, length);
+    if (bitmap == null) {
+      throw new ImageDecoderException(
+          "Could not decode image data with BitmapFactory. (data length = " + data.length + ")");
+    }
+    // BitmapFactory doesn't read the exif header, so we use the ExifInterface to this do ensure the
+    // bitmap is correctly orientated.
+    ExifInterface exifInterface;
+    try (InputStream inputStream = new ByteArrayInputStream(data)) {
+      exifInterface = new ExifInterface(inputStream);
+    } catch (IOException e) {
+      throw new ImageDecoderException(e);
+    }
+    int rotationDegrees = exifInterface.getRotationDegrees();
+    if (rotationDegrees != 0) {
+      Matrix matrix = new Matrix();
+      matrix.postRotate(rotationDegrees);
+      bitmap =
+          Bitmap.createBitmap(
+              bitmap,
+              /* x= */ 0,
+              /* y= */ 0,
+              bitmap.getWidth(),
+              bitmap.getHeight(),
+              matrix,
+              /* filter= */ false);
+    }
+    return bitmap;
+  }
+}

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoder.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.image;
+
+import android.graphics.Bitmap;
+import androidx.annotation.Nullable;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.Decoder;
+import androidx.media3.decoder.DecoderInputBuffer;
+
+/** A {@link Decoder} implementation for images. */
+@UnstableApi
+public interface ImageDecoder
+    extends Decoder<DecoderInputBuffer, ImageOutputBuffer, ImageDecoderException> {
+
+  /**
+   * Queues an {@link DecoderInputBuffer} to the decoder.
+   *
+   * @param inputBuffer The input buffer containing the byte data corresponding to the image(s).
+   * @throws ImageDecoderException If a decoder error has occurred.
+   */
+  @Override
+  void queueInputBuffer(DecoderInputBuffer inputBuffer) throws ImageDecoderException;
+
+  /**
+   * Returns the next decoded {@link Bitmap} in an {@link ImageOutputBuffer}.
+   *
+   * @return The output buffer, or {@code null} if an output buffer isn't available.
+   * @throws ImageDecoderException If a decoder error has occurred.
+   */
+  @Nullable
+  @Override
+  ImageOutputBuffer dequeueOutputBuffer() throws ImageDecoderException;
+}

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageDecoderException.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.image;
+
+import androidx.annotation.Nullable;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.DecoderException;
+
+/** Thrown when an error occurs decoding image data. */
+@UnstableApi
+public final class ImageDecoderException extends DecoderException {
+
+  /**
+   * Creates an instance.
+   *
+   * @param message The detail message for this exception.
+   */
+  public ImageDecoderException(String message) {
+    super(message);
+  }
+
+  /**
+   * Creates an instance.
+   *
+   * @param cause The cause of this exception, or {@code null}.
+   */
+  public ImageDecoderException(@Nullable Throwable cause) {
+    super(cause);
+  }
+
+  /**
+   * Creates an instance.
+   *
+   * @param message The detail message for this exception.
+   * @param cause The cause of this exception, or {@code null}.
+   */
+  public ImageDecoderException(String message, @Nullable Throwable cause) {
+    super(message, cause);
+  }
+}

libraries/exoplayer/src/main/java/androidx/media3/exoplayer/image/ImageOutputBuffer.java

@@ -0,0 +1,28 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.image;
+
+import android.graphics.Bitmap;
+import androidx.annotation.Nullable;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.DecoderOutputBuffer;
+
+/** Output buffer for {@link ImageDecoder}s. */
+@UnstableApi
+public abstract class ImageOutputBuffer extends DecoderOutputBuffer {
+
+  @Nullable public Bitmap bitmap;
+}

libraries/exoplayer/src/test/java/androidx/media3/exoplayer/image/DefaultImageDecoderBufferQueueTest.java

@@ -0,0 +1,115 @@
+/*
+ * Copyright 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package androidx.media3.exoplayer.image;
+
+import static androidx.media3.common.util.Assertions.checkNotNull;
+import static com.google.common.truth.Truth.assertThat;
+
+import android.graphics.Bitmap;
+import androidx.annotation.Nullable;
+import androidx.media3.common.C;
+import androidx.media3.decoder.DecoderInputBuffer;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import java.nio.ByteBuffer;
+import java.util.concurrent.TimeoutException;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Unit tests for {@link DefaultImageDecoder} ensuring the buffer queue system operates correctly.
+ */
+@RunWith(AndroidJUnit4.class)
+public class DefaultImageDecoderBufferQueueTest {
+
+  private static final long TIMEOUT_MS = 5 * C.MICROS_PER_SECOND;
+
+  private DefaultImageDecoder fakeImageDecoder;
+  private Bitmap decodedBitmap1;
+  private Bitmap decodedBitmap2;
+
+  @Before
+  public void setUp() throws Exception {
+    decodedBitmap1 = Bitmap.createBitmap(/* width= */ 1, /* height= */ 1, Bitmap.Config.ARGB_8888);
+    decodedBitmap2 = Bitmap.createBitmap(/* width= */ 2, /* height= */ 2, Bitmap.Config.ARGB_8888);
+    fakeImageDecoder =
+        new DefaultImageDecoder() {
+
+          public int decodeCallCount;
+
+          /** Overrides the decode method to fake it. */
+          @Override
+          protected Bitmap decode(byte[] data, int length) {
+            decodeCallCount++;
+            return decodeCallCount == 1 ? decodedBitmap1 : decodedBitmap2;
+          }
+        };
+  }
+
+  @After
+  public void tearDown() throws Exception {
+    fakeImageDecoder.release();
+  }
+
+  @Test
+  public void decodeIndirectly_returnBitmapAtTheCorrectTimestamp() throws Exception {
+    DecoderInputBuffer inputBuffer = checkNotNull(fakeImageDecoder.dequeueInputBuffer());
+    inputBuffer.timeUs = 2 * C.MILLIS_PER_SECOND;
+    inputBuffer.data = ByteBuffer.wrap(new byte[1]);
+    fakeImageDecoder.queueInputBuffer(inputBuffer);
+    ImageOutputBuffer outputBuffer = getDecodedOutput();
+
+    assertThat(outputBuffer.timeUs).isEqualTo(inputBuffer.timeUs);
+    assertThat(outputBuffer.bitmap).isEqualTo(decodedBitmap1);
+  }
+
+  @Test
+  public void decodeIndirectlyTwice_returnsSecondBitmapAtTheCorrectTimestamp() throws Exception {
+    DecoderInputBuffer inputBuffer1 = checkNotNull(fakeImageDecoder.dequeueInputBuffer());
+    inputBuffer1.timeUs = 0;
+    inputBuffer1.data = ByteBuffer.wrap(new byte[1]);
+    fakeImageDecoder.queueInputBuffer(inputBuffer1);
+    checkNotNull(getDecodedOutput()).release();
+    DecoderInputBuffer inputBuffer2 = checkNotNull(fakeImageDecoder.dequeueInputBuffer());
+    inputBuffer2.timeUs = C.MICROS_PER_SECOND;
+    inputBuffer2.data = ByteBuffer.wrap(new byte[1]);
+    fakeImageDecoder.queueInputBuffer(inputBuffer2);
+
+    ImageOutputBuffer outputBuffer2 = checkNotNull(getDecodedOutput());
+
+    assertThat(outputBuffer2.timeUs).isEqualTo(inputBuffer2.timeUs);
+    assertThat(outputBuffer2.bitmap).isEqualTo(decodedBitmap2);
+  }
+
+  // Polling to see whether the output is available yet since the decode thread doesn't finish
+  // decoding immediately.
+  private ImageOutputBuffer getDecodedOutput() throws Exception {
+    @Nullable ImageOutputBuffer outputBuffer;
+    // Use System.currentTimeMillis() to calculate the wait duration more accurately.
+    long deadlineMs = System.currentTimeMillis() + TIMEOUT_MS;
+    long remainingMs = TIMEOUT_MS;
+    while (remainingMs > 0) {
+      outputBuffer = fakeImageDecoder.dequeueOutputBuffer();
+      if (outputBuffer != null) {
+        return outputBuffer;
+      }
+      Thread.sleep(/* millis= */ 5);
+      remainingMs = deadlineMs - System.currentTimeMillis();
+    }
+    throw new TimeoutException();
+  }
+}