diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 23479c37aa..0b0683550a 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -35,6 +35,8 @@ * Transformer: * Add `SurfaceAssetLoader`, which supports queueing video data to Transformer via a `Surface`. + * `ImageAssetLoader` reports unsupported input via `AssetLoader.onError` + instead of throwing an `IllegalStateException`. * Track Selection: * Extractors: * Allow `Mp4Extractor` and `FragmentedMp4Extractor` to identify H264 diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java index 5220dc7748..d8dc343271 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java @@ -16,9 +16,6 @@ package androidx.media3.transformer; -import static androidx.media3.common.util.Assertions.checkState; - -import android.content.ContentResolver; import android.content.Context; import android.graphics.BitmapFactory; import android.graphics.ColorSpace; @@ -34,9 +31,7 @@ import androidx.media3.datasource.DataSourceBitmapLoader; import androidx.media3.datasource.DefaultDataSource; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.transformer.AssetLoader.CompositionSettings; -import com.google.common.base.Ascii; import com.google.common.util.concurrent.MoreExecutors; -import java.util.Objects; import java.util.concurrent.Executors; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -133,9 +128,9 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { AssetLoader.Listener listener, CompositionSettings compositionSettings) { MediaItem mediaItem = editedMediaItem.mediaItem; - if (isImage(mediaItem.localConfiguration)) { + if (isImage(mediaItem)) { if (imageAssetLoaderFactory == null) { - imageAssetLoaderFactory = new ImageAssetLoader.Factory(bitmapLoader); + imageAssetLoaderFactory = new ImageAssetLoader.Factory(context, bitmapLoader); } return imageAssetLoaderFactory.createAssetLoader( editedMediaItem, looper, listener, compositionSettings); @@ -150,79 +145,8 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory { editedMediaItem, looper, listener, compositionSettings); } - private boolean isImage(@Nullable MediaItem.LocalConfiguration localConfiguration) { - if (localConfiguration == null) { - return false; - } - @Nullable String mimeType = localConfiguration.mimeType; - if (mimeType == null) { - if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) { - ContentResolver cr = context.getContentResolver(); - mimeType = cr.getType(localConfiguration.uri); - } else { - @Nullable String uriPath = localConfiguration.uri.getPath(); - if (uriPath == null) { - return false; - } - int fileExtensionStart = uriPath.lastIndexOf("."); - if (fileExtensionStart >= 0 && fileExtensionStart < uriPath.length() - 1) { - String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart + 1)); - mimeType = getCommonImageMimeTypeFromExtension(extension); - } - } - } - if (mimeType == null) { - return false; - } - if (!MimeTypes.isImage(mimeType)) { - return false; - } - checkState( - bitmapLoader.supportsMimeType(mimeType), - "Image format not supported by given bitmapLoader"); - return true; - } - - @Nullable - private static String getCommonImageMimeTypeFromExtension(String extension) { - switch (extension) { - case "bmp": - case "dib": - return MimeTypes.IMAGE_BMP; - case "heif": - return MimeTypes.IMAGE_HEIF; - case "heic": - return MimeTypes.IMAGE_HEIC; - case "jpg": - case "jpeg": - case "jpe": - case "jif": - case "jfif": - case "jfi": - return MimeTypes.IMAGE_JPEG; - case "png": - return MimeTypes.IMAGE_PNG; - case "webp": - return MimeTypes.IMAGE_WEBP; - case "gif": - return "image/gif"; - case "tiff": - case "tif": - return "image/tiff"; - case "raw": - case "arw": - case "cr2": - case "k25": - return "image/raw"; - case "svg": - case "svgz": - return "image/svg+xml"; - case "ico": - return "image/x-icon"; - case "avif": - return MimeTypes.IMAGE_AVIF; - default: - return null; - } + private boolean isImage(MediaItem mediaItem) { + @Nullable String mimeType = ImageAssetLoader.getImageMimeType(context, mediaItem); + return mimeType != null && MimeTypes.isImage(mimeType); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java index 183b245163..3c6c01b631 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java @@ -25,8 +25,11 @@ import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_SUCCESS; import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_TRY_AGAIN_LATER; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED; +import static com.google.common.util.concurrent.Futures.immediateFailedFuture; import static java.util.concurrent.TimeUnit.MILLISECONDS; +import android.content.ContentResolver; +import android.content.Context; import android.graphics.Bitmap; import android.os.Looper; import androidx.annotation.Nullable; @@ -35,15 +38,18 @@ import androidx.media3.common.ColorInfo; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; +import androidx.media3.common.ParserException; import androidx.media3.common.util.BitmapLoader; import androidx.media3.common.util.ConstantRateTimestampIterator; import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.transformer.SampleConsumer.InputResult; +import com.google.common.base.Ascii; import com.google.common.collect.ImmutableMap; import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import java.util.Objects; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -57,19 +63,20 @@ import java.util.concurrent.ScheduledExecutorService; @UnstableApi public final class ImageAssetLoader implements AssetLoader { - private final boolean retainHdrFromUltraHdrImage; - /** An {@link AssetLoader.Factory} for {@link ImageAssetLoader} instances. */ public static final class Factory implements AssetLoader.Factory { + private final Context context; private final BitmapLoader bitmapLoader; /** * Creates an instance. * + * @param context The {@link Context}. * @param bitmapLoader The {@link BitmapLoader} to use to load and decode images. */ - public Factory(BitmapLoader bitmapLoader) { + public Factory(Context context, BitmapLoader bitmapLoader) { + this.context = context; this.bitmapLoader = bitmapLoader; } @@ -80,15 +87,21 @@ public final class ImageAssetLoader implements AssetLoader { Listener listener, CompositionSettings compositionSettings) { return new ImageAssetLoader( - editedMediaItem, listener, bitmapLoader, compositionSettings.retainHdrFromUltraHdrImage); + context, + editedMediaItem, + listener, + bitmapLoader, + compositionSettings.retainHdrFromUltraHdrImage); } } private static final int QUEUE_BITMAP_INTERVAL_MS = 10; + private final Context context; private final EditedMediaItem editedMediaItem; private final BitmapLoader bitmapLoader; private final Listener listener; + private final boolean retainHdrFromUltraHdrImage; private final ScheduledExecutorService scheduledExecutorService; @Nullable private SampleConsumer sampleConsumer; @@ -97,20 +110,57 @@ public final class ImageAssetLoader implements AssetLoader { private volatile int progress; private ImageAssetLoader( + Context context, EditedMediaItem editedMediaItem, Listener listener, BitmapLoader bitmapLoader, boolean retainHdrFromUltraHdrImage) { - this.retainHdrFromUltraHdrImage = retainHdrFromUltraHdrImage; checkState(editedMediaItem.durationUs != C.TIME_UNSET); checkState(editedMediaItem.frameRate != C.RATE_UNSET_INT); + this.context = context; this.editedMediaItem = editedMediaItem; this.listener = listener; this.bitmapLoader = bitmapLoader; + this.retainHdrFromUltraHdrImage = retainHdrFromUltraHdrImage; scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); progressState = PROGRESS_STATE_NOT_STARTED; } + /** + * Returns the image MIME type corresponding to a {@link MediaItem}. + * + *

This method only supports some common image MIME types. + * + * @param context The {@link Context}. + * @param mediaItem The {@link MediaItem} to inspect. + * @return The MIME type. + */ + @Nullable + public static String getImageMimeType(Context context, MediaItem mediaItem) { + if (mediaItem.localConfiguration == null) { + return null; + } + MediaItem.LocalConfiguration localConfiguration = mediaItem.localConfiguration; + @Nullable String mimeType = localConfiguration.mimeType; + if (mimeType == null) { + if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) { + ContentResolver cr = context.getContentResolver(); + mimeType = cr.getType(localConfiguration.uri); + } else { + @Nullable String uriPath = localConfiguration.uri.getPath(); + if (uriPath == null) { + return null; + } + int fileExtensionStart = uriPath.lastIndexOf("."); + if (fileExtensionStart >= 0 && fileExtensionStart < uriPath.length() - 1) { + String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart + 1)); + mimeType = getCommonImageMimeTypeFromExtension(extension); + } + } + } + return mimeType; + } + @Override // Ignore Future returned by scheduledExecutorService because failures are already handled in the // runnable. @@ -119,10 +169,19 @@ public final class ImageAssetLoader implements AssetLoader { progressState = PROGRESS_STATE_AVAILABLE; listener.onDurationUs(editedMediaItem.durationUs); listener.onTrackCount(1); - MediaItem.LocalConfiguration localConfiguration = - checkNotNull(editedMediaItem.mediaItem.localConfiguration); + ListenableFuture future; - ListenableFuture future = bitmapLoader.loadBitmap(localConfiguration.uri); + @Nullable + String mimeType = ImageAssetLoader.getImageMimeType(context, editedMediaItem.mediaItem); + if (mimeType == null || !bitmapLoader.supportsMimeType(mimeType)) { + future = + immediateFailedFuture( + ParserException.createForUnsupportedContainerFeature( + "Attempted to load a Bitmap from unsupported MIME type: " + mimeType)); + } else { + future = + bitmapLoader.loadBitmap(checkNotNull(editedMediaItem.mediaItem.localConfiguration).uri); + } Futures.addCallback( future, @@ -217,4 +276,47 @@ public final class ImageAssetLoader implements AssetLoader { listener.onError(ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED)); } } + + @Nullable + private static String getCommonImageMimeTypeFromExtension(String extension) { + switch (extension) { + case "bmp": + case "dib": + return MimeTypes.IMAGE_BMP; + case "heif": + return MimeTypes.IMAGE_HEIF; + case "heic": + return MimeTypes.IMAGE_HEIC; + case "jpg": + case "jpeg": + case "jpe": + case "jif": + case "jfif": + case "jfi": + return MimeTypes.IMAGE_JPEG; + case "png": + return MimeTypes.IMAGE_PNG; + case "webp": + return MimeTypes.IMAGE_WEBP; + case "gif": + return "image/gif"; + case "tiff": + case "tif": + return "image/tiff"; + case "raw": + case "arw": + case "cr2": + case "k25": + return "image/raw"; + case "svg": + case "svgz": + return "image/svg+xml"; + case "ico": + return "image/x-icon"; + case "avif": + return MimeTypes.IMAGE_AVIF; + default: + return null; + } + } } diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java index a20f547a16..8b1384568f 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java @@ -18,10 +18,13 @@ package androidx.media3.transformer; import static androidx.media3.test.utils.robolectric.RobolectricUtil.runLooperUntil; import static com.google.common.truth.Truth.assertThat; +import android.content.Context; import android.graphics.Bitmap; import android.os.Looper; +import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; +import androidx.media3.common.ParserException; import androidx.media3.common.util.TimestampIterator; import androidx.media3.datasource.DataSourceBitmapLoader; import androidx.media3.transformer.AssetLoader.CompositionSettings; @@ -102,7 +105,7 @@ public class ImageAssetLoaderTest { } } }; - AssetLoader assetLoader = getAssetLoader(listener); + AssetLoader assetLoader = getAssetLoader(listener, "asset:///media/png/media3test.png"); assetLoader.start(); runLooperUntil( @@ -115,14 +118,63 @@ public class ImageAssetLoaderTest { assertThat(exceptionRef.get()).isNull(); } - private static AssetLoader getAssetLoader(AssetLoader.Listener listener) { + @Test + public void imageAssetLoader_onUnsupportedMimeType_callsListener() throws Exception { + AtomicReference exceptionRef = new AtomicReference<>(); + AssetLoader.Listener listener = + new AssetLoader.Listener() { + + @Override + public void onDurationUs(long durationUs) {} + + @Override + public void onTrackCount(int trackCount) {} + + @Override + public boolean onTrackAdded( + Format inputFormat, @AssetLoader.SupportedOutputTypes int supportedOutputTypes) { + return false; + } + + @Override + public SampleConsumer onOutputFormat(Format format) { + return new FakeSampleConsumer(); + } + + @Override + public void onError(ExportException e) { + exceptionRef.set(e); + } + }; + AssetLoader assetLoader = getAssetLoader(listener, "asset:///media3test.gif"); + + assetLoader.start(); + runLooperUntil( + Looper.myLooper(), + () -> { + ShadowSystemClock.advanceBy(Duration.ofMillis(10)); + return exceptionRef.get() != null; + }); + ParserException parserException = (ParserException) exceptionRef.get().getCause(); + + assertThat(parserException.contentIsMalformed).isFalse(); + assertThat(parserException.dataType).isEqualTo(C.DATA_TYPE_MEDIA); + assertThat(parserException) + .hasMessageThat() + .isEqualTo( + "Attempted to load a Bitmap from unsupported MIME type:" + + " image/gif{contentIsMalformed=false, dataType=1}"); + } + + private static AssetLoader getAssetLoader(AssetLoader.Listener listener, String uri) { + Context context = ApplicationProvider.getApplicationContext(); EditedMediaItem editedMediaItem = - new EditedMediaItem.Builder(MediaItem.fromUri("asset:///media/png/media3test.png")) + new EditedMediaItem.Builder(MediaItem.fromUri(uri)) .setDurationUs(1_000_000) .setFrameRate(30) .build(); return new ImageAssetLoader.Factory( - new DataSourceBitmapLoader(ApplicationProvider.getApplicationContext())) + context, new DataSourceBitmapLoader(ApplicationProvider.getApplicationContext())) .createAssetLoader( editedMediaItem, Looper.myLooper(),