loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Do not checkState based on input data

Browse Source 
App users can choose arbitrary data that might not be
anticipated by developers. Transformer shouldn't `checkState` based on
media data or file type -- report an error for unsupported data instead.

Public API change `ImageAssetLoader` needs to parse MIME type and now accepts
`Context` as parameter.

PiperOrigin-RevId: 660762459
This commit is contained in:
dancho 2024-08-08 03:15:56 -07:00 committed by Copybara-Service
parent 2202397758
commit c1078e3cfa
4 changed files with 173 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -35,6 +35,8 @@
* Transformer:
* Add `SurfaceAssetLoader`, which supports queueing video data to
Transformer via a `Surface`.
* `ImageAssetLoader` reports unsupported input via `AssetLoader.onError`
instead of throwing an `IllegalStateException`.
* Track Selection:
* Extractors:
* Allow `Mp4Extractor` and `FragmentedMp4Extractor` to identify H264

86
libraries/transformer/src/main/java/androidx/media3/transformer/DefaultAssetLoaderFactory.java
View File

@ -16,9 +16,6 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkState;
import android.content.ContentResolver;
import android.content.Context;
import android.graphics.BitmapFactory;
import android.graphics.ColorSpace;
@ -34,9 +31,7 @@ import androidx.media3.datasource.DataSourceBitmapLoader;
import androidx.media3.datasource.DefaultDataSource;
import androidx.media3.exoplayer.source.MediaSource;
import androidx.media3.transformer.AssetLoader.CompositionSettings;
import com.google.common.base.Ascii;
import com.google.common.util.concurrent.MoreExecutors;
import java.util.Objects;
import java.util.concurrent.Executors;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@ -133,9 +128,9 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
AssetLoader.Listener listener,
CompositionSettings compositionSettings) {
MediaItem mediaItem = editedMediaItem.mediaItem;
if (isImage(mediaItem.localConfiguration)) {
if (isImage(mediaItem)) {
if (imageAssetLoaderFactory == null) {
imageAssetLoaderFactory = new ImageAssetLoader.Factory(bitmapLoader);
imageAssetLoaderFactory = new ImageAssetLoader.Factory(context, bitmapLoader);
}
return imageAssetLoaderFactory.createAssetLoader(
editedMediaItem, looper, listener, compositionSettings);
@ -150,79 +145,8 @@ public final class DefaultAssetLoaderFactory implements AssetLoader.Factory {
editedMediaItem, looper, listener, compositionSettings);
}
private boolean isImage(@Nullable MediaItem.LocalConfiguration localConfiguration) {
if (localConfiguration == null) {
return false;
}
@Nullable String mimeType = localConfiguration.mimeType;
if (mimeType == null) {
if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) {
ContentResolver cr = context.getContentResolver();
mimeType = cr.getType(localConfiguration.uri);
} else {
@Nullable String uriPath = localConfiguration.uri.getPath();
if (uriPath == null) {
return false;
}
int fileExtensionStart = uriPath.lastIndexOf(".");
if (fileExtensionStart >= 0 && fileExtensionStart < uriPath.length() - 1) {
String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart + 1));
mimeType = getCommonImageMimeTypeFromExtension(extension);
}
}
}
if (mimeType == null) {
return false;
}
if (!MimeTypes.isImage(mimeType)) {
return false;
}
checkState(
bitmapLoader.supportsMimeType(mimeType),
"Image format not supported by given bitmapLoader");
return true;
}
@Nullable
private static String getCommonImageMimeTypeFromExtension(String extension) {
switch (extension) {
case "bmp":
case "dib":
return MimeTypes.IMAGE_BMP;
case "heif":
return MimeTypes.IMAGE_HEIF;
case "heic":
return MimeTypes.IMAGE_HEIC;
case "jpg":
case "jpeg":
case "jpe":
case "jif":
case "jfif":
case "jfi":
return MimeTypes.IMAGE_JPEG;
case "png":
return MimeTypes.IMAGE_PNG;
case "webp":
return MimeTypes.IMAGE_WEBP;
case "gif":
return "image/gif";
case "tiff":
case "tif":
return "image/tiff";
case "raw":
case "arw":
case "cr2":
case "k25":
return "image/raw";
case "svg":
case "svgz":
return "image/svg+xml";
case "ico":
return "image/x-icon";
case "avif":
return MimeTypes.IMAGE_AVIF;
default:
return null;
}
private boolean isImage(MediaItem mediaItem) {
@Nullable String mimeType = ImageAssetLoader.getImageMimeType(context, mediaItem);
return mimeType != null && MimeTypes.isImage(mimeType);
}
}

118
libraries/transformer/src/main/java/androidx/media3/transformer/ImageAssetLoader.java
View File

@ -25,8 +25,11 @@ import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_SUCCESS;
import static androidx.media3.transformer.SampleConsumer.INPUT_RESULT_TRY_AGAIN_LATER;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
import static com.google.common.util.concurrent.Futures.immediateFailedFuture;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
import android.content.ContentResolver;
import android.content.Context;
import android.graphics.Bitmap;
import android.os.Looper;
import androidx.annotation.Nullable;
@ -35,15 +38,18 @@ import androidx.media3.common.ColorInfo;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.ParserException;
import androidx.media3.common.util.BitmapLoader;
import androidx.media3.common.util.ConstantRateTimestampIterator;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.transformer.SampleConsumer.InputResult;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import java.util.Objects;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@ -57,19 +63,20 @@ import java.util.concurrent.ScheduledExecutorService;
@UnstableApi
public final class ImageAssetLoader implements AssetLoader {
private final boolean retainHdrFromUltraHdrImage;
/** An {@link AssetLoader.Factory} for {@link ImageAssetLoader} instances. */
public static final class Factory implements AssetLoader.Factory {
private final Context context;
private final BitmapLoader bitmapLoader;
/**
* Creates an instance.
*
* @param context The {@link Context}.
* @param bitmapLoader The {@link BitmapLoader} to use to load and decode images.
*/
public Factory(BitmapLoader bitmapLoader) {
public Factory(Context context, BitmapLoader bitmapLoader) {
this.context = context;
this.bitmapLoader = bitmapLoader;
}
@ -80,15 +87,21 @@ public final class ImageAssetLoader implements AssetLoader {
Listener listener,
CompositionSettings compositionSettings) {
return new ImageAssetLoader(
editedMediaItem, listener, bitmapLoader, compositionSettings.retainHdrFromUltraHdrImage);
context,
editedMediaItem,
listener,
bitmapLoader,
compositionSettings.retainHdrFromUltraHdrImage);
}
}
private static final int QUEUE_BITMAP_INTERVAL_MS = 10;
private final Context context;
private final EditedMediaItem editedMediaItem;
private final BitmapLoader bitmapLoader;
private final Listener listener;
private final boolean retainHdrFromUltraHdrImage;
private final ScheduledExecutorService scheduledExecutorService;
@Nullable private SampleConsumer sampleConsumer;
@ -97,20 +110,57 @@ public final class ImageAssetLoader implements AssetLoader {
private volatile int progress;
private ImageAssetLoader(
Context context,
EditedMediaItem editedMediaItem,
Listener listener,
BitmapLoader bitmapLoader,
boolean retainHdrFromUltraHdrImage) {
this.retainHdrFromUltraHdrImage = retainHdrFromUltraHdrImage;
checkState(editedMediaItem.durationUs != C.TIME_UNSET);
checkState(editedMediaItem.frameRate != C.RATE_UNSET_INT);
this.context = context;
this.editedMediaItem = editedMediaItem;
this.listener = listener;
this.bitmapLoader = bitmapLoader;
this.retainHdrFromUltraHdrImage = retainHdrFromUltraHdrImage;
scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
progressState = PROGRESS_STATE_NOT_STARTED;
}
/**
* Returns the image MIME type corresponding to a {@link MediaItem}.
*
* <p>This method only supports some common image MIME types.
*
* @param context The {@link Context}.
* @param mediaItem The {@link MediaItem} to inspect.
* @return The MIME type.
*/
@Nullable
public static String getImageMimeType(Context context, MediaItem mediaItem) {
if (mediaItem.localConfiguration == null) {
return null;
}
MediaItem.LocalConfiguration localConfiguration = mediaItem.localConfiguration;
@Nullable String mimeType = localConfiguration.mimeType;
if (mimeType == null) {
if (Objects.equals(localConfiguration.uri.getScheme(), ContentResolver.SCHEME_CONTENT)) {
ContentResolver cr = context.getContentResolver();
mimeType = cr.getType(localConfiguration.uri);
} else {
@Nullable String uriPath = localConfiguration.uri.getPath();
if (uriPath == null) {
return null;
}
int fileExtensionStart = uriPath.lastIndexOf(".");
if (fileExtensionStart >= 0 && fileExtensionStart < uriPath.length() - 1) {
String extension = Ascii.toLowerCase(uriPath.substring(fileExtensionStart + 1));
mimeType = getCommonImageMimeTypeFromExtension(extension);
}
}
}
return mimeType;
}
@Override
// Ignore Future returned by scheduledExecutorService because failures are already handled in the
// runnable.
@ -119,10 +169,19 @@ public final class ImageAssetLoader implements AssetLoader {
progressState = PROGRESS_STATE_AVAILABLE;
listener.onDurationUs(editedMediaItem.durationUs);
listener.onTrackCount(1);
MediaItem.LocalConfiguration localConfiguration =
checkNotNull(editedMediaItem.mediaItem.localConfiguration);
ListenableFuture<Bitmap> future;
ListenableFuture<Bitmap> future = bitmapLoader.loadBitmap(localConfiguration.uri);
@Nullable
String mimeType = ImageAssetLoader.getImageMimeType(context, editedMediaItem.mediaItem);
if (mimeType == null || !bitmapLoader.supportsMimeType(mimeType)) {
future =
immediateFailedFuture(
ParserException.createForUnsupportedContainerFeature(
"Attempted to load a Bitmap from unsupported MIME type: " + mimeType));
} else {
future =
bitmapLoader.loadBitmap(checkNotNull(editedMediaItem.mediaItem.localConfiguration).uri);
}
Futures.addCallback(
future,
@ -217,4 +276,47 @@ public final class ImageAssetLoader implements AssetLoader {
listener.onError(ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED));
}
}
@Nullable
private static String getCommonImageMimeTypeFromExtension(String extension) {
switch (extension) {
case "bmp":
case "dib":
return MimeTypes.IMAGE_BMP;
case "heif":
return MimeTypes.IMAGE_HEIF;
case "heic":
return MimeTypes.IMAGE_HEIC;
case "jpg":
case "jpeg":
case "jpe":
case "jif":
case "jfif":
case "jfi":
return MimeTypes.IMAGE_JPEG;
case "png":
return MimeTypes.IMAGE_PNG;
case "webp":
return MimeTypes.IMAGE_WEBP;
case "gif":
return "image/gif";
case "tiff":
case "tif":
return "image/tiff";
case "raw":
case "arw":
case "cr2":
case "k25":
return "image/raw";
case "svg":
case "svgz":
return "image/svg+xml";
case "ico":
return "image/x-icon";
case "avif":
return MimeTypes.IMAGE_AVIF;
default:
return null;
}
}
}

60
libraries/transformer/src/test/java/androidx/media3/transformer/ImageAssetLoaderTest.java
View File

@ -18,10 +18,13 @@ package androidx.media3.transformer;
import static androidx.media3.test.utils.robolectric.RobolectricUtil.runLooperUntil;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.graphics.Bitmap;
import android.os.Looper;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.ParserException;
import androidx.media3.common.util.TimestampIterator;
import androidx.media3.datasource.DataSourceBitmapLoader;
import androidx.media3.transformer.AssetLoader.CompositionSettings;
@ -102,7 +105,7 @@ public class ImageAssetLoaderTest {
}
}
};
AssetLoader assetLoader = getAssetLoader(listener);
AssetLoader assetLoader = getAssetLoader(listener, "asset:///media/png/media3test.png");
assetLoader.start();
runLooperUntil(
@ -115,14 +118,63 @@ public class ImageAssetLoaderTest {
assertThat(exceptionRef.get()).isNull();
}
private static AssetLoader getAssetLoader(AssetLoader.Listener listener) {
@Test
public void imageAssetLoader_onUnsupportedMimeType_callsListener() throws Exception {
AtomicReference<Exception> exceptionRef = new AtomicReference<>();
AssetLoader.Listener listener =
new AssetLoader.Listener() {
@Override
public void onDurationUs(long durationUs) {}
@Override
public void onTrackCount(int trackCount) {}
@Override
public boolean onTrackAdded(
Format inputFormat, @AssetLoader.SupportedOutputTypes int supportedOutputTypes) {
return false;
}
@Override
public SampleConsumer onOutputFormat(Format format) {
return new FakeSampleConsumer();
}
@Override
public void onError(ExportException e) {
exceptionRef.set(e);
}
};
AssetLoader assetLoader = getAssetLoader(listener, "asset:///media3test.gif");
assetLoader.start();
runLooperUntil(
Looper.myLooper(),
() -> {
ShadowSystemClock.advanceBy(Duration.ofMillis(10));
return exceptionRef.get() != null;
});
ParserException parserException = (ParserException) exceptionRef.get().getCause();
assertThat(parserException.contentIsMalformed).isFalse();
assertThat(parserException.dataType).isEqualTo(C.DATA_TYPE_MEDIA);
assertThat(parserException)
.hasMessageThat()
.isEqualTo(
"Attempted to load a Bitmap from unsupported MIME type:"
+ " image/gif{contentIsMalformed=false, dataType=1}");
}
private static AssetLoader getAssetLoader(AssetLoader.Listener listener, String uri) {
Context context = ApplicationProvider.getApplicationContext();
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(MediaItem.fromUri("asset:///media/png/media3test.png"))
new EditedMediaItem.Builder(MediaItem.fromUri(uri))
.setDurationUs(1_000_000)
.setFrameRate(30)
.build();
return new ImageAssetLoader.Factory(
new DataSourceBitmapLoader(ApplicationProvider.getApplicationContext()))
context, new DataSourceBitmapLoader(ApplicationProvider.getApplicationContext()))
.createAssetLoader(
editedMediaItem,
Looper.myLooper(),