diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 498877bcc9..105ed420a0 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -20,6 +20,8 @@ `PreloadMediaSource.PreloadControl` implementations to take actions when error occurs. * Transformer: + * Add `SurfaceAssetLoader`, which supports queueing video data to + Transformer via a `Surface`. * Track Selection: * Extractors: * Allow `Mp4Extractor` to identify H264 samples that are not used as diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java index 574946c36b..8ab45f7d39 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java @@ -19,6 +19,7 @@ import static java.lang.annotation.ElementType.TYPE_USE; import android.content.Context; import android.graphics.Bitmap; +import android.graphics.SurfaceTexture; import android.opengl.EGLExt; import android.view.Surface; import androidx.annotation.IntDef; @@ -222,6 +223,14 @@ public interface VideoFrameProcessor { */ void setOnInputFrameProcessedListener(OnInputFrameProcessedListener listener); + /** + * Sets a listener that's called when the {@linkplain #getInputSurface() input surface} is ready + * to use. + */ + void setOnInputSurfaceReadyListener(Runnable listener); + + // TODO: b/351776002 - Call setDefaultBufferSize on the INPUT_TYPE_SURFACE path too and remove + // mentions of the method (which leak an implementation detail) throughout this file. /** * Returns the input {@link Surface}, where {@link VideoFrameProcessor} consumes input frames * from. @@ -230,6 +239,16 @@ public interface VideoFrameProcessor { * VideoFrameProcessor} until {@link #registerInputStream} is called with {@link * #INPUT_TYPE_SURFACE}. * + *

For streams with {@link #INPUT_TYPE_SURFACE}, the returned surface is ready to use + * immediately and will not have a {@linkplain SurfaceTexture#setDefaultBufferSize(int, int) + * default buffer size} set on it. This is suitable for configuring a {@link + * android.media.MediaCodec} decoder. + * + *

For streams with {@link #INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION}, set a listener + * for the surface becoming ready via {@link #setOnInputSurfaceReadyListener(Runnable)} and wait + * for the event before using the returned surface. This is suitable for use with non-decoder + * producers like media projection. + * * @throws UnsupportedOperationException If the {@code VideoFrameProcessor} does not accept * {@linkplain #INPUT_TYPE_SURFACE surface input}. */ diff --git a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java index 7163068f53..8b82433736 100644 --- a/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java +++ b/libraries/common/src/main/java/androidx/media3/common/VideoGraph.java @@ -20,7 +20,7 @@ import androidx.annotation.IntRange; import androidx.annotation.Nullable; import androidx.media3.common.util.UnstableApi; -/** Represents a graph for processing decoded video frames. */ +/** Represents a graph for processing raw video frames. */ @UnstableApi public interface VideoGraph { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java index dcf06ddc7b..c726deb701 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java @@ -76,6 +76,14 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A {@link VideoFrameProcessor} implementation that applies {@link GlEffect} instances using OpenGL * on a background thread. + * + *

When using surface input ({@link #INPUT_TYPE_SURFACE} or {@link + * #INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION}) the surface's format must be supported for + * sampling as an external texture in OpenGL. When a {@link android.media.MediaCodec} decoder is + * writing to the input surface, the default SDR color format is supported. When an {@link + * android.media.ImageWriter} is writing to the input surface, {@link + * android.graphics.PixelFormat#RGBA_8888} is supported for SDR data. Support for other formats may + * be device-dependent. */ @UnstableApi public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @@ -448,6 +456,10 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { @GuardedBy("lock") private boolean registeredFirstInputStream; + @GuardedBy("lock") + @Nullable + private Runnable onInputSurfaceReadyListener; + private final List activeEffects; private final Object lock; private final ColorInfo outputColorInfo; @@ -569,6 +581,17 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { inputSwitcher.setOnInputFrameProcessedListener(listener); } + @Override + public void setOnInputSurfaceReadyListener(Runnable listener) { + synchronized (lock) { + if (inputStreamRegisteredCondition.isOpen()) { + listener.run(); + } else { + onInputSurfaceReadyListener = listener; + } + } + } + @Override public Surface getInputSurface() { return inputSwitcher.getInputSurface(); @@ -992,6 +1015,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor { inputSwitcher.switchToInput(inputStreamInfo.inputType, inputStreamInfo.frameInfo); inputStreamRegisteredCondition.open(); + synchronized (lock) { + if (onInputSurfaceReadyListener != null) { + onInputSurfaceReadyListener.run(); + onInputSurfaceReadyListener = null; + } + } listenerExecutor.execute( () -> listener.onInputStreamRegistered( diff --git a/libraries/test_data/src/test/assets/media/jpeg/london-512.jpg b/libraries/test_data/src/test/assets/media/jpeg/london-512.jpg new file mode 100644 index 0000000000..e71cec58fe Binary files /dev/null and b/libraries/test_data/src/test/assets/media/jpeg/london-512.jpg differ diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java index 0c87049e1f..75a02bee73 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java @@ -205,6 +205,28 @@ public class BitmapPixelTestUtil { plane.getPixelStride()); } + /** + * Copies image data from the specified {@link Bitmap} into the {@link Image}, which must be an + * {@linkplain PixelFormat#RGBA_8888} image. + */ + public static void copyRbga8888BitmapToImage(Bitmap bitmap, Image image) { + assertThat(image.getPlanes()).hasLength(1); + assertThat(image.getFormat()).isEqualTo(PixelFormat.RGBA_8888); + Image.Plane imagePlane = image.getPlanes()[0]; + ByteBuffer imageBuffer = imagePlane.getBuffer(); + for (int y = 0; y < bitmap.getHeight(); y++) { + for (int x = 0; x < bitmap.getWidth(); x++) { + int imageBufferOffset = y * imagePlane.getRowStride() + x * imagePlane.getPixelStride(); + int argbPixel = bitmap.getPixel(x, y); + imageBuffer.position(imageBufferOffset); + imageBuffer.put((byte) ((argbPixel >> 16) & 0xFF)); + imageBuffer.put((byte) ((argbPixel >> 8) & 0xFF)); + imageBuffer.put((byte) (argbPixel & 0xFF)); + imageBuffer.put((byte) ((argbPixel >> 24) & 0xFF)); + } + } + } + public static Bitmap createArgb8888BitmapFromRgba8888ImageBuffer(ImageBuffer imageBuffer) { int[] colors = new int[imageBuffer.width * imageBuffer.height]; for (int y = 0; y < imageBuffer.height; y++) { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SurfaceAssetLoaderTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SurfaceAssetLoaderTest.java new file mode 100644 index 0000000000..3755d0b6d7 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/SurfaceAssetLoaderTest.java @@ -0,0 +1,138 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static org.junit.Assume.assumeTrue; + +import android.content.Context; +import android.graphics.Bitmap; +import android.graphics.PixelFormat; +import android.media.Image; +import android.media.ImageWriter; +import android.os.Handler; +import android.os.Looper; +import android.view.Surface; +import androidx.media3.common.C; +import androidx.media3.common.ColorInfo; +import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.ConditionVariable; +import androidx.media3.common.util.Util; +import androidx.media3.test.utils.BitmapPixelTestUtil; +import androidx.test.core.app.ApplicationProvider; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestName; +import org.junit.runner.RunWith; + +/** End to end instrumentation test for {@link SurfaceAssetLoader} using {@link Transformer}. */ +@RunWith(AndroidJUnit4.class) +public class SurfaceAssetLoaderTest { + + // TODO: b/351776005 - Add HDR-based test case(s). + + private static final String TEST_BITMAP_PATH = "media/jpeg/london-512.jpg"; + private static final long TIMEOUT_MS = 10_000L; // Set to avoid timing out on slow emulators. + + @Rule public final TestName testName = new TestName(); + + private final Context context = ApplicationProvider.getApplicationContext(); + + private String testId; + + @Before + public void setUpTestId() { + testId = testName.getMethodName(); + } + + @Test + public void encodingFromSurface_succeeds() throws Exception { + assumeTrue("ImageWriter with pixel format set requires API 29", Util.SDK_INT >= 29); + + SettableFuture surfaceAssetLoaderSettableFuture = SettableFuture.create(); + SettableFuture surfaceSettableFuture = SettableFuture.create(); + Transformer transformer = + new Transformer.Builder(context) + .setAssetLoaderFactory( + new SurfaceAssetLoader.Factory( + new SurfaceAssetLoader.Callback() { + @Override + public void onSurfaceAssetLoaderCreated( + SurfaceAssetLoader surfaceAssetLoader) { + surfaceAssetLoaderSettableFuture.set(surfaceAssetLoader); + } + + @Override + public void onSurfaceReady(Surface surface, EditedMediaItem editedMediaItem) { + surfaceSettableFuture.set(surface); + } + })) + .build(); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + MediaItem.fromUri(SurfaceAssetLoader.MEDIA_ITEM_URI_SCHEME + ":")) + .build(); + ListenableFuture exportCompletionFuture = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .runAsync(testId, editedMediaItem); + SurfaceAssetLoader surfaceAssetLoader = + surfaceAssetLoaderSettableFuture.get(TIMEOUT_MS, MILLISECONDS); + Bitmap bitmap = BitmapPixelTestUtil.readBitmap(TEST_BITMAP_PATH); + surfaceAssetLoader.setContentFormat( + new Format.Builder() + .setSampleMimeType(MimeTypes.VIDEO_RAW) + .setWidth(bitmap.getWidth()) + .setHeight(bitmap.getHeight()) + .setColorInfo(ColorInfo.SRGB_BT709_FULL) + .build()); + Surface surface = surfaceSettableFuture.get(TIMEOUT_MS, MILLISECONDS); + + int inputFrameCount = 10; + try (ImageWriter imageWriter = + ImageWriter.newInstance(surface, /* maxImages= */ inputFrameCount, PixelFormat.RGBA_8888)) { + ConditionVariable readyForInputCondition = new ConditionVariable(); + imageWriter.setOnImageReleasedListener( + unusedImageWriter -> readyForInputCondition.open(), new Handler(Looper.getMainLooper())); + for (int i = 0; i < inputFrameCount; i++) { + Image image = imageWriter.dequeueInputImage(); + image.setTimestamp(i * C.NANOS_PER_SECOND / 30); + BitmapPixelTestUtil.copyRbga8888BitmapToImage(bitmap, image); + readyForInputCondition.close(); + imageWriter.queueInputImage(image); + // When frames are queued as fast as possible some can be dropped, so throttle input by + // blocking until the previous frame has been released by the downstream pipeline. + if (i > 0) { + assertThat(readyForInputCondition.block(TIMEOUT_MS)).isTrue(); + } + } + } + surfaceAssetLoader.signalEndOfInput(); + + ExportResult exportResult = exportCompletionFuture.get(); + assertThat(exportResult.videoFrameCount).isEqualTo(inputFrameCount); + assertThat(exportResult.width).isEqualTo(bitmap.getWidth()); + assertThat(exportResult.height).isEqualTo(bitmap.getHeight()); + assertThat(exportResult.durationMs).isEqualTo(300); + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java index 3ab19b26a7..14dbe55430 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SampleConsumer.java @@ -133,6 +133,18 @@ public interface SampleConsumer { throw new UnsupportedOperationException(); } + /** + * Sets a listener that's called when the {@linkplain #getInputSurface() input surface} has been + * configured with a default input size, if applicable. + * + *

Should only be used for raw video data when input is provided by the app to a surface. + * + * @param runnable Listener that's called when the input surface is ready. + */ + default void setOnInputSurfaceReadyListener(Runnable runnable) { + throw new UnsupportedOperationException(); + } + /** * Attempts to provide an input texture to the consumer. * diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java index 5103f02e71..beb77fab79 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SequenceAssetLoader.java @@ -482,6 +482,11 @@ import java.util.concurrent.atomic.AtomicInteger; sampleConsumer.setOnInputFrameProcessedListener(listener); } + @Override + public void setOnInputSurfaceReadyListener(Runnable runnable) { + sampleConsumer.setOnInputSurfaceReadyListener(runnable); + } + @Override public @InputResult int queueInputTexture(int texId, long presentationTimeUs) { long globalTimestampUs = totalDurationUs + presentationTimeUs; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceAssetLoader.java b/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceAssetLoader.java new file mode 100644 index 0000000000..380521edf8 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/SurfaceAssetLoader.java @@ -0,0 +1,209 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.transformer.ExportException.ERROR_CODE_UNSPECIFIED; +import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED; + +import android.net.Uri; +import android.os.Handler; +import android.os.Looper; +import android.view.Surface; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.UnstableApi; +import com.google.common.collect.ImmutableMap; +import java.util.Objects; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Asset loader that outputs video data passed to its input {@link Surface}. + * + *

To use this asset loader, pass a callback to the {@linkplain SurfaceAssetLoader.Factory + * factory's} constructor to get access to the underlying asset loader and {@link Surface} to write + * to once they are ready. Then pass the factory to {@link + * Transformer.Builder#setAssetLoaderFactory(AssetLoader.Factory)}. + * + *

The media item passed to transformer must have a URI starting with the scheme {@link + * #MEDIA_ITEM_URI_SCHEME}. + * + *

Call {@link #signalEndOfInput()} when the input stream ends, which will cause the + * transformation to complete. + */ +@UnstableApi +public final class SurfaceAssetLoader implements AssetLoader { + + /** + * URI scheme for creating a {@link MediaItem} that signals that the media is provided from this + * asset loader. + */ + public static final String MEDIA_ITEM_URI_SCHEME = "transformer_surface_asset"; + + /** Callbacks for {@link SurfaceAssetLoader} events. */ + public interface Callback { + /** + * Called when the asset loader has been created. Pass the {@linkplain #setContentFormat(Format) + * content format} to the provided asset loader to trigger surface creation. May be called on + * any thread. + */ + void onSurfaceAssetLoaderCreated(SurfaceAssetLoader surfaceAssetLoader); + + /** + * Called when the input surface is ready to write to. May be called on any thread. + * + * @param surface The {@link Surface} to write to. + * @param editedMediaItem The {@link EditedMediaItem} used to create the associated {@link + * SurfaceAssetLoader}. + */ + void onSurfaceReady(Surface surface, EditedMediaItem editedMediaItem); + } + + /** Factory for {@link SurfaceAssetLoader} instances. */ + public static final class Factory implements AssetLoader.Factory { + + private final Callback callback; + + /** Creates a factory with the specified callback. */ + public Factory(Callback callback) { + this.callback = callback; + } + + @Override + public AssetLoader createAssetLoader( + EditedMediaItem editedMediaItem, + Looper looper, + AssetLoader.Listener listener, + CompositionSettings compositionSettings) { + Uri uri = checkNotNull(editedMediaItem.mediaItem.localConfiguration).uri; + checkState(checkNotNull(uri.getScheme()).equals(MEDIA_ITEM_URI_SCHEME)); + SurfaceAssetLoader surfaceAssetLoader = + new SurfaceAssetLoader(editedMediaItem, looper, listener, callback); + callback.onSurfaceAssetLoaderCreated(surfaceAssetLoader); + return surfaceAssetLoader; + } + } + + private final EditedMediaItem editedMediaItem; + private final AssetLoader.Listener listener; + private final Handler handler; + private final Callback callback; + + private @Transformer.ProgressState int progressState; + + private boolean isStarted; + private boolean isVideoEndOfStreamSignaled; + private @MonotonicNonNull SampleConsumer sampleConsumer; + private @MonotonicNonNull Format contentFormat; + + private SurfaceAssetLoader( + EditedMediaItem editedMediaItem, + Looper looper, + AssetLoader.Listener listener, + Callback callback) { + this.editedMediaItem = editedMediaItem; + this.listener = listener; + this.callback = callback; + handler = new Handler(looper); + progressState = PROGRESS_STATE_NOT_STARTED; + } + + /** + * Sets the video content format, which must have a raw video sample MIME type, width, height and + * color info. May be called on any thread. + */ + public void setContentFormat(Format contentFormat) { + checkArgument(Objects.equals(contentFormat.sampleMimeType, MimeTypes.VIDEO_RAW)); + checkArgument(contentFormat.width != Format.NO_VALUE); + checkArgument(contentFormat.height != Format.NO_VALUE); + checkArgument(checkNotNull(contentFormat.colorInfo).isDataSpaceValid()); + handler.post( + () -> { + this.contentFormat = contentFormat; + try { + maybeFinishPreparation(); + } catch (RuntimeException e) { + listener.onError(ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED)); + } + }); + } + + /** Returns the {@link EditedMediaItem} being loaded by this instance. */ + public EditedMediaItem getEditedMediaItem() { + return editedMediaItem; + } + + /** Signals that no further input frames will be rendered. May be called on any thread. */ + public void signalEndOfInput() { + handler.post( + () -> { + try { + if (!isVideoEndOfStreamSignaled && sampleConsumer != null) { + isVideoEndOfStreamSignaled = true; + sampleConsumer.signalEndOfVideoInput(); + } + } catch (RuntimeException e) { + listener.onError(ExportException.createForAssetLoader(e, ERROR_CODE_UNSPECIFIED)); + } + }); + } + + // AssetLoader implementation. + + @Override + public void start() { + isStarted = true; + maybeFinishPreparation(); + } + + @Override + public @Transformer.ProgressState int getProgress(ProgressHolder progressHolder) { + return progressState; + } + + @Override + public ImmutableMap getDecoderNames() { + return ImmutableMap.of(); + } + + @Override + public void release() { + // Do nothing. + } + + private void maybeFinishPreparation() { + if (!isStarted || contentFormat == null) { + return; + } + listener.onTrackCount(1); + listener.onDurationUs(C.TIME_UNSET); + listener.onTrackAdded(contentFormat, SUPPORTED_OUTPUT_TYPE_DECODED); + try { + sampleConsumer = checkNotNull(listener.onOutputFormat(contentFormat)); + sampleConsumer.setOnInputSurfaceReadyListener( + () -> + callback.onSurfaceReady( + checkNotNull(sampleConsumer).getInputSurface(), editedMediaItem)); + } catch (ExportException e) { + listener.onError(e); + } + progressState = Transformer.PROGRESS_STATE_UNAVAILABLE; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java index 493b2ef6aa..e4fcb6bed2 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java @@ -27,6 +27,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.FrameInfo; +import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.VideoFrameProcessor; @@ -60,11 +61,14 @@ import java.util.concurrent.atomic.AtomicLong; long durationUs, @Nullable Format decodedFormat, boolean isLast) { + boolean isSurfaceAssetLoaderMediaItem = isMediaItemForSurfaceAssetLoader(editedMediaItem); durationUs = editedMediaItem.getDurationAfterEffectsApplied(durationUs); if (decodedFormat != null) { Size decodedSize = getDecodedSize(decodedFormat); videoFrameProcessor.registerInputStream( - getInputType(checkNotNull(decodedFormat.sampleMimeType)), + isSurfaceAssetLoaderMediaItem + ? VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION + : getInputTypeForMimeType(checkNotNull(decodedFormat.sampleMimeType)), createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation), new FrameInfo.Builder( checkNotNull(decodedFormat.colorInfo), @@ -90,6 +94,11 @@ import java.util.concurrent.atomic.AtomicLong; videoFrameProcessor.setOnInputFrameProcessedListener(listener); } + @Override + public void setOnInputSurfaceReadyListener(Runnable runnable) { + videoFrameProcessor.setOnInputSurfaceReadyListener(runnable); + } + @Override public @InputResult int queueInputTexture(int texId, long presentationTimeUs) { return videoFrameProcessor.queueInputTexture(texId, presentationTimeUs) @@ -138,7 +147,7 @@ import java.util.concurrent.atomic.AtomicLong; return effectsWithPresentationBuilder.build(); } - private static @VideoFrameProcessor.InputType int getInputType(String sampleMimeType) { + private static @VideoFrameProcessor.InputType int getInputTypeForMimeType(String sampleMimeType) { if (MimeTypes.isImage(sampleMimeType)) { return INPUT_TYPE_BITMAP; } @@ -150,4 +159,17 @@ import java.util.concurrent.atomic.AtomicLong; } throw new IllegalArgumentException("MIME type not supported " + sampleMimeType); } + + private static boolean isMediaItemForSurfaceAssetLoader(EditedMediaItem editedMediaItem) { + @Nullable + MediaItem.LocalConfiguration localConfiguration = editedMediaItem.mediaItem.localConfiguration; + if (localConfiguration == null) { + return false; + } + @Nullable String scheme = localConfiguration.uri.getScheme(); + if (scheme == null) { + return false; + } + return scheme.equals(SurfaceAssetLoader.MEDIA_ITEM_URI_SCHEME); + } }