diff --git a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java index 8998831524..2ed6026de0 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/PreviewingSingleInputVideoGraph.java @@ -17,7 +17,6 @@ package androidx.media3.effect; import android.content.Context; -import androidx.annotation.Nullable; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; @@ -65,13 +64,6 @@ public final class PreviewingSingleInputVideoGraph extends SingleInputVideoGraph Executor listenerExecutor, List compositionEffects, long initialTimestampOffsetUs) { - @Nullable Presentation presentation = null; - for (int i = 0; i < compositionEffects.size(); i++) { - Effect effect = compositionEffects.get(i); - if (effect instanceof Presentation) { - presentation = (Presentation) effect; - } - } return new PreviewingSingleInputVideoGraph( context, videoFrameProcessorFactory, @@ -79,7 +71,6 @@ public final class PreviewingSingleInputVideoGraph extends SingleInputVideoGraph debugViewProvider, listener, listenerExecutor, - presentation, initialTimestampOffsetUs); } } @@ -91,7 +82,6 @@ public final class PreviewingSingleInputVideoGraph extends SingleInputVideoGraph DebugViewProvider debugViewProvider, Listener listener, Executor listenerExecutor, - @Nullable Presentation presentation, long initialTimestampOffsetUs) { super( context, @@ -103,7 +93,6 @@ public final class PreviewingSingleInputVideoGraph extends SingleInputVideoGraph VideoCompositorSettings.DEFAULT, // Previewing needs frame render timing. /* renderFramesAutomatically= */ false, - presentation, initialTimestampOffsetUs); } diff --git a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java index 9328eab27f..16fbefd815 100644 --- a/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java +++ b/libraries/effect/src/main/java/androidx/media3/effect/SingleInputVideoGraph.java @@ -48,7 +48,6 @@ public abstract class SingleInputVideoGraph implements VideoGraph { private final Executor listenerExecutor; private final boolean renderFramesAutomatically; private final long initialTimestampOffsetUs; - @Nullable private final Presentation presentation; @Nullable private VideoFrameProcessor videoFrameProcessor; @Nullable private SurfaceInfo outputSurfaceInfo; @@ -71,7 +70,6 @@ public abstract class SingleInputVideoGraph implements VideoGraph { Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, boolean renderFramesAutomatically, - @Nullable Presentation presentation, long initialTimestampOffsetUs) { checkState( VideoCompositorSettings.DEFAULT.equals(videoCompositorSettings), @@ -84,7 +82,6 @@ public abstract class SingleInputVideoGraph implements VideoGraph { this.debugViewProvider = debugViewProvider; this.listenerExecutor = listenerExecutor; this.renderFramesAutomatically = renderFramesAutomatically; - this.presentation = presentation; this.initialTimestampOffsetUs = initialTimestampOffsetUs; this.inputIndex = C.INDEX_UNSET; } @@ -203,9 +200,4 @@ public abstract class SingleInputVideoGraph implements VideoGraph { protected long getInitialTimestampOffsetUs() { return initialTimestampOffsetUs; } - - @Nullable - protected Presentation getPresentation() { - return presentation; - } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 3dee4dccec..a8b31d1118 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -597,6 +597,64 @@ public class TransformerEndToEndTest { assertThat(new File(result.filePath).length()).isGreaterThan(0); } + @Test + public void videoEditing_withSingleSequenceAndCompositionEffect_appliesEffect() throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET.videoFormat, + /* outputFormat= */ MP4_ASSET.videoFormat); + Transformer transformer = + new Transformer.Builder(context) + .setEncoderFactory( + new DefaultEncoderFactory.Builder(context).setEnableFallback(false).build()) + .build(); + MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET.uri)); + EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build(); + InputTimestampRecordingShaderProgram timestampRecordingShaderProgram = + new InputTimestampRecordingShaderProgram(); + ImmutableList videoEffects = + ImmutableList.of((GlEffect) (context, useHdr) -> timestampRecordingShaderProgram); + Composition composition = + new Composition.Builder(new EditedMediaItemSequence(editedMediaItem)) + .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects)) + .build(); + + new TransformerAndroidTestRunner.Builder(context, transformer).build().run(testId, composition); + + assertThat(timestampRecordingShaderProgram.getInputTimestampsUs()).isNotEmpty(); + } + + @Test + public void videoEditing_withMultiSequenceAndCompositionEffect_appliesEffect() throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET.videoFormat, + /* outputFormat= */ MP4_ASSET.videoFormat); + Transformer transformer = + new Transformer.Builder(context) + .setEncoderFactory( + new DefaultEncoderFactory.Builder(context).setEnableFallback(false).build()) + .build(); + MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET.uri)); + EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build(); + InputTimestampRecordingShaderProgram timestampRecordingShaderProgram = + new InputTimestampRecordingShaderProgram(); + ImmutableList videoEffects = + ImmutableList.of((GlEffect) (context, useHdr) -> timestampRecordingShaderProgram); + Composition composition = + new Composition.Builder( + new EditedMediaItemSequence(editedMediaItem), + new EditedMediaItemSequence(editedMediaItem)) + .setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects)) + .build(); + + new TransformerAndroidTestRunner.Builder(context, transformer).build().run(testId, composition); + + assertThat(timestampRecordingShaderProgram.getInputTimestampsUs()).isNotEmpty(); + } + @Test public void videoOnly_completesWithConsistentDuration() throws Exception { assumeFormatsSupported( diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerPixelTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerPixelTest.java index 7443829ef7..746be0cb2a 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerPixelTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerPixelTest.java @@ -31,7 +31,6 @@ import android.graphics.Bitmap; import android.graphics.PixelFormat; import android.media.Image; import android.media.ImageReader; -import android.view.SurfaceView; import androidx.media3.common.MediaItem; import androidx.media3.common.util.ConditionVariable; import androidx.media3.common.util.Size; @@ -41,8 +40,6 @@ import androidx.media3.transformer.CompositionPlayer; import androidx.media3.transformer.EditedMediaItem; import androidx.media3.transformer.EditedMediaItemSequence; import androidx.media3.transformer.Effects; -import androidx.media3.transformer.SurfaceTestActivity; -import androidx.test.ext.junit.rules.ActivityScenarioRule; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.common.collect.ImmutableList; import java.util.concurrent.TimeoutException; @@ -67,27 +64,20 @@ public class CompositionPlayerPixelTest { @Rule public final TestName testName = new TestName(); - @Rule - public ActivityScenarioRule rule = - new ActivityScenarioRule<>(SurfaceTestActivity.class); - private final Context context = getInstrumentation().getContext().getApplicationContext(); private @MonotonicNonNull CompositionPlayer player; private @MonotonicNonNull ImageReader outputImageReader; private String testId; - private SurfaceView surfaceView; @Before public void setUp() { - rule.getScenario().onActivity(activity -> surfaceView = activity.getSurfaceView()); testId = testName.getMethodName(); } @After public void tearDown() { - rule.getScenario().close(); getInstrumentation() .runOnMainSync( () -> { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java index 6e3e2f0900..d5860d8bec 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Composition.java @@ -111,8 +111,6 @@ public final class Composition { * *

The default value is {@link Effects#EMPTY}. * - *

This only works with the {@code Presentation} effect. - * * @param effects The {@link Composition} {@link Effects}. * @return This builder. */ diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index cd87cbbf09..c0bb6685e4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -58,7 +58,6 @@ import androidx.media3.common.util.UnstableApi; import androidx.media3.common.util.Util; import androidx.media3.effect.DebugTraceUtil; import androidx.media3.effect.DefaultVideoFrameProcessor; -import androidx.media3.effect.Presentation; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.muxer.Muxer; import com.google.common.collect.ImmutableList; @@ -1018,14 +1017,8 @@ public final class Transformer { * EditedMediaItemSequence}, while the audio format will be determined by the {@code * AudioMediaItem} in the second {@code EditedMediaItemSequence}. * - *

This method is under development. A {@link Composition} must meet the following conditions: - * - *

    - *
  • The video composition {@link Presentation} effect is applied after input streams are - * composited. Other composition effects are ignored. - *
- * - *

{@linkplain EditedMediaItemSequence Sequences} within the {@link Composition} must meet the + *

Some {@linkplain Composition compositions} are not supported yet. More specifically, + * {@linkplain EditedMediaItemSequence Sequences} within the {@link Composition} must meet the * following conditions: * *

    diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java index 5cbb73dd9f..73e47e31e7 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java @@ -27,6 +27,7 @@ import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.VideoGraph; import androidx.media3.effect.MultipleInputVideoGraph; import androidx.media3.effect.VideoCompositorSettings; +import com.google.common.collect.ImmutableList; import java.util.List; import java.util.concurrent.Executor; @@ -99,7 +100,9 @@ import java.util.concurrent.Executor; public GraphInput createInput(int inputIndex) throws VideoFrameProcessingException { registerInput(inputIndex); return new VideoFrameProcessingWrapper( - getProcessor(inputIndex), /* presentation= */ null, getInitialTimestampOffsetUs()); + getProcessor(inputIndex), + /* postProcessingEffects= */ ImmutableList.of(), + getInitialTimestampOffsetUs()); } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java index 0d2cbea29a..39b2962445 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java @@ -20,13 +20,11 @@ import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WIT import static androidx.media3.common.util.Assertions.checkState; import android.content.Context; -import androidx.annotation.Nullable; import androidx.media3.common.ColorInfo; import androidx.media3.common.DebugViewProvider; import androidx.media3.common.Effect; import androidx.media3.common.VideoFrameProcessingException; import androidx.media3.common.VideoFrameProcessor; -import androidx.media3.effect.Presentation; import androidx.media3.effect.SingleInputVideoGraph; import androidx.media3.effect.VideoCompositorSettings; import java.util.List; @@ -60,13 +58,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; List compositionEffects, long initialTimestampOffsetUs, boolean renderFramesAutomatically) { - @Nullable Presentation presentation = null; - for (int i = 0; i < compositionEffects.size(); i++) { - Effect effect = compositionEffects.get(i); - if (effect instanceof Presentation) { - presentation = (Presentation) effect; - } - } return new TransformerSingleInputVideoGraph( context, videoFrameProcessorFactory, @@ -76,11 +67,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; listenerExecutor, videoCompositorSettings, renderFramesAutomatically, - presentation, + compositionEffects, initialTimestampOffsetUs); } } + private final List compositionEffects; private @MonotonicNonNull VideoFrameProcessingWrapper videoFrameProcessingWrapper; private TransformerSingleInputVideoGraph( @@ -92,7 +84,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; Executor listenerExecutor, VideoCompositorSettings videoCompositorSettings, boolean renderFramesAutomatically, - @Nullable Presentation presentation, + List compositionEffects, long initialTimestampOffsetUs) { super( context, @@ -103,8 +95,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; listenerExecutor, videoCompositorSettings, renderFramesAutomatically, - presentation, initialTimestampOffsetUs); + this.compositionEffects = compositionEffects; } @Override @@ -113,7 +105,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; registerInput(inputIndex); videoFrameProcessingWrapper = new VideoFrameProcessingWrapper( - getProcessor(inputIndex), getPresentation(), getInitialTimestampOffsetUs()); + getProcessor(inputIndex), compositionEffects, getInitialTimestampOffsetUs()); return videoFrameProcessingWrapper; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java index 0d726ec215..7cf16ee4f3 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java @@ -132,13 +132,10 @@ public final class TransformerUtil { TransformationRequest transformationRequest, Codec.EncoderFactory encoderFactory, MuxerWrapper muxerWrapper) { - if (composition.sequences.size() > 1 || composition.sequences.get(sequenceIndex).editedMediaItems.size() > 1) { return !composition.transmuxVideo; } - EditedMediaItem firstEditedMediaItem = - composition.sequences.get(sequenceIndex).editedMediaItems.get(0); if (encoderFactory.videoNeedsEncoding()) { return true; } @@ -156,9 +153,15 @@ public final class TransformerUtil { if (inputFormat.pixelWidthHeightRatio != 1f) { return true; } - ImmutableList videoEffects = firstEditedMediaItem.effects.videoEffects; - return !videoEffects.isEmpty() - && maybeCalculateTotalRotationDegreesAppliedInEffects(videoEffects, inputFormat) == -1; + EditedMediaItem firstEditedMediaItem = + composition.sequences.get(sequenceIndex).editedMediaItems.get(0); + ImmutableList combinedEffects = + new ImmutableList.Builder() + .addAll(firstEditedMediaItem.effects.videoEffects) + .addAll(composition.effects.videoEffects) + .build(); + return !combinedEffects.isEmpty() + && maybeCalculateTotalRotationDegreesAppliedInEffects(combinedEffects, inputFormat) == -1; } /** diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java index e4fcb6bed2..2832bb2ff9 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoFrameProcessingWrapper.java @@ -33,7 +33,6 @@ import androidx.media3.common.OnInputFrameProcessedListener; import androidx.media3.common.VideoFrameProcessor; import androidx.media3.common.util.Size; import androidx.media3.common.util.TimestampIterator; -import androidx.media3.effect.Presentation; import com.google.common.collect.ImmutableList; import java.util.List; import java.util.concurrent.atomic.AtomicLong; @@ -41,18 +40,18 @@ import java.util.concurrent.atomic.AtomicLong; /** A wrapper for {@link VideoFrameProcessor} that handles {@link GraphInput} events. */ /* package */ final class VideoFrameProcessingWrapper implements GraphInput { private final VideoFrameProcessor videoFrameProcessor; - private final AtomicLong mediaItemOffsetUs; + private final List postProcessingEffects; private final long initialTimestampOffsetUs; - @Nullable final Presentation presentation; + private final AtomicLong mediaItemOffsetUs; public VideoFrameProcessingWrapper( VideoFrameProcessor videoFrameProcessor, - @Nullable Presentation presentation, + List postProcessingEffects, long initialTimestampOffsetUs) { this.videoFrameProcessor = videoFrameProcessor; - this.mediaItemOffsetUs = new AtomicLong(); + this.postProcessingEffects = postProcessingEffects; this.initialTimestampOffsetUs = initialTimestampOffsetUs; - this.presentation = presentation; + mediaItemOffsetUs = new AtomicLong(); } @Override @@ -65,11 +64,16 @@ import java.util.concurrent.atomic.AtomicLong; durationUs = editedMediaItem.getDurationAfterEffectsApplied(durationUs); if (decodedFormat != null) { Size decodedSize = getDecodedSize(decodedFormat); + ImmutableList combinedEffects = + new ImmutableList.Builder() + .addAll(editedMediaItem.effects.videoEffects) + .addAll(postProcessingEffects) + .build(); videoFrameProcessor.registerInputStream( isSurfaceAssetLoaderMediaItem ? VideoFrameProcessor.INPUT_TYPE_SURFACE_AUTOMATIC_FRAME_REGISTRATION : getInputTypeForMimeType(checkNotNull(decodedFormat.sampleMimeType)), - createEffectListWithPresentation(editedMediaItem.effects.videoEffects, presentation), + combinedEffects, new FrameInfo.Builder( checkNotNull(decodedFormat.colorInfo), decodedSize.getWidth(), @@ -137,16 +141,6 @@ import java.util.concurrent.atomic.AtomicLong; return new Size(decodedWidth, decodedHeight); } - private static ImmutableList createEffectListWithPresentation( - List effects, @Nullable Presentation presentation) { - if (presentation == null) { - return ImmutableList.copyOf(effects); - } - ImmutableList.Builder effectsWithPresentationBuilder = new ImmutableList.Builder<>(); - effectsWithPresentationBuilder.addAll(effects).add(presentation); - return effectsWithPresentationBuilder.build(); - } - private static @VideoFrameProcessor.InputType int getInputTypeForMimeType(String sampleMimeType) { if (MimeTypes.isImage(sampleMimeType)) { return INPUT_TYPE_BITMAP;