diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 216582b94f..0bebc19658 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -59,6 +59,8 @@ import android.util.Pair; import androidx.media3.common.C; import androidx.media3.common.Effect; import androidx.media3.common.Format; +import androidx.media3.common.GlObjectsProvider; +import androidx.media3.common.GlTextureInfo; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.common.OnInputFrameProcessedListener; @@ -77,6 +79,8 @@ import androidx.media3.effect.DefaultGlObjectsProvider; import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.FrameCache; import androidx.media3.effect.GlEffect; +import androidx.media3.effect.GlShaderProgram; +import androidx.media3.effect.PassthroughShaderProgram; import androidx.media3.effect.Presentation; import androidx.media3.effect.RgbFilter; import androidx.media3.effect.ScaleAndRotateTransformation; @@ -96,6 +100,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import java.io.File; import java.nio.ByteBuffer; +import java.util.concurrent.atomic.AtomicInteger; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.Before; import org.junit.Rule; @@ -1335,6 +1340,105 @@ public class TransformerEndToEndTest { assertThat(new File(result.filePath).length()).isGreaterThan(0); } + @Test + public void analyzeAudio_completesSuccessfully() throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT, + /* outputFormat= */ null); + Transformer transformer = ExperimentalAnalyzerModeFactory.buildAnalyzer(context); + AtomicInteger audioBytesSeen = new AtomicInteger(/* initialValue= */ 0); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + MediaItem.fromUri( + Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))) + .setRemoveVideo(true) + .setEffects( + new Effects( + ImmutableList.of(createByteCountingAudioProcessor(audioBytesSeen)), + /* videoEffects= */ ImmutableList.of())) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + assertThat(audioBytesSeen.get()).isEqualTo(2_985_984); + // Confirm no data was written to file. + assertThat(result.exportResult.averageAudioBitrate).isEqualTo(C.RATE_UNSET_INT); + assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET); + } + + @Test + public void analyzeVideo_completesSuccessfully() throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT, + /* outputFormat= */ null); + Transformer transformer = ExperimentalAnalyzerModeFactory.buildAnalyzer(context); + AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0); + // Analysis must be added to item effects because composition effects are not applied to single + // input video. + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + MediaItem.fromUri( + Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))) + .setRemoveAudio(true) + .setEffects( + new Effects( + /* audioProcessors= */ ImmutableList.of(), + ImmutableList.of(createFrameCountingEffect(videoFramesSeen)))) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + assertThat(videoFramesSeen.get()).isEqualTo(932); + // Confirm no data was written to file. + assertThat(result.exportResult.videoFrameCount).isEqualTo(0); + assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET); + } + + @Test + public void analyzeAudioAndVideo_completesSuccessfully() throws Exception { + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT, + /* outputFormat= */ null); + Transformer transformer = ExperimentalAnalyzerModeFactory.buildAnalyzer(context); + AtomicInteger audioBytesSeen = new AtomicInteger(/* initialValue= */ 0); + AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0); + // Analysis must be added to item effects because composition effects are not applied to single + // input video. + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + MediaItem.fromUri( + Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))) + .setEffects( + new Effects( + ImmutableList.of(createByteCountingAudioProcessor(audioBytesSeen)), + ImmutableList.of(createFrameCountingEffect(videoFramesSeen)))) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, editedMediaItem); + + assertThat(audioBytesSeen.get()).isEqualTo(2_985_984); + assertThat(videoFramesSeen.get()).isEqualTo(932); + // Confirm no data was written to file. + assertThat(result.exportResult.averageAudioBitrate).isEqualTo(C.RATE_UNSET_INT); + assertThat(result.exportResult.videoFrameCount).isEqualTo(0); + assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET); + } + @Test public void transcode_withOutputVideoMimeTypeAv1_completesSuccessfully() throws Exception { assumeFormatsSupported( @@ -1628,6 +1732,37 @@ public class TransformerEndToEndTest { return sonic; } + private static AudioProcessor createByteCountingAudioProcessor(AtomicInteger byteCount) { + return new TeeAudioProcessor( + new TeeAudioProcessor.AudioBufferSink() { + @Override + public void flush(int sampleRateHz, int channelCount, @C.PcmEncoding int encoding) {} + + @Override + public void handleBuffer(ByteBuffer buffer) { + byteCount.addAndGet(buffer.remaining()); + } + }); + } + + private static GlEffect createFrameCountingEffect(AtomicInteger frameCount) { + return new GlEffect() { + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) { + return new PassthroughShaderProgram() { + @Override + public void queueInputFrame( + GlObjectsProvider glObjectsProvider, + GlTextureInfo inputTexture, + long presentationTimeUs) { + super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs); + frameCount.incrementAndGet(); + } + }; + } + }; + } + private final class TestTextureAssetLoaderFactory implements AssetLoader.Factory { private final int width; diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java new file mode 100644 index 0000000000..2936ba5c09 --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java @@ -0,0 +1,249 @@ +/* + * Copyright 2024 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkState; + +import android.content.Context; +import android.media.MediaCodec; +import android.media.MediaCodec.BufferInfo; +import android.view.Surface; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.Metadata; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.UnstableApi; +import androidx.media3.decoder.DecoderInputBuffer; +import androidx.media3.exoplayer.video.PlaceholderSurface; +import androidx.media3.muxer.Muxer; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; + +/** + * Factory for creating instances of {@link Transformer} that can be used to analyze media. + * + *

When using {@link Transformer} to analyze decoded data, users should provide their analysis + * effects through the {@link EditedMediaItem#effects}. + * + *

This class is experimental and will be renamed or removed in a future release. + */ +@UnstableApi +public final class ExperimentalAnalyzerModeFactory { + + private ExperimentalAnalyzerModeFactory() {} + + /** + * Builds a {@link Transformer} that runs as an analyzer. + * + *

No encoding or muxing is performed, therefore no data is written to any output files. + * + * @param context The {@link Context}. + * @return The analyzer {@link Transformer}. + */ + public static Transformer buildAnalyzer(Context context) { + return buildAnalyzer(context, new Transformer.Builder(context).build()); + } + + /** + * Builds a {@link Transformer} that runs as an analyzer. + * + *

No encoding or muxing is performed, therefore no data is written to any output files. + * + * @param context The {@link Context}. + * @param transformer The {@link Transformer} to be built upon. + * @return The analyzer {@link Transformer}. + */ + public static Transformer buildAnalyzer(Context context, Transformer transformer) { + return transformer + .buildUpon() + .experimentalSetTrimOptimizationEnabled(false) + .setEncoderFactory(new DroppingEncoder.Factory(context)) + .setMaxDelayBetweenMuxerSamplesMs(C.TIME_UNSET) + .setMuxerFactory( + new NoWriteMuxer.Factory( + /* audioMimeTypes= */ ImmutableList.of(MimeTypes.AUDIO_AAC), + /* videoMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264))) + .setAudioMimeType(MimeTypes.AUDIO_AAC) + .setVideoMimeType(MimeTypes.VIDEO_H264) + .build(); + } + + /** A {@linkplain Codec encoder} implementation that drops input and produces no output. */ + private static final class DroppingEncoder implements Codec { + public static final class Factory implements Codec.EncoderFactory { + private final Context context; + + public Factory(Context context) { + this.context = context; + } + + @Override + public Codec createForAudioEncoding(Format format) { + return new DroppingEncoder(context, format); + } + + @Override + public Codec createForVideoEncoding(Format format) { + return new DroppingEncoder(context, format); + } + } + + private static final String TAG = "DroppingEncoder"; + private static final int INTERNAL_BUFFER_SIZE = 8196; + + private final Context context; + private final Format configurationFormat; + private final ByteBuffer buffer; + + private boolean inputStreamEnded; + + public DroppingEncoder(Context context, Format format) { + this.context = context; + this.configurationFormat = format; + buffer = ByteBuffer.allocateDirect(INTERNAL_BUFFER_SIZE).order(ByteOrder.nativeOrder()); + } + + @Override + public String getName() { + return TAG; + } + + @Override + public Format getConfigurationFormat() { + return configurationFormat; + } + + @Override + public Surface getInputSurface() { + return PlaceholderSurface.newInstance(context, /* secure= */ false); + } + + @Override + @EnsuresNonNullIf(expression = "#1.data", result = true) + public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer) { + if (inputStreamEnded) { + return false; + } + inputBuffer.data = buffer; + return true; + } + + @Override + public void queueInputBuffer(DecoderInputBuffer inputBuffer) { + checkState( + !inputStreamEnded, "Input buffer can not be queued after the input stream has ended."); + if (inputBuffer.isEndOfStream()) { + inputStreamEnded = true; + } + inputBuffer.clear(); + inputBuffer.data = null; + } + + @Override + public void signalEndOfInputStream() { + inputStreamEnded = true; + } + + @Override + @Nullable + public Format getOutputFormat() { + return configurationFormat; + } + + @Override + @Nullable + public ByteBuffer getOutputBuffer() { + return null; + } + + @Override + @Nullable + public BufferInfo getOutputBufferInfo() { + return null; + } + + @Override + public boolean isEnded() { + return inputStreamEnded; + } + + @Override + public void releaseOutputBuffer(boolean render) {} + + @Override + public void releaseOutputBuffer(long renderPresentationTimeUs) {} + + @Override + public void release() {} + } + + /** A {@link Muxer} implementation that does nothing. */ + private static final class NoWriteMuxer implements Muxer { + public static final class Factory implements Muxer.Factory { + + private final ImmutableList audioMimeTypes; + private final ImmutableList videoMimeTypes; + + /** + * Creates an instance. + * + * @param audioMimeTypes The audio {@linkplain MimeTypes mime types} to return in {@link + * #getSupportedSampleMimeTypes(int)}. + * @param videoMimeTypes The video {@linkplain MimeTypes mime types} to return in {@link + * #getSupportedSampleMimeTypes(int)}. + */ + public Factory(ImmutableList audioMimeTypes, ImmutableList videoMimeTypes) { + this.audioMimeTypes = audioMimeTypes; + this.videoMimeTypes = videoMimeTypes; + } + + @Override + public Muxer create(String path) { + return new NoWriteMuxer(); + } + + @Override + public ImmutableList getSupportedSampleMimeTypes(@C.TrackType int trackType) { + if (trackType == C.TRACK_TYPE_AUDIO) { + return audioMimeTypes; + } + if (trackType == C.TRACK_TYPE_VIDEO) { + return videoMimeTypes; + } + return ImmutableList.of(); + } + } + + @Override + public TrackToken addTrack(Format format) { + return new TrackToken() {}; + } + + @Override + public void writeSampleData( + TrackToken trackToken, ByteBuffer data, MediaCodec.BufferInfo bufferInfo) {} + + @Override + public void addMetadataEntry(Metadata.Entry metadataEntry) {} + + @Override + public void close() {} + } +} diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java index 29bdf8f21a..e7bc63ffce 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java @@ -46,6 +46,7 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE; import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -69,11 +70,13 @@ import androidx.media3.common.Effect; import androidx.media3.common.Format; import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; +import androidx.media3.common.audio.AudioProcessor; import androidx.media3.common.audio.SonicAudioProcessor; import androidx.media3.common.audio.ToInt16PcmAudioProcessor; import androidx.media3.effect.Contrast; import androidx.media3.effect.Presentation; import androidx.media3.effect.ScaleAndRotateTransformation; +import androidx.media3.exoplayer.audio.TeeAudioProcessor; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import androidx.media3.exoplayer.source.MediaSource; import androidx.media3.extractor.DefaultExtractorsFactory; @@ -1231,6 +1234,93 @@ public final class MediaItemExportTest { /* originalFileName= */ FILE_AUDIO_VIDEO, /* modifications...= */ "rotated")); } + @Test + public void analyze_audioOnlyWithItemEffect_completesSuccessfully() throws Exception { + removeEncodersAndDecoders(); + addAudioDecoders(MimeTypes.AUDIO_RAW); + addThrowingAudioEncoder(MimeTypes.AUDIO_AAC); + Transformer transformer = + ExperimentalAnalyzerModeFactory.buildAnalyzer( + getApplicationContext(), + new Transformer.Builder(getApplicationContext()) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .build()); + AtomicInteger bytesSeen = new AtomicInteger(0); + EditedMediaItem item = + new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setEffects(createAudioEffects(createByteCountingAudioProcessor(bytesSeen))) + .build(); + + transformer.start(item, outputDir.newFile().getPath()); + ExportResult result = TransformerTestRunner.runLooper(transformer); + + // Confirm that all the data was seen and no output file was created. + assertThat(bytesSeen.get()).isEqualTo(88200); + assertThat(result.fileSizeBytes).isEqualTo(C.LENGTH_UNSET); + } + + @Test + public void analyze_audioOnlyWithCompositionEffect_completesSuccessfully() throws Exception { + removeEncodersAndDecoders(); + addAudioDecoders(MimeTypes.AUDIO_RAW); + addThrowingAudioEncoder(MimeTypes.AUDIO_AAC); + Transformer transformer = + ExperimentalAnalyzerModeFactory.buildAnalyzer( + getApplicationContext(), + new Transformer.Builder(getApplicationContext()) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .build()); + AtomicInteger bytesSeen = new AtomicInteger(0); + Composition composition = + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .build())) + .setEffects(createAudioEffects(createByteCountingAudioProcessor(bytesSeen))) + .build(); + + transformer.start(composition, outputDir.newFile().getPath()); + ExportResult result = TransformerTestRunner.runLooper(transformer); + + // Confirm that all the data was seen and no output file was created. + assertThat(bytesSeen.get()).isEqualTo(88200); + assertThat(result.fileSizeBytes).isEqualTo(C.LENGTH_UNSET); + } + + @Test + public void analyze_audioOnly_itemAndMixerOutputMatch() throws Exception { + removeEncodersAndDecoders(); + addAudioDecoders(MimeTypes.AUDIO_RAW); + addThrowingAudioEncoder(MimeTypes.AUDIO_AAC); + Transformer transformer = + ExperimentalAnalyzerModeFactory.buildAnalyzer( + getApplicationContext(), + new Transformer.Builder(getApplicationContext()) + .setClock(new FakeClock(/* isAutoAdvancing= */ true)) + .build()); + AtomicInteger itemEffectBytesSeen = new AtomicInteger(0); + AtomicInteger compositionEffectBytesSeen = new AtomicInteger(0); + Composition composition = + new Composition.Builder( + new EditedMediaItemSequence( + new EditedMediaItem.Builder( + MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW)) + .setEffects( + createAudioEffects( + createByteCountingAudioProcessor(itemEffectBytesSeen))) + .build())) + .setEffects( + createAudioEffects(createByteCountingAudioProcessor(compositionEffectBytesSeen))) + .build(); + + transformer.start(composition, outputDir.newFile().getPath()); + TransformerTestRunner.runLooper(transformer); + + assertThat(itemEffectBytesSeen.get()).isGreaterThan(0); + assertThat(itemEffectBytesSeen.get()).isEqualTo(compositionEffectBytesSeen.get()); + } + @Test public void getProgress_unknownDuration_returnsConsistentStates() throws Exception { CapturingMuxer.Factory muxerFactory = new CapturingMuxer.Factory(/* handleAudioAsPcm= */ false); @@ -1631,6 +1721,40 @@ public final class MediaItemExportTest { /* modifications...= */ "transmuxed")); } + private static void addThrowingAudioEncoder(String mimeType) { + ShadowMediaCodec.CodecConfig.Codec codec = + new ShadowMediaCodec.CodecConfig.Codec() { + @Override + public void process(ByteBuffer byteBuffer, ByteBuffer byteBuffer1) { + throw new IllegalStateException(); + } + + @Override + public void onConfigured( + MediaFormat format, Surface surface, MediaCrypto crypto, int flags) { + throw new IllegalStateException(); + } + }; + + addAudioEncoders( + new ShadowMediaCodec.CodecConfig( + /* inputBufferSize= */ 100_000, /* outputBufferSize= */ 100_000, codec), + mimeType); + } + + private static AudioProcessor createByteCountingAudioProcessor(AtomicInteger byteCount) { + return new TeeAudioProcessor( + new TeeAudioProcessor.AudioBufferSink() { + @Override + public void flush(int sampleRateHz, int channelCount, @C.PcmEncoding int encoding) {} + + @Override + public void handleBuffer(ByteBuffer buffer) { + byteCount.addAndGet(buffer.remaining()); + } + }); + } + private static final class SlowExtractorsFactory implements ExtractorsFactory { private final long delayBetweenReadsMs;