Add an experimental analyzer mode to Transformer.

PiperOrigin-RevId: 637926059

libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -59,6 +59,8 @@ import android.util.Pair;
 import androidx.media3.common.C;
 import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
+import androidx.media3.common.GlObjectsProvider;
+import androidx.media3.common.GlTextureInfo;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.OnInputFrameProcessedListener;
@@ -77,6 +79,8 @@ import androidx.media3.effect.DefaultGlObjectsProvider;
 import androidx.media3.effect.DefaultVideoFrameProcessor;
 import androidx.media3.effect.FrameCache;
 import androidx.media3.effect.GlEffect;
+import androidx.media3.effect.GlShaderProgram;
+import androidx.media3.effect.PassthroughShaderProgram;
 import androidx.media3.effect.Presentation;
 import androidx.media3.effect.RgbFilter;
 import androidx.media3.effect.ScaleAndRotateTransformation;
@@ -96,6 +100,7 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
 import java.io.File;
 import java.nio.ByteBuffer;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 import org.junit.Before;
 import org.junit.Rule;
@@ -1335,6 +1340,105 @@ public class TransformerEndToEndTest {
     assertThat(new File(result.filePath).length()).isGreaterThan(0);
   }
 
+  @Test
+  public void analyzeAudio_completesSuccessfully() throws Exception {
+    assumeFormatsSupported(
+        context,
+        testId,
+        /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
+        /* outputFormat= */ null);
+    Transformer transformer = ExperimentalAnalyzerModeFactory.buildAnalyzer(context);
+    AtomicInteger audioBytesSeen = new AtomicInteger(/* initialValue= */ 0);
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(
+                MediaItem.fromUri(
+                    Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)))
+            .setRemoveVideo(true)
+            .setEffects(
+                new Effects(
+                    ImmutableList.of(createByteCountingAudioProcessor(audioBytesSeen)),
+                    /* videoEffects= */ ImmutableList.of()))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(audioBytesSeen.get()).isEqualTo(2_985_984);
+    // Confirm no data was written to file.
+    assertThat(result.exportResult.averageAudioBitrate).isEqualTo(C.RATE_UNSET_INT);
+    assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET);
+  }
+
+  @Test
+  public void analyzeVideo_completesSuccessfully() throws Exception {
+    assumeFormatsSupported(
+        context,
+        testId,
+        /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
+        /* outputFormat= */ null);
+    Transformer transformer = ExperimentalAnalyzerModeFactory.buildAnalyzer(context);
+    AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0);
+    // Analysis must be added to item effects because composition effects are not applied to single
+    // input video.
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(
+                MediaItem.fromUri(
+                    Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)))
+            .setRemoveAudio(true)
+            .setEffects(
+                new Effects(
+                    /* audioProcessors= */ ImmutableList.of(),
+                    ImmutableList.of(createFrameCountingEffect(videoFramesSeen))))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(videoFramesSeen.get()).isEqualTo(932);
+    // Confirm no data was written to file.
+    assertThat(result.exportResult.videoFrameCount).isEqualTo(0);
+    assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET);
+  }
+
+  @Test
+  public void analyzeAudioAndVideo_completesSuccessfully() throws Exception {
+    assumeFormatsSupported(
+        context,
+        testId,
+        /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
+        /* outputFormat= */ null);
+    Transformer transformer = ExperimentalAnalyzerModeFactory.buildAnalyzer(context);
+    AtomicInteger audioBytesSeen = new AtomicInteger(/* initialValue= */ 0);
+    AtomicInteger videoFramesSeen = new AtomicInteger(/* initialValue= */ 0);
+    // Analysis must be added to item effects because composition effects are not applied to single
+    // input video.
+    EditedMediaItem editedMediaItem =
+        new EditedMediaItem.Builder(
+                MediaItem.fromUri(
+                    Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)))
+            .setEffects(
+                new Effects(
+                    ImmutableList.of(createByteCountingAudioProcessor(audioBytesSeen)),
+                    ImmutableList.of(createFrameCountingEffect(videoFramesSeen))))
+            .build();
+
+    ExportTestResult result =
+        new TransformerAndroidTestRunner.Builder(context, transformer)
+            .build()
+            .run(testId, editedMediaItem);
+
+    assertThat(audioBytesSeen.get()).isEqualTo(2_985_984);
+    assertThat(videoFramesSeen.get()).isEqualTo(932);
+    // Confirm no data was written to file.
+    assertThat(result.exportResult.averageAudioBitrate).isEqualTo(C.RATE_UNSET_INT);
+    assertThat(result.exportResult.videoFrameCount).isEqualTo(0);
+    assertThat(result.exportResult.fileSizeBytes).isEqualTo(C.LENGTH_UNSET);
+  }
+
   @Test
   public void transcode_withOutputVideoMimeTypeAv1_completesSuccessfully() throws Exception {
     assumeFormatsSupported(
@@ -1628,6 +1732,37 @@ public class TransformerEndToEndTest {
     return sonic;
   }
 
+  private static AudioProcessor createByteCountingAudioProcessor(AtomicInteger byteCount) {
+    return new TeeAudioProcessor(
+        new TeeAudioProcessor.AudioBufferSink() {
+          @Override
+          public void flush(int sampleRateHz, int channelCount, @C.PcmEncoding int encoding) {}
+
+          @Override
+          public void handleBuffer(ByteBuffer buffer) {
+            byteCount.addAndGet(buffer.remaining());
+          }
+        });
+  }
+
+  private static GlEffect createFrameCountingEffect(AtomicInteger frameCount) {
+    return new GlEffect() {
+      @Override
+      public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) {
+        return new PassthroughShaderProgram() {
+          @Override
+          public void queueInputFrame(
+              GlObjectsProvider glObjectsProvider,
+              GlTextureInfo inputTexture,
+              long presentationTimeUs) {
+            super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs);
+            frameCount.incrementAndGet();
+          }
+        };
+      }
+    };
+  }
+
   private final class TestTextureAssetLoaderFactory implements AssetLoader.Factory {
 
     private final int width;

libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java

@@ -0,0 +1,249 @@
+/*
+ * Copyright 2024 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package androidx.media3.transformer;
+
+import static androidx.media3.common.util.Assertions.checkState;
+
+import android.content.Context;
+import android.media.MediaCodec;
+import android.media.MediaCodec.BufferInfo;
+import android.view.Surface;
+import androidx.annotation.Nullable;
+import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.Metadata;
+import androidx.media3.common.MimeTypes;
+import androidx.media3.common.util.UnstableApi;
+import androidx.media3.decoder.DecoderInputBuffer;
+import androidx.media3.exoplayer.video.PlaceholderSurface;
+import androidx.media3.muxer.Muxer;
+import com.google.common.collect.ImmutableList;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf;
+
+/**
+ * Factory for creating instances of {@link Transformer} that can be used to analyze media.
+ *
+ * <p>When using {@link Transformer} to analyze decoded data, users should provide their analysis
+ * effects through the {@link EditedMediaItem#effects}.
+ *
+ * <p>This class is experimental and will be renamed or removed in a future release.
+ */
+@UnstableApi
+public final class ExperimentalAnalyzerModeFactory {
+
+  private ExperimentalAnalyzerModeFactory() {}
+
+  /**
+   * Builds a {@link Transformer} that runs as an analyzer.
+   *
+   * <p>No encoding or muxing is performed, therefore no data is written to any output files.
+   *
+   * @param context The {@link Context}.
+   * @return The analyzer {@link Transformer}.
+   */
+  public static Transformer buildAnalyzer(Context context) {
+    return buildAnalyzer(context, new Transformer.Builder(context).build());
+  }
+
+  /**
+   * Builds a {@link Transformer} that runs as an analyzer.
+   *
+   * <p>No encoding or muxing is performed, therefore no data is written to any output files.
+   *
+   * @param context The {@link Context}.
+   * @param transformer The {@link Transformer} to be built upon.
+   * @return The analyzer {@link Transformer}.
+   */
+  public static Transformer buildAnalyzer(Context context, Transformer transformer) {
+    return transformer
+        .buildUpon()
+        .experimentalSetTrimOptimizationEnabled(false)
+        .setEncoderFactory(new DroppingEncoder.Factory(context))
+        .setMaxDelayBetweenMuxerSamplesMs(C.TIME_UNSET)
+        .setMuxerFactory(
+            new NoWriteMuxer.Factory(
+                /* audioMimeTypes= */ ImmutableList.of(MimeTypes.AUDIO_AAC),
+                /* videoMimeTypes= */ ImmutableList.of(MimeTypes.VIDEO_H264)))
+        .setAudioMimeType(MimeTypes.AUDIO_AAC)
+        .setVideoMimeType(MimeTypes.VIDEO_H264)
+        .build();
+  }
+
+  /** A {@linkplain Codec encoder} implementation that drops input and produces no output. */
+  private static final class DroppingEncoder implements Codec {
+    public static final class Factory implements Codec.EncoderFactory {
+      private final Context context;
+
+      public Factory(Context context) {
+        this.context = context;
+      }
+
+      @Override
+      public Codec createForAudioEncoding(Format format) {
+        return new DroppingEncoder(context, format);
+      }
+
+      @Override
+      public Codec createForVideoEncoding(Format format) {
+        return new DroppingEncoder(context, format);
+      }
+    }
+
+    private static final String TAG = "DroppingEncoder";
+    private static final int INTERNAL_BUFFER_SIZE = 8196;
+
+    private final Context context;
+    private final Format configurationFormat;
+    private final ByteBuffer buffer;
+
+    private boolean inputStreamEnded;
+
+    public DroppingEncoder(Context context, Format format) {
+      this.context = context;
+      this.configurationFormat = format;
+      buffer = ByteBuffer.allocateDirect(INTERNAL_BUFFER_SIZE).order(ByteOrder.nativeOrder());
+    }
+
+    @Override
+    public String getName() {
+      return TAG;
+    }
+
+    @Override
+    public Format getConfigurationFormat() {
+      return configurationFormat;
+    }
+
+    @Override
+    public Surface getInputSurface() {
+      return PlaceholderSurface.newInstance(context, /* secure= */ false);
+    }
+
+    @Override
+    @EnsuresNonNullIf(expression = "#1.data", result = true)
+    public boolean maybeDequeueInputBuffer(DecoderInputBuffer inputBuffer) {
+      if (inputStreamEnded) {
+        return false;
+      }
+      inputBuffer.data = buffer;
+      return true;
+    }
+
+    @Override
+    public void queueInputBuffer(DecoderInputBuffer inputBuffer) {
+      checkState(
+          !inputStreamEnded, "Input buffer can not be queued after the input stream has ended.");
+      if (inputBuffer.isEndOfStream()) {
+        inputStreamEnded = true;
+      }
+      inputBuffer.clear();
+      inputBuffer.data = null;
+    }
+
+    @Override
+    public void signalEndOfInputStream() {
+      inputStreamEnded = true;
+    }
+
+    @Override
+    @Nullable
+    public Format getOutputFormat() {
+      return configurationFormat;
+    }
+
+    @Override
+    @Nullable
+    public ByteBuffer getOutputBuffer() {
+      return null;
+    }
+
+    @Override
+    @Nullable
+    public BufferInfo getOutputBufferInfo() {
+      return null;
+    }
+
+    @Override
+    public boolean isEnded() {
+      return inputStreamEnded;
+    }
+
+    @Override
+    public void releaseOutputBuffer(boolean render) {}
+
+    @Override
+    public void releaseOutputBuffer(long renderPresentationTimeUs) {}
+
+    @Override
+    public void release() {}
+  }
+
+  /** A {@link Muxer} implementation that does nothing. */
+  private static final class NoWriteMuxer implements Muxer {
+    public static final class Factory implements Muxer.Factory {
+
+      private final ImmutableList<String> audioMimeTypes;
+      private final ImmutableList<String> videoMimeTypes;
+
+      /**
+       * Creates an instance.
+       *
+       * @param audioMimeTypes The audio {@linkplain MimeTypes mime types} to return in {@link
+       *     #getSupportedSampleMimeTypes(int)}.
+       * @param videoMimeTypes The video {@linkplain MimeTypes mime types} to return in {@link
+       *     #getSupportedSampleMimeTypes(int)}.
+       */
+      public Factory(ImmutableList<String> audioMimeTypes, ImmutableList<String> videoMimeTypes) {
+        this.audioMimeTypes = audioMimeTypes;
+        this.videoMimeTypes = videoMimeTypes;
+      }
+
+      @Override
+      public Muxer create(String path) {
+        return new NoWriteMuxer();
+      }
+
+      @Override
+      public ImmutableList<String> getSupportedSampleMimeTypes(@C.TrackType int trackType) {
+        if (trackType == C.TRACK_TYPE_AUDIO) {
+          return audioMimeTypes;
+        }
+        if (trackType == C.TRACK_TYPE_VIDEO) {
+          return videoMimeTypes;
+        }
+        return ImmutableList.of();
+      }
+    }
+
+    @Override
+    public TrackToken addTrack(Format format) {
+      return new TrackToken() {};
+    }
+
+    @Override
+    public void writeSampleData(
+        TrackToken trackToken, ByteBuffer data, MediaCodec.BufferInfo bufferInfo) {}
+
+    @Override
+    public void addMetadataEntry(Metadata.Entry metadataEntry) {}
+
+    @Override
+    public void close() {}
+  }
+}

libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java

@@ -46,6 +46,7 @@ import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
 import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
 import static androidx.media3.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE;
 import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
+import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
 import static org.mockito.ArgumentMatchers.any;
@@ -69,11 +70,13 @@ import androidx.media3.common.Effect;
 import androidx.media3.common.Format;
 import androidx.media3.common.MediaItem;
 import androidx.media3.common.MimeTypes;
+import androidx.media3.common.audio.AudioProcessor;
 import androidx.media3.common.audio.SonicAudioProcessor;
 import androidx.media3.common.audio.ToInt16PcmAudioProcessor;
 import androidx.media3.effect.Contrast;
 import androidx.media3.effect.Presentation;
 import androidx.media3.effect.ScaleAndRotateTransformation;
+import androidx.media3.exoplayer.audio.TeeAudioProcessor;
 import androidx.media3.exoplayer.source.DefaultMediaSourceFactory;
 import androidx.media3.exoplayer.source.MediaSource;
 import androidx.media3.extractor.DefaultExtractorsFactory;
@@ -1231,6 +1234,93 @@ public final class MediaItemExportTest {
             /* originalFileName= */ FILE_AUDIO_VIDEO, /* modifications...= */ "rotated"));
   }
 
+  @Test
+  public void analyze_audioOnlyWithItemEffect_completesSuccessfully() throws Exception {
+    removeEncodersAndDecoders();
+    addAudioDecoders(MimeTypes.AUDIO_RAW);
+    addThrowingAudioEncoder(MimeTypes.AUDIO_AAC);
+    Transformer transformer =
+        ExperimentalAnalyzerModeFactory.buildAnalyzer(
+            getApplicationContext(),
+            new Transformer.Builder(getApplicationContext())
+                .setClock(new FakeClock(/* isAutoAdvancing= */ true))
+                .build());
+    AtomicInteger bytesSeen = new AtomicInteger(0);
+    EditedMediaItem item =
+        new EditedMediaItem.Builder(MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW))
+            .setEffects(createAudioEffects(createByteCountingAudioProcessor(bytesSeen)))
+            .build();
+
+    transformer.start(item, outputDir.newFile().getPath());
+    ExportResult result = TransformerTestRunner.runLooper(transformer);
+
+    // Confirm that all the data was seen and no output file was created.
+    assertThat(bytesSeen.get()).isEqualTo(88200);
+    assertThat(result.fileSizeBytes).isEqualTo(C.LENGTH_UNSET);
+  }
+
+  @Test
+  public void analyze_audioOnlyWithCompositionEffect_completesSuccessfully() throws Exception {
+    removeEncodersAndDecoders();
+    addAudioDecoders(MimeTypes.AUDIO_RAW);
+    addThrowingAudioEncoder(MimeTypes.AUDIO_AAC);
+    Transformer transformer =
+        ExperimentalAnalyzerModeFactory.buildAnalyzer(
+            getApplicationContext(),
+            new Transformer.Builder(getApplicationContext())
+                .setClock(new FakeClock(/* isAutoAdvancing= */ true))
+                .build());
+    AtomicInteger bytesSeen = new AtomicInteger(0);
+    Composition composition =
+        new Composition.Builder(
+                new EditedMediaItemSequence(
+                    new EditedMediaItem.Builder(
+                            MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW))
+                        .build()))
+            .setEffects(createAudioEffects(createByteCountingAudioProcessor(bytesSeen)))
+            .build();
+
+    transformer.start(composition, outputDir.newFile().getPath());
+    ExportResult result = TransformerTestRunner.runLooper(transformer);
+
+    // Confirm that all the data was seen and no output file was created.
+    assertThat(bytesSeen.get()).isEqualTo(88200);
+    assertThat(result.fileSizeBytes).isEqualTo(C.LENGTH_UNSET);
+  }
+
+  @Test
+  public void analyze_audioOnly_itemAndMixerOutputMatch() throws Exception {
+    removeEncodersAndDecoders();
+    addAudioDecoders(MimeTypes.AUDIO_RAW);
+    addThrowingAudioEncoder(MimeTypes.AUDIO_AAC);
+    Transformer transformer =
+        ExperimentalAnalyzerModeFactory.buildAnalyzer(
+            getApplicationContext(),
+            new Transformer.Builder(getApplicationContext())
+                .setClock(new FakeClock(/* isAutoAdvancing= */ true))
+                .build());
+    AtomicInteger itemEffectBytesSeen = new AtomicInteger(0);
+    AtomicInteger compositionEffectBytesSeen = new AtomicInteger(0);
+    Composition composition =
+        new Composition.Builder(
+                new EditedMediaItemSequence(
+                    new EditedMediaItem.Builder(
+                            MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_RAW))
+                        .setEffects(
+                            createAudioEffects(
+                                createByteCountingAudioProcessor(itemEffectBytesSeen)))
+                        .build()))
+            .setEffects(
+                createAudioEffects(createByteCountingAudioProcessor(compositionEffectBytesSeen)))
+            .build();
+
+    transformer.start(composition, outputDir.newFile().getPath());
+    TransformerTestRunner.runLooper(transformer);
+
+    assertThat(itemEffectBytesSeen.get()).isGreaterThan(0);
+    assertThat(itemEffectBytesSeen.get()).isEqualTo(compositionEffectBytesSeen.get());
+  }
+
   @Test
   public void getProgress_unknownDuration_returnsConsistentStates() throws Exception {
     CapturingMuxer.Factory muxerFactory = new CapturingMuxer.Factory(/* handleAudioAsPcm= */ false);
@@ -1631,6 +1721,40 @@ public final class MediaItemExportTest {
             /* modifications...= */ "transmuxed"));
   }
 
+  private static void addThrowingAudioEncoder(String mimeType) {
+    ShadowMediaCodec.CodecConfig.Codec codec =
+        new ShadowMediaCodec.CodecConfig.Codec() {
+          @Override
+          public void process(ByteBuffer byteBuffer, ByteBuffer byteBuffer1) {
+            throw new IllegalStateException();
+          }
+
+          @Override
+          public void onConfigured(
+              MediaFormat format, Surface surface, MediaCrypto crypto, int flags) {
+            throw new IllegalStateException();
+          }
+        };
+
+    addAudioEncoders(
+        new ShadowMediaCodec.CodecConfig(
+            /* inputBufferSize= */ 100_000, /* outputBufferSize= */ 100_000, codec),
+        mimeType);
+  }
+
+  private static AudioProcessor createByteCountingAudioProcessor(AtomicInteger byteCount) {
+    return new TeeAudioProcessor(
+        new TeeAudioProcessor.AudioBufferSink() {
+          @Override
+          public void flush(int sampleRateHz, int channelCount, @C.PcmEncoding int encoding) {}
+
+          @Override
+          public void handleBuffer(ByteBuffer buffer) {
+            byteCount.addAndGet(buffer.remaining());
+          }
+        });
+  }
+
   private static final class SlowExtractorsFactory implements ExtractorsFactory {
 
     private final long delayBetweenReadsMs;