Add muxer timer to detect when generating an output sample is too slow

This allows to throw when the Transformer is stuck or is too slow.

PiperOrigin-RevId: 484179037

RELEASENOTES.md

@@ -115,6 +115,9 @@ Release notes
     *   Implement `getDeviceInfo()` to be able to identify `CastPlayer` when
         controlling playback with a `MediaController`
         ([#142](https://github.com/androidx/media/issues/142)).
+*   Transformer:
+    *   Add muxer watchdog timer to detect when generating an output sample is
+        too slow.
 *   Remove deprecated symbols:
     *   Remove `DefaultAudioSink` constructors, use `DefaultAudioSink.Builder`
         instead.

libraries/transformer/src/main/java/androidx/media3/transformer/DefaultMuxer.java

@@ -29,10 +29,27 @@ public final class DefaultMuxer implements Muxer {
 
   /** A {@link Muxer.Factory} for {@link DefaultMuxer}. */
   public static final class Factory implements Muxer.Factory {
+
+    /** The default value returned by {@link #getMaxDelayBetweenSamplesMs()}. */
+    public static final long DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS = 3000;
+
     private final Muxer.Factory muxerFactory;
 
+    /**
+     * Creates an instance with {@link Muxer#getMaxDelayBetweenSamplesMs() maxDelayBetweenSamplesMs}
+     * set to {@link #DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS}.
+     */
     public Factory() {
-      this.muxerFactory = new FrameworkMuxer.Factory();
+      this.muxerFactory = new FrameworkMuxer.Factory(DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS);
+    }
+
+    /**
+     * Creates an instance.
+     *
+     * @param maxDelayBetweenSamplesMs See {@link Muxer#getMaxDelayBetweenSamplesMs()}.
+     */
+    public Factory(long maxDelayBetweenSamplesMs) {
+      this.muxerFactory = new FrameworkMuxer.Factory(maxDelayBetweenSamplesMs);
     }
 
     @Override
@@ -73,4 +90,9 @@ public final class DefaultMuxer implements Muxer {
   public void release(boolean forCancellation) throws MuxerException {
     muxer.release(forCancellation);
   }
+
+  @Override
+  public long getMaxDelayBetweenSamplesMs() {
+    return muxer.getMaxDelayBetweenSamplesMs();
+  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/FrameworkMuxer.java

@@ -55,10 +55,17 @@ import java.nio.ByteBuffer;
 
   /** {@link Muxer.Factory} for {@link FrameworkMuxer}. */
   public static final class Factory implements Muxer.Factory {
+
+    private final long maxDelayBetweenSamplesMs;
+
+    public Factory(long maxDelayBetweenSamplesMs) {
+      this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs;
+    }
+
     @Override
     public FrameworkMuxer create(String path) throws IOException {
       MediaMuxer mediaMuxer = new MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
-      return new FrameworkMuxer(mediaMuxer);
+      return new FrameworkMuxer(mediaMuxer, maxDelayBetweenSamplesMs);
     }
 
     @RequiresApi(26)
@@ -68,7 +75,7 @@ import java.nio.ByteBuffer;
           new MediaMuxer(
               parcelFileDescriptor.getFileDescriptor(),
               MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
-      return new FrameworkMuxer(mediaMuxer);
+      return new FrameworkMuxer(mediaMuxer, maxDelayBetweenSamplesMs);
     }
 
     @Override
@@ -83,13 +90,15 @@ import java.nio.ByteBuffer;
   }
 
   private final MediaMuxer mediaMuxer;
+  private final long maxDelayBetweenSamplesMs;
   private final MediaCodec.BufferInfo bufferInfo;
   private final SparseLongArray trackIndexToLastPresentationTimeUs;
 
   private boolean isStarted;
 
-  private FrameworkMuxer(MediaMuxer mediaMuxer) {
+  private FrameworkMuxer(MediaMuxer mediaMuxer, long maxDelayBetweenSamplesMs) {
     this.mediaMuxer = mediaMuxer;
+    this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs;
     bufferInfo = new MediaCodec.BufferInfo();
     trackIndexToLastPresentationTimeUs = new SparseLongArray();
   }
@@ -183,6 +192,11 @@ import java.nio.ByteBuffer;
     }
   }
 
+  @Override
+  public long getMaxDelayBetweenSamplesMs() {
+    return maxDelayBetweenSamplesMs;
+  }
+
   // Accesses MediaMuxer state via reflection to ensure that muxer resources can be released even
   // if stopping fails.
   @SuppressLint("PrivateApi")

libraries/transformer/src/main/java/androidx/media3/transformer/Muxer.java

@@ -112,4 +112,16 @@ public interface Muxer {
    *     forCancellation} is false.
    */
   void release(boolean forCancellation) throws MuxerException;
+
+  /**
+   * Returns the maximum delay allowed between output samples, in milliseconds, or {@link
+   * C#TIME_UNSET} if there is no maximum.
+   *
+   * <p>This is the maximum delay between samples of any track. They can be of the same or of
+   * different track types.
+   *
+   * <p>This value is used to abort the transformation when the maximum delay is reached. Note that
+   * there is no guarantee that the transformation will be aborted exactly at that time.
+   */
+  long getMaxDelayBetweenSamplesMs();
 }

libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java

@@ -19,6 +19,7 @@ package androidx.media3.transformer;
 import static androidx.media3.common.util.Assertions.checkState;
 import static androidx.media3.common.util.Util.maxValue;
 import static androidx.media3.common.util.Util.minValue;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
 
 import android.util.SparseIntArray;
 import android.util.SparseLongArray;
@@ -29,6 +30,10 @@ import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.Util;
 import com.google.common.collect.ImmutableList;
 import java.nio.ByteBuffer;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
 /**
  * A wrapper around a media muxer.
@@ -47,26 +52,33 @@ import java.nio.ByteBuffer;
 
   private final Muxer muxer;
   private final Muxer.Factory muxerFactory;
+  private final Transformer.AsyncErrorListener asyncErrorListener;
   private final SparseIntArray trackTypeToIndex;
   private final SparseIntArray trackTypeToSampleCount;
   private final SparseLongArray trackTypeToTimeUs;
   private final SparseLongArray trackTypeToBytesWritten;
+  private final ScheduledExecutorService abortScheduledExecutorService;
 
   private int trackCount;
   private int trackFormatCount;
   private boolean isReady;
   private @C.TrackType int previousTrackType;
   private long minTrackTimeUs;
+  private @MonotonicNonNull ScheduledFuture<?> abortScheduledFuture;
+  private boolean isAborted;
 
-  public MuxerWrapper(Muxer muxer, Muxer.Factory muxerFactory) {
+  public MuxerWrapper(
+      Muxer muxer, Muxer.Factory muxerFactory, Transformer.AsyncErrorListener asyncErrorListener) {
     this.muxer = muxer;
     this.muxerFactory = muxerFactory;
+    this.asyncErrorListener = asyncErrorListener;
 
     trackTypeToIndex = new SparseIntArray();
     trackTypeToSampleCount = new SparseIntArray();
     trackTypeToTimeUs = new SparseLongArray();
     trackTypeToBytesWritten = new SparseLongArray();
     previousTrackType = C.TRACK_TYPE_NONE;
+    abortScheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
   }
 
   /**
@@ -131,6 +143,7 @@ import java.nio.ByteBuffer;
     trackFormatCount++;
     if (trackFormatCount == trackCount) {
       isReady = true;
+      resetAbortTimer();
     }
   }
 
@@ -168,6 +181,7 @@ import java.nio.ByteBuffer;
       trackTypeToTimeUs.put(trackType, presentationTimeUs);
     }
 
+    resetAbortTimer();
     muxer.writeSampleData(trackIndex, data, isKeyFrame, presentationTimeUs);
     previousTrackType = trackType;
     return true;
@@ -195,6 +209,7 @@ import java.nio.ByteBuffer;
    */
   public void release(boolean forCancellation) throws Muxer.MuxerException {
     isReady = false;
+    abortScheduledExecutorService.shutdownNow();
     muxer.release(forCancellation);
   }
 
@@ -257,4 +272,31 @@ import java.nio.ByteBuffer;
     }
     return trackTimeUs - minTrackTimeUs <= MAX_TRACK_WRITE_AHEAD_US;
   }
+
+  private void resetAbortTimer() {
+    long maxDelayBetweenSamplesMs = muxer.getMaxDelayBetweenSamplesMs();
+    if (maxDelayBetweenSamplesMs == C.TIME_UNSET) {
+      return;
+    }
+    if (abortScheduledFuture != null) {
+      abortScheduledFuture.cancel(/* mayInterruptIfRunning= */ false);
+    }
+    abortScheduledFuture =
+        abortScheduledExecutorService.schedule(
+            () -> {
+              if (isAborted) {
+                return;
+              }
+              isAborted = true;
+              asyncErrorListener.onTransformationException(
+                  TransformationException.createForMuxer(
+                      new IllegalStateException(
+                          "No output sample written in the last "
+                              + maxDelayBetweenSamplesMs
+                              + " milliseconds. Aborting transformation."),
+                      TransformationException.ERROR_CODE_MUXING_FAILED));
+            },
+            maxDelayBetweenSamplesMs,
+            MILLISECONDS);
+  }
 }

libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java

@@ -726,7 +726,9 @@ public final class Transformer {
     if (player != null) {
       throw new IllegalStateException("There is already a transformation in progress.");
     }
-    MuxerWrapper muxerWrapper = new MuxerWrapper(muxer, muxerFactory);
+    TransformerPlayerListener playerListener = new TransformerPlayerListener(mediaItem, looper);
+    MuxerWrapper muxerWrapper =
+        new MuxerWrapper(muxer, muxerFactory, /* asyncErrorListener= */ playerListener);
     this.muxerWrapper = muxerWrapper;
     DefaultTrackSelector trackSelector = new DefaultTrackSelector(context);
     trackSelector.setParameters(
@@ -743,7 +745,6 @@ public final class Transformer {
                 DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10,
                 DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10)
             .build();
-    TransformerPlayerListener playerListener = new TransformerPlayerListener(mediaItem, looper);
     ExoPlayer.Builder playerBuilder =
         new ExoPlayer.Builder(
                 context,

libraries/transformer/src/test/java/androidx/media3/transformer/TestMuxer.java

@@ -63,6 +63,11 @@ public final class TestMuxer implements Muxer, Dumper.Dumpable {
     muxer.release(forCancellation);
   }
 
+  @Override
+  public long getMaxDelayBetweenSamplesMs() {
+    return muxer.getMaxDelayBetweenSamplesMs();
+  }
+
   // Dumper.Dumpable implementation.
 
   @Override

libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java

@@ -477,6 +477,20 @@ public final class TransformerEndToEndTest {
         .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest);
   }
 
+  @Test
+  public void startTransformation_withUnsetMaxDelayBetweenSamples_completesSuccessfully()
+      throws Exception {
+    Muxer.Factory muxerFactory = new TestMuxerFactory(/* maxDelayBetweenSamplesMs= */ C.TIME_UNSET);
+    Transformer transformer =
+        createTransformerBuilder(/* enableFallback= */ false).setMuxerFactory(muxerFactory).build();
+    MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
+
+    transformer.startTransformation(mediaItem, outputPath);
+    TransformerTestRunner.runUntilCompleted(transformer);
+
+    DumpFileAsserts.assertOutput(context, testMuxer, getDumpFileName(FILE_AUDIO_VIDEO));
+  }
+
   @Test
   public void startTransformation_afterCancellation_completesSuccessfully() throws Exception {
     Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build();
@@ -862,6 +876,10 @@ public final class TransformerEndToEndTest {
       defaultMuxerFactory = new DefaultMuxer.Factory();
     }
 
+    public TestMuxerFactory(long maxDelayBetweenSamplesMs) {
+      defaultMuxerFactory = new DefaultMuxer.Factory(maxDelayBetweenSamplesMs);
+    }
+
     @Override
     public Muxer create(String path) throws IOException {
       testMuxer = new TestMuxer(path, defaultMuxerFactory);