From 16cb5cbc1f6f729d39bf8747e7c0a0f7aea7326d Mon Sep 17 00:00:00 2001 From: kimvde Date: Thu, 27 Oct 2022 07:58:56 +0000 Subject: [PATCH] Add muxer timer to detect when generating an output sample is too slow This allows to throw when the Transformer is stuck or is too slow. PiperOrigin-RevId: 484179037 --- RELEASENOTES.md | 3 ++ .../media3/transformer/DefaultMuxer.java | 24 +++++++++- .../media3/transformer/FrameworkMuxer.java | 20 +++++++-- .../androidx/media3/transformer/Muxer.java | 12 +++++ .../media3/transformer/MuxerWrapper.java | 44 ++++++++++++++++++- .../media3/transformer/Transformer.java | 5 ++- .../media3/transformer/TestMuxer.java | 5 +++ .../transformer/TransformerEndToEndTest.java | 18 ++++++++ 8 files changed, 124 insertions(+), 7 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 0d6bad7a66..d10173ead2 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -115,6 +115,9 @@ Release notes * Implement `getDeviceInfo()` to be able to identify `CastPlayer` when controlling playback with a `MediaController` ([#142](https://github.com/androidx/media/issues/142)). +* Transformer: + * Add muxer watchdog timer to detect when generating an output sample is + too slow. * Remove deprecated symbols: * Remove `DefaultAudioSink` constructors, use `DefaultAudioSink.Builder` instead. diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultMuxer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultMuxer.java index eb76b666f6..16457bbbd6 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultMuxer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/DefaultMuxer.java @@ -29,10 +29,27 @@ public final class DefaultMuxer implements Muxer { /** A {@link Muxer.Factory} for {@link DefaultMuxer}. */ public static final class Factory implements Muxer.Factory { + + /** The default value returned by {@link #getMaxDelayBetweenSamplesMs()}. */ + public static final long DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS = 3000; + private final Muxer.Factory muxerFactory; + /** + * Creates an instance with {@link Muxer#getMaxDelayBetweenSamplesMs() maxDelayBetweenSamplesMs} + * set to {@link #DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS}. + */ public Factory() { - this.muxerFactory = new FrameworkMuxer.Factory(); + this.muxerFactory = new FrameworkMuxer.Factory(DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS); + } + + /** + * Creates an instance. + * + * @param maxDelayBetweenSamplesMs See {@link Muxer#getMaxDelayBetweenSamplesMs()}. + */ + public Factory(long maxDelayBetweenSamplesMs) { + this.muxerFactory = new FrameworkMuxer.Factory(maxDelayBetweenSamplesMs); } @Override @@ -73,4 +90,9 @@ public final class DefaultMuxer implements Muxer { public void release(boolean forCancellation) throws MuxerException { muxer.release(forCancellation); } + + @Override + public long getMaxDelayBetweenSamplesMs() { + return muxer.getMaxDelayBetweenSamplesMs(); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameworkMuxer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameworkMuxer.java index 1ef7f72a7c..ae710c3346 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameworkMuxer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameworkMuxer.java @@ -55,10 +55,17 @@ import java.nio.ByteBuffer; /** {@link Muxer.Factory} for {@link FrameworkMuxer}. */ public static final class Factory implements Muxer.Factory { + + private final long maxDelayBetweenSamplesMs; + + public Factory(long maxDelayBetweenSamplesMs) { + this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs; + } + @Override public FrameworkMuxer create(String path) throws IOException { MediaMuxer mediaMuxer = new MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); - return new FrameworkMuxer(mediaMuxer); + return new FrameworkMuxer(mediaMuxer, maxDelayBetweenSamplesMs); } @RequiresApi(26) @@ -68,7 +75,7 @@ import java.nio.ByteBuffer; new MediaMuxer( parcelFileDescriptor.getFileDescriptor(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); - return new FrameworkMuxer(mediaMuxer); + return new FrameworkMuxer(mediaMuxer, maxDelayBetweenSamplesMs); } @Override @@ -83,13 +90,15 @@ import java.nio.ByteBuffer; } private final MediaMuxer mediaMuxer; + private final long maxDelayBetweenSamplesMs; private final MediaCodec.BufferInfo bufferInfo; private final SparseLongArray trackIndexToLastPresentationTimeUs; private boolean isStarted; - private FrameworkMuxer(MediaMuxer mediaMuxer) { + private FrameworkMuxer(MediaMuxer mediaMuxer, long maxDelayBetweenSamplesMs) { this.mediaMuxer = mediaMuxer; + this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs; bufferInfo = new MediaCodec.BufferInfo(); trackIndexToLastPresentationTimeUs = new SparseLongArray(); } @@ -183,6 +192,11 @@ import java.nio.ByteBuffer; } } + @Override + public long getMaxDelayBetweenSamplesMs() { + return maxDelayBetweenSamplesMs; + } + // Accesses MediaMuxer state via reflection to ensure that muxer resources can be released even // if stopping fails. @SuppressLint("PrivateApi") diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Muxer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Muxer.java index b25270942f..1bf8c4c77c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Muxer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Muxer.java @@ -112,4 +112,16 @@ public interface Muxer { * forCancellation} is false. */ void release(boolean forCancellation) throws MuxerException; + + /** + * Returns the maximum delay allowed between output samples, in milliseconds, or {@link + * C#TIME_UNSET} if there is no maximum. + * + *

This is the maximum delay between samples of any track. They can be of the same or of + * different track types. + * + *

This value is used to abort the transformation when the maximum delay is reached. Note that + * there is no guarantee that the transformation will be aborted exactly at that time. + */ + long getMaxDelayBetweenSamplesMs(); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java index 611e469b06..b96f7f7d53 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java @@ -19,6 +19,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkState; import static androidx.media3.common.util.Util.maxValue; import static androidx.media3.common.util.Util.minValue; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import android.util.SparseIntArray; import android.util.SparseLongArray; @@ -29,6 +30,10 @@ import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Util; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A wrapper around a media muxer. @@ -47,26 +52,33 @@ import java.nio.ByteBuffer; private final Muxer muxer; private final Muxer.Factory muxerFactory; + private final Transformer.AsyncErrorListener asyncErrorListener; private final SparseIntArray trackTypeToIndex; private final SparseIntArray trackTypeToSampleCount; private final SparseLongArray trackTypeToTimeUs; private final SparseLongArray trackTypeToBytesWritten; + private final ScheduledExecutorService abortScheduledExecutorService; private int trackCount; private int trackFormatCount; private boolean isReady; private @C.TrackType int previousTrackType; private long minTrackTimeUs; + private @MonotonicNonNull ScheduledFuture abortScheduledFuture; + private boolean isAborted; - public MuxerWrapper(Muxer muxer, Muxer.Factory muxerFactory) { + public MuxerWrapper( + Muxer muxer, Muxer.Factory muxerFactory, Transformer.AsyncErrorListener asyncErrorListener) { this.muxer = muxer; this.muxerFactory = muxerFactory; + this.asyncErrorListener = asyncErrorListener; trackTypeToIndex = new SparseIntArray(); trackTypeToSampleCount = new SparseIntArray(); trackTypeToTimeUs = new SparseLongArray(); trackTypeToBytesWritten = new SparseLongArray(); previousTrackType = C.TRACK_TYPE_NONE; + abortScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); } /** @@ -131,6 +143,7 @@ import java.nio.ByteBuffer; trackFormatCount++; if (trackFormatCount == trackCount) { isReady = true; + resetAbortTimer(); } } @@ -168,6 +181,7 @@ import java.nio.ByteBuffer; trackTypeToTimeUs.put(trackType, presentationTimeUs); } + resetAbortTimer(); muxer.writeSampleData(trackIndex, data, isKeyFrame, presentationTimeUs); previousTrackType = trackType; return true; @@ -195,6 +209,7 @@ import java.nio.ByteBuffer; */ public void release(boolean forCancellation) throws Muxer.MuxerException { isReady = false; + abortScheduledExecutorService.shutdownNow(); muxer.release(forCancellation); } @@ -257,4 +272,31 @@ import java.nio.ByteBuffer; } return trackTimeUs - minTrackTimeUs <= MAX_TRACK_WRITE_AHEAD_US; } + + private void resetAbortTimer() { + long maxDelayBetweenSamplesMs = muxer.getMaxDelayBetweenSamplesMs(); + if (maxDelayBetweenSamplesMs == C.TIME_UNSET) { + return; + } + if (abortScheduledFuture != null) { + abortScheduledFuture.cancel(/* mayInterruptIfRunning= */ false); + } + abortScheduledFuture = + abortScheduledExecutorService.schedule( + () -> { + if (isAborted) { + return; + } + isAborted = true; + asyncErrorListener.onTransformationException( + TransformationException.createForMuxer( + new IllegalStateException( + "No output sample written in the last " + + maxDelayBetweenSamplesMs + + " milliseconds. Aborting transformation."), + TransformationException.ERROR_CODE_MUXING_FAILED)); + }, + maxDelayBetweenSamplesMs, + MILLISECONDS); + } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 8f5fa0fda4..af2c8a7721 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -726,7 +726,9 @@ public final class Transformer { if (player != null) { throw new IllegalStateException("There is already a transformation in progress."); } - MuxerWrapper muxerWrapper = new MuxerWrapper(muxer, muxerFactory); + TransformerPlayerListener playerListener = new TransformerPlayerListener(mediaItem, looper); + MuxerWrapper muxerWrapper = + new MuxerWrapper(muxer, muxerFactory, /* asyncErrorListener= */ playerListener); this.muxerWrapper = muxerWrapper; DefaultTrackSelector trackSelector = new DefaultTrackSelector(context); trackSelector.setParameters( @@ -743,7 +745,6 @@ public final class Transformer { DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10, DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10) .build(); - TransformerPlayerListener playerListener = new TransformerPlayerListener(mediaItem, looper); ExoPlayer.Builder playerBuilder = new ExoPlayer.Builder( context, diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TestMuxer.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TestMuxer.java index d134fd2b40..24a59ae3c6 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TestMuxer.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TestMuxer.java @@ -63,6 +63,11 @@ public final class TestMuxer implements Muxer, Dumper.Dumpable { muxer.release(forCancellation); } + @Override + public long getMaxDelayBetweenSamplesMs() { + return muxer.getMaxDelayBetweenSamplesMs(); + } + // Dumper.Dumpable implementation. @Override diff --git a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java index 4ab9a2a006..92f4386c60 100644 --- a/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/test/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -477,6 +477,20 @@ public final class TransformerEndToEndTest { .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest); } + @Test + public void startTransformation_withUnsetMaxDelayBetweenSamples_completesSuccessfully() + throws Exception { + Muxer.Factory muxerFactory = new TestMuxerFactory(/* maxDelayBetweenSamplesMs= */ C.TIME_UNSET); + Transformer transformer = + createTransformerBuilder(/* enableFallback= */ false).setMuxerFactory(muxerFactory).build(); + MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); + + transformer.startTransformation(mediaItem, outputPath); + TransformerTestRunner.runUntilCompleted(transformer); + + DumpFileAsserts.assertOutput(context, testMuxer, getDumpFileName(FILE_AUDIO_VIDEO)); + } + @Test public void startTransformation_afterCancellation_completesSuccessfully() throws Exception { Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build(); @@ -862,6 +876,10 @@ public final class TransformerEndToEndTest { defaultMuxerFactory = new DefaultMuxer.Factory(); } + public TestMuxerFactory(long maxDelayBetweenSamplesMs) { + defaultMuxerFactory = new DefaultMuxer.Factory(maxDelayBetweenSamplesMs); + } + @Override public Muxer create(String path) throws IOException { testMuxer = new TestMuxer(path, defaultMuxerFactory);