From 376ee77f52bed47de54c6478b4006f1b25a543d0 Mon Sep 17 00:00:00 2001 From: kimvde Date: Thu, 27 Oct 2022 07:58:56 +0000 Subject: [PATCH] Add muxer timer to detect when generating an output sample is too slow This allows to throw when the Transformer is stuck or is too slow. PiperOrigin-RevId: 484179037 --- .../exoplayer2/transformer/DefaultMuxer.java | 24 +++++++++- .../transformer/FrameworkMuxer.java | 20 +++++++-- .../android/exoplayer2/transformer/Muxer.java | 12 +++++ .../exoplayer2/transformer/MuxerWrapper.java | 44 ++++++++++++++++++- .../exoplayer2/transformer/Transformer.java | 5 ++- .../exoplayer2/transformer/TestMuxer.java | 5 +++ .../transformer/TransformerEndToEndTest.java | 18 ++++++++ 7 files changed, 121 insertions(+), 7 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultMuxer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultMuxer.java index 58f658b771..de4afce318 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultMuxer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/DefaultMuxer.java @@ -27,10 +27,27 @@ public final class DefaultMuxer implements Muxer { /** A {@link Muxer.Factory} for {@link DefaultMuxer}. */ public static final class Factory implements Muxer.Factory { + + /** The default value returned by {@link #getMaxDelayBetweenSamplesMs()}. */ + public static final long DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS = 3000; + private final Muxer.Factory muxerFactory; + /** + * Creates an instance with {@link Muxer#getMaxDelayBetweenSamplesMs() maxDelayBetweenSamplesMs} + * set to {@link #DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS}. + */ public Factory() { - this.muxerFactory = new FrameworkMuxer.Factory(); + this.muxerFactory = new FrameworkMuxer.Factory(DEFAULT_MAX_DELAY_BETWEEN_SAMPLES_MS); + } + + /** + * Creates an instance. + * + * @param maxDelayBetweenSamplesMs See {@link Muxer#getMaxDelayBetweenSamplesMs()}. + */ + public Factory(long maxDelayBetweenSamplesMs) { + this.muxerFactory = new FrameworkMuxer.Factory(maxDelayBetweenSamplesMs); } @Override @@ -71,4 +88,9 @@ public final class DefaultMuxer implements Muxer { public void release(boolean forCancellation) throws MuxerException { muxer.release(forCancellation); } + + @Override + public long getMaxDelayBetweenSamplesMs() { + return muxer.getMaxDelayBetweenSamplesMs(); + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameworkMuxer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameworkMuxer.java index e76cb11f18..be82c9536f 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameworkMuxer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameworkMuxer.java @@ -55,10 +55,17 @@ import java.nio.ByteBuffer; /** {@link Muxer.Factory} for {@link FrameworkMuxer}. */ public static final class Factory implements Muxer.Factory { + + private final long maxDelayBetweenSamplesMs; + + public Factory(long maxDelayBetweenSamplesMs) { + this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs; + } + @Override public FrameworkMuxer create(String path) throws IOException { MediaMuxer mediaMuxer = new MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); - return new FrameworkMuxer(mediaMuxer); + return new FrameworkMuxer(mediaMuxer, maxDelayBetweenSamplesMs); } @RequiresApi(26) @@ -68,7 +75,7 @@ import java.nio.ByteBuffer; new MediaMuxer( parcelFileDescriptor.getFileDescriptor(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4); - return new FrameworkMuxer(mediaMuxer); + return new FrameworkMuxer(mediaMuxer, maxDelayBetweenSamplesMs); } @Override @@ -83,13 +90,15 @@ import java.nio.ByteBuffer; } private final MediaMuxer mediaMuxer; + private final long maxDelayBetweenSamplesMs; private final MediaCodec.BufferInfo bufferInfo; private final SparseLongArray trackIndexToLastPresentationTimeUs; private boolean isStarted; - private FrameworkMuxer(MediaMuxer mediaMuxer) { + private FrameworkMuxer(MediaMuxer mediaMuxer, long maxDelayBetweenSamplesMs) { this.mediaMuxer = mediaMuxer; + this.maxDelayBetweenSamplesMs = maxDelayBetweenSamplesMs; bufferInfo = new MediaCodec.BufferInfo(); trackIndexToLastPresentationTimeUs = new SparseLongArray(); } @@ -183,6 +192,11 @@ import java.nio.ByteBuffer; } } + @Override + public long getMaxDelayBetweenSamplesMs() { + return maxDelayBetweenSamplesMs; + } + // Accesses MediaMuxer state via reflection to ensure that muxer resources can be released even // if stopping fails. @SuppressLint("PrivateApi") diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Muxer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Muxer.java index 59933fca0e..37e1c8e318 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Muxer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Muxer.java @@ -110,4 +110,16 @@ public interface Muxer { * forCancellation} is false. */ void release(boolean forCancellation) throws MuxerException; + + /** + * Returns the maximum delay allowed between output samples, in milliseconds, or {@link + * C#TIME_UNSET} if there is no maximum. + * + *

This is the maximum delay between samples of any track. They can be of the same or of + * different track types. + * + *

This value is used to abort the transformation when the maximum delay is reached. Note that + * there is no guarantee that the transformation will be aborted exactly at that time. + */ + long getMaxDelayBetweenSamplesMs(); } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java index 817586d03e..01f6283f7a 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/MuxerWrapper.java @@ -19,6 +19,7 @@ package com.google.android.exoplayer2.transformer; import static com.google.android.exoplayer2.util.Assertions.checkState; import static com.google.android.exoplayer2.util.Util.maxValue; import static com.google.android.exoplayer2.util.Util.minValue; +import static java.util.concurrent.TimeUnit.MILLISECONDS; import android.util.SparseIntArray; import android.util.SparseLongArray; @@ -29,6 +30,10 @@ import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; /** * A wrapper around a media muxer. @@ -47,26 +52,33 @@ import java.nio.ByteBuffer; private final Muxer muxer; private final Muxer.Factory muxerFactory; + private final Transformer.AsyncErrorListener asyncErrorListener; private final SparseIntArray trackTypeToIndex; private final SparseIntArray trackTypeToSampleCount; private final SparseLongArray trackTypeToTimeUs; private final SparseLongArray trackTypeToBytesWritten; + private final ScheduledExecutorService abortScheduledExecutorService; private int trackCount; private int trackFormatCount; private boolean isReady; private @C.TrackType int previousTrackType; private long minTrackTimeUs; + private @MonotonicNonNull ScheduledFuture abortScheduledFuture; + private boolean isAborted; - public MuxerWrapper(Muxer muxer, Muxer.Factory muxerFactory) { + public MuxerWrapper( + Muxer muxer, Muxer.Factory muxerFactory, Transformer.AsyncErrorListener asyncErrorListener) { this.muxer = muxer; this.muxerFactory = muxerFactory; + this.asyncErrorListener = asyncErrorListener; trackTypeToIndex = new SparseIntArray(); trackTypeToSampleCount = new SparseIntArray(); trackTypeToTimeUs = new SparseLongArray(); trackTypeToBytesWritten = new SparseLongArray(); previousTrackType = C.TRACK_TYPE_NONE; + abortScheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); } /** @@ -131,6 +143,7 @@ import java.nio.ByteBuffer; trackFormatCount++; if (trackFormatCount == trackCount) { isReady = true; + resetAbortTimer(); } } @@ -168,6 +181,7 @@ import java.nio.ByteBuffer; trackTypeToTimeUs.put(trackType, presentationTimeUs); } + resetAbortTimer(); muxer.writeSampleData(trackIndex, data, isKeyFrame, presentationTimeUs); previousTrackType = trackType; return true; @@ -195,6 +209,7 @@ import java.nio.ByteBuffer; */ public void release(boolean forCancellation) throws Muxer.MuxerException { isReady = false; + abortScheduledExecutorService.shutdownNow(); muxer.release(forCancellation); } @@ -257,4 +272,31 @@ import java.nio.ByteBuffer; } return trackTimeUs - minTrackTimeUs <= MAX_TRACK_WRITE_AHEAD_US; } + + private void resetAbortTimer() { + long maxDelayBetweenSamplesMs = muxer.getMaxDelayBetweenSamplesMs(); + if (maxDelayBetweenSamplesMs == C.TIME_UNSET) { + return; + } + if (abortScheduledFuture != null) { + abortScheduledFuture.cancel(/* mayInterruptIfRunning= */ false); + } + abortScheduledFuture = + abortScheduledExecutorService.schedule( + () -> { + if (isAborted) { + return; + } + isAborted = true; + asyncErrorListener.onTransformationException( + TransformationException.createForMuxer( + new IllegalStateException( + "No output sample written in the last " + + maxDelayBetweenSamplesMs + + " milliseconds. Aborting transformation."), + TransformationException.ERROR_CODE_MUXING_FAILED)); + }, + maxDelayBetweenSamplesMs, + MILLISECONDS); + } } diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index be7f002383..855f624a36 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -724,7 +724,9 @@ public final class Transformer { if (player != null) { throw new IllegalStateException("There is already a transformation in progress."); } - MuxerWrapper muxerWrapper = new MuxerWrapper(muxer, muxerFactory); + TransformerPlayerListener playerListener = new TransformerPlayerListener(mediaItem, looper); + MuxerWrapper muxerWrapper = + new MuxerWrapper(muxer, muxerFactory, /* asyncErrorListener= */ playerListener); this.muxerWrapper = muxerWrapper; DefaultTrackSelector trackSelector = new DefaultTrackSelector(context); trackSelector.setParameters( @@ -741,7 +743,6 @@ public final class Transformer { DEFAULT_BUFFER_FOR_PLAYBACK_MS / 10, DEFAULT_BUFFER_FOR_PLAYBACK_AFTER_REBUFFER_MS / 10) .build(); - TransformerPlayerListener playerListener = new TransformerPlayerListener(mediaItem, looper); ExoPlayer.Builder playerBuilder = new ExoPlayer.Builder( context, diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TestMuxer.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TestMuxer.java index 11f454a56d..61712e39e7 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TestMuxer.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TestMuxer.java @@ -63,6 +63,11 @@ public final class TestMuxer implements Muxer, Dumper.Dumpable { muxer.release(forCancellation); } + @Override + public long getMaxDelayBetweenSamplesMs() { + return muxer.getMaxDelayBetweenSamplesMs(); + } + // Dumper.Dumpable implementation. @Override diff --git a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java index ab7a727ca2..5500e46dc4 100644 --- a/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java +++ b/library/transformer/src/test/java/com/google/android/exoplayer2/transformer/TransformerEndToEndTest.java @@ -477,6 +477,20 @@ public final class TransformerEndToEndTest { .onFallbackApplied(mediaItem, originalTransformationRequest, fallbackTransformationRequest); } + @Test + public void startTransformation_withUnsetMaxDelayBetweenSamples_completesSuccessfully() + throws Exception { + Muxer.Factory muxerFactory = new TestMuxerFactory(/* maxDelayBetweenSamplesMs= */ C.TIME_UNSET); + Transformer transformer = + createTransformerBuilder(/* enableFallback= */ false).setMuxerFactory(muxerFactory).build(); + MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO); + + transformer.startTransformation(mediaItem, outputPath); + TransformerTestRunner.runUntilCompleted(transformer); + + DumpFileAsserts.assertOutput(context, testMuxer, getDumpFileName(FILE_AUDIO_VIDEO)); + } + @Test public void startTransformation_afterCancellation_completesSuccessfully() throws Exception { Transformer transformer = createTransformerBuilder(/* enableFallback= */ false).build(); @@ -862,6 +876,10 @@ public final class TransformerEndToEndTest { defaultMuxerFactory = new DefaultMuxer.Factory(); } + public TestMuxerFactory(long maxDelayBetweenSamplesMs) { + defaultMuxerFactory = new DefaultMuxer.Factory(maxDelayBetweenSamplesMs); + } + @Override public Muxer create(String path) throws IOException { testMuxer = new TestMuxer(path, defaultMuxerFactory);