loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle muxing with timestamps offset from zero in wrapper

Browse Source 
Sources (for example media projection) can populate the `Surface` from
`SurfaceAssetLoader` with timestamps that don't start from zero. But
`MuxerWrapper` assumes the latest sample timestamp can be used as the duration
when it calculates average bitrate and notifies its listener.

This can cause a crash because the calculated average bitrate can be zero if
the denominator duration is large enough.

Use the max minus first sample timestamp across tracks instead to get the
duration.

Side note: the large timestamps from the surface texture when using media
projection arrive unchanged (apart from conversion from ns to us) in effect
implementations and in the muxer wrapper (and are passed to the underlying
muxer). The outputs of media3 muxer and the framework muxer are similar.

PiperOrigin-RevId: 652422674
This commit is contained in:
andrewlewis 2024-07-15 03:27:55 -07:00 committed by Copybara-Service
parent c510ab81bb
commit 4b7cc80593
2 changed files with 85 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
libraries/transformer/src/androidTest/java/androidx/media3/transformer/SurfaceAssetLoaderTest.java
View File

@ -135,4 +135,77 @@ public class SurfaceAssetLoaderTest {
assertThat(exportResult.height).isEqualTo(bitmap.getHeight());
assertThat(exportResult.durationMs).isEqualTo(300);
}
@Test
public void encodingFromSurface_withLargeTimestamps_succeeds() throws Exception {
assumeTrue("ImageWriter with pixel format set requires API 29", Util.SDK_INT >= 29);
SettableFuture<SurfaceAssetLoader> surfaceAssetLoaderSettableFuture = SettableFuture.create();
SettableFuture<Surface> surfaceSettableFuture = SettableFuture.create();
Transformer transformer =
new Transformer.Builder(context)
.setAssetLoaderFactory(
new SurfaceAssetLoader.Factory(
new SurfaceAssetLoader.Callback() {
@Override
public void onSurfaceAssetLoaderCreated(
SurfaceAssetLoader surfaceAssetLoader) {
surfaceAssetLoaderSettableFuture.set(surfaceAssetLoader);
}
@Override
public void onSurfaceReady(Surface surface, EditedMediaItem editedMediaItem) {
surfaceSettableFuture.set(surface);
}
}))
.build();
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(
MediaItem.fromUri(SurfaceAssetLoader.MEDIA_ITEM_URI_SCHEME + ":"))
.build();
ListenableFuture<ExportResult> exportCompletionFuture =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.runAsync(testId, editedMediaItem);
SurfaceAssetLoader surfaceAssetLoader =
surfaceAssetLoaderSettableFuture.get(TIMEOUT_MS, MILLISECONDS);
Bitmap bitmap = BitmapPixelTestUtil.readBitmap(TEST_BITMAP_PATH);
surfaceAssetLoader.setContentFormat(
new Format.Builder()
.setSampleMimeType(MimeTypes.VIDEO_RAW)
.setWidth(bitmap.getWidth())
.setHeight(bitmap.getHeight())
.setColorInfo(ColorInfo.SRGB_BT709_FULL)
.build());
Surface surface = surfaceSettableFuture.get(TIMEOUT_MS, MILLISECONDS);
int inputFrameCount = 10;
try (ImageWriter imageWriter =
ImageWriter.newInstance(surface, /* maxImages= */ inputFrameCount, PixelFormat.RGBA_8888)) {
ConditionVariable readyForInputCondition = new ConditionVariable();
imageWriter.setOnImageReleasedListener(
unusedImageWriter -> readyForInputCondition.open(), new Handler(Looper.getMainLooper()));
for (int i = 0; i < inputFrameCount; i++) {
Image image = imageWriter.dequeueInputImage();
// Add a large base offset in nanoseconds.
image.setTimestamp(3_020_642_044_930_642L + i * C.NANOS_PER_SECOND / 30);
BitmapPixelTestUtil.copyRbga8888BitmapToImage(bitmap, image);
readyForInputCondition.close();
imageWriter.queueInputImage(image);
// When frames are queued as fast as possible some can be dropped, so throttle input by
// blocking until the previous frame has been released by the downstream pipeline.
if (i > 0) {
assertThat(readyForInputCondition.block(TIMEOUT_MS)).isTrue();
}
}
}
surfaceAssetLoader.signalEndOfInput();
ExportResult exportResult = exportCompletionFuture.get();
assertThat(exportResult.videoFrameCount).isEqualTo(inputFrameCount);
assertThat(exportResult.width).isEqualTo(bitmap.getWidth());
assertThat(exportResult.height).isEqualTo(bitmap.getHeight());
assertThat(exportResult.durationMs).isEqualTo(300);
}
}

15
libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java
View File

@ -30,6 +30,7 @@ import static androidx.media3.effect.DebugTraceUtil.EVENT_CAN_WRITE_SAMPLE;
import static androidx.media3.effect.DebugTraceUtil.EVENT_INPUT_ENDED;
import static androidx.media3.effect.DebugTraceUtil.EVENT_OUTPUT_ENDED;
import static java.lang.Math.max;
import static java.lang.Math.min;
import static java.lang.annotation.ElementType.TYPE_USE;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
@ -161,6 +162,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private boolean isEnded;
private @C.TrackType int previousTrackType;
private long minTrackTimeUs;
private long minEndedTrackTimeUs;
private long maxEndedTrackTimeUs;
private @MonotonicNonNull ScheduledFuture<?> abortScheduledFuture;
private boolean isAborted;
@ -214,6 +216,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
trackTypeToInfo = new SparseArray<>();
previousTrackType = C.TRACK_TYPE_NONE;
firstVideoPresentationTimeUs = C.TIME_UNSET;
minEndedTrackTimeUs = Long.MAX_VALUE;
abortScheduledExecutorService = Util.newSingleThreadScheduledExecutor(TIMER_THREAD_NAME);
bufferInfo = new BufferInfo();
}
@ -562,6 +565,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
return false;
}
if (trackInfo.sampleCount == 0) {
trackInfo.startTimeUs = presentationTimeUs;
}
trackInfo.sampleCount++;
trackInfo.bytesWritten += data.remaining();
trackInfo.timeUs = max(trackInfo.timeUs, presentationTimeUs);
@ -597,6 +603,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
TrackInfo trackInfo = trackTypeToInfo.get(trackType);
minEndedTrackTimeUs = max(0, min(minEndedTrackTimeUs, trackInfo.startTimeUs));
maxEndedTrackTimeUs = max(maxEndedTrackTimeUs, trackInfo.timeUs);
listener.onTrackEnded(
trackType, trackInfo.format, trackInfo.getAverageBitrate(), trackInfo.sampleCount);
@ -621,10 +628,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
long durationMs = usToMs(maxEndedTrackTimeUs - minEndedTrackTimeUs);
if (muxerMode == MUXER_MODE_MUX_PARTIAL
&& muxedPartialVideo
&& (muxedPartialAudio || trackCount == 1)) {
listener.onEnded(usToMs(maxEndedTrackTimeUs), getCurrentOutputSizeBytes());
listener.onEnded(durationMs, getCurrentOutputSizeBytes());
if (abortScheduledFuture != null) {
abortScheduledFuture.cancel(/* mayInterruptIfRunning= */ false);
}
@ -632,7 +640,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
if (isEnded) {
listener.onEnded(usToMs(maxEndedTrackTimeUs), getCurrentOutputSizeBytes());
listener.onEnded(durationMs, getCurrentOutputSizeBytes());
abortScheduledExecutorService.shutdownNow();
}
}
@ -775,6 +783,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
public final Format format;
public final TrackToken trackToken;
public long startTimeUs;
public long bytesWritten;
public int sampleCount;
public long timeUs;
@ -799,7 +808,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Util.scaleLargeTimestamp(
/* timestamp= */ bytesWritten,
/* multiplier= */ C.BITS_PER_BYTE * C.MICROS_PER_SECOND,
/* divisor= */ timeUs);
/* divisor= */ timeUs - startTimeUs);
}
}
}