loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix minor timestamp handling issue

Browse Source 
- Video release should check for buffer timestamp (which is renderer-offsetted), rather than the frame timestamp
- ImageRenderer should report ended after all of it's outputs are released, rather than when finished consuming its input.

Add tests for timestamp handling

PiperOrigin-RevId: 642587290
This commit is contained in:
claincly 2024-06-12 05:40:28 -07:00 committed by Copybara-Service
parent 8bd6e5d10a
commit f2bdc08b24
5 changed files with 460 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
libraries/exoplayer/src/main/java/androidx/media3/exoplayer/video/CompositingVideoSinkProvider.java
View File

@ -590,8 +590,7 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi
public boolean isEnded() {
return isInitialized()
&& finalBufferPresentationTimeUs != C.TIME_UNSET
&& CompositingVideoSinkProvider.this.hasReleasedFrame(
finalBufferPresentationTimeUs + inputBufferTimestampAdjustmentUs);
&& CompositingVideoSinkProvider.this.hasReleasedFrame(finalBufferPresentationTimeUs);
}
@Override
@ -755,7 +754,9 @@ public final class CompositingVideoSinkProvider implements VideoSinkProvider, Vi
// the state of the iterator.
TimestampIterator copyTimestampIterator = timestampIterator.copyOf();
long bufferPresentationTimeUs = copyTimestampIterator.next();
long lastBufferPresentationTimeUs = copyTimestampIterator.getLastTimestampUs();
// TimestampIterator generates frame time.
long lastBufferPresentationTimeUs =
copyTimestampIterator.getLastTimestampUs() - inputBufferTimestampAdjustmentUs;
checkState(lastBufferPresentationTimeUs != C.TIME_UNSET);
maybeSetStreamOffsetChange(bufferPresentationTimeUs);
this.lastBufferPresentationTimeUs = lastBufferPresentationTimeUs;

46
libraries/transformer/src/androidTest/java/androidx/media3/transformer/InputTimestampRecordingShaderProgram.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Copyright 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.effect.PassthroughShaderProgram;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
/** A {@link PassthroughShaderProgram} that records the input timestamps. */
public class InputTimestampRecordingShaderProgram extends PassthroughShaderProgram {
private final List<Long> inputTimestampsUs;
/** Creates an instance. */
public InputTimestampRecordingShaderProgram() {
inputTimestampsUs = new ArrayList<>();
}
/** Returns the captured timestamps, in microseconds. */
public ImmutableList<Long> getInputTimestampsUs() {
return ImmutableList.copyOf(inputTimestampsUs);
}
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs);
inputTimestampsUs.add(presentationTimeUs);
}
}

398
libraries/transformer/src/androidTest/java/androidx/media3/transformer/VideoTimestampConsistencyTest.java Normal file
View File

@ -0,0 +1,398 @@
/*
* Copyright 2024 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Util.usToMs;
import static com.google.common.truth.Truth.assertThat;
import android.app.Instrumentation;
import android.content.Context;
import android.view.SurfaceView;
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
import androidx.media3.effect.GlEffect;
import androidx.test.ext.junit.rules.ActivityScenarioRule;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.platform.app.InstrumentationRegistry;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import java.util.List;
import org.junit.After;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
/**
* A test that guarantees the timestamp is handled identically between {@link CompositionPlayer} and
* {@link Transformer}.
*/
@RunWith(AndroidJUnit4.class)
public class VideoTimestampConsistencyTest {
private static final long TEST_TIMEOUT_MS = 10_000;
private static final String MP4_ASSET = "asset:///media/mp4/sample.mp4";
private static final long MP4_ASSET_DURATION_US = 1_024_000L;
private static final ImmutableList<Long> MP4_ASSET_FRAME_TIMESTAMPS_US =
ImmutableList.of(
0L, 33366L, 66733L, 100100L, 133466L, 166833L, 200200L, 233566L, 266933L, 300300L,
333666L, 367033L, 400400L, 433766L, 467133L, 500500L, 533866L, 567233L, 600600L, 633966L,
667333L, 700700L, 734066L, 767433L, 800800L, 834166L, 867533L, 900900L, 934266L, 967633L);
private static final String IMAGE_ASSET = "asset:///media/jpeg/white-1x1.jpg";
private static final ImmutableList<Long> IMAGE_TIMESTAMPS_US_500_MS_30_FPS =
ImmutableList.of(
0L, 33333L, 66667L, 100000L, 133333L, 166667L, 200000L, 233333L, 266667L, 300000L,
333333L, 366667L, 400000L, 433333L, 466667L);
@Rule public final TestName testName = new TestName();
@Rule
public ActivityScenarioRule<SurfaceTestActivity> rule =
new ActivityScenarioRule<>(SurfaceTestActivity.class);
private final Instrumentation instrumentation = InstrumentationRegistry.getInstrumentation();
private final Context applicationContext = instrumentation.getContext().getApplicationContext();
private CompositionPlayer compositionPlayer;
private SurfaceView surfaceView;
@Before
public void setupSurfaces() {
rule.getScenario().onActivity(activity -> surfaceView = activity.getSurfaceView());
}
@After
public void closeActivity() {
rule.getScenario().close();
}
@Test
public void oneImageComposition_timestampsAreConsistent() throws Exception {
long imageDurationUs = 500_000L;
EditedMediaItem image =
new EditedMediaItem.Builder(
new MediaItem.Builder()
.setUri(IMAGE_ASSET)
.setImageDurationMs(usToMs(imageDurationUs))
.build())
.setDurationUs(imageDurationUs)
.setFrameRate(30)
.build();
compareTimestamps(ImmutableList.of(image), IMAGE_TIMESTAMPS_US_500_MS_30_FPS);
}
@Test
public void oneVideoComposition_timestampsAreConsistent() throws Exception {
EditedMediaItem video =
new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET))
.setDurationUs(MP4_ASSET_DURATION_US)
.build();
compareTimestamps(ImmutableList.of(video), MP4_ASSET_FRAME_TIMESTAMPS_US);
}
@Test
public void twoVideosComposition_clippingTheFirst_timestampsAreConsistent() throws Exception {
// TODO - b/341279499: Add test that clips the second media.
long clippedStartUs = 500_000L;
EditedMediaItem video1 =
new EditedMediaItem.Builder(
MediaItem.fromUri(MP4_ASSET)
.buildUpon()
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(usToMs(clippedStartUs))
.build())
.build())
.setDurationUs(MP4_ASSET_DURATION_US)
.build();
EditedMediaItem video2 =
new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET))
.setDurationUs(MP4_ASSET_DURATION_US)
.build();
ImmutableList<Long> expectedTimestamps =
new ImmutableList.Builder<Long>()
.addAll(getClippedTimestamps(MP4_ASSET_FRAME_TIMESTAMPS_US, clippedStartUs))
.addAll(
Lists.transform(
MP4_ASSET_FRAME_TIMESTAMPS_US,
timestampUs -> ((MP4_ASSET_DURATION_US - clippedStartUs) + timestampUs)))
.build();
compareTimestamps(ImmutableList.of(video1, video2), expectedTimestamps);
}
@Test
public void twoVideosComposition_timestampsAreConsistent() throws Exception {
EditedMediaItem video1 =
new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET))
.setDurationUs(MP4_ASSET_DURATION_US)
.build();
EditedMediaItem video2 =
new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET))
.setDurationUs(MP4_ASSET_DURATION_US)
.build();
ImmutableList<Long> expectedTimestamps =
new ImmutableList.Builder<Long>()
.addAll(MP4_ASSET_FRAME_TIMESTAMPS_US)
.addAll(
Lists.transform(
MP4_ASSET_FRAME_TIMESTAMPS_US,
timestampUs -> (MP4_ASSET_DURATION_US + timestampUs)))
.build();
compareTimestamps(ImmutableList.of(video1, video2), expectedTimestamps);
}
@Test
public void twoImagesComposition_timestampsAreConsistent() throws Exception {
long imageDurationUs = 500_000L;
EditedMediaItem image1 =
new EditedMediaItem.Builder(
new MediaItem.Builder()
.setUri(IMAGE_ASSET)
.setImageDurationMs(usToMs(imageDurationUs))
.build())
.setDurationUs(imageDurationUs)
.setFrameRate(30)
.build();
EditedMediaItem image2 =
new EditedMediaItem.Builder(
new MediaItem.Builder()
.setUri(IMAGE_ASSET)
.setImageDurationMs(usToMs(imageDurationUs))
.build())
.setDurationUs(imageDurationUs)
.setFrameRate(30)
.build();
ImmutableList<Long> expectedTimestamps =
new ImmutableList.Builder<Long>()
.addAll(IMAGE_TIMESTAMPS_US_500_MS_30_FPS)
// The offset timestamps for image2.
.addAll(
Lists.transform(
IMAGE_TIMESTAMPS_US_500_MS_30_FPS,
timestampUs -> (timestampUs + imageDurationUs)))
.build();
compareTimestamps(ImmutableList.of(image1, image2), expectedTimestamps);
}
@Test
public void imageThenVideoComposition_timestampsAreConsistent() throws Exception {
long imageDurationUs = 500_000L;
EditedMediaItem image =
new EditedMediaItem.Builder(
new MediaItem.Builder()
.setUri(IMAGE_ASSET)
.setImageDurationMs(usToMs(imageDurationUs))
.build())
.setDurationUs(imageDurationUs)
.setFrameRate(30)
.build();
EditedMediaItem video =
new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET))
.setDurationUs(MP4_ASSET_DURATION_US)
.build();
ImmutableList<Long> expectedTimestamps =
new ImmutableList.Builder<Long>()
.addAll(IMAGE_TIMESTAMPS_US_500_MS_30_FPS)
.addAll(
Lists.transform(
MP4_ASSET_FRAME_TIMESTAMPS_US, timestampUs -> (timestampUs + imageDurationUs)))
.build();
compareTimestamps(ImmutableList.of(image, video), expectedTimestamps);
}
@Test
public void videoThenImageComposition_timestampsAreConsistent() throws Exception {
long imageDurationUs = 500_000L;
EditedMediaItem video =
new EditedMediaItem.Builder(MediaItem.fromUri(MP4_ASSET))
.setDurationUs(MP4_ASSET_DURATION_US)
.build();
EditedMediaItem image =
new EditedMediaItem.Builder(
new MediaItem.Builder()
.setUri(IMAGE_ASSET)
.setImageDurationMs(usToMs(imageDurationUs))
.build())
.setDurationUs(imageDurationUs)
.setFrameRate(30)
.build();
ImmutableList<Long> expectedTimestamps =
new ImmutableList.Builder<Long>()
.addAll(MP4_ASSET_FRAME_TIMESTAMPS_US)
.addAll(
Lists.transform(
IMAGE_TIMESTAMPS_US_500_MS_30_FPS,
timestampUs -> (MP4_ASSET_DURATION_US + timestampUs)))
.build();
compareTimestamps(ImmutableList.of(video, image), expectedTimestamps);
}
@Test
public void videoThenImageComposition_clippingVideo_timestampsAreConsistent() throws Exception {
long clippedStartUs = 500_000L;
long imageDurationUs = 500_000L;
EditedMediaItem video =
new EditedMediaItem.Builder(
MediaItem.fromUri(MP4_ASSET)
.buildUpon()
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(usToMs(clippedStartUs))
.build())
.build())
.setDurationUs(MP4_ASSET_DURATION_US)
.build();
EditedMediaItem image =
new EditedMediaItem.Builder(
new MediaItem.Builder()
.setUri(IMAGE_ASSET)
.setImageDurationMs(usToMs(imageDurationUs))
.build())
.setDurationUs(imageDurationUs)
.setFrameRate(30)
.build();
ImmutableList<Long> expectedTimestamps =
new ImmutableList.Builder<Long>()
.addAll(getClippedTimestamps(MP4_ASSET_FRAME_TIMESTAMPS_US, clippedStartUs))
.addAll(
Lists.transform(
IMAGE_TIMESTAMPS_US_500_MS_30_FPS,
timestampUs -> ((MP4_ASSET_DURATION_US - clippedStartUs) + timestampUs)))
.build();
compareTimestamps(ImmutableList.of(video, image), expectedTimestamps);
}
private void compareTimestamps(List<EditedMediaItem> mediaItems, List<Long> expectedTimestamps)
throws Exception {
ImmutableList<Long> timestampsFromCompositionPlayer =
getTimestampsFromCompositionPlayer(mediaItems);
ImmutableList<Long> timestampsFromTransformer = getTimestampsFromTransformer(mediaItems);
assertThat(timestampsFromCompositionPlayer).isEqualTo(timestampsFromTransformer);
assertThat(timestampsFromTransformer).isEqualTo(expectedTimestamps);
}
private ImmutableList<Long> getTimestampsFromTransformer(List<EditedMediaItem> editedMediaItems)
throws Exception {
InputTimestampRecordingShaderProgram timestampRecordingShaderProgram =
new InputTimestampRecordingShaderProgram();
ImmutableList<EditedMediaItem> timestampRecordingEditedMediaItems =
prependVideoEffects(
editedMediaItems,
/* effects= */ ImmutableList.of(
(GlEffect) (context, useHdr) -> timestampRecordingShaderProgram));
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(
applicationContext, new Transformer.Builder(applicationContext).build())
.build()
.run(
/* testId= */ testName.getMethodName(),
new Composition.Builder(
new EditedMediaItemSequence(timestampRecordingEditedMediaItems))
.experimentalSetForceAudioTrack(true)
.build());
return timestampRecordingShaderProgram.getInputTimestampsUs();
}
private ImmutableList<Long> getTimestampsFromCompositionPlayer(
List<EditedMediaItem> editedMediaItems) throws Exception {
PlayerTestListener compositionPlayerListener = new PlayerTestListener(TEST_TIMEOUT_MS);
InputTimestampRecordingShaderProgram timestampRecordingShaderProgram =
new InputTimestampRecordingShaderProgram();
ImmutableList<EditedMediaItem> timestampRecordingEditedMediaItems =
prependVideoEffects(
editedMediaItems,
/* effects= */ ImmutableList.of(
(GlEffect) (context, useHdr) -> timestampRecordingShaderProgram));
instrumentation.runOnMainSync(
() -> {
compositionPlayer = new CompositionPlayer.Builder(applicationContext).build();
// Set a surface on the player even though there is no UI on this test. We need a surface
// otherwise the player will skip/drop video frames.
compositionPlayer.setVideoSurfaceView(surfaceView);
compositionPlayer.addListener(compositionPlayerListener);
compositionPlayer.setComposition(
new Composition.Builder(
new EditedMediaItemSequence(timestampRecordingEditedMediaItems))
.experimentalSetForceAudioTrack(true)
.build());
compositionPlayer.prepare();
compositionPlayer.play();
});
compositionPlayerListener.waitUntilPlayerEnded();
instrumentation.runOnMainSync(() -> compositionPlayer.release());
return timestampRecordingShaderProgram.getInputTimestampsUs();
}
private static ImmutableList<EditedMediaItem> prependVideoEffects(
List<EditedMediaItem> editedMediaItems, List<Effect> effects) {
ImmutableList.Builder<EditedMediaItem> prependedItems = new ImmutableList.Builder<>();
for (EditedMediaItem editedMediaItem : editedMediaItems) {
prependedItems.add(
editedMediaItem
.buildUpon()
.setEffects(
new Effects(
editedMediaItem.effects.audioProcessors,
new ImmutableList.Builder<Effect>()
.addAll(effects)
.addAll(editedMediaItem.effects.videoEffects)
.build()))
.build());
}
return prependedItems.build();
}
private static ImmutableList<Long> getClippedTimestamps(List<Long> timestamps, long clipStartUs) {
ImmutableList.Builder<Long> clippedTimestamps = new ImmutableList.Builder<>();
for (Long timestamp : timestamps) {
if (timestamp < clipStartUs) {
continue;
}
clippedTimestamps.add(timestamp - clipStartUs);
}
return clippedTimestamps.build();
}
}

23
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/performance/CompositionPlayerSeekTest.java
View File

@ -26,19 +26,18 @@ import androidx.media3.common.GlObjectsProvider;
import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.MediaItem;
import androidx.media3.effect.GlEffect;
import androidx.media3.effect.PassthroughShaderProgram;
import androidx.media3.transformer.Composition;
import androidx.media3.transformer.CompositionPlayer;
import androidx.media3.transformer.EditedMediaItem;
import androidx.media3.transformer.EditedMediaItemSequence;
import androidx.media3.transformer.Effects;
import androidx.media3.transformer.InputTimestampRecordingShaderProgram;
import androidx.media3.transformer.PlayerTestListener;
import androidx.media3.transformer.SurfaceTestActivity;
import androidx.test.ext.junit.rules.ActivityScenarioRule;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.platform.app.InstrumentationRegistry;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
@ -88,10 +87,9 @@ public class CompositionPlayerSeekTest {
});
}
// TODO: b/320244483 - Add tests that seek into the middle of the sequence.
@Test
public void seekToZero_singleSequenceOfTwoVideos() throws Exception {
PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS * 1000);
PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS);
InputTimestampRecordingShaderProgram inputTimestampRecordingShaderProgram =
new InputTimestampRecordingShaderProgram();
EditedMediaItem video =
@ -180,7 +178,7 @@ public class CompositionPlayerSeekTest {
1958266L,
1991633L);
assertThat(inputTimestampRecordingShaderProgram.timestampsUs)
assertThat(inputTimestampRecordingShaderProgram.getInputTimestampsUs())
// Seeked after the first playback ends, so the timestamps are repeated twice.
.containsExactlyElementsIn(
new ImmutableList.Builder<Long>()
@ -192,7 +190,7 @@ public class CompositionPlayerSeekTest {
@Test
public void seekToZero_after15framesInSingleSequenceOfTwoVideos() throws Exception {
PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS * 1000);
PlayerTestListener listener = new PlayerTestListener(TEST_TIMEOUT_MS);
ResettableCountDownLatch framesReceivedLatch = new ResettableCountDownLatch(15);
AtomicBoolean shaderProgramShouldBlockInput = new AtomicBoolean();
@ -331,7 +329,7 @@ public class CompositionPlayerSeekTest {
1958266L,
1991633L);
assertThat(inputTimestampRecordingShaderProgram.timestampsUs)
assertThat(inputTimestampRecordingShaderProgram.getInputTimestampsUs())
.containsExactlyElementsIn(expectedTimestampsUs)
.inOrder();
}
@ -343,17 +341,6 @@ public class CompositionPlayerSeekTest {
.build();
}
private static class InputTimestampRecordingShaderProgram extends PassthroughShaderProgram {
public final ArrayList<Long> timestampsUs = new ArrayList<>();
@Override
public void queueInputFrame(
GlObjectsProvider glObjectsProvider, GlTextureInfo inputTexture, long presentationTimeUs) {
super.queueInputFrame(glObjectsProvider, inputTexture, presentationTimeUs);
timestampsUs.add(presentationTimeUs);
}
}
private static final class ResettableCountDownLatch {
private CountDownLatch latch;

7
libraries/transformer/src/main/java/androidx/media3/transformer/SequencePlayerRenderersWrapper.java
View File

@ -351,6 +351,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
videoSink.onRendererDisabled();
}
@Override
public boolean isEnded() {
return super.isEnded()
&& videoSink.isEnded()
&& (timestampIterator == null || !timestampIterator.hasNext());
}
@Override
public boolean isReady() {
// If the renderer was enabled with mayRenderStartOfStream set to false, meaning the image