loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix edge case: no frame between trim position and the next sync sample

Browse Source 
PiperOrigin-RevId: 664841893
This commit is contained in:
claincly 2024-08-19 09:27:56 -07:00 committed by Copybara-Service
parent 63b45b7503
commit 677f8ad9f4
6 changed files with 93 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java
View File

@ -484,6 +484,19 @@ public final class Mp4Extractor implements Extractor, SeekMap {
}
}
/**
* Returns the list of sample timestamps of a {@code trackId}, in microseconds.
*
* @param trackId The id of the track to get the sample timestamps.
* @return The corresponding sample timestmaps of the track.
*/
public long[] getSampleTimestampsUs(int trackId) {
if (tracks.length <= trackId) {
return new long[0];
}
return tracks[trackId].sampleTable.timestampsUs;
}
// Private methods.
private void enterReadingAtomHeaderState() {

BIN
libraries/test_data/src/test/assets/media/mp4/trim_optimization_failure.mp4 Normal file
View File

Binary file not shown.

13
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -581,6 +581,19 @@ public final class AndroidTestUtil {
.build())
.build();
// From b/357743907.
public static final AssetInfo MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO =
new AssetInfo.Builder("asset:///media/mp4/trim_optimization_failure.mp4")
.setVideoFormat(
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setWidth(518)
.setHeight(488)
.setFrameRate(29.882f)
.setCodecs("avc1.640034")
.build())
.build();
// The 7 HIGHMOTION files are H264 and AAC.
public static final AssetInfo MP4_REMOTE_1280W_720H_5_SECOND_HIGHMOTION =

42
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -21,6 +21,7 @@ import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat;
import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP3_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_SHORTER_AUDIO;
@ -664,6 +665,47 @@ public class TransformerEndToEndTest {
assertThat(format.rotationDegrees).isEqualTo(90);
}
@Test
public void clippedMedia_trimOptimizationEnabledAndTrimFromCloseToKeyFrame_succeeds()
throws Exception {
// This test covers the case where there's no frame between the trim point and the next sync
// sample. The frame has to be further than roughly 25ms apart.
assumeFormatsSupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO.videoFormat,
/* outputFormat= */ MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO.videoFormat);
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
// The previous sample is at 1137 and the next sample (which is a sync sample) is at 1171.
long clippingStartMs = 1138;
long clippingEndMs = 5601;
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(Uri.parse(MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO.uri))
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(1138)
.setEndPositionMs(5601)
.build())
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, mediaItem);
assertThat(result.exportResult.optimizationResult)
.isEqualTo(OPTIMIZATION_ABANDONED_KEYFRAME_PLACEMENT_OPTIMAL_FOR_TRIM);
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
assertThat(result.exportResult.videoConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED);
assertThat(result.exportResult.audioConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED);
assertThat(new File(result.filePath).length()).isGreaterThan(0);
}
@Test
public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch()
throws Exception {

23
libraries/transformer/src/main/java/androidx/media3/transformer/Mp4Info.java
View File

@ -24,6 +24,7 @@ import androidx.media3.common.C;
import androidx.media3.common.DataReader;
import androidx.media3.common.Format;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.Util;
import androidx.media3.datasource.DataSourceUtil;
import androidx.media3.datasource.DataSpec;
import androidx.media3.datasource.DefaultDataSource;
@ -65,6 +66,9 @@ import org.checkerframework.checker.nullness.qual.Nullable;
*/
public final long firstSyncSampleTimestampUsAfterTimeUs;
/** Whether the first sample at or after {@code timeUs} is a sync sample. */
public final boolean isFirstVideoSampleAfterTimeUsSyncSample;
/** The video {@link Format} or {@code null} if there is no video track. */
public final @Nullable Format videoFormat;
@ -75,11 +79,13 @@ import org.checkerframework.checker.nullness.qual.Nullable;
long durationUs,
long lastSyncSampleTimestampUs,
long firstSyncSampleTimestampUsAfterTimeUs,
boolean isFirstVideoSampleAfterTimeUsSyncSample,
@Nullable Format videoFormat,
@Nullable Format audioFormat) {
this.durationUs = durationUs;
this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs;
this.firstSyncSampleTimestampUsAfterTimeUs = firstSyncSampleTimestampUsAfterTimeUs;
this.isFirstVideoSampleAfterTimeUsSyncSample = isFirstVideoSampleAfterTimeUsSyncSample;
this.videoFormat = videoFormat;
this.audioFormat = audioFormat;
}
@ -143,6 +149,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
long durationUs = mp4Extractor.getDurationUs();
long lastSyncSampleTimestampUs = C.TIME_UNSET;
long firstSyncSampleTimestampUsAfterTimeUs = C.TIME_UNSET;
boolean isFirstSampleAfterTimeUsSyncSample = false;
@Nullable Format videoFormat = null;
if (extractorOutput.videoTrackId != C.INDEX_UNSET) {
ExtractorOutputImpl.TrackOutputImpl videoTrackOutput =
@ -164,6 +171,21 @@ import org.checkerframework.checker.nullness.qual.Nullable;
} else { // There is no sync sample after timeUs
firstSyncSampleTimestampUsAfterTimeUs = C.TIME_END_OF_SOURCE;
}
long[] trackTimestampsUs =
mp4Extractor.getSampleTimestampsUs(extractorOutput.videoTrackId);
int indexOfTrackTimestampUsAfterTimeUs =
Util.binarySearchCeil(
trackTimestampsUs, timeUs, /* inclusive= */ true, /* stayInBounds= */ false);
if (indexOfTrackTimestampUsAfterTimeUs < trackTimestampsUs.length) {
// Has found an element that is greater or equal to timeUs.
long firstTrackTimestampUsAfterTimeUs =
trackTimestampsUs[indexOfTrackTimestampUsAfterTimeUs];
if (firstTrackTimestampUsAfterTimeUs == firstSyncSampleTimestampUsAfterTimeUs) {
isFirstSampleAfterTimeUsSyncSample = true;
}
}
}
}
@ -178,6 +200,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
durationUs,
lastSyncSampleTimestampUs,
firstSyncSampleTimestampUsAfterTimeUs,
isFirstSampleAfterTimeUsSyncSample,
videoFormat,
audioFormat);
} finally {

3
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -1502,7 +1502,8 @@ public final class Transformer {
AAC_LC_AUDIO_SAMPLE_COUNT, mp4Info.audioFormat.sampleRate);
}
if (mp4Info.firstSyncSampleTimestampUsAfterTimeUs - trimStartTimeUs
<= maxEncodedAudioBufferDurationUs) {
<= maxEncodedAudioBufferDurationUs
|| mp4Info.isFirstVideoSampleAfterTimeUsSyncSample) {
Transformer.this.composition =
buildUponCompositionForTrimOptimization(
composition,