diff --git a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java index dd96df587f..fe64215925 100644 --- a/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java +++ b/libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java @@ -484,6 +484,19 @@ public final class Mp4Extractor implements Extractor, SeekMap { } } + /** + * Returns the list of sample timestamps of a {@code trackId}, in microseconds. + * + * @param trackId The id of the track to get the sample timestamps. + * @return The corresponding sample timestmaps of the track. + */ + public long[] getSampleTimestampsUs(int trackId) { + if (tracks.length <= trackId) { + return new long[0]; + } + return tracks[trackId].sampleTable.timestampsUs; + } + // Private methods. private void enterReadingAtomHeaderState() { diff --git a/libraries/test_data/src/test/assets/media/mp4/trim_optimization_failure.mp4 b/libraries/test_data/src/test/assets/media/mp4/trim_optimization_failure.mp4 new file mode 100644 index 0000000000..83baaae6aa Binary files /dev/null and b/libraries/test_data/src/test/assets/media/mp4/trim_optimization_failure.mp4 differ diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java index 24792bd37c..ee50b4f7d2 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java @@ -581,6 +581,19 @@ public final class AndroidTestUtil { .build()) .build(); + // From b/357743907. + public static final AssetInfo MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO = + new AssetInfo.Builder("asset:///media/mp4/trim_optimization_failure.mp4") + .setVideoFormat( + new Format.Builder() + .setSampleMimeType(VIDEO_H264) + .setWidth(518) + .setHeight(488) + .setFrameRate(29.882f) + .setCodecs("avc1.640034") + .build()) + .build(); + // The 7 HIGHMOTION files are H264 and AAC. public static final AssetInfo MP4_REMOTE_1280W_720H_5_SECOND_HIGHMOTION = diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index 26ae3087e2..0bf899e057 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -21,6 +21,7 @@ import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat; import static androidx.media3.transformer.AndroidTestUtil.JPG_ASSET; import static androidx.media3.transformer.AndroidTestUtil.MP3_ASSET; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_SHORTER_AUDIO; @@ -664,6 +665,47 @@ public class TransformerEndToEndTest { assertThat(format.rotationDegrees).isEqualTo(90); } + @Test + public void clippedMedia_trimOptimizationEnabledAndTrimFromCloseToKeyFrame_succeeds() + throws Exception { + // This test covers the case where there's no frame between the trim point and the next sync + // sample. The frame has to be further than roughly 25ms apart. + assumeFormatsSupported( + context, + testId, + /* inputFormat= */ MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO.videoFormat, + /* outputFormat= */ MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO.videoFormat); + + Transformer transformer = + new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build(); + + // The previous sample is at 1137 and the next sample (which is a sync sample) is at 1171. + long clippingStartMs = 1138; + long clippingEndMs = 5601; + + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(Uri.parse(MP4_ASSET_PHOTOS_TRIM_OPTIMIZATION_VIDEO.uri)) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(1138) + .setEndPositionMs(5601) + .build()) + .build(); + + ExportTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, mediaItem); + + assertThat(result.exportResult.optimizationResult) + .isEqualTo(OPTIMIZATION_ABANDONED_KEYFRAME_PLACEMENT_OPTIMAL_FOR_TRIM); + assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs); + assertThat(result.exportResult.videoConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED); + assertThat(result.exportResult.audioConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED); + assertThat(new File(result.filePath).length()).isGreaterThan(0); + } + @Test public void clippedMedia_trimOptimizationEnabled_fallbackToNormalExportUponFormatMismatch() throws Exception { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Mp4Info.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Mp4Info.java index 394e228fee..e027abede1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Mp4Info.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Mp4Info.java @@ -24,6 +24,7 @@ import androidx.media3.common.C; import androidx.media3.common.DataReader; import androidx.media3.common.Format; import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; import androidx.media3.datasource.DataSourceUtil; import androidx.media3.datasource.DataSpec; import androidx.media3.datasource.DefaultDataSource; @@ -65,6 +66,9 @@ import org.checkerframework.checker.nullness.qual.Nullable; */ public final long firstSyncSampleTimestampUsAfterTimeUs; + /** Whether the first sample at or after {@code timeUs} is a sync sample. */ + public final boolean isFirstVideoSampleAfterTimeUsSyncSample; + /** The video {@link Format} or {@code null} if there is no video track. */ public final @Nullable Format videoFormat; @@ -75,11 +79,13 @@ import org.checkerframework.checker.nullness.qual.Nullable; long durationUs, long lastSyncSampleTimestampUs, long firstSyncSampleTimestampUsAfterTimeUs, + boolean isFirstVideoSampleAfterTimeUsSyncSample, @Nullable Format videoFormat, @Nullable Format audioFormat) { this.durationUs = durationUs; this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs; this.firstSyncSampleTimestampUsAfterTimeUs = firstSyncSampleTimestampUsAfterTimeUs; + this.isFirstVideoSampleAfterTimeUsSyncSample = isFirstVideoSampleAfterTimeUsSyncSample; this.videoFormat = videoFormat; this.audioFormat = audioFormat; } @@ -143,6 +149,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; long durationUs = mp4Extractor.getDurationUs(); long lastSyncSampleTimestampUs = C.TIME_UNSET; long firstSyncSampleTimestampUsAfterTimeUs = C.TIME_UNSET; + boolean isFirstSampleAfterTimeUsSyncSample = false; @Nullable Format videoFormat = null; if (extractorOutput.videoTrackId != C.INDEX_UNSET) { ExtractorOutputImpl.TrackOutputImpl videoTrackOutput = @@ -164,6 +171,21 @@ import org.checkerframework.checker.nullness.qual.Nullable; } else { // There is no sync sample after timeUs firstSyncSampleTimestampUsAfterTimeUs = C.TIME_END_OF_SOURCE; } + + long[] trackTimestampsUs = + mp4Extractor.getSampleTimestampsUs(extractorOutput.videoTrackId); + + int indexOfTrackTimestampUsAfterTimeUs = + Util.binarySearchCeil( + trackTimestampsUs, timeUs, /* inclusive= */ true, /* stayInBounds= */ false); + if (indexOfTrackTimestampUsAfterTimeUs < trackTimestampsUs.length) { + // Has found an element that is greater or equal to timeUs. + long firstTrackTimestampUsAfterTimeUs = + trackTimestampsUs[indexOfTrackTimestampUsAfterTimeUs]; + if (firstTrackTimestampUsAfterTimeUs == firstSyncSampleTimestampUsAfterTimeUs) { + isFirstSampleAfterTimeUsSyncSample = true; + } + } } } @@ -178,6 +200,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; durationUs, lastSyncSampleTimestampUs, firstSyncSampleTimestampUsAfterTimeUs, + isFirstSampleAfterTimeUsSyncSample, videoFormat, audioFormat); } finally { diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 9467b30454..cd87cbbf09 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -1502,7 +1502,8 @@ public final class Transformer { AAC_LC_AUDIO_SAMPLE_COUNT, mp4Info.audioFormat.sampleRate); } if (mp4Info.firstSyncSampleTimestampUsAfterTimeUs - trimStartTimeUs - <= maxEncodedAudioBufferDurationUs) { + <= maxEncodedAudioBufferDurationUs + || mp4Info.isFirstVideoSampleAfterTimeUsSyncSample) { Transformer.this.composition = buildUponCompositionForTrimOptimization( composition,