From 0b88e09a82e800f3d72c137c439f075670eb2676 Mon Sep 17 00:00:00 2001 From: sheenachhabra Date: Thu, 21 Dec 2023 04:42:59 -0800 Subject: [PATCH] Improve durationsVuForStts method implementation Issues with the current implementation 1. The implementation is unnecessarily complicated and can be easily simplified.To make all the tracks start from the same time, its only the first sample that require some timestamp adjustments but the current implementation shifts all the timestamps. Since method calculates the `sample duration`, shifting all the timestamps has no effect as such. 2. The implementation always forced first sample to start at 0. But when we want to use same method for `Fragmented MP4` then it will look inaccurate as we will call this method for different `fragments` and each `fragment` will not start from 0 presentation time. Although the output will be same since this method returns `duration` and not the `timestamps`. 3. As per previous implementation if there is just one sample then its duration is made equals to its presentation time, which looks incorrect. With new changes, if a single sample is passed then its duration will always be 0 irrespective of specified last sample duration behaviour. PiperOrigin-RevId: 592826612 --- .../java/androidx/media3/muxer/Boxes.java | 86 ++++++++++--------- .../media3/muxer/Mp4MoovStructure.java | 2 +- .../java/androidx/media3/muxer/BoxesTest.java | 60 +++---------- 3 files changed, 59 insertions(+), 89 deletions(-) diff --git a/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java b/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java index 9d76711142..8c6edc3acf 100644 --- a/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java +++ b/libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java @@ -23,6 +23,7 @@ import static androidx.media3.muxer.ColorUtils.MEDIAFORMAT_TRANSFER_TO_MP4_TRANS import static androidx.media3.muxer.Mp4Utils.MVHD_TIMEBASE; import android.media.MediaCodec; +import android.media.MediaCodec.BufferInfo; import androidx.annotation.Nullable; import androidx.media3.common.C; import androidx.media3.common.ColorInfo; @@ -345,7 +346,7 @@ import java.util.Locale; } String locationString = - String.format(Locale.US, "%+.4f%+.4f/", location.latitude, location.longitude); + Util.formatInvariant("%+.4f%+.4f/", location.latitude, location.longitude); ByteBuffer xyzBoxContents = ByteBuffer.allocate(locationString.length() + 2 + 2); xyzBoxContents.putShort((short) (xyzBoxContents.capacity() - 4)); @@ -580,53 +581,54 @@ import java.util.Locale; } /** - * Converts sample presentation times (in microseconds) to sample durations (in timebase units) - * that will go into the stts box. + * Converts sample presentation times (in microseconds) to sample durations (in timebase units). * - *

ISO/IEC 14496-12: 8.6.1.3.1 recommends each track starts at 0. Therefore, the first sample - * presentation timestamp is set to 0 and the duration of that sample may be larger as a result. + *

All the tracks must start from the same time. If all the tracks do not start from the same + * time, then the caller must pass the minimum presentation timestamp across all tracks to be set + * for the first sample. As a result, the duration of that first sample may be larger. * - * @param writtenSamples All the written samples. - * @param minInputPresentationTimestampUs The global minimum presentation timestamp which needs to - * be subtracted from each sample's presentation timestamp. + * @param samplesInfo A list of {@linkplain BufferInfo sample info}. + * @param firstSamplePresentationTimeUs The presentation timestamp to override the first sample's + * presentation timestamp, in microseconds. This should be the minimum presentation timestamp + * across all tracks if the {@code samplesInfo} contains the first sample of the track. + * Otherwise this should be equal to the presentation timestamp of first sample present in the + * {@code samplesInfo} list. * @param videoUnitTimescale The timescale of the track. * @param lastDurationBehavior The behaviour for the last sample duration. * @return A list of all the sample durations. */ // TODO: b/280084657 - Add support for setting last sample duration. - public static List durationsVuForStts( - List writtenSamples, - long minInputPresentationTimestampUs, + public static List convertPresentationTimestampsToDurationsVu( + List samplesInfo, + long firstSamplePresentationTimeUs, int videoUnitTimescale, @Mp4Muxer.LastFrameDurationBehavior int lastDurationBehavior) { - List durationsVu = new ArrayList<>(); + List durationsVu = new ArrayList<>(samplesInfo.size()); - long currentTimeVu = 0L; - - for (int sampleId = 0; sampleId < writtenSamples.size(); sampleId++) { - long samplePtsUs = writtenSamples.get(sampleId).presentationTimeUs; - long sampleSpanEndsAtUs = - sampleId == writtenSamples.size() - 1 - ? samplePtsUs - : writtenSamples.get(sampleId + 1).presentationTimeUs; - - sampleSpanEndsAtUs -= minInputPresentationTimestampUs; - - long sampleSpanEndsAtVu = Mp4Utils.vuFromUs(sampleSpanEndsAtUs, videoUnitTimescale); - - long durationVu = sampleSpanEndsAtVu - currentTimeVu; - currentTimeVu = sampleSpanEndsAtVu; - - if (durationVu >= Integer.MAX_VALUE) { - throw new IllegalArgumentException( - String.format(Locale.US, "Timestamp delta %d doesn't fit into an int", durationVu)); - } - - durationsVu.add(durationVu); + if (samplesInfo.isEmpty()) { + return durationsVu; } - adjustLastSampleDuration(durationsVu, lastDurationBehavior); + long currentSampleTimeUs = firstSamplePresentationTimeUs; + for (int nextSampleId = 1; nextSampleId < samplesInfo.size(); nextSampleId++) { + long nextSampleTimeUs = samplesInfo.get(nextSampleId).presentationTimeUs; + // TODO: b/316158030 - First calculate the duration and then convert us to vu to avoid + // rounding error. + long currentSampleDurationVu = + Mp4Utils.vuFromUs(nextSampleTimeUs, videoUnitTimescale) + - Mp4Utils.vuFromUs(currentSampleTimeUs, videoUnitTimescale); + if (currentSampleDurationVu > Integer.MAX_VALUE) { + throw new IllegalArgumentException( + String.format( + Locale.US, "Timestamp delta %d doesn't fit into an int", currentSampleDurationVu)); + } + durationsVu.add(currentSampleDurationVu); + currentSampleTimeUs = nextSampleTimeUs; + } + // Default duration for the last sample. + durationsVu.add(0L); + adjustLastSampleDuration(durationsVu, lastDurationBehavior); return durationsVu; } @@ -803,16 +805,16 @@ import java.util.Locale; return BoxUtils.wrapBoxesIntoBox("ftyp", boxBytes); } + // TODO: b/317117431 - Change this method to getLastSampleDuration(). /** Adjusts the duration of the very last sample if needed. */ private static void adjustLastSampleDuration( List durationsToBeAdjustedVu, @Mp4Muxer.LastFrameDurationBehavior int behavior) { - // Technically, MP4 files store not timestamps but frame durations. Thus, if we interpret - // timestamps as the start of frames then it's not obvious what's the duration of the very - // last frame should be. If our samples follow each other in roughly regular intervals (e.g. in - // a normal, 30 fps video), it makes sense to assume that the last sample will last the same ~33 - // ms as the all the other ones before. On the other hand, if we have just a few, irregularly - // spaced frames, with duplication, the entire duration of the video will increase, creating - // abnormal gaps. + // Technically, MP4 file stores frame durations, not timestamps. If a frame starts at a + // given timestamp then the duration of the last frame is not obvious. If samples follow each + // other in roughly regular intervals (e.g. in a normal, 30 fps video), it can be safely assumed + // that the last sample will have same duration (~33ms) as other samples. On the other hand, if + // there are just a few, irregularly spaced frames, with duplication, the entire duration of the + // video will increase, creating abnormal gaps. if (durationsToBeAdjustedVu.size() <= 2) { // Nothing to duplicate if there are 0 or 1 entries. diff --git a/libraries/muxer/src/main/java/androidx/media3/muxer/Mp4MoovStructure.java b/libraries/muxer/src/main/java/androidx/media3/muxer/Mp4MoovStructure.java index 921a41c487..dee215f30c 100644 --- a/libraries/muxer/src/main/java/androidx/media3/muxer/Mp4MoovStructure.java +++ b/libraries/muxer/src/main/java/androidx/media3/muxer/Mp4MoovStructure.java @@ -72,7 +72,7 @@ import org.checkerframework.checker.nullness.qual.PolyNull; // Generate the sample durations to calculate the total duration for tkhd box. List sampleDurationsVu = - Boxes.durationsVuForStts( + Boxes.convertPresentationTimestampsToDurationsVu( track.writtenSamples(), minInputPtsUs, track.videoUnitTimebase(), diff --git a/libraries/muxer/src/test/java/androidx/media3/muxer/BoxesTest.java b/libraries/muxer/src/test/java/androidx/media3/muxer/BoxesTest.java index aebce1c5da..22fa860fd9 100644 --- a/libraries/muxer/src/test/java/androidx/media3/muxer/BoxesTest.java +++ b/libraries/muxer/src/test/java/androidx/media3/muxer/BoxesTest.java @@ -371,14 +371,14 @@ public class BoxesTest { @Test public void - getDurationsVuForStts_singleSampleAtZeroTimestamp_lastFrameDurationShort_returnsSingleZeroLengthSample() { + convertPresentationTimestampsToDurationsVu_singleSampleAtZeroTimestamp_returnsSampleLengthEqualsZero() { List sampleBufferInfos = createBufferInfoListWithSamplePresentationTimestamps(0L); List durationsVu = - Boxes.durationsVuForStts( + Boxes.convertPresentationTimestampsToDurationsVu( sampleBufferInfos, - /* minInputPresentationTimestampUs= */ 0L, + /* firstSamplePresentationTimeUs= */ 0L, VU_TIMEBASE, LAST_FRAME_DURATION_BEHAVIOR_INSERT_SHORT_FRAME); @@ -387,62 +387,30 @@ public class BoxesTest { @Test public void - getDurationsVuForStts_singleSampleAtZeroTimestamp_lastFrameDurationDuplicate_returnsSingleZeroLengthSample() { + convertPresentationTimestampsToDurationsVu_singleSampleAtNonZeroTimestamp_returnsSampleLengthEqualsZero() { List sampleBufferInfos = - createBufferInfoListWithSamplePresentationTimestamps(0L); + createBufferInfoListWithSamplePresentationTimestamps(5_000L); List durationsVu = - Boxes.durationsVuForStts( + Boxes.convertPresentationTimestampsToDurationsVu( sampleBufferInfos, - /* minInputPresentationTimestampUs= */ 0L, + /* firstSamplePresentationTimeUs= */ 0L, VU_TIMEBASE, - LAST_FRAME_DURATION_BEHAVIOR_DUPLICATE_PREV_DURATION); + LAST_FRAME_DURATION_BEHAVIOR_INSERT_SHORT_FRAME); assertThat(durationsVu).containsExactly(0L); } @Test public void - getDurationsVuForStts_singleSampleAtNonZeroTimestamp_lastFrameDurationShort_returnsSampleLengthEqualsTimestamp() { - List sampleBufferInfos = - createBufferInfoListWithSamplePresentationTimestamps(5_000L); - - List durationsVu = - Boxes.durationsVuForStts( - sampleBufferInfos, - /* minInputPresentationTimestampUs= */ 0L, - VU_TIMEBASE, - LAST_FRAME_DURATION_BEHAVIOR_INSERT_SHORT_FRAME); - - assertThat(durationsVu).containsExactly(500L); - } - - @Test - public void - getDurationsVuForStts_singleSampleAtNonZeroTimestamp_lastFrameDurationDuplicate_returnsSampleLengthEqualsTimestamp() { - List sampleBufferInfos = - createBufferInfoListWithSamplePresentationTimestamps(5_000L); - - List durationsVu = - Boxes.durationsVuForStts( - sampleBufferInfos, - /* minInputPresentationTimestampUs= */ 0L, - VU_TIMEBASE, - LAST_FRAME_DURATION_BEHAVIOR_DUPLICATE_PREV_DURATION); - - assertThat(durationsVu).containsExactly(500L); - } - - @Test - public void - getDurationsVuForStts_differentSampleDurations_lastFrameDurationShort_returnsLastSampleOfZeroDuration() { + convertPresentationTimestampsToDurationsVu_differentSampleDurations_lastFrameDurationShort_returnsLastSampleOfZeroDuration() { List sampleBufferInfos = createBufferInfoListWithSamplePresentationTimestamps(0L, 30_000L, 80_000L); List durationsVu = - Boxes.durationsVuForStts( + Boxes.convertPresentationTimestampsToDurationsVu( sampleBufferInfos, - /* minInputPresentationTimestampUs= */ 0L, + /* firstSamplePresentationTimeUs= */ 0L, VU_TIMEBASE, LAST_FRAME_DURATION_BEHAVIOR_INSERT_SHORT_FRAME); @@ -451,14 +419,14 @@ public class BoxesTest { @Test public void - getDurationsVuForStts_differentSampleDurations_lastFrameDurationDuplicate_returnsLastSampleOfDuplicateDuration() { + convertPresentationTimestampsToDurationsVu_differentSampleDurations_lastFrameDurationDuplicate_returnsLastSampleOfDuplicateDuration() { List sampleBufferInfos = createBufferInfoListWithSamplePresentationTimestamps(0L, 30_000L, 80_000L); List durationsVu = - Boxes.durationsVuForStts( + Boxes.convertPresentationTimestampsToDurationsVu( sampleBufferInfos, - /* minInputPresentationTimestampUs= */ 0L, + /* firstSamplePresentationTimeUs= */ 0L, VU_TIMEBASE, LAST_FRAME_DURATION_BEHAVIOR_DUPLICATE_PREV_DURATION);