loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve durationsVuForStts method implementation

Browse Source 
Issues with the current implementation
1. The implementation is unnecessarily complicated and can be
easily simplified.To make all the tracks start from the same time,
its only the first sample that require some timestamp adjustments
but the current implementation shifts all the timestamps. Since method
calculates the `sample duration`, shifting all the timestamps has no effect
as such.
2. The implementation always forced first sample to start at 0. But when we
want to use same method for `Fragmented MP4` then it will look inaccurate
as we will call this method for different `fragments` and each `fragment`
will not start from 0 presentation time. Although the output will be same
since this method returns `duration` and not the `timestamps`.
3. As per previous implementation if there is just one sample then
its duration is made equals to its presentation time, which looks incorrect.
With new changes, if a single sample is passed then its duration will always
be 0 irrespective of specified last sample duration behaviour.

PiperOrigin-RevId: 592826612
This commit is contained in:
sheenachhabra 2023-12-21 04:42:59 -08:00 committed by Copybara-Service
parent 250fc80419
commit 0b88e09a82
3 changed files with 59 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
libraries/muxer/src/main/java/androidx/media3/muxer/Boxes.java
View File

@ -23,6 +23,7 @@ import static androidx.media3.muxer.ColorUtils.MEDIAFORMAT_TRANSFER_TO_MP4_TRANS
import static androidx.media3.muxer.Mp4Utils.MVHD_TIMEBASE;
import android.media.MediaCodec;
import android.media.MediaCodec.BufferInfo;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.ColorInfo;
@ -345,7 +346,7 @@ import java.util.Locale;
}
String locationString =
String.format(Locale.US, "%+.4f%+.4f/", location.latitude, location.longitude);
Util.formatInvariant("%+.4f%+.4f/", location.latitude, location.longitude);
ByteBuffer xyzBoxContents = ByteBuffer.allocate(locationString.length() + 2 + 2);
xyzBoxContents.putShort((short) (xyzBoxContents.capacity() - 4));
@ -580,53 +581,54 @@ import java.util.Locale;
}
/**
* Converts sample presentation times (in microseconds) to sample durations (in timebase units)
* that will go into the stts box.
* Converts sample presentation times (in microseconds) to sample durations (in timebase units).
*
* <p>ISO/IEC 14496-12: 8.6.1.3.1 recommends each track starts at 0. Therefore, the first sample
* presentation timestamp is set to 0 and the duration of that sample may be larger as a result.
* <p>All the tracks must start from the same time. If all the tracks do not start from the same
* time, then the caller must pass the minimum presentation timestamp across all tracks to be set
* for the first sample. As a result, the duration of that first sample may be larger.
*
* @param writtenSamples All the written samples.
* @param minInputPresentationTimestampUs The global minimum presentation timestamp which needs to
* be subtracted from each sample's presentation timestamp.
* @param samplesInfo A list of {@linkplain BufferInfo sample info}.
* @param firstSamplePresentationTimeUs The presentation timestamp to override the first sample's
* presentation timestamp, in microseconds. This should be the minimum presentation timestamp
* across all tracks if the {@code samplesInfo} contains the first sample of the track.
* Otherwise this should be equal to the presentation timestamp of first sample present in the
* {@code samplesInfo} list.
* @param videoUnitTimescale The timescale of the track.
* @param lastDurationBehavior The behaviour for the last sample duration.
* @return A list of all the sample durations.
*/
// TODO: b/280084657 - Add support for setting last sample duration.
public static List<Long> durationsVuForStts(
List<MediaCodec.BufferInfo> writtenSamples,
long minInputPresentationTimestampUs,
public static List<Long> convertPresentationTimestampsToDurationsVu(
List<BufferInfo> samplesInfo,
long firstSamplePresentationTimeUs,
int videoUnitTimescale,
@Mp4Muxer.LastFrameDurationBehavior int lastDurationBehavior) {
List<Long> durationsVu = new ArrayList<>();
List<Long> durationsVu = new ArrayList<>(samplesInfo.size());
long currentTimeVu = 0L;
for (int sampleId = 0; sampleId < writtenSamples.size(); sampleId++) {
long samplePtsUs = writtenSamples.get(sampleId).presentationTimeUs;
long sampleSpanEndsAtUs =
sampleId == writtenSamples.size() - 1
? samplePtsUs
: writtenSamples.get(sampleId + 1).presentationTimeUs;
sampleSpanEndsAtUs -= minInputPresentationTimestampUs;
long sampleSpanEndsAtVu = Mp4Utils.vuFromUs(sampleSpanEndsAtUs, videoUnitTimescale);
long durationVu = sampleSpanEndsAtVu - currentTimeVu;
currentTimeVu = sampleSpanEndsAtVu;
if (durationVu >= Integer.MAX_VALUE) {
throw new IllegalArgumentException(
String.format(Locale.US, "Timestamp delta %d doesn't fit into an int", durationVu));
}
durationsVu.add(durationVu);
if (samplesInfo.isEmpty()) {
return durationsVu;
}
adjustLastSampleDuration(durationsVu, lastDurationBehavior);
long currentSampleTimeUs = firstSamplePresentationTimeUs;
for (int nextSampleId = 1; nextSampleId < samplesInfo.size(); nextSampleId++) {
long nextSampleTimeUs = samplesInfo.get(nextSampleId).presentationTimeUs;
// TODO: b/316158030 - First calculate the duration and then convert us to vu to avoid
// rounding error.
long currentSampleDurationVu =
Mp4Utils.vuFromUs(nextSampleTimeUs, videoUnitTimescale)
- Mp4Utils.vuFromUs(currentSampleTimeUs, videoUnitTimescale);
if (currentSampleDurationVu > Integer.MAX_VALUE) {
throw new IllegalArgumentException(
String.format(
Locale.US, "Timestamp delta %d doesn't fit into an int", currentSampleDurationVu));
}
durationsVu.add(currentSampleDurationVu);
currentSampleTimeUs = nextSampleTimeUs;
}
// Default duration for the last sample.
durationsVu.add(0L);
adjustLastSampleDuration(durationsVu, lastDurationBehavior);
return durationsVu;
}
@ -803,16 +805,16 @@ import java.util.Locale;
return BoxUtils.wrapBoxesIntoBox("ftyp", boxBytes);
}
// TODO: b/317117431 - Change this method to getLastSampleDuration().
/** Adjusts the duration of the very last sample if needed. */
private static void adjustLastSampleDuration(
List<Long> durationsToBeAdjustedVu, @Mp4Muxer.LastFrameDurationBehavior int behavior) {
// Technically, MP4 files store not timestamps but frame durations. Thus, if we interpret
// timestamps as the start of frames then it's not obvious what's the duration of the very
// last frame should be. If our samples follow each other in roughly regular intervals (e.g. in
// a normal, 30 fps video), it makes sense to assume that the last sample will last the same ~33
// ms as the all the other ones before. On the other hand, if we have just a few, irregularly
// spaced frames, with duplication, the entire duration of the video will increase, creating
// abnormal gaps.
// Technically, MP4 file stores frame durations, not timestamps. If a frame starts at a
// given timestamp then the duration of the last frame is not obvious. If samples follow each
// other in roughly regular intervals (e.g. in a normal, 30 fps video), it can be safely assumed
// that the last sample will have same duration (~33ms) as other samples. On the other hand, if
// there are just a few, irregularly spaced frames, with duplication, the entire duration of the
// video will increase, creating abnormal gaps.
if (durationsToBeAdjustedVu.size() <= 2) {
// Nothing to duplicate if there are 0 or 1 entries.

2
libraries/muxer/src/main/java/androidx/media3/muxer/Mp4MoovStructure.java
View File

@ -72,7 +72,7 @@ import org.checkerframework.checker.nullness.qual.PolyNull;
// Generate the sample durations to calculate the total duration for tkhd box.
List<Long> sampleDurationsVu =
Boxes.durationsVuForStts(
Boxes.convertPresentationTimestampsToDurationsVu(
track.writtenSamples(),
minInputPtsUs,
track.videoUnitTimebase(),

60
libraries/muxer/src/test/java/androidx/media3/muxer/BoxesTest.java
View File

@ -371,14 +371,14 @@ public class BoxesTest {
@Test
public void
getDurationsVuForStts_singleSampleAtZeroTimestamp_lastFrameDurationShort_returnsSingleZeroLengthSample() {
convertPresentationTimestampsToDurationsVu_singleSampleAtZeroTimestamp_returnsSampleLengthEqualsZero() {
List<MediaCodec.BufferInfo> sampleBufferInfos =
createBufferInfoListWithSamplePresentationTimestamps(0L);
List<Long> durationsVu =
Boxes.durationsVuForStts(
Boxes.convertPresentationTimestampsToDurationsVu(
sampleBufferInfos,
/* minInputPresentationTimestampUs= */ 0L,
/* firstSamplePresentationTimeUs= */ 0L,
VU_TIMEBASE,
LAST_FRAME_DURATION_BEHAVIOR_INSERT_SHORT_FRAME);
@ -387,62 +387,30 @@ public class BoxesTest {
@Test
public void
getDurationsVuForStts_singleSampleAtZeroTimestamp_lastFrameDurationDuplicate_returnsSingleZeroLengthSample() {
convertPresentationTimestampsToDurationsVu_singleSampleAtNonZeroTimestamp_returnsSampleLengthEqualsZero() {
List<MediaCodec.BufferInfo> sampleBufferInfos =
createBufferInfoListWithSamplePresentationTimestamps(0L);
createBufferInfoListWithSamplePresentationTimestamps(5_000L);
List<Long> durationsVu =
Boxes.durationsVuForStts(
Boxes.convertPresentationTimestampsToDurationsVu(
sampleBufferInfos,
/* minInputPresentationTimestampUs= */ 0L,
/* firstSamplePresentationTimeUs= */ 0L,
VU_TIMEBASE,
LAST_FRAME_DURATION_BEHAVIOR_DUPLICATE_PREV_DURATION);
LAST_FRAME_DURATION_BEHAVIOR_INSERT_SHORT_FRAME);
assertThat(durationsVu).containsExactly(0L);
}
@Test
public void
getDurationsVuForStts_singleSampleAtNonZeroTimestamp_lastFrameDurationShort_returnsSampleLengthEqualsTimestamp() {
List<MediaCodec.BufferInfo> sampleBufferInfos =
createBufferInfoListWithSamplePresentationTimestamps(5_000L);
List<Long> durationsVu =
Boxes.durationsVuForStts(
sampleBufferInfos,
/* minInputPresentationTimestampUs= */ 0L,
VU_TIMEBASE,
LAST_FRAME_DURATION_BEHAVIOR_INSERT_SHORT_FRAME);
assertThat(durationsVu).containsExactly(500L);
}
@Test
public void
getDurationsVuForStts_singleSampleAtNonZeroTimestamp_lastFrameDurationDuplicate_returnsSampleLengthEqualsTimestamp() {
List<MediaCodec.BufferInfo> sampleBufferInfos =
createBufferInfoListWithSamplePresentationTimestamps(5_000L);
List<Long> durationsVu =
Boxes.durationsVuForStts(
sampleBufferInfos,
/* minInputPresentationTimestampUs= */ 0L,
VU_TIMEBASE,
LAST_FRAME_DURATION_BEHAVIOR_DUPLICATE_PREV_DURATION);
assertThat(durationsVu).containsExactly(500L);
}
@Test
public void
getDurationsVuForStts_differentSampleDurations_lastFrameDurationShort_returnsLastSampleOfZeroDuration() {
convertPresentationTimestampsToDurationsVu_differentSampleDurations_lastFrameDurationShort_returnsLastSampleOfZeroDuration() {
List<MediaCodec.BufferInfo> sampleBufferInfos =
createBufferInfoListWithSamplePresentationTimestamps(0L, 30_000L, 80_000L);
List<Long> durationsVu =
Boxes.durationsVuForStts(
Boxes.convertPresentationTimestampsToDurationsVu(
sampleBufferInfos,
/* minInputPresentationTimestampUs= */ 0L,
/* firstSamplePresentationTimeUs= */ 0L,
VU_TIMEBASE,
LAST_FRAME_DURATION_BEHAVIOR_INSERT_SHORT_FRAME);
@ -451,14 +419,14 @@ public class BoxesTest {
@Test
public void
getDurationsVuForStts_differentSampleDurations_lastFrameDurationDuplicate_returnsLastSampleOfDuplicateDuration() {
convertPresentationTimestampsToDurationsVu_differentSampleDurations_lastFrameDurationDuplicate_returnsLastSampleOfDuplicateDuration() {
List<MediaCodec.BufferInfo> sampleBufferInfos =
createBufferInfoListWithSamplePresentationTimestamps(0L, 30_000L, 80_000L);
List<Long> durationsVu =
Boxes.durationsVuForStts(
Boxes.convertPresentationTimestampsToDurationsVu(
sampleBufferInfos,
/* minInputPresentationTimestampUs= */ 0L,
/* firstSamplePresentationTimeUs= */ 0L,
VU_TIMEBASE,
LAST_FRAME_DURATION_BEHAVIOR_DUPLICATE_PREV_DURATION);