loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add Format.cueReplacementBehavior

Browse Source 
Previously any `CuesWithTiming.durationUs` could be `TIME_UNSET`,
meaning it should be replaced by the next `CuesWithTiming` instance
(instead of being merged if the durations overlap, which is currently
the expected behavior for all `CuesWithTiming` with a 'real' duration).

This technically allowed a single subtitle track to include a mixture of
`CuesWithTiming` that should be merged, and some that should be
replaced. This is not actually needed for any of the subtitle formats
currently supported by ExoPlayer - in all cases a format expects either
all cues to be merged, or each cue to replace the previous one.

Supporting this mixture of merging and replacing in `TextRenderer` ended
up being very complicated, and it seemed a bit pointless since it's not
actually needed. This change means a given subtitle track either merges
**all** cues (meaning `CuesWithTiming.durationUs = C.TIME_UNSET` is not
allowed), or **every** cue is replaced by the next one (meaning
`CuesWithTiming.durationUs` may be set (to allow for cues to 'time out',
needed for CEA-608), or may be `TIME_UNSET`).

This value will be used in a subsequent change that adds cue-merging
support to `TextRenderer`.

PiperOrigin-RevId: 565028066
This commit is contained in:
ibaker 2023-09-13 06:06:58 -07:00 committed by Copybara-Service
parent 282171cb6f
commit abaf3e7aa1
13 changed files with 149 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
libraries/common/src/main/java/androidx/media3/common/Format.java
View File

@ -15,13 +15,20 @@
*/
package androidx.media3.common;
import static java.lang.annotation.ElementType.TYPE_USE;
import android.os.Bundle;
import androidx.annotation.IntDef;
import androidx.annotation.Nullable;
import androidx.media3.common.util.BundleableUtil;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import com.google.common.base.Joiner;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@ -172,6 +179,7 @@ public final class Format implements Bundleable {
// Text specific.
private int accessibilityChannel;
@UnstableApi private @CueReplacementBehavior int cueReplacementBehavior;
// Image specific
@ -201,6 +209,7 @@ public final class Format implements Bundleable {
pcmEncoding = NO_VALUE;
// Text specific.
accessibilityChannel = NO_VALUE;
cueReplacementBehavior = CUE_REPLACEMENT_BEHAVIOR_MERGE;
// Image specific.
tileCountHorizontal = NO_VALUE;
tileCountVertical = NO_VALUE;
@ -248,6 +257,7 @@ public final class Format implements Bundleable {
this.encoderPadding = format.encoderPadding;
// Text specific.
this.accessibilityChannel = format.accessibilityChannel;
this.cueReplacementBehavior = format.cueReplacementBehavior;
// Image specific.
this.tileCountHorizontal = format.tileCountHorizontal;
this.tileCountVertical = format.tileCountVertical;
@ -626,6 +636,19 @@ public final class Format implements Bundleable {
return this;
}
/**
* Sets {@link Format#cueReplacementBehavior}. The default value is {@link
* #CUE_REPLACEMENT_BEHAVIOR_MERGE}.
*
* @param cueReplacementBehavior The {@link Format.CueReplacementBehavior}.
* @return The builder.
*/
@CanIgnoreReturnValue
public Builder setCueReplacementBehavior(@CueReplacementBehavior int cueReplacementBehavior) {
this.cueReplacementBehavior = cueReplacementBehavior;
return this;
}
// Image specific.
/**
@ -673,6 +696,36 @@ public final class Format implements Bundleable {
}
}
/**
* The replacement behaviors for consecutive samples in a {@linkplain C#TRACK_TYPE_TEXT text
* track} of type {@link MimeTypes#APPLICATION_MEDIA3_CUES}.
*/
@UnstableApi
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({
CUE_REPLACEMENT_BEHAVIOR_MERGE,
CUE_REPLACEMENT_BEHAVIOR_REPLACE,
})
public @interface CueReplacementBehavior {}
/**
* Subsequent cues should be merged with any previous cues that should still be shown on screen.
*
* <p>Tracks with this behavior must not contain samples with an {@linkplain C#TIME_UNSET unset}
* duration.
*/
@UnstableApi public static final int CUE_REPLACEMENT_BEHAVIOR_MERGE = 1;
/**
* Subsequent cues should replace all previous cues.
*
* <p>Tracks with this behavior may contain samples with an {@linkplain C#TIME_UNSET unset}
* duration (but the duration may also be set to a 'real' value).
*/
@UnstableApi public static final int CUE_REPLACEMENT_BEHAVIOR_REPLACE = 2;
/** A value for various fields to indicate that the field's value is unknown or not applicable. */
public static final int NO_VALUE = -1;
@ -847,6 +900,12 @@ public final class Format implements Bundleable {
/** The Accessibility channel, or {@link #NO_VALUE} if not known or applicable. */
@UnstableApi public final int accessibilityChannel;
/**
* The replacement behavior that should be followed when handling consecutive samples in a
* {@linkplain C#TRACK_TYPE_TEXT text track} of type {@link MimeTypes#APPLICATION_MEDIA3_CUES}.
*/
@UnstableApi public final @CueReplacementBehavior int cueReplacementBehavior;
// Image specific.
/**
@ -908,6 +967,7 @@ public final class Format implements Bundleable {
encoderPadding = builder.encoderPadding == NO_VALUE ? 0 : builder.encoderPadding;
// Text specific.
accessibilityChannel = builder.accessibilityChannel;
cueReplacementBehavior = builder.cueReplacementBehavior;
// Image specific.
tileCountHorizontal = builder.tileCountHorizontal;
tileCountVertical = builder.tileCountVertical;

10
libraries/extractor/src/main/java/androidx/media3/extractor/text/CuesWithTiming.java
View File

@ -17,6 +17,7 @@
package androidx.media3.extractor.text;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.text.Cue;
import androidx.media3.common.util.UnstableApi;
import com.google.common.collect.ImmutableList;
@ -41,9 +42,12 @@ public class CuesWithTiming {
* The duration for which {@link #cues} should be shown on screen, in microseconds, or {@link
* C#TIME_UNSET} if not known.
*
* <p>If this value is set then {@link #cues} from multiple instances may be shown on the screen
* simultaneously (if their durations overlap). If this value is {@link C#TIME_UNSET} then {@link
* #cues} should be shown on the screen until the {@link #startTimeUs} of the next instance.
* <p>If {@link Format#cueReplacementBehavior} is {@link Format#CUE_REPLACEMENT_BEHAVIOR_MERGE}
* then cues from multiple instances will be shown on screen simultaneously if their start times
* and durations overlap.
*
* <p>{@link C#TIME_UNSET} is only permitted if the {@link Format#cueReplacementBehavior} of the
* current track is {@link Format#CUE_REPLACEMENT_BEHAVIOR_REPLACE}.
*/
public final long durationUs;

15
libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java
View File

@ -16,9 +16,12 @@
package androidx.media3.extractor.text;
import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Format.CueReplacementBehavior;
import androidx.media3.common.util.UnstableApi;
import java.util.List;
@ -95,4 +98,16 @@ public interface SubtitleParser {
* <p>The default implementation is a no-op.
*/
default void reset() {}
/**
* Returns the {@link CueReplacementBehavior} for consecutive {@link CuesWithTiming} emitted by
* this implementation.
*
* <p>A given instance must always return the same value from this method.
*
* <p>The default implementation returns {@link Format#CUE_REPLACEMENT_BEHAVIOR_MERGE}.
*/
default @CueReplacementBehavior int getCueReplacementBehavior() {
return CUE_REPLACEMENT_BEHAVIOR_MERGE;
}
}

4
libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java
View File

@ -26,6 +26,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.DataReader;
import androidx.media3.common.Format;
import androidx.media3.common.Format.CueReplacementBehavior;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.Util;
@ -86,6 +87,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
if (currentSubtitleParser == null) {
delegate.format(format);
} else {
@CueReplacementBehavior
int nextCuesBehavior = currentSubtitleParser.getCueReplacementBehavior();
delegate.format(
format
.buildUpon()
@ -94,6 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
// Reset this value to the default. All non-default timestamp adjustments are done
// below in sampleMetadata() and there are no 'subsamples' after transcoding.
.setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE)
.setCueReplacementBehavior(nextCuesBehavior)
.build());
}
}

7
libraries/extractor/src/main/java/androidx/media3/extractor/text/dvb/DvbParser.java
View File

@ -26,6 +26,8 @@ import android.graphics.PorterDuffXfermode;
import android.util.SparseArray;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Format.CueReplacementBehavior;
import androidx.media3.common.text.Cue;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.ParsableBitArray;
@ -128,6 +130,11 @@ public final class DvbParser implements SubtitleParser {
subtitleService.reset();
}
@Override
public @CueReplacementBehavior int getCueReplacementBehavior() {
return Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
}
@Override
public ImmutableList<CuesWithTiming> parse(byte[] data, int offset, int length) {
ParsableBitArray dataBitArray = new ParsableBitArray(data, /* limit= */ offset + length);

7
libraries/extractor/src/main/java/androidx/media3/extractor/text/pgs/PgsParser.java
View File

@ -20,6 +20,8 @@ import static java.lang.Math.min;
import android.graphics.Bitmap;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Format.CueReplacementBehavior;
import androidx.media3.common.text.Cue;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
@ -53,6 +55,11 @@ public final class PgsParser implements SubtitleParser {
cueBuilder = new CueBuilder();
}
@Override
public @CueReplacementBehavior int getCueReplacementBehavior() {
return Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
}
@Override
public ImmutableList<CuesWithTiming> parse(byte[] data, int offset, int length) {
buffer.reset(data, /* limit= */ offset + length);

7
libraries/extractor/src/main/java/androidx/media3/extractor/text/tx3g/Tx3gParser.java
View File

@ -29,6 +29,8 @@ import android.text.style.TypefaceSpan;
import android.text.style.UnderlineSpan;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Format.CueReplacementBehavior;
import androidx.media3.common.text.Cue;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.ParsableByteArray;
@ -122,6 +124,11 @@ public final class Tx3gParser implements SubtitleParser {
}
}
@Override
public @CueReplacementBehavior int getCueReplacementBehavior() {
return Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
}
@Override
public ImmutableList<CuesWithTiming> parse(byte[] data, int offset, int length) {
parsableByteArray.reset(data, /* limit= */ offset + length);

7
libraries/extractor/src/main/java/androidx/media3/extractor/text/webvtt/Mp4WebvttParser.java
View File

@ -19,6 +19,8 @@ import static androidx.media3.common.util.Assertions.checkArgument;
import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.Format.CueReplacementBehavior;
import androidx.media3.common.text.Cue;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.UnstableApi;
@ -52,6 +54,11 @@ public final class Mp4WebvttParser implements SubtitleParser {
parsableByteArray = new ParsableByteArray();
}
@Override
public @CueReplacementBehavior int getCueReplacementBehavior() {
return Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
}
@Override
public ImmutableList<CuesWithTiming> parse(byte[] data, int offset, int length) {
parsableByteArray.reset(data, /* limit= */ offset + length);

7
libraries/extractor/src/test/java/androidx/media3/extractor/text/ssa/SsaParserTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.extractor.text.ssa;
import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
@ -60,6 +61,12 @@ public final class SsaParserTest {
private static final String STYLE_UNDERLINE = "media/ssa/style_underline";
private static final String STYLE_STRIKEOUT = "media/ssa/style_strikeout";
@Test
public void cuesReplacementBehaviorIsMerge() throws IOException {
SsaParser parser = new SsaParser();
assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_MERGE);
}
@Test
public void parseEmpty() throws IOException {
SsaParser parser = new SsaParser();

7
libraries/extractor/src/test/java/androidx/media3/extractor/text/subrip/SubripParserTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.extractor.text.subrip;
import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
import static com.google.common.truth.Truth.assertThat;
import androidx.media3.common.text.Cue;
@ -48,6 +49,12 @@ public final class SubripParserTest {
private static final String TYPICAL_NO_HOURS_AND_MILLIS =
"media/subrip/typical_no_hours_and_millis";
@Test
public void cueReplacementBehaviorIsMerge() {
SubripParser parser = new SubripParser();
assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_MERGE);
}
@Test
public void parseEmpty() throws IOException {
SubripParser parser = new SubripParser();

7
libraries/extractor/src/test/java/androidx/media3/extractor/text/tx3g/Tx3gParserTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.extractor.text.tx3g;
import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
import static androidx.media3.test.utils.truth.SpannedSubject.assertThat;
import static com.google.common.truth.Truth.assertThat;
@ -55,6 +56,12 @@ public final class Tx3gParserTest {
private static final String INITIALIZATION_ALL_DEFAULTS =
"media/tx3g/initialization_all_defaults";
@Test
public void cueReplacementBehaviorIsReplace() {
Tx3gParser parser = new Tx3gParser(/* initializationData= */ ImmutableList.of());
assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_REPLACE);
}
@Test
public void parseNoSubtitle() throws Exception {
Tx3gParser parser = new Tx3gParser(ImmutableList.of());

7
libraries/extractor/src/test/java/androidx/media3/extractor/text/webvtt/Mp4WebvttParserTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.extractor.text.webvtt;
import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
@ -159,6 +160,12 @@ public final class Mp4WebvttParserTest {
@Rule public final Expect expect = Expect.create();
@Test
public void cueReplacementBehaviorIsReplace() {
Mp4WebvttParser parser = new Mp4WebvttParser();
assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_REPLACE);
}
// Positive tests.
@Test

7
libraries/extractor/src/test/java/androidx/media3/extractor/text/webvtt/WebvttParserTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.extractor.text.webvtt;
import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
import static androidx.media3.test.utils.truth.SpannedSubject.assertThat;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
@ -65,6 +66,12 @@ public class WebvttParserTest {
@Rule public final Expect expect = Expect.create();
@Test
public void cueReplacementBehaviorIsMerge() throws IOException {
WebvttParser parser = new WebvttParser();
assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_MERGE);
}
@Test
public void parseEmpty() throws IOException {
WebvttParser parser = new WebvttParser();