Add Format.cueReplacementBehavior

Previously any `CuesWithTiming.durationUs` could be `TIME_UNSET`,
meaning it should be replaced by the next `CuesWithTiming` instance
(instead of being merged if the durations overlap, which is currently
the expected behavior for all `CuesWithTiming` with a 'real' duration).

This technically allowed a single subtitle track to include a mixture of
`CuesWithTiming` that should be merged, and some that should be
replaced. This is not actually needed for any of the subtitle formats
currently supported by ExoPlayer - in all cases a format expects either
all cues to be merged, or each cue to replace the previous one.

Supporting this mixture of merging and replacing in `TextRenderer` ended
up being very complicated, and it seemed a bit pointless since it's not
actually needed. This change means a given subtitle track either merges
**all** cues (meaning `CuesWithTiming.durationUs = C.TIME_UNSET` is not
allowed), or **every** cue is replaced by the next one (meaning
`CuesWithTiming.durationUs` may be set (to allow for cues to 'time out',
needed for CEA-608), or may be `TIME_UNSET`).

This value will be used in a subsequent change that adds cue-merging
support to `TextRenderer`.

PiperOrigin-RevId: 565028066

libraries/common/src/main/java/androidx/media3/common/Format.java

@@ -15,13 +15,20 @@
  */
 package androidx.media3.common;
 
+import static java.lang.annotation.ElementType.TYPE_USE;
+
 import android.os.Bundle;
+import androidx.annotation.IntDef;
 import androidx.annotation.Nullable;
 import androidx.media3.common.util.BundleableUtil;
 import androidx.media3.common.util.UnstableApi;
 import androidx.media3.common.util.Util;
 import com.google.common.base.Joiner;
 import com.google.errorprone.annotations.CanIgnoreReturnValue;
+import java.lang.annotation.Documented;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -172,6 +179,7 @@ public final class Format implements Bundleable {
     // Text specific.
 
     private int accessibilityChannel;
+    @UnstableApi private @CueReplacementBehavior int cueReplacementBehavior;
 
     // Image specific
 
@@ -201,6 +209,7 @@ public final class Format implements Bundleable {
       pcmEncoding = NO_VALUE;
       // Text specific.
       accessibilityChannel = NO_VALUE;
+      cueReplacementBehavior = CUE_REPLACEMENT_BEHAVIOR_MERGE;
       // Image specific.
       tileCountHorizontal = NO_VALUE;
       tileCountVertical = NO_VALUE;
@@ -248,6 +257,7 @@ public final class Format implements Bundleable {
       this.encoderPadding = format.encoderPadding;
       // Text specific.
       this.accessibilityChannel = format.accessibilityChannel;
+      this.cueReplacementBehavior = format.cueReplacementBehavior;
       // Image specific.
       this.tileCountHorizontal = format.tileCountHorizontal;
       this.tileCountVertical = format.tileCountVertical;
@@ -626,6 +636,19 @@ public final class Format implements Bundleable {
       return this;
     }
 
+    /**
+     * Sets {@link Format#cueReplacementBehavior}. The default value is {@link
+     * #CUE_REPLACEMENT_BEHAVIOR_MERGE}.
+     *
+     * @param cueReplacementBehavior The {@link Format.CueReplacementBehavior}.
+     * @return The builder.
+     */
+    @CanIgnoreReturnValue
+    public Builder setCueReplacementBehavior(@CueReplacementBehavior int cueReplacementBehavior) {
+      this.cueReplacementBehavior = cueReplacementBehavior;
+      return this;
+    }
+
     // Image specific.
 
     /**
@@ -673,6 +696,36 @@ public final class Format implements Bundleable {
     }
   }
 
+  /**
+   * The replacement behaviors for consecutive samples in a {@linkplain C#TRACK_TYPE_TEXT text
+   * track} of type {@link MimeTypes#APPLICATION_MEDIA3_CUES}.
+   */
+  @UnstableApi
+  @Documented
+  @Retention(RetentionPolicy.SOURCE)
+  @Target(TYPE_USE)
+  @IntDef({
+    CUE_REPLACEMENT_BEHAVIOR_MERGE,
+    CUE_REPLACEMENT_BEHAVIOR_REPLACE,
+  })
+  public @interface CueReplacementBehavior {}
+
+  /**
+   * Subsequent cues should be merged with any previous cues that should still be shown on screen.
+   *
+   * <p>Tracks with this behavior must not contain samples with an {@linkplain C#TIME_UNSET unset}
+   * duration.
+   */
+  @UnstableApi public static final int CUE_REPLACEMENT_BEHAVIOR_MERGE = 1;
+
+  /**
+   * Subsequent cues should replace all previous cues.
+   *
+   * <p>Tracks with this behavior may contain samples with an {@linkplain C#TIME_UNSET unset}
+   * duration (but the duration may also be set to a 'real' value).
+   */
+  @UnstableApi public static final int CUE_REPLACEMENT_BEHAVIOR_REPLACE = 2;
+
   /** A value for various fields to indicate that the field's value is unknown or not applicable. */
   public static final int NO_VALUE = -1;
 
@@ -847,6 +900,12 @@ public final class Format implements Bundleable {
   /** The Accessibility channel, or {@link #NO_VALUE} if not known or applicable. */
   @UnstableApi public final int accessibilityChannel;
 
+  /**
+   * The replacement behavior that should be followed when handling consecutive samples in a
+   * {@linkplain C#TRACK_TYPE_TEXT text track} of type {@link MimeTypes#APPLICATION_MEDIA3_CUES}.
+   */
+  @UnstableApi public final @CueReplacementBehavior int cueReplacementBehavior;
+
   // Image specific.
 
   /**
@@ -908,6 +967,7 @@ public final class Format implements Bundleable {
     encoderPadding = builder.encoderPadding == NO_VALUE ? 0 : builder.encoderPadding;
     // Text specific.
     accessibilityChannel = builder.accessibilityChannel;
+    cueReplacementBehavior = builder.cueReplacementBehavior;
     // Image specific.
     tileCountHorizontal = builder.tileCountHorizontal;
     tileCountVertical = builder.tileCountVertical;

libraries/extractor/src/main/java/androidx/media3/extractor/text/CuesWithTiming.java

@@ -17,6 +17,7 @@
 package androidx.media3.extractor.text;
 
 import androidx.media3.common.C;
+import androidx.media3.common.Format;
 import androidx.media3.common.text.Cue;
 import androidx.media3.common.util.UnstableApi;
 import com.google.common.collect.ImmutableList;
@@ -41,9 +42,12 @@ public class CuesWithTiming {
    * The duration for which {@link #cues} should be shown on screen, in microseconds, or {@link
    * C#TIME_UNSET} if not known.
    *
-   * <p>If this value is set then {@link #cues} from multiple instances may be shown on the screen
+   * <p>If {@link Format#cueReplacementBehavior} is {@link Format#CUE_REPLACEMENT_BEHAVIOR_MERGE}
-   * simultaneously (if their durations overlap). If this value is {@link C#TIME_UNSET} then {@link
+   * then cues from multiple instances will be shown on screen simultaneously if their start times
-   * #cues} should be shown on the screen until the {@link #startTimeUs} of the next instance.
+   * and durations overlap.
+   *
+   * <p>{@link C#TIME_UNSET} is only permitted if the {@link Format#cueReplacementBehavior} of the
+   * current track is {@link Format#CUE_REPLACEMENT_BEHAVIOR_REPLACE}.
    */
   public final long durationUs;
 

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleParser.java

@@ -16,9 +16,12 @@
 
 package androidx.media3.extractor.text;
 
+import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
+
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.Format;
+import androidx.media3.common.Format.CueReplacementBehavior;
 import androidx.media3.common.util.UnstableApi;
 import java.util.List;
 
@@ -95,4 +98,16 @@ public interface SubtitleParser {
    * <p>The default implementation is a no-op.
    */
   default void reset() {}
+
+  /**
+   * Returns the {@link CueReplacementBehavior} for consecutive {@link CuesWithTiming} emitted by
+   * this implementation.
+   *
+   * <p>A given instance must always return the same value from this method.
+   *
+   * <p>The default implementation returns {@link Format#CUE_REPLACEMENT_BEHAVIOR_MERGE}.
+   */
+  default @CueReplacementBehavior int getCueReplacementBehavior() {
+    return CUE_REPLACEMENT_BEHAVIOR_MERGE;
+  }
 }

libraries/extractor/src/main/java/androidx/media3/extractor/text/SubtitleTranscodingTrackOutput.java

@@ -26,6 +26,7 @@ import androidx.annotation.Nullable;
 import androidx.media3.common.C;
 import androidx.media3.common.DataReader;
 import androidx.media3.common.Format;
+import androidx.media3.common.Format.CueReplacementBehavior;
 import androidx.media3.common.MimeTypes;
 import androidx.media3.common.util.ParsableByteArray;
 import androidx.media3.common.util.Util;
@@ -86,6 +87,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     if (currentSubtitleParser == null) {
       delegate.format(format);
     } else {
+      @CueReplacementBehavior
+      int nextCuesBehavior = currentSubtitleParser.getCueReplacementBehavior();
       delegate.format(
           format
               .buildUpon()
@@ -94,6 +97,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
               // Reset this value to the default. All non-default timestamp adjustments are done
               // below in sampleMetadata() and there are no 'subsamples' after transcoding.
               .setSubsampleOffsetUs(Format.OFFSET_SAMPLE_RELATIVE)
+              .setCueReplacementBehavior(nextCuesBehavior)
               .build());
     }
   }

libraries/extractor/src/main/java/androidx/media3/extractor/text/dvb/DvbParser.java

@@ -26,6 +26,8 @@ import android.graphics.PorterDuffXfermode;
 import android.util.SparseArray;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.Format.CueReplacementBehavior;
 import androidx.media3.common.text.Cue;
 import androidx.media3.common.util.Log;
 import androidx.media3.common.util.ParsableBitArray;
@@ -128,6 +130,11 @@ public final class DvbParser implements SubtitleParser {
     subtitleService.reset();
   }
 
+  @Override
+  public @CueReplacementBehavior int getCueReplacementBehavior() {
+    return Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
+  }
+
   @Override
   public ImmutableList<CuesWithTiming> parse(byte[] data, int offset, int length) {
     ParsableBitArray dataBitArray = new ParsableBitArray(data, /* limit= */ offset + length);

libraries/extractor/src/main/java/androidx/media3/extractor/text/pgs/PgsParser.java

@@ -20,6 +20,8 @@ import static java.lang.Math.min;
 import android.graphics.Bitmap;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.Format.CueReplacementBehavior;
 import androidx.media3.common.text.Cue;
 import androidx.media3.common.util.ParsableByteArray;
 import androidx.media3.common.util.UnstableApi;
@@ -53,6 +55,11 @@ public final class PgsParser implements SubtitleParser {
     cueBuilder = new CueBuilder();
   }
 
+  @Override
+  public @CueReplacementBehavior int getCueReplacementBehavior() {
+    return Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
+  }
+
   @Override
   public ImmutableList<CuesWithTiming> parse(byte[] data, int offset, int length) {
     buffer.reset(data, /* limit= */ offset + length);

libraries/extractor/src/main/java/androidx/media3/extractor/text/tx3g/Tx3gParser.java

@@ -29,6 +29,8 @@ import android.text.style.TypefaceSpan;
 import android.text.style.UnderlineSpan;
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.Format.CueReplacementBehavior;
 import androidx.media3.common.text.Cue;
 import androidx.media3.common.util.Log;
 import androidx.media3.common.util.ParsableByteArray;
@@ -122,6 +124,11 @@ public final class Tx3gParser implements SubtitleParser {
     }
   }
 
+  @Override
+  public @CueReplacementBehavior int getCueReplacementBehavior() {
+    return Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
+  }
+
   @Override
   public ImmutableList<CuesWithTiming> parse(byte[] data, int offset, int length) {
     parsableByteArray.reset(data, /* limit= */ offset + length);

libraries/extractor/src/main/java/androidx/media3/extractor/text/webvtt/Mp4WebvttParser.java

@@ -19,6 +19,8 @@ import static androidx.media3.common.util.Assertions.checkArgument;
 
 import androidx.annotation.Nullable;
 import androidx.media3.common.C;
+import androidx.media3.common.Format;
+import androidx.media3.common.Format.CueReplacementBehavior;
 import androidx.media3.common.text.Cue;
 import androidx.media3.common.util.ParsableByteArray;
 import androidx.media3.common.util.UnstableApi;
@@ -52,6 +54,11 @@ public final class Mp4WebvttParser implements SubtitleParser {
     parsableByteArray = new ParsableByteArray();
   }
 
+  @Override
+  public @CueReplacementBehavior int getCueReplacementBehavior() {
+    return Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
+  }
+
   @Override
   public ImmutableList<CuesWithTiming> parse(byte[] data, int offset, int length) {
     parsableByteArray.reset(data, /* limit= */ offset + length);

libraries/extractor/src/test/java/androidx/media3/extractor/text/ssa/SsaParserTest.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.extractor.text.ssa;
 
+import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
 import static com.google.common.truth.Truth.assertThat;
 import static com.google.common.truth.Truth.assertWithMessage;
 
@@ -60,6 +61,12 @@ public final class SsaParserTest {
   private static final String STYLE_UNDERLINE = "media/ssa/style_underline";
   private static final String STYLE_STRIKEOUT = "media/ssa/style_strikeout";
 
+  @Test
+  public void cuesReplacementBehaviorIsMerge() throws IOException {
+    SsaParser parser = new SsaParser();
+    assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_MERGE);
+  }
+
   @Test
   public void parseEmpty() throws IOException {
     SsaParser parser = new SsaParser();

libraries/extractor/src/test/java/androidx/media3/extractor/text/subrip/SubripParserTest.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.extractor.text.subrip;
 
+import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
 import static com.google.common.truth.Truth.assertThat;
 
 import androidx.media3.common.text.Cue;
@@ -48,6 +49,12 @@ public final class SubripParserTest {
   private static final String TYPICAL_NO_HOURS_AND_MILLIS =
       "media/subrip/typical_no_hours_and_millis";
 
+  @Test
+  public void cueReplacementBehaviorIsMerge() {
+    SubripParser parser = new SubripParser();
+    assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_MERGE);
+  }
+
   @Test
   public void parseEmpty() throws IOException {
     SubripParser parser = new SubripParser();

libraries/extractor/src/test/java/androidx/media3/extractor/text/tx3g/Tx3gParserTest.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.extractor.text.tx3g;
 
+import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
 import static androidx.media3.test.utils.truth.SpannedSubject.assertThat;
 import static com.google.common.truth.Truth.assertThat;
 
@@ -55,6 +56,12 @@ public final class Tx3gParserTest {
   private static final String INITIALIZATION_ALL_DEFAULTS =
       "media/tx3g/initialization_all_defaults";
 
+  @Test
+  public void cueReplacementBehaviorIsReplace() {
+    Tx3gParser parser = new Tx3gParser(/* initializationData= */ ImmutableList.of());
+    assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_REPLACE);
+  }
+
   @Test
   public void parseNoSubtitle() throws Exception {
     Tx3gParser parser = new Tx3gParser(ImmutableList.of());

libraries/extractor/src/test/java/androidx/media3/extractor/text/webvtt/Mp4WebvttParserTest.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.extractor.text.webvtt;
 
+import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_REPLACE;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
 
@@ -159,6 +160,12 @@ public final class Mp4WebvttParserTest {
 
   @Rule public final Expect expect = Expect.create();
 
+  @Test
+  public void cueReplacementBehaviorIsReplace() {
+    Mp4WebvttParser parser = new Mp4WebvttParser();
+    assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_REPLACE);
+  }
+
   // Positive tests.
 
   @Test

libraries/extractor/src/test/java/androidx/media3/extractor/text/webvtt/WebvttParserTest.java

@@ -15,6 +15,7 @@
  */
 package androidx.media3.extractor.text.webvtt;
 
+import static androidx.media3.common.Format.CUE_REPLACEMENT_BEHAVIOR_MERGE;
 import static androidx.media3.test.utils.truth.SpannedSubject.assertThat;
 import static com.google.common.truth.Truth.assertThat;
 import static org.junit.Assert.assertThrows;
@@ -65,6 +66,12 @@ public class WebvttParserTest {
 
   @Rule public final Expect expect = Expect.create();
 
+  @Test
+  public void cueReplacementBehaviorIsMerge() throws IOException {
+    WebvttParser parser = new WebvttParser();
+    assertThat(parser.getCueReplacementBehavior()).isEqualTo(CUE_REPLACEMENT_BEHAVIOR_MERGE);
+  }
+
   @Test
   public void parseEmpty() throws IOException {
     WebvttParser parser = new WebvttParser();