Merge pull request #10793 from fraunhoferfokus:dash-thumbnail-support

PiperOrigin-RevId: 506261584

RELEASENOTES.md

@@ -37,6 +37,9 @@
         `Subtitle.getEventTime` if a subtitle file contains no cues.
     *   SubRip: Add support for UTF-16 files if they start with a byte order
         mark.
+*   DASH:
+    *   Add full parsing for image adaptation sets, including tile counts
+        ([#3752](https://github.com/google/ExoPlayer/issues/3752)).
 *   UI:
     *   Fix the deprecated
         `PlayerView.setControllerVisibilityListener(PlayerControlView.VisibilityListener)`

libraries/common/src/main/java/androidx/media3/common/Format.java

@@ -105,6 +105,13 @@ import java.util.UUID;
  * <ul>
  *   <li>{@link #accessibilityChannel}
  * </ul>
+ *
+ * <h2 id="image-formats">Fields relevant to image formats</h2>
+ *
+ * <ul>
+ *   <li>{@link #tileCountHorizontal}
+ *   <li>{@link #tileCountVertical}
+ * </ul>
  */
 public final class Format implements Bundleable {
 
@@ -165,6 +172,11 @@ public final class Format implements Bundleable {
 
     private int accessibilityChannel;
 
+    // Image specific
+
+    private int tileCountHorizontal;
+    private int tileCountVertical;
+
     // Provided by the source.
 
     private @C.CryptoType int cryptoType;
@@ -188,6 +200,9 @@ public final class Format implements Bundleable {
       pcmEncoding = NO_VALUE;
       // Text specific.
       accessibilityChannel = NO_VALUE;
+      // Image specific.
+      tileCountHorizontal = NO_VALUE;
+      tileCountVertical = NO_VALUE;
       // Provided by the source.
       cryptoType = C.CRYPTO_TYPE_NONE;
     }
@@ -232,6 +247,9 @@ public final class Format implements Bundleable {
       this.encoderPadding = format.encoderPadding;
       // Text specific.
       this.accessibilityChannel = format.accessibilityChannel;
+      // Image specific.
+      this.tileCountHorizontal = format.tileCountHorizontal;
+      this.tileCountVertical = format.tileCountVertical;
       // Provided by the source.
       this.cryptoType = format.cryptoType;
     }
@@ -607,6 +625,32 @@ public final class Format implements Bundleable {
       return this;
     }
 
+    // Image specific.
+
+    /**
+     * Sets {@link Format#tileCountHorizontal}. The default value is {@link #NO_VALUE}.
+     *
+     * @param tileCountHorizontal The {@link Format#accessibilityChannel}.
+     * @return The builder.
+     */
+    @CanIgnoreReturnValue
+    public Builder setTileCountHorizontal(int tileCountHorizontal) {
+      this.tileCountHorizontal = tileCountHorizontal;
+      return this;
+    }
+
+    /**
+     * Sets {@link Format#tileCountVertical}. The default value is {@link #NO_VALUE}.
+     *
+     * @param tileCountVertical The {@link Format#accessibilityChannel}.
+     * @return The builder.
+     */
+    @CanIgnoreReturnValue
+    public Builder setTileCountVertical(int tileCountVertical) {
+      this.tileCountVertical = tileCountVertical;
+      return this;
+    }
+
     // Provided by source.
 
     /**
@@ -779,6 +823,15 @@ public final class Format implements Bundleable {
   /** The Accessibility channel, or {@link #NO_VALUE} if not known or applicable. */
   @UnstableApi public final int accessibilityChannel;
 
+  // Image specific.
+
+  /**
+   * The number of horizontal tiles in an image, or {@link #NO_VALUE} if not known or applicable.
+   */
+  @UnstableApi public final int tileCountHorizontal;
+  /** The number of vertical tiles in an image, or {@link #NO_VALUE} if not known or applicable. */
+  @UnstableApi public final int tileCountVertical;
+
   // Provided by source.
 
   /**
@@ -1008,6 +1061,9 @@ public final class Format implements Bundleable {
     encoderPadding = builder.encoderPadding == NO_VALUE ? 0 : builder.encoderPadding;
     // Text specific.
     accessibilityChannel = builder.accessibilityChannel;
+    // Image specific.
+    tileCountHorizontal = builder.tileCountHorizontal;
+    tileCountVertical = builder.tileCountVertical;
     // Provided by source.
     if (builder.cryptoType == C.CRYPTO_TYPE_NONE && drmInitData != null) {
       // Encrypted content cannot use CRYPTO_TYPE_NONE.
@@ -1268,6 +1324,9 @@ public final class Format implements Bundleable {
       result = 31 * result + encoderPadding;
       // Text specific.
       result = 31 * result + accessibilityChannel;
+      // Image specific.
+      result = 31 * result + tileCountHorizontal;
+      result = 31 * result + tileCountVertical;
       // Provided by the source.
       result = 31 * result + cryptoType;
       hashCode = result;
@@ -1304,6 +1363,8 @@ public final class Format implements Bundleable {
         && encoderDelay == other.encoderDelay
         && encoderPadding == other.encoderPadding
         && accessibilityChannel == other.accessibilityChannel
+        && tileCountHorizontal == other.tileCountHorizontal
+        && tileCountVertical == other.tileCountVertical
         && cryptoType == other.cryptoType
         && Float.compare(frameRate, other.frameRate) == 0
         && Float.compare(pixelWidthHeightRatio, other.pixelWidthHeightRatio) == 0
@@ -1500,6 +1561,8 @@ public final class Format implements Bundleable {
   private static final String FIELD_ENCODER_PADDING = Util.intToStringMaxRadix(27);
   private static final String FIELD_ACCESSIBILITY_CHANNEL = Util.intToStringMaxRadix(28);
   private static final String FIELD_CRYPTO_TYPE = Util.intToStringMaxRadix(29);
+  private static final String FIELD_TILE_COUNT_HORIZONTAL = Util.intToStringMaxRadix(30);
+  private static final String FIELD_TILE_COUNT_VERTICAL = Util.intToStringMaxRadix(31);
 
   @UnstableApi
   @Override
@@ -1557,6 +1620,9 @@ public final class Format implements Bundleable {
     bundle.putInt(FIELD_ENCODER_PADDING, encoderPadding);
     // Text specific.
     bundle.putInt(FIELD_ACCESSIBILITY_CHANNEL, accessibilityChannel);
+    // Image specific.
+    bundle.putInt(FIELD_TILE_COUNT_HORIZONTAL, tileCountHorizontal);
+    bundle.putInt(FIELD_TILE_COUNT_VERTICAL, tileCountVertical);
     // Source specific.
     bundle.putInt(FIELD_CRYPTO_TYPE, cryptoType);
     return bundle;
@@ -1621,6 +1687,10 @@ public final class Format implements Bundleable {
         // Text specific.
         .setAccessibilityChannel(
             bundle.getInt(FIELD_ACCESSIBILITY_CHANNEL, DEFAULT.accessibilityChannel))
+        // Image specific.
+        .setTileCountHorizontal(
+            bundle.getInt(FIELD_TILE_COUNT_HORIZONTAL, DEFAULT.tileCountHorizontal))
+        .setTileCountVertical(bundle.getInt(FIELD_TILE_COUNT_VERTICAL, DEFAULT.tileCountVertical))
         // Source specific.
         .setCryptoType(bundle.getInt(FIELD_CRYPTO_TYPE, DEFAULT.cryptoType));
 

libraries/common/src/test/java/androidx/media3/common/FormatTest.java

@@ -111,6 +111,8 @@ public final class FormatTest {
         .setEncoderPadding(1002)
         .setAccessibilityChannel(2)
         .setCryptoType(C.CRYPTO_TYPE_CUSTOM_BASE)
+        .setTileCountHorizontal(20)
+        .setTileCountVertical(40)
         .build();
   }
 

libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/DashMediaSource.java

@@ -1058,9 +1058,11 @@ public final class DashMediaSource extends BaseMediaSource {
     for (int i = 0; i < period.adaptationSets.size(); i++) {
       AdaptationSet adaptationSet = period.adaptationSets.get(i);
       List<Representation> representations = adaptationSet.representations;
-      // Exclude text adaptation sets from duration calculations, if we have at least one audio
+      // Exclude other adaptation sets from duration calculations, if we have at least one audio or
-      // or video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029
+      // video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029.
-      if ((haveAudioVideoAdaptationSets && adaptationSet.type == C.TRACK_TYPE_TEXT)
+      boolean adaptationSetIsNotAudioVideo =
+          adaptationSet.type != C.TRACK_TYPE_AUDIO && adaptationSet.type != C.TRACK_TYPE_VIDEO;
+      if ((haveAudioVideoAdaptationSets && adaptationSetIsNotAudioVideo)
           || representations.isEmpty()) {
         continue;
       }
@@ -1090,9 +1092,11 @@ public final class DashMediaSource extends BaseMediaSource {
     for (int i = 0; i < period.adaptationSets.size(); i++) {
       AdaptationSet adaptationSet = period.adaptationSets.get(i);
       List<Representation> representations = adaptationSet.representations;
-      // Exclude text adaptation sets from duration calculations, if we have at least one audio
+      // Exclude other adaptation sets from duration calculations, if we have at least one audio or
-      // or video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029
+      // video adaptation set. See: https://github.com/google/ExoPlayer/issues/4029
-      if ((haveAudioVideoAdaptationSets && adaptationSet.type == C.TRACK_TYPE_TEXT)
+      boolean adaptationSetIsNotAudioVideo =
+          adaptationSet.type != C.TRACK_TYPE_AUDIO && adaptationSet.type != C.TRACK_TYPE_VIDEO;
+      if ((haveAudioVideoAdaptationSets && adaptationSetIsNotAudioVideo)
           || representations.isEmpty()) {
         continue;
       }

libraries/exoplayer_dash/src/main/java/androidx/media3/exoplayer/dash/manifest/DashManifestParser.java

@@ -557,7 +557,9 @@ public class DashManifestParser extends DefaultHandler
                 ? C.TRACK_TYPE_VIDEO
                 : MimeTypes.BASE_TYPE_TEXT.equals(contentType)
                     ? C.TRACK_TYPE_TEXT
-                    : C.TRACK_TYPE_UNKNOWN;
+                    : MimeTypes.BASE_TYPE_IMAGE.equals(contentType)
+                        ? C.TRACK_TYPE_IMAGE
+                        : C.TRACK_TYPE_UNKNOWN;
   }
 
   /**
@@ -810,6 +812,7 @@ public class DashManifestParser extends DefaultHandler
     roleFlags |= parseRoleFlagsFromAccessibilityDescriptors(accessibilityDescriptors);
     roleFlags |= parseRoleFlagsFromProperties(essentialProperties);
     roleFlags |= parseRoleFlagsFromProperties(supplementalProperties);
+    @Nullable Pair<Integer, Integer> tileCounts = parseTileCountFromProperties(essentialProperties);
 
     Format.Builder formatBuilder =
         new Format.Builder()
@@ -820,7 +823,9 @@ public class DashManifestParser extends DefaultHandler
             .setPeakBitrate(bitrate)
             .setSelectionFlags(selectionFlags)
             .setRoleFlags(roleFlags)
-            .setLanguage(language);
+            .setLanguage(language)
+            .setTileCountHorizontal(tileCounts != null ? tileCounts.first : Format.NO_VALUE)
+            .setTileCountVertical(tileCounts != null ? tileCounts.second : Format.NO_VALUE);
 
     if (MimeTypes.isVideo(sampleMimeType)) {
       formatBuilder.setWidth(width).setHeight(height).setFrameRate(frameRate);
@@ -1629,6 +1634,41 @@ public class DashManifestParser extends DefaultHandler
     return attributeValue.split(",");
   }
 
+  // Thumbnail tile information parsing
+
+  /**
+   * Parses given descriptors for thumbnail tile information.
+   *
+   * @param essentialProperties List of descriptors that contain thumbnail tile information.
+   * @return A pair of Integer values, where the first is the count of horizontal tiles and the
+   *     second is the count of vertical tiles, or null if no thumbnail tile information is found.
+   */
+  @Nullable
+  protected Pair<Integer, Integer> parseTileCountFromProperties(
+      List<Descriptor> essentialProperties) {
+    for (int i = 0; i < essentialProperties.size(); i++) {
+      Descriptor descriptor = essentialProperties.get(i);
+      if ((Ascii.equalsIgnoreCase("http://dashif.org/thumbnail_tile", descriptor.schemeIdUri)
+              || Ascii.equalsIgnoreCase(
+                  "http://dashif.org/guidelines/thumbnail_tile", descriptor.schemeIdUri))
+          && descriptor.value != null) {
+        String size = descriptor.value;
+        String[] sizeSplit = Util.split(size, "x");
+        if (sizeSplit.length != 2) {
+          continue;
+        }
+        try {
+          int tileCountHorizontal = Integer.parseInt(sizeSplit[0]);
+          int tileCountVertical = Integer.parseInt(sizeSplit[1]);
+          return Pair.create(tileCountHorizontal, tileCountVertical);
+        } catch (NumberFormatException e) {
+          // Ignore property if it's malformed.
+        }
+      }
+    }
+    return null;
+  }
+
   // Utility methods.
 
   /**

libraries/exoplayer_dash/src/test/java/androidx/media3/exoplayer/dash/manifest/DashManifestParserTest.java

@@ -252,11 +252,19 @@ public class DashManifestParserTest {
                 ApplicationProvider.getApplicationContext(), SAMPLE_MPD_IMAGES));
 
     AdaptationSet adaptationSet = manifest.getPeriod(0).adaptationSets.get(0);
-    Format format = adaptationSet.representations.get(0).format;
+    Format format0 = adaptationSet.representations.get(0).format;
+    Format format1 = adaptationSet.representations.get(1).format;
 
-    assertThat(format.sampleMimeType).isEqualTo("image/jpeg");
+    assertThat(format0.sampleMimeType).isEqualTo("image/jpeg");
-    assertThat(format.width).isEqualTo(320);
+    assertThat(format0.width).isEqualTo(320);
-    assertThat(format.height).isEqualTo(180);
+    assertThat(format0.height).isEqualTo(180);
+    assertThat(format0.tileCountHorizontal).isEqualTo(12);
+    assertThat(format0.tileCountVertical).isEqualTo(16);
+    assertThat(format1.sampleMimeType).isEqualTo("image/jpeg");
+    assertThat(format1.width).isEqualTo(640);
+    assertThat(format1.height).isEqualTo(360);
+    assertThat(format1.tileCountHorizontal).isEqualTo(2);
+    assertThat(format1.tileCountVertical).isEqualTo(4);
   }
 
   @Test

libraries/test_data/src/test/assets/media/mpd/sample_mpd_images

@@ -4,7 +4,10 @@
   <AdaptationSet id="3" mimeType="image/jpeg" contentType="image">
     <SegmentTemplate media="$RepresentationID$/tile_$Number$.jpg" duration="100" startNumber="1"/>
     <Representation bandwidth="1234" id="images_320x180" width="320" height="180">
-      <EssentialProperty schemeIdUri="http://dashif.org/thumbnail_tile" value="title"/>
+      <EssentialProperty schemeIdUri="http://dashif.org/thumbnail_tile" value="12x16"/>
+    </Representation>
+    <Representation bandwidth="2345" id="images_640x360" width="640" height="360">
+      <EssentialProperty schemeIdUri="http://dashif.org/guidelines/thumbnail_tile" value="2x4"/>
     </Representation>
   </AdaptationSet>
  </Period>