Reading average and peak bitrates from esds boxes.

This provides better compatibility with MediaExtractor, which does read these fields; we also need them for being able to mux file contents into another mp4 file.

Also, there is a minor refactor included so that we have an actual type for esds box contents instead of a pair.

PiperOrigin-RevId: 438673825

RELEASENOTES.md

@@ -24,6 +24,7 @@
         Media Rating Council (MRC) recommendations.
 *   Extractors:
     *   Matroska: Parse `DiscardPadding` for Opus tracks.
+    *   Parse bitrates from `esds` boxes.
 *   UI:
     *   Fix delivery of events to `OnClickListener`s set on `PlayerView` and
         `LegacyPlayerView`, in the case that `useController=false`

libraries/common/src/main/java/androidx/media3/common/util/MediaFormatUtil.java

@@ -47,6 +47,19 @@ public final class MediaFormatUtil {
   // The constant value must not be changed, because it's also set by the framework MediaParser API.
   public static final String KEY_PCM_ENCODING_EXTENDED = "exo-pcm-encoding-int";
 
+  /**
+   * The {@link MediaFormat} key for the maximum bitrate in bits per second.
+   *
+   * <p>The associated value is an integer.
+   *
+   * <p>The key string constant is the same as {@code MediaFormat#KEY_MAX_BITRATE}. Values for it
+   * are already returned by the framework MediaExtractor; the key is a hidden field in {@code
+   * MediaFormat} though, which is why it's being replicated here.
+   */
+  // The constant value must not be changed, because it's also set by the framework MediaParser and
+  // MediaExtractor APIs.
+  public static final String KEY_MAX_BIT_RATE = "max-bitrate";
+
   private static final int MAX_POWER_OF_TWO_INT = 1 << 30;
 
   /**
@@ -63,6 +76,7 @@ public final class MediaFormatUtil {
   public static MediaFormat createMediaFormatFromFormat(Format format) {
     MediaFormat result = new MediaFormat();
     maybeSetInteger(result, MediaFormat.KEY_BIT_RATE, format.bitrate);
+    maybeSetInteger(result, KEY_MAX_BIT_RATE, format.peakBitrate);
     maybeSetInteger(result, MediaFormat.KEY_CHANNEL_COUNT, format.channelCount);
 
     maybeSetColorInfo(result, format.colorInfo);

libraries/extractor/src/main/java/androidx/media3/extractor/mp4/AtomParsers.java

@@ -1116,6 +1116,7 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
     @Nullable String codecs = null;
     @Nullable byte[] projectionData = null;
     @C.StereoMode int stereoMode = Format.NO_VALUE;
+    @Nullable EsdsData esdsData = null;
 
     // HDR related metadata.
     @C.ColorSpace int colorSpace = Format.NO_VALUE;
@@ -1210,10 +1211,9 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
         mimeType = MimeTypes.VIDEO_H263;
       } else if (childAtomType == Atom.TYPE_esds) {
         ExtractorUtil.checkContainerInput(mimeType == null, /* message= */ null);
-        Pair<@NullableType String, byte @NullableType []> mimeTypeAndInitializationDataBytes =
+        esdsData = parseEsdsFromParent(parent, childStartPosition);
-            parseEsdsFromParent(parent, childStartPosition);
+        mimeType = esdsData.mimeType;
-        mimeType = mimeTypeAndInitializationDataBytes.first;
+        @Nullable byte[] initializationDataBytes = esdsData.initializationData;
-        @Nullable byte[] initializationDataBytes = mimeTypeAndInitializationDataBytes.second;
         if (initializationDataBytes != null) {
           initializationData = ImmutableList.of(initializationDataBytes);
         }
@@ -1301,6 +1301,11 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
               colorTransfer,
               hdrStaticInfo != null ? hdrStaticInfo.array() : null));
     }
+
+    if (esdsData != null) {
+      formatBuilder.setAverageBitrate(esdsData.bitrate).setPeakBitrate(esdsData.peakBitrate);
+    }
+
     out.format = formatBuilder.build();
   }
 
@@ -1391,6 +1396,7 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
     int sampleRateMlp = 0;
     @C.PcmEncoding int pcmEncoding = Format.NO_VALUE;
     @Nullable String codecs = null;
+    @Nullable EsdsData esdsData = null;
 
     if (quickTimeSoundDescriptionVersion == 0 || quickTimeSoundDescriptionVersion == 1) {
       channelCount = parent.readUnsignedShort();
@@ -1507,10 +1513,9 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
                 ? childPosition
                 : findBoxPosition(parent, Atom.TYPE_esds, childPosition, childAtomSize);
         if (esdsAtomPosition != C.POSITION_UNSET) {
-          Pair<@NullableType String, byte @NullableType []> mimeTypeAndInitializationData =
+          esdsData = parseEsdsFromParent(parent, esdsAtomPosition);
-              parseEsdsFromParent(parent, esdsAtomPosition);
+          mimeType = esdsData.mimeType;
-          mimeType = mimeTypeAndInitializationData.first;
+          @Nullable byte[] initializationDataBytes = esdsData.initializationData;
-          @Nullable byte[] initializationDataBytes = mimeTypeAndInitializationData.second;
           if (initializationDataBytes != null) {
             if (MimeTypes.AUDIO_AAC.equals(mimeType)) {
               // Update sampleRate and channelCount from the AudioSpecificConfig initialization
@@ -1591,7 +1596,7 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
     }
 
     if (out.format == null && mimeType != null) {
-      out.format =
+      Format.Builder formatBuilder =
           new Format.Builder()
               .setId(trackId)
               .setSampleMimeType(mimeType)
@@ -1601,8 +1606,13 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
               .setPcmEncoding(pcmEncoding)
               .setInitializationData(initializationData)
               .setDrmInitData(drmInitData)
-              .setLanguage(language)
+              .setLanguage(language);
-              .build();
+
+      if (esdsData != null) {
+        formatBuilder.setAverageBitrate(esdsData.bitrate).setPeakBitrate(esdsData.peakBitrate);
+      }
+
+      out.format = formatBuilder.build();
     }
   }
 
@@ -1637,8 +1647,7 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
   }
 
   /** Returns codec-specific initialization data contained in an esds box. */
-  private static Pair<@NullableType String, byte @NullableType []> parseEsdsFromParent(
+  private static EsdsData parseEsdsFromParent(ParsableByteArray parent, int position) {
-      ParsableByteArray parent, int position) {
     parent.setPosition(position + Atom.HEADER_SIZE + 4);
     // Start of the ES_Descriptor (defined in ISO/IEC 14496-1)
     parent.skipBytes(1); // ES_Descriptor tag
@@ -1666,17 +1675,29 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
     if (MimeTypes.AUDIO_MPEG.equals(mimeType)
         || MimeTypes.AUDIO_DTS.equals(mimeType)
         || MimeTypes.AUDIO_DTS_HD.equals(mimeType)) {
-      return Pair.create(mimeType, null);
+      return new EsdsData(
+          mimeType,
+          /* initializationData= */ null,
+          /* bitrate= */ Format.NO_VALUE,
+          /* peakBitrate= */ Format.NO_VALUE);
     }
 
-    parent.skipBytes(12);
+    parent.skipBytes(4);
+    int peakBitrate = parent.readUnsignedIntToInt();
+    int bitrate = parent.readUnsignedIntToInt();
 
     // Start of the DecoderSpecificInfo.
     parent.skipBytes(1); // DecoderSpecificInfo tag
     int initializationDataSize = parseExpandableClassSize(parent);
     byte[] initializationData = new byte[initializationDataSize];
     parent.readBytes(initializationData, 0, initializationDataSize);
-    return Pair.create(mimeType, initializationData);
+
+    // Skipping zero values as unknown.
+    return new EsdsData(
+        mimeType,
+        /* initializationData= */ initializationData,
+        /* bitrate= */ bitrate > 0 ? bitrate : Format.NO_VALUE,
+        /* peakBitrate= */ peakBitrate > 0 ? peakBitrate : Format.NO_VALUE);
   }
 
   /**
@@ -1918,6 +1939,25 @@ import org.checkerframework.checker.nullness.compatqual.NullableType;
     }
   }
 
+  /** Data parsed from an esds box. */
+  private static final class EsdsData {
+    private final @NullableType String mimeType;
+    private final byte @NullableType [] initializationData;
+    private final int bitrate;
+    private final int peakBitrate;
+
+    public EsdsData(
+        @NullableType String mimeType,
+        byte @NullableType [] initializationData,
+        int bitrate,
+        int peakBitrate) {
+      this.mimeType = mimeType;
+      this.initializationData = initializationData;
+      this.bitrate = bitrate;
+      this.peakBitrate = peakBitrate;
+    }
+  }
+
   /** A box containing sample sizes (e.g. stsz, stz2). */
   private interface SampleSizeBox {
 

libraries/test_data/src/test/assets/extractordumps/mp4/sample.mp4.0.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 9529
   sample count = 45
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample.mp4.1.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 7464
   sample count = 33
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample.mp4.2.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 4019
   sample count = 18
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample.mp4.3.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 470
   sample count = 3
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample.mp4.unknown_length.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 9529
   sample count = 45
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented.mp4.0.dump

@@ -139,6 +139,7 @@ track 1:
   total output bytes = 18257
   sample count = 46
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented.mp4.unknown_length.dump

@@ -139,6 +139,7 @@ track 1:
   total output bytes = 18257
   sample count = 46
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented_seekable.mp4.0.dump

@@ -142,6 +142,8 @@ track 1:
   total output bytes = 18257
   sample count = 46
   format 0:
+    averageBitrate = 136736
+    peakBitrate = 145976
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented_seekable.mp4.1.dump

@@ -142,6 +142,8 @@ track 1:
   total output bytes = 13359
   sample count = 31
   format 0:
+    averageBitrate = 136736
+    peakBitrate = 145976
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented_seekable.mp4.2.dump

@@ -142,6 +142,8 @@ track 1:
   total output bytes = 6804
   sample count = 16
   format 0:
+    averageBitrate = 136736
+    peakBitrate = 145976
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented_seekable.mp4.3.dump

@@ -142,6 +142,8 @@ track 1:
   total output bytes = 10
   sample count = 1
   format 0:
+    averageBitrate = 136736
+    peakBitrate = 145976
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented_seekable.mp4.unknown_length.dump

@@ -142,6 +142,8 @@ track 1:
   total output bytes = 18257
   sample count = 46
   format 0:
+    averageBitrate = 136736
+    peakBitrate = 145976
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented_sei.mp4.0.dump

@@ -139,6 +139,7 @@ track 1:
   total output bytes = 18257
   sample count = 46
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_fragmented_sei.mp4.unknown_length.dump

@@ -139,6 +139,7 @@ track 1:
   total output bytes = 18257
   sample count = 46
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_mdat_too_long.mp4.0.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 9529
   sample count = 45
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_mdat_too_long.mp4.1.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 7464
   sample count = 33
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_mdat_too_long.mp4.2.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 4019
   sample count = 18
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_mdat_too_long.mp4.3.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 470
   sample count = 3
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_mdat_too_long.mp4.unknown_length.dump

@@ -144,6 +144,7 @@ track 1:
   total output bytes = 9529
   sample count = 45
   format 0:
+    peakBitrate = 200000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_partially_fragmented.mp4.0.dump

@@ -139,6 +139,8 @@ track 1:
   total output bytes = 10107
   sample count = 45
   format 0:
+    averageBitrate = 1254
+    peakBitrate = 69000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_partially_fragmented.mp4.unknown_length.dump

@@ -139,6 +139,8 @@ track 1:
   total output bytes = 10107
   sample count = 45
   format 0:
+    averageBitrate = 1254
+    peakBitrate = 69000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_color_info.mp4.0.dump

@@ -65,6 +65,7 @@ track 1:
   total output bytes = 5993
   sample count = 15
   format 0:
+    peakBitrate = 192000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_color_info.mp4.1.dump

@@ -65,6 +65,7 @@ track 1:
   total output bytes = 4794
   sample count = 11
   format 0:
+    peakBitrate = 192000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_color_info.mp4.2.dump

@@ -65,6 +65,7 @@ track 1:
   total output bytes = 3001
   sample count = 7
   format 0:
+    peakBitrate = 192000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_color_info.mp4.3.dump

@@ -65,6 +65,7 @@ track 1:
   total output bytes = 1074
   sample count = 3
   format 0:
+    peakBitrate = 192000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_color_info.mp4.unknown_length.dump

@@ -65,6 +65,7 @@ track 1:
   total output bytes = 5993
   sample count = 15
   format 0:
+    peakBitrate = 192000
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.0.dump

@@ -265,6 +265,8 @@ track 1:
   total output bytes = 16638
   sample count = 44
   format 0:
+    averageBitrate = 130279
+    peakBitrate = 130279
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.1.dump

@@ -265,6 +265,8 @@ track 1:
   total output bytes = 11156
   sample count = 30
   format 0:
+    averageBitrate = 130279
+    peakBitrate = 130279
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.2.dump

@@ -265,6 +265,8 @@ track 1:
   total output bytes = 5567
   sample count = 15
   format 0:
+    averageBitrate = 130279
+    peakBitrate = 130279
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.3.dump

@@ -265,6 +265,8 @@ track 1:
   total output bytes = 374
   sample count = 1
   format 0:
+    averageBitrate = 130279
+    peakBitrate = 130279
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/extractordumps/mp4/sample_with_colr_mdcv_and_clli.mp4.unknown_length.dump

@@ -265,6 +265,8 @@ track 1:
   total output bytes = 16638
   sample count = 44
   format 0:
+    averageBitrate = 130279
+    peakBitrate = 130279
     id = 2
     sampleMimeType = audio/mp4a-latm
     codecs = mp4a.40.2

libraries/test_data/src/test/assets/transformerdumps/mp4/sample.mp4.dump

@@ -1,5 +1,6 @@
 containerMimeType = video/mp4
 format 0:
+  peakBitrate = 200000
   id = 2
   sampleMimeType = audio/mp4a-latm
   codecs = mp4a.40.2

libraries/test_data/src/test/assets/transformerdumps/mp4/sample.mp4.novideo.dump

@@ -1,5 +1,6 @@
 containerMimeType = video/mp4
 format 0:
+  peakBitrate = 200000
   id = 2
   sampleMimeType = audio/mp4a-latm
   codecs = mp4a.40.2