Support seeking based on MLLT metadata

Issue: #3241

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=217252254

RELEASENOTES.md

@@ -21,6 +21,9 @@
   ([#4788](https://github.com/google/ExoPlayer/issues/4788)).
 * SubRip: Add support for alignment tags, and remove tags from the displayed
   captions ([#4306](https://github.com/google/ExoPlayer/issues/4306)).
+* Audio:
+  * Support seeking based on MLLT metadata
+    ([#3241](https://github.com/google/ExoPlayer/issues/3241)).
 * Fix issue where buffered position is not updated correctly when transitioning
   between periods
   ([#4899](https://github.com/google/ExoPlayer/issues/4899)).

library/core/src/main/java/com/google/android/exoplayer2/extractor/GaplessInfoHolder.java

@@ -18,7 +18,6 @@ package com.google.android.exoplayer2.extractor;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.metadata.Metadata;
 import com.google.android.exoplayer2.metadata.id3.CommentFrame;
-import com.google.android.exoplayer2.metadata.id3.Id3Decoder.FramePredicate;
 import com.google.android.exoplayer2.metadata.id3.InternalFrame;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -28,15 +27,6 @@ import java.util.regex.Pattern;
  */
 public final class GaplessInfoHolder {
 
-  /**
-   * A {@link FramePredicate} suitable for use when decoding {@link Metadata} that will be passed to
-   * {@link #setFromMetadata(Metadata)}. Only frames that might contain gapless playback information
-   * are decoded.
-   */
-  public static final FramePredicate GAPLESS_INFO_ID3_FRAME_PREDICATE =
-      (majorVersion, id0, id1, id2, id3) ->
-          id0 == 'C' && id1 == 'O' && id2 == 'M' && (id3 == 'M' || majorVersion == 2);
-
   private static final String GAPLESS_DOMAIN = "com.apple.iTunes";
   private static final String GAPLESS_DESCRIPTION = "iTunSMPB";
   private static final Pattern GAPLESS_COMMENT_PATTERN =

library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/MlltSeeker.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.extractor.mp3;
+
+import android.util.Pair;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.extractor.SeekPoint;
+import com.google.android.exoplayer2.metadata.id3.MlltFrame;
+import com.google.android.exoplayer2.util.Util;
+
+/** MP3 seeker that uses metadata from an {@link MlltFrame}. */
+/* package */ final class MlltSeeker implements Mp3Extractor.Seeker {
+
+  /**
+   * Returns an {@link MlltSeeker} for seeking in the stream.
+   *
+   * @param firstFramePosition The position of the start of the first frame in the stream.
+   * @param mlltFrame The MLLT frame with seeking metadata.
+   * @return An {@link MlltSeeker} for seeking in the stream.
+   */
+  public static MlltSeeker create(long firstFramePosition, MlltFrame mlltFrame) {
+    int referenceCount = mlltFrame.bytesDeviations.length;
+    long[] referencePositions = new long[1 + referenceCount];
+    long[] referenceTimesMs = new long[1 + referenceCount];
+    referencePositions[0] = firstFramePosition;
+    referenceTimesMs[0] = 0;
+    long position = firstFramePosition;
+    long timeMs = 0;
+    for (int i = 1; i <= referenceCount; i++) {
+      position += mlltFrame.bytesBetweenReference + mlltFrame.bytesDeviations[i - 1];
+      timeMs += mlltFrame.millisecondsBetweenReference + mlltFrame.millisecondsDeviations[i - 1];
+      referencePositions[i] = position;
+      referenceTimesMs[i] = timeMs;
+    }
+    return new MlltSeeker(referencePositions, referenceTimesMs);
+  }
+
+  private final long[] referencePositions;
+  private final long[] referenceTimesMs;
+  private final long durationUs;
+
+  private MlltSeeker(long[] referencePositions, long[] referenceTimesMs) {
+    this.referencePositions = referencePositions;
+    this.referenceTimesMs = referenceTimesMs;
+    // Use the last reference point as the duration, as extrapolating variable bitrate at the end of
+    // the stream may give a large error.
+    durationUs = C.msToUs(referenceTimesMs[referenceTimesMs.length - 1]);
+  }
+
+  @Override
+  public boolean isSeekable() {
+    return true;
+  }
+
+  @Override
+  public SeekPoints getSeekPoints(long timeUs) {
+    timeUs = Util.constrainValue(timeUs, 0, durationUs);
+    Pair<Long, Long> timeMsAndPosition =
+        linearlyInterpolate(C.usToMs(timeUs), referenceTimesMs, referencePositions);
+    timeUs = C.msToUs(timeMsAndPosition.first);
+    long position = timeMsAndPosition.second;
+    return new SeekPoints(new SeekPoint(timeUs, position));
+  }
+
+  @Override
+  public long getTimeUs(long position) {
+    Pair<Long, Long> positionAndTimeMs =
+        linearlyInterpolate(position, referencePositions, referenceTimesMs);
+    return C.msToUs(positionAndTimeMs.second);
+  }
+
+  @Override
+  public long getDurationUs() {
+    return durationUs;
+  }
+
+  /**
+   * Given a set of reference points as coordinates in {@code xReferences} and {@code yReferences}
+   * and an x-axis value, linearly interpolates between corresponding reference points to give a
+   * y-axis value.
+   *
+   * @param x The x-axis value for which a y-axis value is needed.
+   * @param xReferences x coordinates of reference points.
+   * @param yReferences y coordinates of reference points.
+   * @return The linearly interpolated y-axis value.
+   */
+  private static Pair<Long, Long> linearlyInterpolate(
+      long x, long[] xReferences, long[] yReferences) {
+    int previousReferenceIndex =
+        Util.binarySearchFloor(xReferences, x, /* inclusive= */ true, /* stayInBounds= */ true);
+    long xPreviousReference = xReferences[previousReferenceIndex];
+    long yPreviousReference = yReferences[previousReferenceIndex];
+    int nextReferenceIndex = previousReferenceIndex + 1;
+    if (nextReferenceIndex == xReferences.length) {
+      return Pair.create(xPreviousReference, yPreviousReference);
+    } else {
+      long xNextReference = xReferences[nextReferenceIndex];
+      long yNextReference = yReferences[nextReferenceIndex];
+      double proportion =
+          xNextReference == xPreviousReference
+              ? 0.0
+              : ((double) x - xPreviousReference) / (xNextReference - xPreviousReference);
+      long y = (long) (proportion * (yNextReference - yPreviousReference)) + yPreviousReference;
+      return Pair.create(x, y);
+    }
+  }
+
+}

library/core/src/main/java/com/google/android/exoplayer2/extractor/mp3/Mp3Extractor.java

@@ -16,6 +16,7 @@
 package com.google.android.exoplayer2.extractor.mp3;
 
 import android.support.annotation.IntDef;
+import android.support.annotation.Nullable;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.ParserException;
@@ -31,6 +32,8 @@ import com.google.android.exoplayer2.extractor.SeekMap;
 import com.google.android.exoplayer2.extractor.TrackOutput;
 import com.google.android.exoplayer2.metadata.Metadata;
 import com.google.android.exoplayer2.metadata.id3.Id3Decoder;
+import com.google.android.exoplayer2.metadata.id3.Id3Decoder.FramePredicate;
+import com.google.android.exoplayer2.metadata.id3.MlltFrame;
 import com.google.android.exoplayer2.util.ParsableByteArray;
 import com.google.android.exoplayer2.util.Util;
 import java.io.EOFException;
@@ -68,6 +71,12 @@ public final class Mp3Extractor implements Extractor {
    */
   public static final int FLAG_DISABLE_ID3_METADATA = 2;
 
+  /** Predicate that matches ID3 frames containing only required gapless/seeking metadata. */
+  private static final FramePredicate REQUIRED_ID3_FRAME_PREDICATE =
+      (majorVersion, id0, id1, id2, id3) ->
+          ((id0 == 'C' && id1 == 'O' && id2 == 'M' && (id3 == 'M' || majorVersion == 2))
+              || (id0 == 'M' && id1 == 'L' && id2 == 'L' && (id3 == 'T' || majorVersion == 2)));
+
   /**
    * The maximum number of bytes to search when synchronizing, before giving up.
    */
@@ -174,7 +183,15 @@ public final class Mp3Extractor implements Extractor {
       }
     }
     if (seeker == null) {
-      seeker = maybeReadSeekFrame(input);
+      // Read past any seek frame and set the seeker based on metadata or a seek frame. Metadata
+      // takes priority as it can provide greater precision.
+      Seeker seekFrameSeeker = maybeReadSeekFrame(input);
+      Seeker metadataSeeker = maybeHandleSeekMetadata(metadata, input.getPosition());
+      if (metadataSeeker != null) {
+        seeker = metadataSeeker;
+      } else if (seekFrameSeeker != null) {
+        seeker = seekFrameSeeker;
+      }
       if (seeker == null
           || (!seeker.isSeekable() && (flags & FLAG_ENABLE_CONSTANT_BITRATE_SEEKING) != 0)) {
         seeker = getConstantBitrateSeeker(input);
@@ -253,11 +270,11 @@ public final class Mp3Extractor implements Extractor {
     int searchLimitBytes = sniffing ? MAX_SNIFF_BYTES : MAX_SYNC_BYTES;
     input.resetPeekPosition();
     if (input.getPosition() == 0) {
-      // We need to parse enough ID3 metadata to retrieve any gapless playback information even
-      // if ID3 metadata parsing is disabled.
-      boolean onlyDecodeGaplessInfoFrames = (flags & FLAG_DISABLE_ID3_METADATA) != 0;
+      // We need to parse enough ID3 metadata to retrieve any gapless/seeking playback information
+      // even if ID3 metadata parsing is disabled.
+      boolean parseAllId3Frames = (flags & FLAG_DISABLE_ID3_METADATA) == 0;
       Id3Decoder.FramePredicate id3FramePredicate =
-          onlyDecodeGaplessInfoFrames ? GaplessInfoHolder.GAPLESS_INFO_ID3_FRAME_PREDICATE : null;
+          parseAllId3Frames ? null : REQUIRED_ID3_FRAME_PREDICATE;
       metadata = id3Peeker.peekId3Data(input, id3FramePredicate);
       if (metadata != null) {
         gaplessInfoHolder.setFromMetadata(metadata);
@@ -401,6 +418,20 @@ public final class Mp3Extractor implements Extractor {
     return SEEK_HEADER_UNSET;
   }
 
+  @Nullable
+  private static MlltSeeker maybeHandleSeekMetadata(Metadata metadata, long firstFramePosition) {
+    if (metadata != null) {
+      int length = metadata.length();
+      for (int i = 0; i < length; i++) {
+        Metadata.Entry entry = metadata.get(i);
+        if (entry instanceof MlltFrame) {
+          return MlltSeeker.create(firstFramePosition, (MlltFrame) entry);
+        }
+      }
+    }
+    return null;
+  }
+
   /**
    * {@link SeekMap} that also allows mapping from position (byte offset) back to time, which can be
    * used to work out the new sample basis timestamp after seeking and resynchronization.

library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/Id3Decoder.java

@@ -21,6 +21,7 @@ import com.google.android.exoplayer2.metadata.Metadata;
 import com.google.android.exoplayer2.metadata.MetadataDecoder;
 import com.google.android.exoplayer2.metadata.MetadataInputBuffer;
 import com.google.android.exoplayer2.util.Log;
+import com.google.android.exoplayer2.util.ParsableBitArray;
 import com.google.android.exoplayer2.util.ParsableByteArray;
 import com.google.android.exoplayer2.util.Util;
 import java.io.UnsupportedEncodingException;
@@ -382,6 +383,8 @@ public final class Id3Decoder implements MetadataDecoder {
       } else if (frameId0 == 'C' && frameId1 == 'T' && frameId2 == 'O' && frameId3 == 'C') {
         frame = decodeChapterTOCFrame(id3Data, frameSize, majorVersion, unsignedIntFrameSizeHack,
             frameHeaderSize, framePredicate);
+      } else if (frameId0 == 'M' && frameId1 == 'L' && frameId2 == 'L' && frameId3 == 'T') {
+        frame = decodeMlltFrame(id3Data, frameSize);
       } else {
         String id = getFrameId(majorVersion, frameId0, frameId1, frameId2, frameId3);
         frame = decodeBinaryFrame(id3Data, frameSize, id);
@@ -662,6 +665,36 @@ public final class Id3Decoder implements MetadataDecoder {
     return new ChapterTocFrame(elementId, isRoot, isOrdered, children, subFrameArray);
   }
 
+  private static MlltFrame decodeMlltFrame(ParsableByteArray id3Data, int frameSize) {
+    // See ID3v2.4.0 native frames subsection 4.6.
+    int mpegFramesBetweenReference = id3Data.readUnsignedShort();
+    int bytesBetweenReference = id3Data.readUnsignedInt24();
+    int millisecondsBetweenReference = id3Data.readUnsignedInt24();
+    int bitsForBytesDeviation = id3Data.readUnsignedByte();
+    int bitsForMillisecondsDeviation = id3Data.readUnsignedByte();
+
+    ParsableBitArray references = new ParsableBitArray();
+    references.reset(id3Data);
+    int referencesBits = 8 * (frameSize - 10);
+    int bitsPerReference = bitsForBytesDeviation + bitsForMillisecondsDeviation;
+    int referencesCount = referencesBits / bitsPerReference;
+    int[] bytesDeviations = new int[referencesCount];
+    int[] millisecondsDeviations = new int[referencesCount];
+    for (int i = 0; i < referencesCount; i++) {
+      int bytesDeviation = references.readBits(bitsForBytesDeviation);
+      int millisecondsDeviation = references.readBits(bitsForMillisecondsDeviation);
+      bytesDeviations[i] = bytesDeviation;
+      millisecondsDeviations[i] = millisecondsDeviation;
+    }
+
+    return new MlltFrame(
+        mpegFramesBetweenReference,
+        bytesBetweenReference,
+        millisecondsBetweenReference,
+        bytesDeviations,
+        millisecondsDeviations);
+  }
+
   private static BinaryFrame decodeBinaryFrame(ParsableByteArray id3Data, int frameSize,
       String id) {
     byte[] frame = new byte[frameSize];

library/core/src/main/java/com/google/android/exoplayer2/metadata/id3/MlltFrame.java

@@ -0,0 +1,112 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.metadata.id3;
+
+import android.os.Parcel;
+import android.support.annotation.Nullable;
+import java.util.Arrays;
+
+/** MPEG location lookup table frame. */
+public final class MlltFrame extends Id3Frame {
+
+  public static final String ID = "MLLT";
+
+  public final int mpegFramesBetweenReference;
+  public final int bytesBetweenReference;
+  public final int millisecondsBetweenReference;
+  public final int[] bytesDeviations;
+  public final int[] millisecondsDeviations;
+
+  public MlltFrame(
+      int mpegFramesBetweenReference,
+      int bytesBetweenReference,
+      int millisecondsBetweenReference,
+      int[] bytesDeviations,
+      int[] millisecondsDeviations) {
+    super(ID);
+    this.mpegFramesBetweenReference = mpegFramesBetweenReference;
+    this.bytesBetweenReference = bytesBetweenReference;
+    this.millisecondsBetweenReference = millisecondsBetweenReference;
+    this.bytesDeviations = bytesDeviations;
+    this.millisecondsDeviations = millisecondsDeviations;
+  }
+
+  /* package */ MlltFrame(Parcel in) {
+    super(ID);
+    this.mpegFramesBetweenReference = in.readInt();
+    this.bytesBetweenReference = in.readInt();
+    this.millisecondsBetweenReference = in.readInt();
+    this.bytesDeviations = in.createIntArray();
+    this.millisecondsDeviations = in.createIntArray();
+  }
+
+  @Override
+  public boolean equals(@Nullable Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null || getClass() != obj.getClass()) {
+      return false;
+    }
+    MlltFrame other = (MlltFrame) obj;
+    return mpegFramesBetweenReference == other.mpegFramesBetweenReference
+        && bytesBetweenReference == other.bytesBetweenReference
+        && millisecondsBetweenReference == other.millisecondsBetweenReference
+        && Arrays.equals(bytesDeviations, other.bytesDeviations)
+        && Arrays.equals(millisecondsDeviations, other.millisecondsDeviations);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = 17;
+    result = 31 * result + mpegFramesBetweenReference;
+    result = 31 * result + bytesBetweenReference;
+    result = 31 * result + millisecondsBetweenReference;
+    result = 31 * result + Arrays.hashCode(bytesDeviations);
+    result = 31 * result + Arrays.hashCode(millisecondsDeviations);
+    return result;
+  }
+
+  // Parcelable implementation.
+
+  @Override
+  public void writeToParcel(Parcel dest, int flags) {
+    dest.writeInt(mpegFramesBetweenReference);
+    dest.writeInt(bytesBetweenReference);
+    dest.writeInt(millisecondsBetweenReference);
+    dest.writeIntArray(bytesDeviations);
+    dest.writeIntArray(millisecondsDeviations);
+  }
+
+  @Override
+  public int describeContents() {
+    return 0;
+  }
+
+  public static final Creator<MlltFrame> CREATOR =
+      new Creator<MlltFrame>() {
+
+        @Override
+        public MlltFrame createFromParcel(Parcel in) {
+          return new MlltFrame(in);
+        }
+
+        @Override
+        public MlltFrame[] newArray(int size) {
+          return new MlltFrame[size];
+        }
+      };
+}

library/core/src/test/java/com/google/android/exoplayer2/metadata/id3/MlltFrameTest.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.metadata.id3;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import android.os.Parcel;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+
+/** Test for {@link MlltFrame}. */
+@RunWith(RobolectricTestRunner.class)
+public final class MlltFrameTest {
+
+  @Test
+  public void testParcelable() {
+    MlltFrame mlltFrameToParcel =
+        new MlltFrame(
+            /* mpegFramesBetweenReference= */ 1,
+            /* bytesBetweenReference= */ 1,
+            /* millisecondsBetweenReference= */ 1,
+            /* bytesDeviations= */ new int[] {1, 2},
+            /* millisecondsDeviations= */ new int[] {1, 2});
+
+    Parcel parcel = Parcel.obtain();
+    mlltFrameToParcel.writeToParcel(parcel, 0);
+    parcel.setDataPosition(0);
+
+    MlltFrame mlltFrameFromParcel = MlltFrame.CREATOR.createFromParcel(parcel);
+    assertThat(mlltFrameFromParcel).isEqualTo(mlltFrameToParcel);
+
+    parcel.recycle();
+  }
+
+}