From ebb72e358f8bc6dd3fa39d3e7bce46269641074c Mon Sep 17 00:00:00 2001 From: ibaker Date: Thu, 15 Aug 2019 12:09:48 +0100 Subject: [PATCH] Support unwrapping nested Metadata messages in MetadataRenderer Initially this supports ID3-in-EMSG, but can also be used to support SCTE35-in-EMSG too. PiperOrigin-RevId: 263535925 --- .../android/exoplayer2/metadata/Metadata.java | 26 ++- .../exoplayer2/metadata/MetadataRenderer.java | 46 +++++- .../metadata/emsg/EventMessage.java | 22 +++ .../metadata/MetadataRendererTest.java | 153 ++++++++++++++++++ 4 files changed, 239 insertions(+), 8 deletions(-) create mode 100644 library/core/src/test/java/com/google/android/exoplayer2/metadata/MetadataRendererTest.java diff --git a/library/core/src/main/java/com/google/android/exoplayer2/metadata/Metadata.java b/library/core/src/main/java/com/google/android/exoplayer2/metadata/Metadata.java index dbc1114bd5..35702da576 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/metadata/Metadata.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/metadata/Metadata.java @@ -18,6 +18,7 @@ package com.google.android.exoplayer2.metadata; import android.os.Parcel; import android.os.Parcelable; import androidx.annotation.Nullable; +import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.util.Util; import java.util.Arrays; import java.util.List; @@ -28,10 +29,27 @@ import org.checkerframework.checker.nullness.compatqual.NullableType; */ public final class Metadata implements Parcelable { - /** - * A metadata entry. - */ - public interface Entry extends Parcelable {} + /** A metadata entry. */ + public interface Entry extends Parcelable { + + /** + * Returns the {@link Format} that can be used to decode the wrapped metadata in {@link + * #getWrappedMetadataBytes()}, or null if this Entry doesn't contain wrapped metadata. + */ + @Nullable + default Format getWrappedMetadataFormat() { + return null; + } + + /** + * Returns the bytes of the wrapped metadata in this Entry, or null if it doesn't contain + * wrapped metadata. + */ + @Nullable + default byte[] getWrappedMetadataBytes() { + return null; + } + } private final Entry[] entries; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java index 0fc0a85104..0dc0dc6096 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/metadata/MetadataRenderer.java @@ -27,7 +27,9 @@ import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.FormatHolder; import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Util; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; /** * A renderer for metadata. @@ -123,12 +125,18 @@ public final class MetadataRenderer extends BaseRenderer implements Callback { } else { buffer.subsampleOffsetUs = subsampleOffsetUs; buffer.flip(); - int index = (pendingMetadataIndex + pendingMetadataCount) % MAX_PENDING_METADATA_COUNT; Metadata metadata = decoder.decode(buffer); if (metadata != null) { - pendingMetadata[index] = metadata; - pendingMetadataTimestamps[index] = buffer.timeUs; - pendingMetadataCount++; + List entries = new ArrayList<>(metadata.length()); + decodeWrappedMetadata(metadata, entries); + if (!entries.isEmpty()) { + Metadata expandedMetadata = new Metadata(entries); + int index = + (pendingMetadataIndex + pendingMetadataCount) % MAX_PENDING_METADATA_COUNT; + pendingMetadata[index] = expandedMetadata; + pendingMetadataTimestamps[index] = buffer.timeUs; + pendingMetadataCount++; + } } } } else if (result == C.RESULT_FORMAT_READ) { @@ -144,6 +152,36 @@ public final class MetadataRenderer extends BaseRenderer implements Callback { } } + /** + * Iterates through {@code metadata.entries} and checks each one to see if contains wrapped + * metadata. If it does, then we recursively decode the wrapped metadata. If it doesn't (recursion + * base-case), we add the {@link Metadata.Entry} to {@code decodedEntries} (output parameter). + */ + private void decodeWrappedMetadata(Metadata metadata, List decodedEntries) { + for (int i = 0; i < metadata.length(); i++) { + Format wrappedMetadataFormat = metadata.get(i).getWrappedMetadataFormat(); + if (wrappedMetadataFormat != null && decoderFactory.supportsFormat(wrappedMetadataFormat)) { + MetadataDecoder wrappedMetadataDecoder = + decoderFactory.createDecoder(wrappedMetadataFormat); + // wrappedMetadataFormat != null so wrappedMetadataBytes must be non-null too. + byte[] wrappedMetadataBytes = + Assertions.checkNotNull(metadata.get(i).getWrappedMetadataBytes()); + buffer.clear(); + buffer.ensureSpaceForWrite(wrappedMetadataBytes.length); + buffer.data.put(wrappedMetadataBytes); + buffer.flip(); + @Nullable Metadata innerMetadata = wrappedMetadataDecoder.decode(buffer); + if (innerMetadata != null) { + // The decoding succeeded, so we'll try another level of unwrapping. + decodeWrappedMetadata(innerMetadata, decodedEntries); + } + } else { + // Entry doesn't contain any wrapped metadata, so output it directly. + decodedEntries.add(metadata.get(i)); + } + } + } + @Override protected void onDisabled() { flushPendingMetadata(); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/metadata/emsg/EventMessage.java b/library/core/src/main/java/com/google/android/exoplayer2/metadata/emsg/EventMessage.java index ca1e390181..c9e9d54093 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/metadata/emsg/EventMessage.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/metadata/emsg/EventMessage.java @@ -20,7 +20,10 @@ import static com.google.android.exoplayer2.util.Util.castNonNull; import android.os.Parcel; import android.os.Parcelable; import androidx.annotation.Nullable; +import androidx.annotation.VisibleForTesting; +import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.metadata.Metadata; +import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; import java.util.Arrays; @@ -29,6 +32,13 @@ import java.util.Arrays; */ public final class EventMessage implements Metadata.Entry { + @VisibleForTesting + public static final String ID3_SCHEME_ID = "https://developer.apple.com/streaming/emsg-id3"; + + private static final Format ID3_FORMAT = + Format.createSampleFormat( + /* id= */ null, MimeTypes.APPLICATION_ID3, Format.OFFSET_SAMPLE_RELATIVE); + /** * The message scheme. */ @@ -81,6 +91,18 @@ public final class EventMessage implements Metadata.Entry { messageData = castNonNull(in.createByteArray()); } + @Override + @Nullable + public Format getWrappedMetadataFormat() { + return ID3_SCHEME_ID.equals(schemeIdUri) ? ID3_FORMAT : null; + } + + @Override + @Nullable + public byte[] getWrappedMetadataBytes() { + return ID3_SCHEME_ID.equals(schemeIdUri) ? messageData : null; + } + @Override public int hashCode() { if (hashCode == 0) { diff --git a/library/core/src/test/java/com/google/android/exoplayer2/metadata/MetadataRendererTest.java b/library/core/src/test/java/com/google/android/exoplayer2/metadata/MetadataRendererTest.java new file mode 100644 index 0000000000..4de8bb76cc --- /dev/null +++ b/library/core/src/test/java/com/google/android/exoplayer2/metadata/MetadataRendererTest.java @@ -0,0 +1,153 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package com.google.android.exoplayer2.metadata; + +import static com.google.common.truth.Truth.assertThat; +import static java.nio.charset.StandardCharsets.ISO_8859_1; +import static java.nio.charset.StandardCharsets.UTF_8; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.android.exoplayer2.ExoPlaybackException; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.metadata.emsg.EventMessage; +import com.google.android.exoplayer2.metadata.emsg.EventMessageEncoder; +import com.google.android.exoplayer2.metadata.id3.TextInformationFrame; +import com.google.android.exoplayer2.testutil.FakeSampleStream; +import com.google.android.exoplayer2.testutil.TestUtil; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +/** Tests for {@link MetadataRenderer}. */ +@RunWith(AndroidJUnit4.class) +public class MetadataRendererTest { + + private static final Format EMSG_FORMAT = + Format.createSampleFormat(null, MimeTypes.APPLICATION_EMSG, Format.OFFSET_SAMPLE_RELATIVE); + + private final EventMessageEncoder eventMessageEncoder = new EventMessageEncoder(); + + @Test + public void decodeMetadata() throws Exception { + EventMessage emsg = + new EventMessage( + "urn:test-scheme-id", + /* value= */ "", + /* durationMs= */ 1, + /* id= */ 0, + "Test data".getBytes(UTF_8)); + + List metadata = runRenderer(eventMessageEncoder.encode(emsg)); + + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).length()).isEqualTo(1); + assertThat(metadata.get(0).get(0)).isEqualTo(emsg); + } + + @Test + public void decodeMetadata_handlesWrappedMetadata() throws Exception { + EventMessage emsg = + new EventMessage( + EventMessage.ID3_SCHEME_ID, + /* value= */ "", + /* durationMs= */ 1, + /* id= */ 0, + encodeTxxxId3Frame("Test description", "Test value")); + + List metadata = runRenderer(eventMessageEncoder.encode(emsg)); + + assertThat(metadata).hasSize(1); + assertThat(metadata.get(0).length()).isEqualTo(1); + TextInformationFrame expectedId3Frame = + new TextInformationFrame("TXXX", "Test description", "Test value"); + assertThat(metadata.get(0).get(0)).isEqualTo(expectedId3Frame); + } + + @Test + public void decodeMetadata_skipsMalformedWrappedMetadata() throws Exception { + EventMessage emsg = + new EventMessage( + EventMessage.ID3_SCHEME_ID, + /* value= */ "", + /* durationMs= */ 1, + /* id= */ 0, + "Not a real ID3 tag".getBytes(ISO_8859_1)); + + List metadata = runRenderer(eventMessageEncoder.encode(emsg)); + + assertThat(metadata).isEmpty(); + } + + private static List runRenderer(byte[] input) throws ExoPlaybackException { + List metadata = new ArrayList<>(); + MetadataRenderer renderer = new MetadataRenderer(metadata::add, /* outputLooper= */ null); + renderer.replaceStream( + new Format[] {EMSG_FORMAT}, + new FakeSampleStream(EMSG_FORMAT, /* eventDispatcher= */ null, input), + /* offsetUs= */ 0L); + renderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); // Read the format + renderer.render(/* positionUs= */ 0, /* elapsedRealtimeUs= */ 0); // Read the data + + return Collections.unmodifiableList(metadata); + } + + /** + * Builds an ID3v2 tag containing a single 'user defined text information frame' (id='TXXX') with + * {@code description} and {@code value}. + * + * + */ + private static byte[] encodeTxxxId3Frame(String description, String value) { + byte[] id3FrameData = + TestUtil.joinByteArrays( + "TXXX".getBytes(ISO_8859_1), // ID for a 'user defined text information frame' + TestUtil.createByteArray(0, 0, 0, 0), // Frame size (set later) + TestUtil.createByteArray(0, 0), // Frame flags + TestUtil.createByteArray(0), // Character encoding = ISO-8859-1 + description.getBytes(ISO_8859_1), + TestUtil.createByteArray(0), // String null terminator + value.getBytes(ISO_8859_1), + TestUtil.createByteArray(0)); // String null terminator + int frameSizeIndex = 7; + int frameSize = id3FrameData.length - 10; + Assertions.checkArgument( + frameSize < 128, "frameSize must fit in 7 bits to avoid synch-safe encoding: " + frameSize); + id3FrameData[frameSizeIndex] = (byte) frameSize; + + byte[] id3Bytes = + TestUtil.joinByteArrays( + "ID3".getBytes(ISO_8859_1), // identifier + TestUtil.createByteArray(0x04, 0x00), // version + TestUtil.createByteArray(0), // Tag flags + TestUtil.createByteArray(0, 0, 0, 0), // Tag size (set later) + id3FrameData); + int tagSizeIndex = 9; + int tagSize = id3Bytes.length - 10; + Assertions.checkArgument( + tagSize < 128, "tagSize must fit in 7 bits to avoid synch-safe encoding: " + tagSize); + id3Bytes[tagSizeIndex] = (byte) tagSize; + return id3Bytes; + } +}