loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-04 14:10:40 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Use Mp4WebvttDecoder for WebVTT content in DASH MP4 containers

Browse Source 
This was broken by 74a9d8f680
because DashManifestParser switched to setting Format.sampleMimeType to
text/vtt while SubtitleDecoderFactory was still expecting
application/x-mp4-vtt. This change teaches SubtitleDecoderFactory to
check both Format.containerMimeType and Format.sampleMimeType.

I'll investigate a follow-up change to remove
MimeTypes.APPLICATION_MP4VTT completely (it's currently still used in
AtomParsers).

Issue: #7985
PiperOrigin-RevId: 334771672
This commit is contained in:
ibaker 2020-10-01 10:22:06 +01:00 committed by Oliver Woodman
parent 9ce2ac8a8c
commit b8c8ce0ee0
9 changed files with 229 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
RELEASENOTES.md
View File

@ -11,6 +11,8 @@
([#7866](https://github.com/google/ExoPlayer/issues/7866)).
* Text:
* Add support for `\h` SSA/ASS style override code (non-breaking space).
* Fix WebVTT subtitles in MP4 containers in DASH streams
([#7985](https://github.com/google/ExoPlayer/issues/7985)).
* UI:
* Do not require subtitleButton in custom layouts of StyledPlayerView
([#7962](https://github.com/google/ExoPlayer/issues/7962)).

16
library/core/src/main/java/com/google/android/exoplayer2/text/SubtitleDecoderFactory.java
View File

@ -91,11 +91,15 @@ public interface SubtitleDecoderFactory {
@Override
public SubtitleDecoder createDecoder(Format format) {
@Nullable String mimeType = format.sampleMimeType;
if (mimeType != null) {
switch (mimeType) {
@Nullable String sampleMimeType = format.sampleMimeType;
if (sampleMimeType != null) {
switch (sampleMimeType) {
case MimeTypes.TEXT_VTT:
return new WebvttDecoder();
if (MimeTypes.APPLICATION_MP4.equals(format.containerMimeType)) {
return new Mp4WebvttDecoder();
} else {
return new WebvttDecoder();
}
case MimeTypes.TEXT_SSA:
return new SsaDecoder(format.initializationData);
case MimeTypes.APPLICATION_MP4VTT:
@ -109,7 +113,7 @@ public interface SubtitleDecoderFactory {
case MimeTypes.APPLICATION_CEA608:
case MimeTypes.APPLICATION_MP4CEA608:
return new Cea608Decoder(
mimeType,
sampleMimeType,
format.accessibilityChannel,
Cea608Decoder.MIN_DATA_CHANNEL_TIMEOUT_MS);
case MimeTypes.APPLICATION_CEA708:
@ -123,7 +127,7 @@ public interface SubtitleDecoderFactory {
}
}
throw new IllegalArgumentException(
"Attempted to create decoder for unsupported MIME type: " + mimeType);
"Attempted to create decoder for unsupported MIME type: " + sampleMimeType);
}
};
}

1
library/dash/build.gradle
View File

@ -36,6 +36,7 @@ dependencies {
compileOnly 'org.checkerframework:checker-compat-qual:' + checkerframeworkCompatVersion
compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion
implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion
testImplementation project(modulePrefix + 'robolectricutils')
testImplementation project(modulePrefix + 'testutils')
testImplementation 'org.robolectric:robolectric:' + robolectricVersion
}

73
library/dash/src/test/java/com/google/android/exoplayer2/e2etest/DashPlaybackTest.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Copyright (C) 2020 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.google.android.exoplayer2.e2etest;
import static com.google.android.exoplayer2.util.Assertions.checkNotNull;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.android.exoplayer2.MediaItem;
import com.google.android.exoplayer2.Player;
import com.google.android.exoplayer2.SimpleExoPlayer;
import com.google.android.exoplayer2.robolectric.PlaybackOutput;
import com.google.android.exoplayer2.robolectric.ShadowMediaCodecConfig;
import com.google.android.exoplayer2.testutil.AutoAdvancingFakeClock;
import com.google.android.exoplayer2.testutil.DumpFileAsserts;
import com.google.android.exoplayer2.testutil.TestExoPlayer;
import com.google.android.exoplayer2.trackselection.DefaultTrackSelector;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.annotation.Config;
/** End-to-end tests using DASH samples. */
// TODO(b/143232359): Remove once https://issuetracker.google.com/143232359 is resolved.
@Config(sdk = 29)
@RunWith(AndroidJUnit4.class)
public final class DashPlaybackTest {
@Rule
public ShadowMediaCodecConfig mediaCodecConfig =
ShadowMediaCodecConfig.forAllSupportedMimeTypes();
// https://github.com/google/ExoPlayer/issues/7985
@Test
public void webvttInMp4() throws Exception {
SimpleExoPlayer player =
new SimpleExoPlayer.Builder(ApplicationProvider.getApplicationContext())
.setClock(new AutoAdvancingFakeClock())
.build();
player.setVideoSurface(new Surface(new SurfaceTexture(/* texName= */ 1)));
PlaybackOutput playbackOutput = PlaybackOutput.register(player, mediaCodecConfig);
// Ensure the subtitle track is selected.
DefaultTrackSelector trackSelector =
checkNotNull((DefaultTrackSelector) player.getTrackSelector());
trackSelector.setParameters(trackSelector.buildUponParameters().setPreferredTextLanguage("en"));
player.setMediaItem(MediaItem.fromUri("asset:///media/dash/webvtt-in-mp4/sample.mpd"));
player.prepare();
player.play();
TestExoPlayer.runUntilPlaybackState(player, Player.STATE_ENDED);
player.release();
DumpFileAsserts.assertOutput(
ApplicationProvider.getApplicationContext(),
playbackOutput,
"playbackdumps/dash/webvtt-in-mp4.dump");
}
}

63
robolectricutils/src/main/java/com/google/android/exoplayer2/robolectric/PlaybackOutput.java
View File

@ -15,12 +15,17 @@
*/
package com.google.android.exoplayer2.robolectric;
import android.graphics.Bitmap;
import androidx.annotation.Nullable;
import com.google.android.exoplayer2.SimpleExoPlayer;
import com.google.android.exoplayer2.metadata.Metadata;
import com.google.android.exoplayer2.testutil.Dumper;
import com.google.android.exoplayer2.text.Cue;
import com.google.android.exoplayer2.util.Assertions;
import com.google.android.exoplayer2.util.Util;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -35,17 +40,19 @@ public final class PlaybackOutput implements Dumper.Dumpable {
private final ShadowMediaCodecConfig codecConfig;
// TODO: Add support for subtitles too
private final List<Metadata> metadatas;
private final List<List<Cue>> subtitles;
private PlaybackOutput(SimpleExoPlayer player, ShadowMediaCodecConfig codecConfig) {
this.codecConfig = codecConfig;
metadatas = Collections.synchronizedList(new ArrayList<>());
subtitles = Collections.synchronizedList(new ArrayList<>());
// TODO: Consider passing playback position into MetadataOutput and TextOutput. Calling
// player.getCurrentPosition() inside onMetadata/Cues will likely be non-deterministic
// because renderer-thread != playback-thread.
player.addMetadataOutput(metadatas::add);
player.addTextOutput(subtitles::add);
}
/**
@ -74,6 +81,7 @@ public final class PlaybackOutput implements Dumper.Dumpable {
}
dumpMetadata(dumper);
dumpSubtitles(dumper);
}
private void dumpMetadata(Dumper dumper) {
@ -91,4 +99,57 @@ public final class PlaybackOutput implements Dumper.Dumpable {
}
dumper.endBlock();
}
private void dumpSubtitles(Dumper dumper) {
if (subtitles.isEmpty()) {
return;
}
dumper.startBlock("TextOutput");
for (int i = 0; i < subtitles.size(); i++) {
dumper.startBlock("Subtitle[" + i + "]");
List<Cue> subtitle = subtitles.get(i);
if (subtitle.isEmpty()) {
dumper.add("Cues", ImmutableList.of());
}
for (int j = 0; j < subtitle.size(); j++) {
dumper.startBlock("Cue[" + j + "]");
Cue cue = subtitle.get(j);
dumpIfNotEqual(dumper, "text", cue.text, null);
dumpIfNotEqual(dumper, "textAlignment", cue.textAlignment, null);
dumpBitmap(dumper, cue.bitmap);
dumpIfNotEqual(dumper, "line", cue.line, Cue.DIMEN_UNSET);
dumpIfNotEqual(dumper, "lineType", cue.lineType, Cue.TYPE_UNSET);
dumpIfNotEqual(dumper, "lineAnchor", cue.lineAnchor, Cue.TYPE_UNSET);
dumpIfNotEqual(dumper, "position", cue.position, Cue.DIMEN_UNSET);
dumpIfNotEqual(dumper, "positionAnchor", cue.positionAnchor, Cue.TYPE_UNSET);
dumpIfNotEqual(dumper, "size", cue.size, Cue.DIMEN_UNSET);
dumpIfNotEqual(dumper, "bitmapHeight", cue.bitmapHeight, Cue.DIMEN_UNSET);
if (cue.windowColorSet) {
dumper.add("cue.windowColor", cue.windowColor);
}
dumpIfNotEqual(dumper, "textSizeType", cue.textSizeType, Cue.TYPE_UNSET);
dumpIfNotEqual(dumper, "textSize", cue.textSize, Cue.DIMEN_UNSET);
dumpIfNotEqual(dumper, "verticalType", cue.verticalType, Cue.TYPE_UNSET);
dumper.endBlock();
}
dumper.endBlock();
}
dumper.endBlock();
}
private static void dumpIfNotEqual(
Dumper dumper, String field, @Nullable Object actual, @Nullable Object comparison) {
if (!Util.areEqual(actual, comparison)) {
dumper.add(field, actual);
}
}
private static void dumpBitmap(Dumper dumper, @Nullable Bitmap bitmap) {
if (bitmap == null) {
return;
}
byte[] bytes = new byte[bitmap.getByteCount()];
bitmap.copyPixelsToBuffer(ByteBuffer.wrap(bytes));
dumper.add("bitmap", bytes);
}
}

23
testdata/src/test/assets/media/dash/webvtt-in-mp4/sample.mpd vendored Normal file
View File

@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--Generated with https://github.com/google/shaka-packager version v2.4.1-c731217-release-->
<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT1.0010000467300415S">
<Period id="0">
<AdaptationSet id="0" contentType="text" subsegmentAlignment="true" lang="eng">
<Role schemeIdUri="urn:mpeg:dash:role:2011" value="subtitle"/>
<Representation id="0" bandwidth="314" codecs="wvtt" mimeType="application/mp4">
<BaseURL>sample.text.mp4</BaseURL>
<SegmentBase indexRange="727-770" timescale="1000">
<Initialization range="0-726"/>
</SegmentBase>
</Representation>
</AdaptationSet>
<AdaptationSet id="1" contentType="video" width="1080" height="720" frameRate="30000/1001" subsegmentAlignment="true" par="3:2">
<Representation id="1" bandwidth="721967" codecs="avc1.64001f" mimeType="video/mp4" sar="1:1">
<BaseURL>sample.video.mp4</BaseURL>
<SegmentBase indexRange="862-905" timescale="30000">
<Initialization range="0-861"/>
</SegmentBase>
</Representation>
</AdaptationSet>
</Period>
</MPD>

BIN
testdata/src/test/assets/media/dash/webvtt-in-mp4/sample.text.mp4 vendored Normal file
View File

Binary file not shown.

BIN
testdata/src/test/assets/media/dash/webvtt-in-mp4/sample.video.mp4 vendored Normal file
View File

Binary file not shown.

58
testdata/src/test/assets/playbackdumps/dash/webvtt-in-mp4.dump vendored Normal file
View File

@ -0,0 +1,58 @@
MediaCodec (video/avc):
buffers.length = 31
buffers[0] = length 36692, hash D216076E
buffers[1] = length 5312, hash D45D3CA0
buffers[2] = length 599, hash 1BE7812D
buffers[3] = length 7735, hash 4490F110
buffers[4] = length 987, hash 560B5036
buffers[5] = length 673, hash ED7CD8C7
buffers[6] = length 523, hash 3020DF50
buffers[7] = length 6061, hash 736C72B2
buffers[8] = length 992, hash FE132F23
buffers[9] = length 623, hash 5B2C1816
buffers[10] = length 421, hash 742E69C1
buffers[11] = length 4899, hash F72F86A1
buffers[12] = length 568, hash 519A8E50
buffers[13] = length 620, hash 3990AA39
buffers[14] = length 5450, hash F06EC4AA
buffers[15] = length 1051, hash 92DFA63A
buffers[16] = length 874, hash 69587FB4
buffers[17] = length 781, hash 36BE495B
buffers[18] = length 4725, hash AC0C8CD3
buffers[19] = length 1022, hash 5D8BFF34
buffers[20] = length 790, hash 99413A99
buffers[21] = length 610, hash 5E129290
buffers[22] = length 2751, hash 769974CB
buffers[23] = length 745, hash B78A477A
buffers[24] = length 621, hash CF741E7A
buffers[25] = length 505, hash 1DB4894E
buffers[26] = length 1268, hash C15348DC
buffers[27] = length 880, hash C2DE85D0
buffers[28] = length 530, hash C98BC6A8
buffers[29] = length 568, hash 4FE5C8EA
buffers[30] = length 0, hash 1
TextOutput:
Subtitle[0]:
Cues = []
Subtitle[1]:
Cue[0]:
text = This is the first subtitle.
textAlignment = ALIGN_CENTER
lineType = 1
lineAnchor = 0
position = 0.5
positionAnchor = 1
size = 1.0
Subtitle[2]:
Cues = []
Subtitle[3]:
Cue[0]:
text = This is the second subtitle.
textAlignment = ALIGN_CENTER
lineType = 1
lineAnchor = 0
position = 0.5
positionAnchor = 1
size = 1.0
Subtitle[4]:
Cues = []