loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Plumb SubtitleParser.Factory into Mp4Extractor

Browse Source 
Mp4Extractor will no longer be wrapped in SubtitleTranscodingExtractor, but instead use SubtitleTranscodingExtractorOutput under the hood.

FLAG_EMIT_RAW_SUBTITLE_DATA flag will be used to toggle between subtitle parsing during extraction (before the sample queue) or during decoding (after the sample queue).

PiperOrigin-RevId: 597221831
This commit is contained in:
jbibik 2024-01-10 05:32:46 -08:00 committed by Copybara-Service
parent 51d60e1f3a
commit d6ef48fff8
5 changed files with 157 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
libraries/extractor/src/main/java/androidx/media3/extractor/DefaultExtractorsFactory.java
View File

@ -264,7 +264,7 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory {
/**
* Sets flags for {@link Mp4Extractor} instances created by the factory.
*
* @see Mp4Extractor#Mp4Extractor(int)
* @see Mp4Extractor#Mp4Extractor(SubtitleParser.Factory, int)
* @param flags The flags to use.
* @return The factory, for convenience.
*/
@ -440,6 +440,7 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory {
result[i] =
textTrackTranscodingEnabled
&& !(extractor.getUnderlyingImplementation() instanceof FragmentedMp4Extractor)
&& !(extractor.getUnderlyingImplementation() instanceof Mp4Extractor)
? new SubtitleTranscodingExtractor(extractor, subtitleParserFactory)
: extractor;
}
@ -509,7 +510,13 @@ public final class DefaultExtractorsFactory implements ExtractorsFactory {
| (textTrackTranscodingEnabled
? 0
: FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA)));
extractors.add(new Mp4Extractor(mp4Flags));
extractors.add(
new Mp4Extractor(
subtitleParserFactory,
mp4Flags
| (textTrackTranscodingEnabled
? 0
: Mp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA)));
break;
case FileTypes.OGG:
extractors.add(new OggExtractor());

5
libraries/extractor/src/main/java/androidx/media3/extractor/jpeg/JpegMotionPhotoExtractor.java
View File

@ -36,6 +36,7 @@ import androidx.media3.extractor.SeekMap;
import androidx.media3.extractor.TrackOutput;
import androidx.media3.extractor.metadata.mp4.MotionPhotoMetadata;
import androidx.media3.extractor.mp4.Mp4Extractor;
import androidx.media3.extractor.text.SubtitleParser;
import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
@ -245,7 +246,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} else {
input.resetPeekPosition();
if (mp4Extractor == null) {
mp4Extractor = new Mp4Extractor(FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE);
mp4Extractor =
new Mp4Extractor(
SubtitleParser.Factory.UNSUPPORTED, FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE);
}
mp4ExtractorStartOffsetExtractorInput =
new StartOffsetExtractorInput(input, mp4StartPosition);

56
libraries/extractor/src/main/java/androidx/media3/extractor/mp4/Mp4Extractor.java
View File

@ -50,6 +50,8 @@ import androidx.media3.extractor.TrueHdSampleRechunker;
import androidx.media3.extractor.metadata.mp4.MotionPhotoMetadata;
import androidx.media3.extractor.metadata.mp4.SlowMotionData;
import androidx.media3.extractor.mp4.Atom.ContainerAtom;
import androidx.media3.extractor.text.SubtitleParser;
import androidx.media3.extractor.text.SubtitleTranscodingExtractorOutput;
import java.io.IOException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
@ -64,9 +66,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@UnstableApi
public final class Mp4Extractor implements Extractor, SeekMap {
/** Factory for {@link Mp4Extractor} instances. */
/**
* @deprecated Use {@link #newFactory(SubtitleParser.Factory)} instead.
*/
@Deprecated
public static final ExtractorsFactory FACTORY = () -> new Extractor[] {new Mp4Extractor()};
/**
* Creates a factory for {@link Mp4Extractor} instances with the provided {@link
* SubtitleParser.Factory}.
*/
public static ExtractorsFactory newFactory(SubtitleParser.Factory subtitleParserFactory) {
return () -> new Extractor[] {new Mp4Extractor(subtitleParserFactory)};
}
/**
* Flags controlling the behavior of the extractor. Possible flag values are {@link
* #FLAG_WORKAROUND_IGNORE_EDIT_LISTS}, {@link #FLAG_READ_MOTION_PHOTO_METADATA} and {@link
@ -81,7 +94,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
FLAG_WORKAROUND_IGNORE_EDIT_LISTS,
FLAG_READ_MOTION_PHOTO_METADATA,
FLAG_READ_SEF_DATA,
FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE
FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE,
FLAG_EMIT_RAW_SUBTITLE_DATA
})
public @interface Flags {}
@ -109,6 +123,8 @@ public final class Mp4Extractor implements Extractor, SeekMap {
*/
public static final int FLAG_MARK_FIRST_VIDEO_TRACK_WITH_MAIN_ROLE = 1 << 3;
public static final int FLAG_EMIT_RAW_SUBTITLE_DATA = 1 << 4;
/** Parser states. */
@Documented
@Retention(RetentionPolicy.SOURCE)
@ -149,6 +165,7 @@ public final class Mp4Extractor implements Extractor, SeekMap {
*/
private static final long MAXIMUM_READ_AHEAD_BYTES_STREAM = 10 * 1024 * 1024;
private final SubtitleParser.Factory subtitleParserFactory;
private final @Flags int flags;
// Temporary arrays.
@ -183,18 +200,42 @@ public final class Mp4Extractor implements Extractor, SeekMap {
private @FileType int fileType;
@Nullable private MotionPhotoMetadata motionPhotoMetadata;
/** Creates a new extractor for unfragmented MP4 streams. */
/**
* @deprecated Use {@link #Mp4Extractor(SubtitleParser.Factory)} instead
*/
@Deprecated
public Mp4Extractor() {
this(/* flags= */ 0);
this(SubtitleParser.Factory.UNSUPPORTED, /* flags= */ FLAG_EMIT_RAW_SUBTITLE_DATA);
}
/**
* Creates a new extractor for unfragmented MP4 streams.
*
* @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
* extraction.
*/
public Mp4Extractor(SubtitleParser.Factory subtitleParserFactory) {
this(subtitleParserFactory, /* flags= */ 0);
}
/**
* @deprecated Use {@link #Mp4Extractor(SubtitleParser.Factory, int)} instead
*/
@Deprecated
public Mp4Extractor(@Flags int flags) {
this(SubtitleParser.Factory.UNSUPPORTED, flags);
}
/**
* Creates a new extractor for unfragmented MP4 streams, using the specified flags to control the
* extractor's behavior.
*
* @param subtitleParserFactory The {@link SubtitleParser.Factory} for parsing subtitles during
* extraction.
* @param flags Flags that control the extractor's behavior.
*/
public Mp4Extractor(@Flags int flags) {
public Mp4Extractor(SubtitleParser.Factory subtitleParserFactory, @Flags int flags) {
this.subtitleParserFactory = subtitleParserFactory;
this.flags = flags;
parserState =
((flags & FLAG_READ_SEF_DATA) != 0) ? STATE_READING_SEF : STATE_READING_ATOM_HEADER;
@ -218,7 +259,10 @@ public final class Mp4Extractor implements Extractor, SeekMap {
@Override
public void init(ExtractorOutput output) {
extractorOutput = output;
extractorOutput =
(flags & FLAG_EMIT_RAW_SUBTITLE_DATA) == 0
? new SubtitleTranscodingExtractorOutput(output, subtitleParserFactory)
: output;
}
@Override

2
libraries/extractor/src/test/java/androidx/media3/extractor/DefaultExtractorsFactoryTest.java
View File

@ -177,7 +177,7 @@ public final class DefaultExtractorsFactoryTest {
}
assertThat(aviExtractor).isInstanceOf(SubtitleTranscodingExtractor.class);
assertThat(matroskaExtractor).isInstanceOf(SubtitleTranscodingExtractor.class);
assertThat(mp4Extractor).isInstanceOf(SubtitleTranscodingExtractor.class);
assertThat(mp4Extractor).isNotInstanceOf(SubtitleTranscodingExtractor.class);
assertThat(fragmentedMp4Extractor).isNotInstanceOf(SubtitleTranscodingExtractor.class);
assertThat(tsExtractor).isInstanceOf(SubtitleTranscodingExtractor.class);
}

118
libraries/extractor/src/test/java/androidx/media3/extractor/mp4/Mp4ExtractorTest.java
View File

@ -15,8 +15,13 @@
*/
package androidx.media3.extractor.mp4;
import static androidx.media3.extractor.mp4.FragmentedMp4Extractor.FLAG_EMIT_RAW_SUBTITLE_DATA;
import androidx.media3.extractor.text.DefaultSubtitleParserFactory;
import androidx.media3.extractor.text.SubtitleParser;
import androidx.media3.test.utils.ExtractorAsserts;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.ParameterizedRobolectricTestRunner;
@ -27,22 +32,36 @@ import org.robolectric.ParameterizedRobolectricTestRunner.Parameters;
@RunWith(ParameterizedRobolectricTestRunner.class)
public final class Mp4ExtractorTest {
@Parameters(name = "{0}")
public static ImmutableList<ExtractorAsserts.SimulationConfig> params() {
return ExtractorAsserts.configs();
@Parameters(name = "{0},subtitlesParsedDuringExtraction={1}")
public static List<Object[]> params() {
List<Object[]> parameterList = new ArrayList<>();
for (ExtractorAsserts.SimulationConfig config : ExtractorAsserts.configs()) {
parameterList.add(new Object[] {config, /* subtitlesParsedDuringExtraction */ true});
parameterList.add(new Object[] {config, /* subtitlesParsedDuringExtraction */ false});
}
return parameterList;
}
@Parameter public ExtractorAsserts.SimulationConfig simulationConfig;
@Parameter(0)
public ExtractorAsserts.SimulationConfig simulationConfig;
@Parameter(1)
public boolean subtitlesParsedDuringExtraction;
@Test
public void mp4Sample() throws Exception {
ExtractorAsserts.assertBehavior(Mp4Extractor::new, "media/mp4/sample.mp4", simulationConfig);
ExtractorAsserts.assertBehavior(
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithSlowMotionMetadata() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_android_slow_motion.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_android_slow_motion.mp4",
simulationConfig);
}
/**
@ -52,55 +71,73 @@ public final class Mp4ExtractorTest {
@Test
public void mp4SampleWithMdatTooLong() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_mdat_too_long.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_mdat_too_long.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithAc3Track() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_ac3.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_ac3.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithAc4Track() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_ac4.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_ac4.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithEac3Track() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_eac3.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_eac3.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithEac3jocTrack() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_eac3joc.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_eac3joc.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithOpusTrack() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_opus.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_opus.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithMha1Track() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_mpegh_mha1.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_mpegh_mha1.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithMhm1Track() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_mpegh_mhm1.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_mpegh_mhm1.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithColorInfo() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_with_color_info.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_with_color_info.mp4",
simulationConfig);
}
/**
@ -111,26 +148,32 @@ public final class Mp4ExtractorTest {
@Test
public void mp4Sample18ByteNclxColr() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_18byte_nclx_colr.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_18byte_nclx_colr.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithDolbyTrueHDTrack() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_dthd.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_dthd.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithColrMdcvAndClli() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_with_colr_mdcv_and_clli.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_with_colr_mdcv_and_clli.mp4",
simulationConfig);
}
/** Test case for supporting original QuickTime specification [Internal: b/297137302]. */
@Test
public void mp4SampleWithOriginalQuicktimeSpecification() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new,
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_with_original_quicktime_specification.mov",
simulationConfig);
}
@ -138,30 +181,55 @@ public final class Mp4ExtractorTest {
@Test
public void mp4SampleWithAv1c() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_with_av1c.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_with_av1c.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithMhm1BlCicp1Track() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_mhm1_bl_cicp1.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_mhm1_bl_cicp1.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithMhm1LcBlCicp1Track() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_mhm1_lcbl_cicp1.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_mhm1_lcbl_cicp1.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithMhm1BlConfigChangeTrack() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_mhm1_bl_configchange.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_mhm1_bl_configchange.mp4",
simulationConfig);
}
@Test
public void mp4SampleWithMhm1LcBlConfigChangeTrack() throws Exception {
ExtractorAsserts.assertBehavior(
Mp4Extractor::new, "media/mp4/sample_mhm1_lcbl_configchange.mp4", simulationConfig);
getExtractorFactory(subtitlesParsedDuringExtraction),
"media/mp4/sample_mhm1_lcbl_configchange.mp4",
simulationConfig);
}
private static ExtractorAsserts.ExtractorFactory getExtractorFactory(
boolean subtitlesParsedDuringExtraction) {
SubtitleParser.Factory subtitleParserFactory;
@Mp4Extractor.Flags int flags;
if (subtitlesParsedDuringExtraction) {
subtitleParserFactory = new DefaultSubtitleParserFactory();
flags = 0;
} else {
subtitleParserFactory = SubtitleParser.Factory.UNSUPPORTED;
flags = FLAG_EMIT_RAW_SUBTITLE_DATA;
}
return () -> new Mp4Extractor(subtitleParserFactory, flags);
}
}