loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Abandon trim optimization when transcoding effects are set

Browse Source 
PiperOrigin-RevId: 588839072
This commit is contained in:
tofunmi 2023-12-07 10:36:47 -08:00 committed by Copybara-Service
parent ab798659d9
commit c5c8e988e8
6 changed files with 132 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -28,9 +28,11 @@ import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects;
import static androidx.media3.transformer.AndroidTestUtil.generateTextureFromBitmap;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_SUCCEEDED;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
import android.content.Context;
@ -478,6 +480,38 @@ public class TransformerEndToEndTest {
assertThat(result.exportResult.durationMs).isAtMost(2000);
}
@Test
public void videoEditing_trimOptimizationEnabled_fallbackToNormalExport() throws Exception {
String testId = "videoEditing_trimOptimizationEnabled_fallbackToNormalExport";
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
// The trim optimization is only guaranteed to work on emulator for this file.
assumeTrue(isRunningOnEmulator());
// MediaCodec returns a segmentation fault fails at this SDK level on emulators.
assumeFalse(Util.SDK_INT == 26);
MediaItem mediaItem =
new MediaItem.Builder()
.setUri("asset:///media/mp4/crow_emulator_transformer_output.mp4")
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(500)
.setEndPositionMs(2500)
.build())
.build();
ImmutableList<Effect> videoEffects = ImmutableList.of(Presentation.createForHeight(480));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem)
.setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects))
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_ABANDONED);
}
@Test
public void videoEncoderFormatUnsupported_completesWithError() throws Exception {
String testId = "videoEncoderFormatUnsupported_completesWithError";

24
libraries/transformer/src/main/java/androidx/media3/transformer/Mp4MetadataInfo.java
View File

@ -64,15 +64,20 @@ import org.checkerframework.checker.nullness.qual.Nullable;
/** The video {@link Format} or {@code null} if there is no video track. */
public final @Nullable Format videoFormat;
/** The audio {@link Format} or {@code null} if there is no audio track. */
public final @Nullable Format audioFormat;
private Mp4MetadataInfo(
long durationUs,
long lastSyncSampleTimestampUs,
long firstSyncSampleTimestampUsAfterTimeUs,
@Nullable Format videoFormat) {
@Nullable Format videoFormat,
@Nullable Format audioFormat) {
this.durationUs = durationUs;
this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs;
this.firstSyncSampleTimestampUsAfterTimeUs = firstSyncSampleTimestampUsAfterTimeUs;
this.videoFormat = videoFormat;
this.audioFormat = audioFormat;
}
/**
@ -129,11 +134,11 @@ import org.checkerframework.checker.nullness.qual.Nullable;
throw new IllegalStateException("The MP4 file is invalid");
}
}
long durationUs = mp4Extractor.getDurationUs();
long lastSyncSampleTimestampUs = C.TIME_UNSET;
long firstSyncSampleTimestampUsAfterTimeUs = C.TIME_UNSET;
@Nullable Format videoFormat = null;
if (extractorOutput.videoTrackId != C.INDEX_UNSET) {
ExtractorOutputImpl.TrackOutputImpl videoTrackOutput =
checkNotNull(extractorOutput.trackTypeToTrackOutput.get(C.TRACK_TYPE_VIDEO));
@ -155,11 +160,20 @@ import org.checkerframework.checker.nullness.qual.Nullable;
}
}
}
@Nullable Format audioFormat = null;
if (extractorOutput.audioTrackId != C.INDEX_UNSET) {
ExtractorOutputImpl.TrackOutputImpl audioTrackOutput =
checkNotNull(extractorOutput.trackTypeToTrackOutput.get(C.TRACK_TYPE_AUDIO));
audioFormat = checkNotNull(audioTrackOutput.format);
}
return new Mp4MetadataInfo(
durationUs,
lastSyncSampleTimestampUs,
firstSyncSampleTimestampUsAfterTimeUs,
videoFormat);
videoFormat,
audioFormat);
} finally {
DataSourceUtil.closeQuietly(dataSource);
mp4Extractor.release();
@ -168,12 +182,14 @@ import org.checkerframework.checker.nullness.qual.Nullable;
private static final class ExtractorOutputImpl implements ExtractorOutput {
public int videoTrackId;
public int audioTrackId;
public boolean seekMapInitialized;
final Map<Integer, TrackOutputImpl> trackTypeToTrackOutput;
public ExtractorOutputImpl() {
videoTrackId = C.INDEX_UNSET;
audioTrackId = C.INDEX_UNSET;
trackTypeToTrackOutput = new HashMap<>();
}
@ -181,6 +197,8 @@ import org.checkerframework.checker.nullness.qual.Nullable;
public TrackOutput track(int id, @C.TrackType int type) {
if (type == C.TRACK_TYPE_VIDEO) {
videoTrackId = id;
} else if (type == C.TRACK_TYPE_AUDIO) {
audioTrackId = id;
}
@Nullable TrackOutputImpl trackOutput = trackTypeToTrackOutput.get(type);

48
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -22,6 +22,8 @@ import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.transformer.Composition.HDR_MODE_EXPERIMENTAL_FORCE_INTERPRET_HDR_AS_SDR;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo;
import static androidx.media3.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes;
import static java.lang.annotation.ElementType.TYPE_USE;
@ -301,14 +303,13 @@ public final class Transformer {
* <li>Progress updates will be unavailable.
* </ul>
*
* <p>{@link ExportResult#optimizationResult} will indicate whether the optimization was
* applied.
*
* <p>This process relies on the given {@linkplain #setEncoderFactory EncoderFactory} providing
* the right encoder level and profiles when transcoding, so that the transcoded and transmuxed
* segments of the file can be stitched together. If the file segments can't be stitched
* together, the {@linkplain #start(Composition, String) export operation} will throw an
* exception.
* together, Transformer throw away any progress and proceed with unoptimized export instead.
*
* <p>The {@link ExportResult#optimizationResult} will indicate whether the optimization was
* applied.
*
* @param enabled Whether to enable trim optimization.
* @return This builder.
@ -1257,7 +1258,32 @@ public final class Transformer {
processFullInput();
return;
}
remuxingMuxerWrapper =
new MuxerWrapper(
checkNotNull(outputFilePath),
muxerFactory,
componentListener,
MuxerWrapper.MUXER_MODE_MUX_PARTIAL);
if (shouldTranscodeVideo(
checkNotNull(mp4MetadataInfo.videoFormat),
composition,
/* sequenceIndex= */ 0,
transformationRequest,
encoderFactory,
remuxingMuxerWrapper)
|| (mp4MetadataInfo.audioFormat != null
&& shouldTranscodeAudio(
mp4MetadataInfo.audioFormat,
composition,
/* sequenceIndex= */ 0,
transformationRequest,
encoderFactory,
remuxingMuxerWrapper))) {
remuxingMuxerWrapper = null;
exportResultBuilder.setOptimizationResult(OPTIMIZATION_ABANDONED);
processFullInput();
return;
}
Transformer.this.mp4MetadataInfo = mp4MetadataInfo;
Composition trancodeComposition =
buildNewCompositionWithClipTimes(
@ -1267,17 +1293,9 @@ public final class Transformer {
mp4MetadataInfo.durationUs,
/* startsAtKeyFrame= */ false);
// TODO: b/304476154 - Check for cases where we shouldTranscode anyway and proceed with
// normal export instead.
remuxingMuxerWrapper =
new MuxerWrapper(
checkNotNull(outputFilePath),
muxerFactory,
componentListener,
MuxerWrapper.MUXER_MODE_MUX_PARTIAL);
startInternal(
trancodeComposition,
remuxingMuxerWrapper,
checkNotNull(remuxingMuxerWrapper),
componentListener,
/* initialTimestampOffsetUs= */ 0);
}

19
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -45,6 +45,7 @@ import androidx.annotation.Nullable;
import androidx.media3.common.C;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.util.Clock;
@ -692,12 +693,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} else if (trackType == C.TRACK_TYPE_VIDEO) {
shouldTranscode =
shouldTranscodeVideo(
inputFormat,
composition,
sequenceIndex,
transformationRequest,
encoderFactory,
muxerWrapper);
inputFormat,
composition,
sequenceIndex,
transformationRequest,
encoderFactory,
muxerWrapper)
|| clippingRequiresTranscode(firstEditedMediaItem.mediaItem);
}
checkState(!shouldTranscode || assetLoaderCanOutputDecoded);
@ -706,6 +708,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
}
}
private static boolean clippingRequiresTranscode(MediaItem mediaItem) {
return mediaItem.clippingConfiguration.startPositionMs > 0
&& !mediaItem.clippingConfiguration.startsAtKeyFrame;
}
/** Tracks the inputs and outputs of {@link AssetLoader AssetLoaders}. */
private static final class AssetLoaderInputTracker {
private final List<SequenceMetadata> sequencesMetadata;

4
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java
View File

@ -125,10 +125,6 @@ import com.google.common.collect.ImmutableList;
}
EditedMediaItem firstEditedMediaItem =
composition.sequences.get(sequenceIndex).editedMediaItems.get(0);
if (firstEditedMediaItem.mediaItem.clippingConfiguration.startPositionMs > 0
&& !firstEditedMediaItem.mediaItem.clippingConfiguration.startsAtKeyFrame) {
return true;
}
if (encoderFactory.videoNeedsEncoding()) {
return true;
}

32
libraries/transformer/src/test/java/androidx/media3/transformer/Mp4MetadataInfoTest.java
View File

@ -15,6 +15,8 @@
*/
package androidx.media3.transformer;
import static androidx.media3.common.MimeTypes.AUDIO_AAC;
import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;
@ -125,7 +127,7 @@ public class Mp4MetadataInfoTest {
}
@Test
public void videoFormat_outputsFormatObjectWithCorrectInitializationData() throws IOException {
public void videoFormat_outputsFormatObjectWithCorrectRelevantFormatData() throws IOException {
String mp4FilePath = "asset:///media/mp4/sample.mp4";
Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath);
byte[] expectedCsd0 = {
@ -139,6 +141,11 @@ public class Mp4MetadataInfoTest {
assertThat(actualFormat).isNotNull();
assertThat(actualFormat.initializationData.get(0)).isEqualTo(expectedCsd0);
assertThat(actualFormat.initializationData.get(1)).isEqualTo(expectedCsd1);
assertThat(actualFormat.sampleMimeType).isEqualTo(VIDEO_H264);
assertThat(actualFormat.width).isEqualTo(1080);
assertThat(actualFormat.height).isEqualTo(720);
assertThat(actualFormat.rotationDegrees).isEqualTo(0);
assertThat(actualFormat.pixelWidthHeightRatio).isEqualTo(1);
}
@Test
@ -148,4 +155,27 @@ public class Mp4MetadataInfoTest {
assertThat(mp4MetadataInfo.videoFormat).isNull();
}
@Test
public void audioFormat_outputsFormatObjectWithCorrectRelevantFormatData() throws IOException {
String mp4FilePath = "asset:///media/mp4/sample.mp4";
Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath);
byte[] expectedCsd0 = {18, 8};
Format actualFormat = mp4MetadataInfo.audioFormat;
assertThat(actualFormat).isNotNull();
assertThat(actualFormat.sampleMimeType).isEqualTo(AUDIO_AAC);
assertThat(actualFormat.channelCount).isEqualTo(1);
assertThat(actualFormat.sampleRate).isEqualTo(44100);
assertThat(actualFormat.initializationData.get(0)).isEqualTo(expectedCsd0);
}
@Test
public void audioFormat_videoOnlyMp4File_outputsNull() throws IOException {
String mp4FilePath = "asset:///media/mp4/sample_18byte_nclx_colr.mp4";
Mp4MetadataInfo mp4MetadataInfo = Mp4MetadataInfo.create(context, mp4FilePath);
assertThat(mp4MetadataInfo.audioFormat).isNull();
}
}