loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-04-30 06:46:50 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Move setting the muxerWrapper rotation out of shouldTranscodeVideo()

Browse Source 
Before supporting transmuxing when both no op effects and regular rotations are set, move setting the muxerWrapper rotation out of shouldTranscodeVideo() to ensure the muxerWrapper rotation is only set at the appropriate times.

This cl also ensures the state between the muxerWrapper and the list of video effects is consistent by clearing the list of videoEffects in trim optimization. If trim optimisation is being applied, then EditedMediItem.effects.videoEffects only contains no-op effects or regular rotations that get be applied in the muxer wrapper. Therefore, we should clear the list of video effects to ensure that no effect gets applied twice.

PiperOrigin-RevId: 604292052
This commit is contained in:
tofunmi 2024-02-05 05:50:43 -08:00 committed by Copybara-Service
parent 029071a342
commit 49c6d25106
8 changed files with 10286 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10001
libraries/test_data/src/test/assets/transformerdumps/mp4/sample_with_increasing_timestamps_320w_240h.mp4/rotated.dump Normal file
View File

File diff suppressed because it is too large Load Diff

65
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -70,6 +70,7 @@ import androidx.media3.effect.RgbFilter;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.media3.effect.TimestampWrapper;
import androidx.media3.exoplayer.audio.TeeAudioProcessor;
import androidx.media3.test.utils.FileUtil;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
@ -87,6 +88,7 @@ import org.junit.runner.RunWith;
public class TransformerEndToEndTest {
private final Context context = ApplicationProvider.getApplicationContext();
private volatile @MonotonicNonNull TextureAssetLoader textureAssetLoader;
@Test
@ -484,6 +486,55 @@ public class TransformerEndToEndTest {
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
}
@Test
public void
clippedAndRotatedMedia_trimOptimizationEnabledButFormatsMismatch_fallsbackWithCorrectOrientationOutput()
throws Exception {
String testId =
"clippedAndRotatedMedia_trimOptimizationEnabledButFormatsMismatch_fallsbackWithCorrectOrientationOutput";
if (AndroidTestUtil.skipAndLogIfFormatsUnsupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT,
/* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT)) {
return;
}
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
long clippingStartMs = 10_000;
long clippingEndMs = 13_000;
// The format for this file cannot be encoded on phones, so it will trigger trim optimization
// fallback. This is because its csd doesn't match any known phone decoder.
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(clippingStartMs)
.setEndPositionMs(clippingEndMs)
.build())
.build();
ImmutableList<Effect> videoEffects =
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build());
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.optimizationResult)
.isEqualTo(OPTIMIZATION_FAILED_FORMAT_MISMATCH);
assertThat(result.exportResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
Format format = FileUtil.retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO);
// The video is transcoded, so the rotation is performed in the VideoFrameProcessor.
// The output video is portrait, but Transformer's default setup encodes videos landscape.
assertThat(format.rotationDegrees).isEqualTo(0);
}
@Test
public void
clippedMedia_trimOptimizationEnabled_noKeyFrameBetweenClipTimes_fallbackToNormalExport()
@ -591,9 +642,10 @@ public class TransformerEndToEndTest {
@Test
public void
clippedMedia_trimOptimizationEnabled_audioRemovedAndRotated_completesWithOptimizationApplied()
clippedMediaAudioRemovedAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation()
throws Exception {
String testId = "clippedMedia_trimOptimizationEnabled_completesWithOptimizationApplied";
String testId =
"clippedMediaAudioRemovedAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation";
if (!isRunningOnEmulator() || Util.SDK_INT != 33) {
// The trim optimization is only guaranteed to work on emulator for this (emulator-transcoded)
// file.
@ -615,7 +667,7 @@ public class TransformerEndToEndTest {
new Effects(
/* audioProcessors= */ ImmutableList.of(),
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build()));
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build()));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setRemoveAudio(true).setEffects(effects).build();
@ -626,6 +678,13 @@ public class TransformerEndToEndTest {
assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_SUCCEEDED);
assertThat(result.exportResult.durationMs).isAtMost(2000);
Format format = FileUtil.retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO);
// The video is trim-optimized, so the rotation is performed in MuxerWrapper.
// The MuxerWrapper rotation is clockwise while the ScaleAndRotateTransformation rotation
// is counterclockwise.
// Manually verified that the video has correct rotation.
assertThat(format.rotationDegrees).isEqualTo(180);
}
@Test

33
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -25,9 +25,10 @@ import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_OT
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_ABANDONED_TRIM_AND_TRANSCODING_TRANSFORMATION_REQUESTED;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_EXTRACTION_FAILED;
import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_FORMAT_MISMATCH;
import static androidx.media3.transformer.TransformerUtil.maybeSetMuxerWrapperAdditionalRotationDegrees;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo;
import static androidx.media3.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes;
import static androidx.media3.transformer.TransmuxTranscodeHelper.buildUponCompositionForTrimOptimization;
import static java.lang.Math.round;
import static java.lang.annotation.ElementType.TYPE_USE;
@ -1359,14 +1360,14 @@ public final class Transformer {
private void processMediaBeforeFirstSyncSampleAfterTrimStartTime() {
transformerState = TRANSFORMER_STATE_PROCESS_MEDIA_START;
MediaItem firstMediaItem =
checkNotNull(composition).sequences.get(0).editedMediaItems.get(0).mediaItem;
long trimStartTimeUs = firstMediaItem.clippingConfiguration.startPositionUs;
long trimEndTimeUs = firstMediaItem.clippingConfiguration.endPositionUs;
EditedMediaItem firstEditedMediaItem =
checkNotNull(composition).sequences.get(0).editedMediaItems.get(0);
long trimStartTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.startPositionUs;
long trimEndTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.endPositionUs;
ListenableFuture<Mp4Info> getMp4InfoFuture =
TransmuxTranscodeHelper.getMp4Info(
context,
checkNotNull(firstMediaItem.localConfiguration).uri.toString(),
checkNotNull(firstEditedMediaItem.mediaItem.localConfiguration).uri.toString(),
trimStartTimeUs);
Futures.addCallback(
getMp4InfoFuture,
@ -1397,12 +1398,13 @@ public final class Transformer {
if (mp4Info.firstSyncSampleTimestampUsAfterTimeUs - trimStartTimeUs
<= maxEncodedAudioBufferDurationUs) {
Transformer.this.composition =
buildNewCompositionWithClipTimes(
buildUponCompositionForTrimOptimization(
composition,
mp4Info.firstSyncSampleTimestampUsAfterTimeUs,
firstMediaItem.clippingConfiguration.endPositionUs,
trimEndTimeUs,
mp4Info.durationUs,
/* startsAtKeyFrame= */ true);
/* startsAtKeyFrame= */ true,
/* clearVideoEffects= */ false);
exportResultBuilder.setOptimizationResult(
OPTIMIZATION_ABANDONED_KEYFRAME_PLACEMENT_OPTIMAL_FOR_TRIM);
processFullInput();
@ -1437,14 +1439,16 @@ public final class Transformer {
return;
}
Transformer.this.mediaItemInfo = mp4Info;
maybeSetMuxerWrapperAdditionalRotationDegrees(
remuxingMuxerWrapper, firstEditedMediaItem.effects.videoEffects);
Composition trancodeComposition =
buildNewCompositionWithClipTimes(
buildUponCompositionForTrimOptimization(
composition,
trimStartTimeUs,
mp4Info.firstSyncSampleTimestampUsAfterTimeUs,
mp4Info.durationUs,
/* startsAtKeyFrame= */ false);
/* startsAtKeyFrame= */ false,
/* clearVideoEffects= */ true);
startInternal(
trancodeComposition,
checkNotNull(remuxingMuxerWrapper),
@ -1476,12 +1480,13 @@ public final class Transformer {
long trimStartTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.startPositionUs;
long trimEndTimeUs = firstEditedMediaItem.mediaItem.clippingConfiguration.endPositionUs;
Composition transmuxComposition =
buildNewCompositionWithClipTimes(
buildUponCompositionForTrimOptimization(
composition,
mediaItemInfo.firstSyncSampleTimestampUsAfterTimeUs,
trimEndTimeUs,
mediaItemInfo.durationUs,
/* startsAtKeyFrame= */ true);
/* startsAtKeyFrame= */ true,
/* clearVideoEffects= */ true);
checkNotNull(remuxingMuxerWrapper);
remuxingMuxerWrapper.changeToAppendMode();
startInternal(

7
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -28,6 +28,7 @@ import static androidx.media3.transformer.ExportException.ERROR_CODE_MUXING_FAIL
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
import static androidx.media3.transformer.TransformerUtil.getProcessedTrackType;
import static androidx.media3.transformer.TransformerUtil.maybeSetMuxerWrapperAdditionalRotationDegrees;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo;
import static java.lang.Math.max;
@ -553,6 +554,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
boolean shouldTranscode =
shouldTranscode(firstAssetLoaderInputFormat, supportedOutputTypes);
if (!shouldTranscode
&& getProcessedTrackType(firstAssetLoaderInputFormat.sampleMimeType)
== TRACK_TYPE_VIDEO) {
maybeSetMuxerWrapperAdditionalRotationDegrees(
muxerWrapper, firstEditedMediaItem.effects.videoEffects);
}
assetLoaderInputTracker.setShouldTranscode(trackType, shouldTranscode);
return shouldTranscode;
}

32
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerUtil.java
View File

@ -145,7 +145,7 @@ import com.google.common.collect.ImmutableList;
ImmutableList<Effect> videoEffects = firstEditedMediaItem.effects.videoEffects;
return !videoEffects.isEmpty()
&& !areVideoEffectsAllNoOp(videoEffects, inputFormat)
&& !hasOnlyRegularRotationEffect(videoEffects, muxerWrapper);
&& getRegularRotationDegrees(videoEffects) == -1;
}
/**
@ -173,27 +173,43 @@ import com.google.common.collect.ImmutableList;
return true;
}
private static boolean hasOnlyRegularRotationEffect(
ImmutableList<Effect> videoEffects, MuxerWrapper muxerWrapper) {
private static float getRegularRotationDegrees(ImmutableList<Effect> videoEffects) {
if (videoEffects.size() != 1) {
return false;
return -1;
}
Effect videoEffect = videoEffects.get(0);
if (!(videoEffect instanceof ScaleAndRotateTransformation)) {
return false;
return -1;
}
ScaleAndRotateTransformation scaleAndRotateTransformation =
(ScaleAndRotateTransformation) videoEffect;
if (scaleAndRotateTransformation.scaleX != 1f || scaleAndRotateTransformation.scaleY != 1f) {
return false;
return -1;
}
float rotationDegrees = scaleAndRotateTransformation.rotationDegrees;
if (rotationDegrees == 90f || rotationDegrees == 180f || rotationDegrees == 270f) {
return rotationDegrees;
}
return -1;
}
// TODO: b/322146331 - Support setting MuxerWrapper#setAdditionalRotationDegrees(int) for
// videoEffects lists that are a mix of no-ops and rotations.
/**
* Sets {@linkplain MuxerWrapper#setAdditionalRotationDegrees(int) the additionalRotationDegrees}
* on the given {@link MuxerWrapper} if the given {@code videoEffects} only contains one regular
* rotation effect. A regular rotation is a rotation divisible by 90 degrees.
*/
public static void maybeSetMuxerWrapperAdditionalRotationDegrees(
MuxerWrapper muxerWrapper, ImmutableList<Effect> videoEffects) {
float rotationDegrees = getRegularRotationDegrees(videoEffects);
if (rotationDegrees == -1) {
return;
}
if (rotationDegrees == 90f || rotationDegrees == 180f || rotationDegrees == 270f) {
// The MuxerWrapper rotation is clockwise while the ScaleAndRotateTransformation rotation
// is counterclockwise.
muxerWrapper.setAdditionalRotationDegrees(360 - Math.round(rotationDegrees));
return true;
}
return false;
}
}

12
libraries/transformer/src/main/java/androidx/media3/transformer/TransmuxTranscodeHelper.java
View File

@ -75,12 +75,13 @@ import java.util.List;
return mp4InfoSettableFuture;
}
public static Composition buildNewCompositionWithClipTimes(
public static Composition buildUponCompositionForTrimOptimization(
Composition oldComposition,
long startTimeUs,
long endTimeUs,
long mediaDurationUs,
boolean startsAtKeyFrame) {
boolean startsAtKeyFrame,
boolean clearVideoEffects) {
EditedMediaItem firstEditedMediaItem = oldComposition.sequences.get(0).editedMediaItems.get(0);
MediaItem.ClippingConfiguration clippingConfiguration =
@ -96,11 +97,18 @@ import java.util.List;
.buildUpon()
.setClippingConfiguration(clippingConfiguration)
.build();
Effects effects =
clearVideoEffects
? new Effects(
firstEditedMediaItem.effects.audioProcessors,
/* videoEffects= */ ImmutableList.of())
: firstEditedMediaItem.effects;
EditedMediaItem editedMediaItem =
firstEditedMediaItem
.buildUpon()
.setMediaItem(mediaItem)
.setDurationUs(mediaDurationUs)
.setEffects(effects)
.build();
return oldComposition

33
libraries/transformer/src/test/java/androidx/media3/transformer/MediaItemExportTest.java
View File

@ -178,6 +178,39 @@ public final class MediaItemExportTest {
getDumpFileName(/* originalFileName= */ FILE_AUDIO_VIDEO_INCREASING_TIMESTAMPS_15S));
}
@Test
public void
start_trimOptimizationEnabled_clippingConfigurationUnsetAndRotated_outputMatchesOriginalRotated()
throws Exception {
Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false)
.experimentalSetTrimOptimizationEnabled(true)
.build();
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO_INCREASING_TIMESTAMPS_15S)
.build();
ImmutableList<Effect> videoEffects =
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build());
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
transformer.start(editedMediaItem, outputDir.newFile().getPath());
ExportResult exportResult = TransformerTestRunner.runLooper(transformer);
assertThat(exportResult.optimizationResult)
.isEqualTo(OPTIMIZATION_ABANDONED_KEYFRAME_PLACEMENT_OPTIMAL_FOR_TRIM);
// Asserts against file generated when experimentalSetTrimOptimizationEnabled is set to false.
DumpFileAsserts.assertOutput(
context,
muxerFactory.getCreatedMuxer(),
getDumpFileName(
/* originalFileName= */ FILE_AUDIO_VIDEO_INCREASING_TIMESTAMPS_15S,
/* modifications...= */ "rotated"));
}
@Test
public void start_trimOptimizationEnabled_withClippingStartAtKeyFrame_completesSuccessfully()
throws Exception {

130
libraries/transformer/src/test/java/androidx/media3/transformer/TransformerUtilTest.java Normal file
View File

@ -0,0 +1,130 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.MimeTypes.VIDEO_H264;
import static androidx.media3.transformer.MuxerWrapper.MUXER_MODE_DEFAULT;
import static androidx.media3.transformer.TestUtil.ASSET_URI_PREFIX;
import static androidx.media3.transformer.TestUtil.FILE_AUDIO_VIDEO;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo;
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
import static com.google.common.truth.Truth.assertThat;
import androidx.media3.common.C;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.effect.GlEffect;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.ScaleAndRotateTransformation;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
/** Unit tests for {@link TransformerUtil}. */
@RunWith(AndroidJUnit4.class)
public final class TransformerUtilTest {
@Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder();
public static final Format FORMAT =
new Format.Builder()
.setSampleMimeType(VIDEO_H264)
.setWidth(1080)
.setHeight(720)
.setFrameRate(29.97f)
.setCodecs("avc1.64001F")
.build();
@Test
public void shouldTranscodeVideo_regularRotationAndTranscodingPresentation_returnsTrue()
throws Exception {
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
GlEffect regularRotation =
new ScaleAndRotateTransformation.Builder().setRotationDegrees(90).build();
ImmutableList<Effect> videoEffects =
ImmutableList.of(regularRotation, Presentation.createForHeight(FORMAT.height));
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
Composition composition =
new Composition.Builder(new EditedMediaItemSequence(editedMediaItem)).build();
MuxerWrapper muxerWrapper =
new MuxerWrapper(
temporaryFolder.newFile().getPath(),
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false);
assertThat(
shouldTranscodeVideo(
FORMAT,
composition,
/* sequenceIndex= */ 0,
new TransformationRequest.Builder().build(),
new DefaultEncoderFactory.Builder(getApplicationContext()).build(),
muxerWrapper))
.isTrue();
}
@Test
public void shouldTranscodeVideo_irregularRotationAndPresentation_returnsTrue() throws Exception {
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
GlEffect irregularRotation =
new ScaleAndRotateTransformation.Builder().setRotationDegrees(45).build();
ImmutableList<Effect> videoEffects =
ImmutableList.of(
irregularRotation, Presentation.createForHeight(FORMAT.height), irregularRotation);
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
Composition composition =
new Composition.Builder(new EditedMediaItemSequence(editedMediaItem)).build();
MuxerWrapper muxerWrapper =
new MuxerWrapper(
temporaryFolder.newFile().getPath(),
new DefaultMuxer.Factory(),
new NoOpMuxerListenerImpl(),
MUXER_MODE_DEFAULT,
/* dropSamplesBeforeFirstVideoSample= */ false);
assertThat(
shouldTranscodeVideo(
FORMAT,
composition,
/* sequenceIndex= */ 0,
new TransformationRequest.Builder().build(),
new DefaultEncoderFactory.Builder(getApplicationContext()).build(),
muxerWrapper))
.isTrue();
}
private static final class NoOpMuxerListenerImpl implements MuxerWrapper.Listener {
@Override
public void onTrackEnded(
@C.TrackType int trackType, Format format, int averageBitrate, int sampleCount) {}
@Override
public void onEnded(long durationMs, long fileSizeBytes) {}
@Override
public void onError(ExportException exportException) {}
}
}