loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Have VideoSampleExporter output orientation match input

Browse Source 
This should enable trim optimization to work correctly in more cases.

PiperOrigin-RevId: 611096958
This commit is contained in:
Googler 2024-02-28 07:34:31 -08:00 committed by Copybara-Service
parent 55bfe4f95c
commit 08993b6fb1
7 changed files with 119 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
libraries/common/src/main/java/androidx/media3/common/util/Util.java
View File

@ -501,6 +501,7 @@ public final class Util {
String deviceName = Ascii.toLowerCase(Util.DEVICE);
return deviceName.contains("emulator")
|| deviceName.contains("emu64a")
|| deviceName.contains("emu64x")
|| deviceName.contains("generic");
}

BIN
libraries/test_data/src/test/assets/media/mp4/internal_emulator_transformer_output_180_rotated.mp4 Normal file
View File

Binary file not shown.

BIN
libraries/test_data/src/test/assets/media/mp4/internal_emulator_transformer_output_270_rotated.mp4 Normal file
View File

Binary file not shown.

6
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -72,6 +72,12 @@ public final class AndroidTestUtil {
public static final String MP4_TRIM_OPTIMIZATION_URI_STRING =
"asset:///media/mp4/internal_emulator_transformer_output.mp4";
public static final String MP4_TRIM_OPTIMIZATION_270_URI_STRING =
"asset:///media/mp4/internal_emulator_transformer_output_270_rotated.mp4";
public static final String MP4_TRIM_OPTIMIZATION_180_URI_STRING =
"asset:///media/mp4/internal_emulator_transformer_output_180_rotated.mp4";
public static final String MP4_TRIM_OPTIMIZATION_PIXEL_URI_STRING =
"asset:///media/mp4/pixel7_videoOnly_cleaned.mp4";

76
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -25,6 +25,8 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_180_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_270_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_TRIM_OPTIMIZATION_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.PNG_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.createOpenGlObjects;
@ -704,6 +706,80 @@ public class TransformerEndToEndTest {
assertThat(result.exportResult.audioConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED);
}
@Test
public void
clippedMedia_trimOptimizationEnabled_inputFileRotated270_completesWithOptimizationApplied()
throws Exception {
if (!isRunningOnEmulator() || Util.SDK_INT < 33) {
// The trim optimization is only guaranteed to work on emulator for this (emulator-transcoded)
// file.
recordTestSkipped(context, testId, /* reason= */ "SDK 33 Emulator only test");
assumeTrue(false);
}
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(MP4_TRIM_OPTIMIZATION_270_URI_STRING)
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(500)
.setEndPositionMs(2500)
.build())
.build();
EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_SUCCEEDED);
assertThat(result.exportResult.durationMs).isAtMost(2000);
assertThat(result.exportResult.videoConversionProcess)
.isEqualTo(CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED);
assertThat(result.exportResult.audioConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED);
Format format = retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO);
assertThat(format.rotationDegrees).isEqualTo(270);
}
@Test
public void
clippedMedia_trimOptimizationEnabled_inputFileRotated180_completesWithOptimizationApplied()
throws Exception {
if (!isRunningOnEmulator() || Util.SDK_INT < 33) {
// The trim optimization is only guaranteed to work on emulator for this (emulator-transcoded)
// file.
recordTestSkipped(context, testId, /* reason= */ "SDK 33 Emulator only test");
assumeTrue(false);
}
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(MP4_TRIM_OPTIMIZATION_180_URI_STRING)
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(500)
.setEndPositionMs(2500)
.build())
.build();
EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.optimizationResult).isEqualTo(OPTIMIZATION_SUCCEEDED);
assertThat(result.exportResult.durationMs).isAtMost(2000);
assertThat(result.exportResult.videoConversionProcess)
.isEqualTo(CONVERSION_PROCESS_TRANSMUXED_AND_TRANSCODED);
assertThat(result.exportResult.audioConversionProcess).isEqualTo(CONVERSION_PROCESS_TRANSMUXED);
Format format = retrieveTrackFormat(context, result.filePath, C.TRACK_TYPE_VIDEO);
assertThat(format.rotationDegrees).isEqualTo(180);
}
@Test
public void
clippedMediaAudioRemovedNoOpEffectAndRotated_trimOptimizationEnabled_completedWithOptimizationAppliedAndCorrectOrientation()

39
libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java
View File

@ -367,6 +367,22 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
checkArgument(
trackType == C.TRACK_TYPE_AUDIO || trackType == C.TRACK_TYPE_VIDEO,
"Unsupported track format: " + sampleMimeType);
if (trackType == C.TRACK_TYPE_VIDEO) {
format =
format
.buildUpon()
.setRotationDegrees((format.rotationDegrees + additionalRotationDegrees) % 360)
.build();
if (muxerMode == MUXER_MODE_MUX_PARTIAL) {
List<byte[]> mostCompatibleInitializationData =
getMostComatibleInitializationData(format, checkNotNull(appendVideoFormat));
if (mostCompatibleInitializationData == null) {
throw new AppendTrackFormatException("Switching to MUXER_MODE_APPEND will fail.");
}
format = format.buildUpon().setInitializationData(mostCompatibleInitializationData).build();
}
}
if (muxerMode == MUXER_MODE_APPEND) {
if (trackType == C.TRACK_TYPE_VIDEO) {
checkState(contains(trackTypeToInfo, C.TRACK_TYPE_VIDEO));
@ -392,6 +408,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
throw new AppendTrackFormatException(
"Video format mismatch - height: " + existingFormat.height + " != " + format.height);
}
if (existingFormat.rotationDegrees != format.rotationDegrees) {
throw new AppendTrackFormatException(
"Video format mismatch - rotationDegrees: "
+ existingFormat.rotationDegrees
+ " != "
+ format.rotationDegrees);
}
// The initialization data of the existing format is already compatible with
// appendVideoFormat.
if (!format.initializationDataEquals(checkNotNull(appendVideoFormat))) {
@ -439,22 +462,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
checkState(
!contains(trackTypeToInfo, trackType), "There is already a track of type " + trackType);
if (trackType == C.TRACK_TYPE_VIDEO) {
format =
format
.buildUpon()
.setRotationDegrees((format.rotationDegrees + additionalRotationDegrees) % 360)
.build();
if (muxerMode == MUXER_MODE_MUX_PARTIAL) {
List<byte[]> mostCompatibleInitializationData =
getMostComatibleInitializationData(format, checkNotNull(appendVideoFormat));
if (mostCompatibleInitializationData == null) {
throw new AppendTrackFormatException("Switching to MUXER_MODE_APPEND will fail.");
}
format = format.buildUpon().setInitializationData(mostCompatibleInitializationData).build();
}
}
ensureMuxerInitialized();
TrackInfo trackInfo = new TrackInfo(format, muxer.addTrack(format));
trackTypeToInfo.put(trackType, trackInfo);

13
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java
View File

@ -301,6 +301,10 @@ import org.checkerframework.dataflow.qual.Pure;
// frame before encoding, so the encoded frame's width >= height, and sets
// rotationDegrees in the output Format to ensure the frame is displayed in the correct
// orientation.
// VideoGraph rotates the decoded video frames counter-clockwise by outputRotationDegrees.
// Instruct the muxer to signal clockwise rotation by outputRotationDegrees.
// When both VideoGraph and muxer rotations are applied, the video will be displayed the right
// way up.
if (requestedWidth < requestedHeight) {
int temp = requestedWidth;
requestedWidth = requestedHeight;
@ -308,6 +312,15 @@ import org.checkerframework.dataflow.qual.Pure;
outputRotationDegrees = 90;
}
// Try to match the inputFormat's rotation, but preserve landscape mode.
// This is a best-effort attempt to preserve input video properties
// (helpful for trim optimization), but is not guaranteed to work when effects are applied.
if (inputFormat.rotationDegrees % 180 == outputRotationDegrees % 180) {
outputRotationDegrees = inputFormat.rotationDegrees;
}
// Rotation is handled by this class. The encoder must see a landscape video with zero
// degrees rotation.
Format requestedEncoderFormat =
new Format.Builder()
.setWidth(requestedWidth)