From fe3831c5b42e9d92ba2847be48b47c0a32130644 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Fri, 27 May 2022 10:22:04 +0000 Subject: [PATCH] Fix handling clipping in transformer renderers. Decode-only video frames (needed when the frame at / first frame after the clipping start is not a key frame) need to be decoded but not passed to the frame processor chain or encoder. The clipping start offset needs to be removed from the frame timestamps in the passthrough and video pipelines. There are no changes needed for this in the audio pipeline, as it doesn't use the input timestamps -- it uses its own timestamps derived from the buffer sizes instead. Also add demo option to try this out. #minor-release PiperOrigin-RevId: 451353609 --- .../transformer/ConfigurationActivity.java | 37 ++++++++++++++ .../demo/transformer/TransformerActivity.java | 21 +++++++- .../res/layout/configuration_activity.xml | 10 ++++ .../src/main/res/layout/trim_options.xml | 48 +++++++++++++++++++ .../src/main/res/values/strings.xml | 2 + .../TransformerAndroidTestRunner.java | 25 ++++++---- .../transformer/TransformerEndToEndTest.java | 33 ++++++++++++- .../RepeatedTranscodeTransformationTest.java | 8 ++-- .../mh/SetFrameEditTransformationTest.java | 5 +- .../transformer/mh/TranscodeQualityTest.java | 17 +++++-- .../transformer/mh/TransformationTest.java | 16 ++++--- .../mh/analysis/BitrateAnalysisTest.java | 4 +- .../EncoderPerformanceAnalysisTest.java | 3 +- .../PassthroughSamplePipeline.java | 4 ++ .../transformer/TransformationRequest.java | 3 ++ .../media3/transformer/Transformer.java | 7 +++ .../transformer/TransformerAudioRenderer.java | 3 +- .../transformer/TransformerBaseRenderer.java | 2 + .../transformer/TransformerVideoRenderer.java | 8 +++- .../VideoTranscodingSamplePipeline.java | 35 ++++++++++++-- 20 files changed, 257 insertions(+), 34 deletions(-) create mode 100644 demos/transformer/src/main/res/layout/trim_options.xml diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index fc4fe1b9ca..026f396091 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -32,6 +32,7 @@ import android.widget.TextView; import androidx.annotation.Nullable; import androidx.appcompat.app.AlertDialog; import androidx.appcompat.app.AppCompatActivity; +import androidx.media3.common.C; import androidx.media3.common.MimeTypes; import androidx.media3.common.util.Util; import com.google.android.material.slider.RangeSlider; @@ -55,6 +56,8 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String SCALE_X = "scale_x"; public static final String SCALE_Y = "scale_y"; public static final String ROTATE_DEGREES = "rotate_degrees"; + public static final String TRIM_START_MS = "trim_start_ms"; + public static final String TRIM_END_MS = "trim_end_ms"; public static final String ENABLE_FALLBACK = "enable_fallback"; public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping"; public static final String ENABLE_HDR_EDITING = "enable_hdr_editing"; @@ -115,12 +118,15 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull Spinner resolutionHeightSpinner; private @MonotonicNonNull Spinner scaleSpinner; private @MonotonicNonNull Spinner rotateSpinner; + private @MonotonicNonNull CheckBox trimCheckBox; private @MonotonicNonNull CheckBox enableFallbackCheckBox; private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox; private @MonotonicNonNull CheckBox enableHdrEditingCheckBox; private @MonotonicNonNull Button selectDemoEffectsButton; private boolean @MonotonicNonNull [] demoEffectsSelections; private int inputUriPosition; + private long trimStartMs; + private long trimEndMs; private float periodicVignetteCenterX; private float periodicVignetteCenterY; private float periodicVignetteInnerRadius; @@ -188,6 +194,11 @@ public final class ConfigurationActivity extends AppCompatActivity { rotateSpinner.setAdapter(rotateAdapter); rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "60", "90", "180"); + trimCheckBox = findViewById(R.id.trim_checkbox); + trimCheckBox.setOnCheckedChangeListener(this::selectTrimBounds); + trimStartMs = C.TIME_UNSET; + trimEndMs = C.TIME_UNSET; + enableFallbackCheckBox = findViewById(R.id.enable_fallback_checkbox); enableRequestSdrToneMappingCheckBox = findViewById(R.id.request_sdr_tone_mapping_checkbox); enableRequestSdrToneMappingCheckBox.setEnabled(isRequestSdrToneMappingSupported()); @@ -224,6 +235,7 @@ public final class ConfigurationActivity extends AppCompatActivity { "resolutionHeightSpinner", "scaleSpinner", "rotateSpinner", + "trimCheckBox", "enableFallbackCheckBox", "enableRequestSdrToneMappingCheckBox", "enableHdrEditingCheckBox", @@ -258,6 +270,10 @@ public final class ConfigurationActivity extends AppCompatActivity { if (!SAME_AS_INPUT_OPTION.equals(selectedRotate)) { bundle.putFloat(ROTATE_DEGREES, Float.parseFloat(selectedRotate)); } + if (trimCheckBox.isChecked()) { + bundle.putLong(TRIM_START_MS, trimStartMs); + bundle.putLong(TRIM_END_MS, trimEndMs); + } bundle.putBoolean(ENABLE_FALLBACK, enableFallbackCheckBox.isChecked()); bundle.putBoolean( ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked()); @@ -295,6 +311,27 @@ public final class ConfigurationActivity extends AppCompatActivity { .show(); } + private void selectTrimBounds(View view, boolean isChecked) { + if (!isChecked) { + return; + } + View dialogView = getLayoutInflater().inflate(R.layout.trim_options, /* root= */ null); + RangeSlider radiusRangeSlider = + checkNotNull(dialogView.findViewById(R.id.trim_bounds_range_slider)); + radiusRangeSlider.setValues(0f, 60f); // seconds + new AlertDialog.Builder(/* context= */ this) + .setView(dialogView) + .setPositiveButton( + android.R.string.ok, + (DialogInterface dialogInterface, int i) -> { + List radiusRange = radiusRangeSlider.getValues(); + trimStartMs = 1000 * radiusRange.get(0).longValue(); + trimEndMs = 1000 * radiusRange.get(1).longValue(); + }) + .create() + .show(); + } + @RequiresNonNull("selectedFileTextView") private void selectFileInDialog(DialogInterface dialog, int which) { inputUriPosition = which; diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 72d4e9eb6a..594459e315 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -152,9 +152,10 @@ public final class TransformerActivity extends AppCompatActivity { externalCacheFile = createExternalCacheFile("transformer-output.mp4"); String filePath = externalCacheFile.getAbsolutePath(); @Nullable Bundle bundle = intent.getExtras(); + MediaItem mediaItem = createMediaItem(bundle, uri); Transformer transformer = createTransformer(bundle, filePath); transformationStopwatch.start(); - transformer.startTransformation(MediaItem.fromUri(uri), filePath); + transformer.startTransformation(mediaItem, filePath); this.transformer = transformer; } catch (IOException e) { throw new IllegalStateException(e); @@ -181,6 +182,24 @@ public final class TransformerActivity extends AppCompatActivity { }); } + private MediaItem createMediaItem(@Nullable Bundle bundle, Uri uri) { + MediaItem.Builder mediaItemBuilder = new MediaItem.Builder().setUri(uri); + if (bundle != null) { + long trimStartMs = + bundle.getLong(ConfigurationActivity.TRIM_START_MS, /* defaultValue= */ C.TIME_UNSET); + long trimEndMs = + bundle.getLong(ConfigurationActivity.TRIM_END_MS, /* defaultValue= */ C.TIME_UNSET); + if (trimStartMs != C.TIME_UNSET && trimEndMs != C.TIME_UNSET) { + mediaItemBuilder.setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(trimStartMs) + .setEndPositionMs(trimEndMs) + .build()); + } + } + return mediaItemBuilder.build(); + } + // Create a cache file, resetting it if it already exists. private File createExternalCacheFile(String fileName) throws IOException { File file = new File(getExternalCacheDir(), fileName); diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index 3af465719a..2879d6a637 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -159,6 +159,16 @@ android:layout_gravity="right|center_vertical" android:gravity="right" /> + + + + diff --git a/demos/transformer/src/main/res/layout/trim_options.xml b/demos/transformer/src/main/res/layout/trim_options.xml new file mode 100644 index 0000000000..cf2de0f310 --- /dev/null +++ b/demos/transformer/src/main/res/layout/trim_options.xml @@ -0,0 +1,48 @@ + + + + + + + + + + + diff --git a/demos/transformer/src/main/res/values/strings.xml b/demos/transformer/src/main/res/values/strings.xml index 98dc42ecb8..50ac310080 100644 --- a/demos/transformer/src/main/res/values/strings.xml +++ b/demos/transformer/src/main/res/values/strings.xml @@ -27,6 +27,7 @@ Scale video Rotate video (degrees) Enable fallback + Trim Request SDR tone-mapping (API 31+) [Experimental] HDR editing Add demo effects @@ -42,4 +43,5 @@ Center X Center Y Radius range + Bounds in seconds diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java index d52c178a6b..286c858e5b 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java @@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static java.util.concurrent.TimeUnit.SECONDS; import android.content.Context; -import android.net.Uri; import android.view.Surface; import androidx.annotation.Nullable; import androidx.media3.common.Format; @@ -179,17 +178,17 @@ public class TransformerAndroidTestRunner { * cache. * * @param testId A unique identifier for the transformer test run. - * @param uriString The uri (as a {@link String}) of the file to transform. + * @param mediaItem The {@link MediaItem} to transform. * @return The {@link TransformationTestResult}. * @throws Exception The cause of the transformation not completing. */ - public TransformationTestResult run(String testId, String uriString) throws Exception { + public TransformationTestResult run(String testId, MediaItem mediaItem) throws Exception { JSONObject resultJson = new JSONObject(); if (inputValues != null) { resultJson.put("inputValues", JSONObject.wrap(inputValues)); } try { - TransformationTestResult transformationTestResult = runInternal(testId, uriString); + TransformationTestResult transformationTestResult = runInternal(testId, mediaItem); resultJson.put("transformationResult", transformationTestResult.asJsonObject()); if (!suppressAnalysisExceptions && transformationTestResult.analysisException != null) { throw transformationTestResult.analysisException; @@ -208,7 +207,7 @@ public class TransformerAndroidTestRunner { * Transforms the {@code uriString}. * * @param testId An identifier for the test. - * @param uriString The uri (as a {@link String}) of the file to transform. + * @param mediaItem The {@link MediaItem} to transform. * @return The {@link TransformationTestResult}. * @throws IOException If an error occurs opening the output file for writing * @throws TimeoutException If the transformation takes longer than the {@link #timeoutSeconds}. @@ -218,8 +217,14 @@ public class TransformerAndroidTestRunner { * @throws IllegalArgumentException If the path is invalid. * @throws IllegalStateException If an unexpected exception occurs when starting a transformation. */ - private TransformationTestResult runInternal(String testId, String uriString) + private TransformationTestResult runInternal(String testId, MediaItem mediaItem) throws InterruptedException, IOException, TimeoutException, TransformationException { + if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET) + && calculateSsim) { + throw new UnsupportedOperationException( + "SSIM calculation is not supported for clipped inputs."); + } + AtomicReference<@NullableType TransformationException> transformationExceptionReference = new AtomicReference<>(); AtomicReference<@NullableType Exception> unexpectedExceptionReference = new AtomicReference<>(); @@ -249,15 +254,13 @@ public class TransformerAndroidTestRunner { }) .build(); - Uri uri = Uri.parse(uriString); File outputVideoFile = AndroidTestUtil.createExternalCacheFile(context, /* fileName= */ testId + "-output.mp4"); InstrumentationRegistry.getInstrumentation() .runOnMainSync( () -> { try { - testTransformer.startTransformation( - MediaItem.fromUri(uri), outputVideoFile.getAbsolutePath()); + testTransformer.startTransformation(mediaItem, outputVideoFile.getAbsolutePath()); // Catch all exceptions to report. Exceptions thrown here and not caught will NOT // propagate. } catch (Exception e) { @@ -300,7 +303,9 @@ public class TransformerAndroidTestRunner { if (calculateSsim) { double ssim = SsimHelper.calculate( - context, /* referenceVideoPath= */ uriString, outputVideoFile.getPath()); + context, + /* referenceVideoPath= */ checkNotNull(mediaItem.localConfiguration).uri.toString(), + outputVideoFile.getPath()); resultBuilder.setSsim(ssim); } } catch (InterruptedException interruptedException) { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java index d1a9182e87..c7844fd9e1 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java @@ -16,9 +16,12 @@ package androidx.media3.transformer; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING; import static com.google.common.truth.Truth.assertThat; import android.content.Context; +import android.net.Uri; +import androidx.media3.common.MediaItem; import androidx.test.core.app.ApplicationProvider; import androidx.test.ext.junit.runners.AndroidJUnit4; import org.junit.Test; @@ -50,7 +53,7 @@ public class TransformerEndToEndTest { .build() .run( /* testId= */ "videoEditing_completesWithConsistentFrameCount", - MP4_ASSET_URI_STRING); + MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))); assertThat(result.transformationResult.videoFrameCount).isEqualTo(expectedFrameCount); } @@ -71,8 +74,34 @@ public class TransformerEndToEndTest { TransformationTestResult result = new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(/* testId= */ "videoOnly_completesWithConsistentDuration", MP4_ASSET_URI_STRING); + .run( + /* testId= */ "videoOnly_completesWithConsistentDuration", + MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))); assertThat(result.transformationResult.durationMs).isEqualTo(expectedDurationMs); } + + @Test + public void clippedMedia_completesWithClippedDuration() throws Exception { + Context context = ApplicationProvider.getApplicationContext(); + Transformer transformer = new Transformer.Builder(context).build(); + long clippingStartMs = 10_000; + long clippingEndMs = 11_000; + MediaItem mediaItem = + new MediaItem.Builder() + .setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(clippingStartMs) + .setEndPositionMs(clippingEndMs) + .build()) + .build(); + + TransformationTestResult result = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(/* testId= */ "clippedMedia_completesWithClippedDuration", mediaItem); + + assertThat(result.transformationResult.durationMs).isAtMost(clippingEndMs - clippingStartMs); + } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java index 1baa4d4098..e3a12b5b8c 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java @@ -19,6 +19,8 @@ import static androidx.media3.common.util.Assertions.checkNotNull; import static com.google.common.truth.Truth.assertWithMessage; import android.content.Context; +import android.net.Uri; +import androidx.media3.common.MediaItem; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.TransformationRequest; import androidx.media3.transformer.TransformationTestResult; @@ -56,7 +58,7 @@ public final class RepeatedTranscodeTransformationTest { TransformationTestResult testResult = transformerRunner.run( /* testId= */ "repeatedTranscode_givesConsistentLengthOutput_" + i, - AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING); + MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING))); differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes)); } @@ -86,7 +88,7 @@ public final class RepeatedTranscodeTransformationTest { TransformationTestResult testResult = transformerRunner.run( /* testId= */ "repeatedTranscodeNoAudio_givesConsistentLengthOutput_" + i, - AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING); + MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING))); differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes)); } @@ -115,7 +117,7 @@ public final class RepeatedTranscodeTransformationTest { TransformationTestResult testResult = transformerRunner.run( /* testId= */ "repeatedTranscodeNoVideo_givesConsistentLengthOutput_" + i, - AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING); + MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING))); differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes)); } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java index 92bb7b3d16..1322935959 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java @@ -18,6 +18,8 @@ package androidx.media3.transformer.mh; import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING; import android.content.Context; +import android.net.Uri; +import androidx.media3.common.MediaItem; import androidx.media3.transformer.TransformationRequest; import androidx.media3.transformer.Transformer; import androidx.media3.transformer.TransformerAndroidTestRunner; @@ -41,6 +43,7 @@ public class SetFrameEditTransformationTest { new TransformerAndroidTestRunner.Builder(context, transformer) .build() .run( - /* testId= */ "SetFrameEditTransform", MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + /* testId= */ "SetFrameEditTransform", + MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java index d42d85ee92..d0e62e3301 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java @@ -19,6 +19,8 @@ package androidx.media3.transformer.mh; import static com.google.common.truth.Truth.assertThat; import android.content.Context; +import android.net.Uri; +import androidx.media3.common.MediaItem; import androidx.media3.common.MimeTypes; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.TransformationRequest; @@ -58,7 +60,10 @@ public final class TranscodeQualityTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run( + testId, + MediaItem.fromUri( + Uri.parse(AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); if (result.ssim != TransformationTestResult.SSIM_UNSET) { assertThat(result.ssim).isGreaterThan(0.90); @@ -92,7 +97,10 @@ public final class TranscodeQualityTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run( + testId, + MediaItem.fromUri( + Uri.parse(AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); if (result.ssim != TransformationTestResult.SSIM_UNSET) { assertThat(result.ssim).isGreaterThan(0.90); @@ -121,7 +129,10 @@ public final class TranscodeQualityTest { .build() .run( testId, - AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING); + MediaItem.fromUri( + Uri.parse( + AndroidTestUtil + .MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))); if (result.ssim != TransformationTestResult.SSIM_UNSET) { assertThat(result.ssim).isGreaterThan(0.90); diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java index a2d3b654d9..bbc9aa29a1 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java @@ -24,6 +24,8 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_REMOTE_4K60_PORTRA import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import android.content.Context; +import android.net.Uri; +import androidx.media3.common.MediaItem; import androidx.media3.common.util.Util; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.DefaultEncoderFactory; @@ -54,7 +56,7 @@ public class TransformationTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @Test @@ -65,7 +67,7 @@ public class TransformationTest { // No need to calculate SSIM because no decode/encoding, so input frames match output frames. new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @Test @@ -84,7 +86,7 @@ public class TransformationTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @Test @@ -106,7 +108,7 @@ public class TransformationTest { .setCalculateSsim(true) .setTimeoutSeconds(180) .build() - .run(testId, MP4_REMOTE_4K60_PORTRAIT_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_REMOTE_4K60_PORTRAIT_URI_STRING))); } @Test @@ -121,7 +123,7 @@ public class TransformationTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setCalculateSsim(true) .build() - .run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING))); } @Test @@ -135,7 +137,7 @@ public class TransformationTest { .build(); new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(testId, MP4_ASSET_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING))); } @Test @@ -159,6 +161,6 @@ public class TransformationTest { .build(); new TransformerAndroidTestRunner.Builder(context, transformer) .build() - .run(testId, MP4_ASSET_SEF_URI_STRING); + .run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_SEF_URI_STRING))); } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java index e2fc11eb40..3d31d43718 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java @@ -19,6 +19,8 @@ import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR; import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR; import android.content.Context; +import android.net.Uri; +import androidx.media3.common.MediaItem; import androidx.media3.common.util.Assertions; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.DefaultEncoderFactory; @@ -116,6 +118,6 @@ public class BitrateAnalysisTest { .setInputValues(inputValues) .setCalculateSsim(true) .build() - .run(testId, fileUri); + .run(testId, MediaItem.fromUri(Uri.parse(fileUri))); } } diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java index e83a1ff18d..1f379e6653 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java @@ -22,6 +22,7 @@ import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped; import android.content.Context; import android.media.MediaFormat; import android.net.Uri; +import androidx.media3.common.MediaItem; import androidx.media3.common.util.Util; import androidx.media3.transformer.AndroidTestUtil; import androidx.media3.transformer.DefaultEncoderFactory; @@ -136,6 +137,6 @@ public class EncoderPerformanceAnalysisTest { new TransformerAndroidTestRunner.Builder(context, transformer) .setInputValues(inputValues) .build() - .run(testId, fileUri); + .run(testId, MediaItem.fromUri(Uri.parse(fileUri))); } } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java index f2387ace36..a8bbc8f577 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java @@ -25,14 +25,17 @@ import androidx.media3.decoder.DecoderInputBuffer; private final DecoderInputBuffer buffer; private final Format format; + private final long outputPresentationTimeOffsetUs; private boolean hasPendingBuffer; public PassthroughSamplePipeline( Format format, + long outputPresentationTimeOffsetUs, TransformationRequest transformationRequest, FallbackListener fallbackListener) { this.format = format; + this.outputPresentationTimeOffsetUs = outputPresentationTimeOffsetUs; buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); hasPendingBuffer = false; fallbackListener.onTransformationRequestFinalized(transformationRequest); @@ -46,6 +49,7 @@ import androidx.media3.decoder.DecoderInputBuffer; @Override public void queueInputBuffer() { + buffer.timeUs -= outputPresentationTimeOffsetUs; hasPendingBuffer = true; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java index 0441f9619d..0d114287e1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java @@ -90,6 +90,9 @@ public final class TransformationRequest { * Mp4Extractor#FLAG_READ_SEF_DATA} is set on the {@link Mp4Extractor} used. Otherwise, the slow * motion metadata will be ignored and the input won't be flattened. * + *

Using slow motion flattening together with {@link + * androidx.media3.common.MediaItem.ClippingConfiguration} is not supported yet. + * * @param flattenForSlowMotion Whether to flatten for slow motion. * @return This builder. */ diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 6cea6addbe..41acc4b6a0 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -61,6 +61,7 @@ import androidx.media3.extractor.DefaultExtractorsFactory; import androidx.media3.extractor.mp4.Mp4Extractor; import com.google.common.collect.ImmutableList; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -685,6 +686,12 @@ public final class Transformer { * @throws IOException If an error occurs opening the output file for writing. */ public void startTransformation(MediaItem mediaItem, String path) throws IOException { + if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET) + && transformationRequest.flattenForSlowMotion) { + // TODO(b/233986762): Support clipping with SEF flattening. + throw new UnsupportedEncodingException( + "Clipping is not supported when slow motion flattening is requested"); + } startTransformation(mediaItem, muxerFactory.create(path, containerMimeType)); } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java index 29b39076cf..271fc81ccd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java @@ -70,7 +70,8 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData; Format inputFormat = checkNotNull(formatHolder.format); if (shouldPassthrough(inputFormat)) { samplePipeline = - new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); + new PassthroughSamplePipeline( + inputFormat, startPositionOffsetUs, transformationRequest, fallbackListener); } else { samplePipeline = new AudioTranscodingSamplePipeline( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java index d6e6c8b645..2bc4cafac1 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java @@ -45,6 +45,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; protected boolean muxerWrapperTrackAdded; protected boolean muxerWrapperTrackEnded; protected long streamOffsetUs; + protected long startPositionOffsetUs; protected @MonotonicNonNull SamplePipeline samplePipeline; public TransformerBaseRenderer( @@ -109,6 +110,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; @Override protected final void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) { this.streamOffsetUs = offsetUs; + this.startPositionOffsetUs = startPositionUs - offsetUs; } @Override diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java index e0bf43732c..94bae3a888 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java @@ -86,12 +86,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; Format inputFormat = checkNotNull(formatHolder.format); if (shouldPassthrough(inputFormat)) { samplePipeline = - new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener); + new PassthroughSamplePipeline( + inputFormat, startPositionOffsetUs, transformationRequest, fallbackListener); } else { samplePipeline = new VideoTranscodingSamplePipeline( context, inputFormat, + startPositionOffsetUs, transformationRequest, effects, decoderFactory, @@ -108,6 +110,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } private boolean shouldPassthrough(Format inputFormat) { + // TODO(b/233988291): Use passthrough pipeline if the clipping start is a key-frame. + if (startPositionOffsetUs != 0) { + return false; + } if (encoderFactory.videoNeedsEncoding()) { return false; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java index 534ae5d667..fa0ce4cdc4 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java @@ -27,6 +27,7 @@ import androidx.media3.common.Format; import androidx.media3.common.util.Util; import androidx.media3.decoder.DecoderInputBuffer; import com.google.common.collect.ImmutableList; +import java.util.ArrayList; import java.util.List; import org.checkerframework.dataflow.qual.Pure; @@ -35,9 +36,12 @@ import org.checkerframework.dataflow.qual.Pure; */ /* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline { private final int outputRotationDegrees; + private final long outputPresentationTimeOffsetUs; + private final int maxPendingFrameCount; + private final DecoderInputBuffer decoderInputBuffer; private final Codec decoder; - private final int maxPendingFrameCount; + private final ArrayList decodeOnlyPresentationTimestamps; private final FrameProcessorChain frameProcessorChain; @@ -49,6 +53,7 @@ import org.checkerframework.dataflow.qual.Pure; public VideoTranscodingSamplePipeline( Context context, Format inputFormat, + long outputPresentationTimeOffsetUs, TransformationRequest transformationRequest, ImmutableList effects, Codec.DecoderFactory decoderFactory, @@ -58,10 +63,12 @@ import org.checkerframework.dataflow.qual.Pure; FrameProcessorChain.Listener frameProcessorChainListener, Transformer.DebugViewProvider debugViewProvider) throws TransformationException { + this.outputPresentationTimeOffsetUs = outputPresentationTimeOffsetUs; decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); encoderOutputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED); + decodeOnlyPresentationTimestamps = new ArrayList<>(); // The decoder rotates encoded frames for display by inputFormat.rotationDegrees. int decodedWidth = @@ -148,6 +155,9 @@ import org.checkerframework.dataflow.qual.Pure; @Override public void queueInputBuffer() throws TransformationException { + if (decoderInputBuffer.isDecodeOnly()) { + decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs); + } decoder.queueInputBuffer(decoderInputBuffer); } @@ -192,7 +202,7 @@ import org.checkerframework.dataflow.qual.Pure; return null; } MediaCodec.BufferInfo bufferInfo = checkNotNull(encoder.getOutputBufferInfo()); - encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs; + encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs - outputPresentationTimeOffsetUs; encoderOutputBuffer.setFlags(bufferInfo.flags); return encoderOutputBuffer; } @@ -251,10 +261,16 @@ import org.checkerframework.dataflow.qual.Pure; * @throws TransformationException If a problem occurs while processing the frame. */ private boolean maybeProcessDecoderOutput() throws TransformationException { - if (decoder.getOutputBufferInfo() == null) { + @Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo(); + if (decoderOutputBufferInfo == null) { return false; } + if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) { + decoder.releaseOutputBuffer(/* render= */ false); + return true; + } + if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT && frameProcessorChain.getPendingFrameCount() == maxPendingFrameCount) { return false; @@ -264,4 +280,17 @@ import org.checkerframework.dataflow.qual.Pure; decoder.releaseOutputBuffer(/* render= */ true); return true; } + + private boolean isDecodeOnlyBuffer(long presentationTimeUs) { + // We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would + // box presentationTimeUs, creating a Long object that would need to be garbage collected. + int size = decodeOnlyPresentationTimestamps.size(); + for (int i = 0; i < size; i++) { + if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) { + decodeOnlyPresentationTimestamps.remove(i); + return true; + } + } + return false; + } }