loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix handling clipping in transformer renderers.

Browse Source 
Decode-only video frames (needed when the frame at / first frame after the
clipping start is not a key frame) need to be decoded but not passed to
the frame processor chain or encoder.

The clipping start offset needs to be removed from the frame timestamps
in the passthrough and video pipelines.
There are no changes needed for this in the audio pipeline, as it doesn't
use the input timestamps -- it uses its own timestamps derived from the
buffer sizes instead.

Also add demo option to try this out.

#minor-release

PiperOrigin-RevId: 451353609
This commit is contained in:
hschlueter 2022-05-27 10:22:04 +00:00 committed by Marc Baechinger
parent 821615cea0
commit fe3831c5b4
20 changed files with 257 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java
View File

@ -32,6 +32,7 @@ import android.widget.TextView;
import androidx.annotation.Nullable;
import androidx.appcompat.app.AlertDialog;
import androidx.appcompat.app.AppCompatActivity;
import androidx.media3.common.C;
import androidx.media3.common.MimeTypes;
import androidx.media3.common.util.Util;
import com.google.android.material.slider.RangeSlider;
@ -55,6 +56,8 @@ public final class ConfigurationActivity extends AppCompatActivity {
public static final String SCALE_X = "scale_x";
public static final String SCALE_Y = "scale_y";
public static final String ROTATE_DEGREES = "rotate_degrees";
public static final String TRIM_START_MS = "trim_start_ms";
public static final String TRIM_END_MS = "trim_end_ms";
public static final String ENABLE_FALLBACK = "enable_fallback";
public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping";
public static final String ENABLE_HDR_EDITING = "enable_hdr_editing";
@ -115,12 +118,15 @@ public final class ConfigurationActivity extends AppCompatActivity {
private @MonotonicNonNull Spinner resolutionHeightSpinner;
private @MonotonicNonNull Spinner scaleSpinner;
private @MonotonicNonNull Spinner rotateSpinner;
private @MonotonicNonNull CheckBox trimCheckBox;
private @MonotonicNonNull CheckBox enableFallbackCheckBox;
private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox;
private @MonotonicNonNull CheckBox enableHdrEditingCheckBox;
private @MonotonicNonNull Button selectDemoEffectsButton;
private boolean @MonotonicNonNull [] demoEffectsSelections;
private int inputUriPosition;
private long trimStartMs;
private long trimEndMs;
private float periodicVignetteCenterX;
private float periodicVignetteCenterY;
private float periodicVignetteInnerRadius;
@ -188,6 +194,11 @@ public final class ConfigurationActivity extends AppCompatActivity {
rotateSpinner.setAdapter(rotateAdapter);
rotateAdapter.addAll(SAME_AS_INPUT_OPTION, "0", "10", "45", "60", "90", "180");
trimCheckBox = findViewById(R.id.trim_checkbox);
trimCheckBox.setOnCheckedChangeListener(this::selectTrimBounds);
trimStartMs = C.TIME_UNSET;
trimEndMs = C.TIME_UNSET;
enableFallbackCheckBox = findViewById(R.id.enable_fallback_checkbox);
enableRequestSdrToneMappingCheckBox = findViewById(R.id.request_sdr_tone_mapping_checkbox);
enableRequestSdrToneMappingCheckBox.setEnabled(isRequestSdrToneMappingSupported());
@ -224,6 +235,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"resolutionHeightSpinner",
"scaleSpinner",
"rotateSpinner",
"trimCheckBox",
"enableFallbackCheckBox",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
@ -258,6 +270,10 @@ public final class ConfigurationActivity extends AppCompatActivity {
if (!SAME_AS_INPUT_OPTION.equals(selectedRotate)) {
bundle.putFloat(ROTATE_DEGREES, Float.parseFloat(selectedRotate));
}
if (trimCheckBox.isChecked()) {
bundle.putLong(TRIM_START_MS, trimStartMs);
bundle.putLong(TRIM_END_MS, trimEndMs);
}
bundle.putBoolean(ENABLE_FALLBACK, enableFallbackCheckBox.isChecked());
bundle.putBoolean(
ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked());
@ -295,6 +311,27 @@ public final class ConfigurationActivity extends AppCompatActivity {
.show();
}
private void selectTrimBounds(View view, boolean isChecked) {
if (!isChecked) {
return;
}
View dialogView = getLayoutInflater().inflate(R.layout.trim_options, /* root= */ null);
RangeSlider radiusRangeSlider =
checkNotNull(dialogView.findViewById(R.id.trim_bounds_range_slider));
radiusRangeSlider.setValues(0f, 60f); // seconds
new AlertDialog.Builder(/* context= */ this)
.setView(dialogView)
.setPositiveButton(
android.R.string.ok,
(DialogInterface dialogInterface, int i) -> {
List<Float> radiusRange = radiusRangeSlider.getValues();
trimStartMs = 1000 * radiusRange.get(0).longValue();
trimEndMs = 1000 * radiusRange.get(1).longValue();
})
.create()
.show();
}
@RequiresNonNull("selectedFileTextView")
private void selectFileInDialog(DialogInterface dialog, int which) {
inputUriPosition = which;

21
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -152,9 +152,10 @@ public final class TransformerActivity extends AppCompatActivity {
externalCacheFile = createExternalCacheFile("transformer-output.mp4");
String filePath = externalCacheFile.getAbsolutePath();
@Nullable Bundle bundle = intent.getExtras();
MediaItem mediaItem = createMediaItem(bundle, uri);
Transformer transformer = createTransformer(bundle, filePath);
transformationStopwatch.start();
transformer.startTransformation(MediaItem.fromUri(uri), filePath);
transformer.startTransformation(mediaItem, filePath);
this.transformer = transformer;
} catch (IOException e) {
throw new IllegalStateException(e);
@ -181,6 +182,24 @@ public final class TransformerActivity extends AppCompatActivity {
});
}
private MediaItem createMediaItem(@Nullable Bundle bundle, Uri uri) {
MediaItem.Builder mediaItemBuilder = new MediaItem.Builder().setUri(uri);
if (bundle != null) {
long trimStartMs =
bundle.getLong(ConfigurationActivity.TRIM_START_MS, /* defaultValue= */ C.TIME_UNSET);
long trimEndMs =
bundle.getLong(ConfigurationActivity.TRIM_END_MS, /* defaultValue= */ C.TIME_UNSET);
if (trimStartMs != C.TIME_UNSET && trimEndMs != C.TIME_UNSET) {
mediaItemBuilder.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(trimStartMs)
.setEndPositionMs(trimEndMs)
.build());
}
}
return mediaItemBuilder.build();
}
// Create a cache file, resetting it if it already exists.
private File createExternalCacheFile(String fileName) throws IOException {
File file = new File(getExternalCacheDir(), fileName);

10
demos/transformer/src/main/res/layout/configuration_activity.xml
View File

@ -159,6 +159,16 @@
android:layout_gravity="right|center_vertical"
android:gravity="right" />
</TableRow>
<TableRow
android:layout_weight="1"
android:gravity="center_vertical" >
<TextView
android:id="@+id/trim"
android:text="@string/trim" />
<CheckBox
android:id="@+id/trim_checkbox"
android:layout_gravity="right" />
</TableRow>
<TableRow
android:layout_weight="1"
android:gravity="center_vertical" >

48
demos/transformer/src/main/res/layout/trim_options.xml Normal file
View File

@ -0,0 +1,48 @@
<?xml version="1.0" encoding="utf-8"?>
<!--
~ Copyright 2022 The Android Open Source Project
~
~ Licensed under the Apache License, Version 2.0 (the "License");
~ you may not use this file except in compliance with the License.
~ You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->
<androidx.constraintlayout.widget.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:app="http://schemas.android.com/apk/res-auto"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="fill_parent"
android:layout_height="fill_parent"
tools:context=".ConfigurationActivity">
<TableLayout
android:layout_width="fill_parent"
android:layout_height="wrap_content"
android:stretchColumns="1"
android:layout_marginTop="32dp"
android:measureWithLargestChild="true"
android:paddingLeft="24dp"
android:paddingRight="12dp"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toTopOf="parent"
app:layout_constraintBottom_toBottomOf="parent">
<TableRow
android:layout_weight="1"
android:gravity="center_vertical" >
<TextView
android:text="@string/trim_range" />
<com.google.android.material.slider.RangeSlider
android:id="@+id/trim_bounds_range_slider"
android:valueFrom="0.0"
android:valueTo="60.0"
android:layout_gravity="right"/>
</TableRow>
</TableLayout>
</androidx.constraintlayout.widget.ConstraintLayout>

2
demos/transformer/src/main/res/values/strings.xml
View File

@ -27,6 +27,7 @@
<string name="scale" translatable="false">Scale video</string>
<string name="rotate" translatable="false">Rotate video (degrees)</string>
<string name="enable_fallback" translatable="false">Enable fallback</string>
<string name="trim" translatable="false">Trim</string>
<string name="request_sdr_tone_mapping" translatable="false">Request SDR tone-mapping (API 31+)</string>
<string name="hdr_editing" translatable="false">[Experimental] HDR editing</string>
<string name="select_demo_effects" translatable="false">Add demo effects</string>
@ -42,4 +43,5 @@
<string name="center_x">Center X</string>
<string name="center_y">Center Y</string>
<string name="radius_range">Radius range</string>
<string name="trim_range">Bounds in seconds</string>
</resources>

25
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java
View File

@ -19,7 +19,6 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static java.util.concurrent.TimeUnit.SECONDS;
import android.content.Context;
import android.net.Uri;
import android.view.Surface;
import androidx.annotation.Nullable;
import androidx.media3.common.Format;
@ -179,17 +178,17 @@ public class TransformerAndroidTestRunner {
* cache.
*
* @param testId A unique identifier for the transformer test run.
* @param uriString The uri (as a {@link String}) of the file to transform.
* @param mediaItem The {@link MediaItem} to transform.
* @return The {@link TransformationTestResult}.
* @throws Exception The cause of the transformation not completing.
*/
public TransformationTestResult run(String testId, String uriString) throws Exception {
public TransformationTestResult run(String testId, MediaItem mediaItem) throws Exception {
JSONObject resultJson = new JSONObject();
if (inputValues != null) {
resultJson.put("inputValues", JSONObject.wrap(inputValues));
}
try {
TransformationTestResult transformationTestResult = runInternal(testId, uriString);
TransformationTestResult transformationTestResult = runInternal(testId, mediaItem);
resultJson.put("transformationResult", transformationTestResult.asJsonObject());
if (!suppressAnalysisExceptions && transformationTestResult.analysisException != null) {
throw transformationTestResult.analysisException;
@ -208,7 +207,7 @@ public class TransformerAndroidTestRunner {
* Transforms the {@code uriString}.
*
* @param testId An identifier for the test.
* @param uriString The uri (as a {@link String}) of the file to transform.
* @param mediaItem The {@link MediaItem} to transform.
* @return The {@link TransformationTestResult}.
* @throws IOException If an error occurs opening the output file for writing
* @throws TimeoutException If the transformation takes longer than the {@link #timeoutSeconds}.
@ -218,8 +217,14 @@ public class TransformerAndroidTestRunner {
* @throws IllegalArgumentException If the path is invalid.
* @throws IllegalStateException If an unexpected exception occurs when starting a transformation.
*/
private TransformationTestResult runInternal(String testId, String uriString)
private TransformationTestResult runInternal(String testId, MediaItem mediaItem)
throws InterruptedException, IOException, TimeoutException, TransformationException {
if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET)
&& calculateSsim) {
throw new UnsupportedOperationException(
"SSIM calculation is not supported for clipped inputs.");
}
AtomicReference<@NullableType TransformationException> transformationExceptionReference =
new AtomicReference<>();
AtomicReference<@NullableType Exception> unexpectedExceptionReference = new AtomicReference<>();
@ -249,15 +254,13 @@ public class TransformerAndroidTestRunner {
})
.build();
Uri uri = Uri.parse(uriString);
File outputVideoFile =
AndroidTestUtil.createExternalCacheFile(context, /* fileName= */ testId + "-output.mp4");
InstrumentationRegistry.getInstrumentation()
.runOnMainSync(
() -> {
try {
testTransformer.startTransformation(
MediaItem.fromUri(uri), outputVideoFile.getAbsolutePath());
testTransformer.startTransformation(mediaItem, outputVideoFile.getAbsolutePath());
// Catch all exceptions to report. Exceptions thrown here and not caught will NOT
// propagate.
} catch (Exception e) {
@ -300,7 +303,9 @@ public class TransformerAndroidTestRunner {
if (calculateSsim) {
double ssim =
SsimHelper.calculate(
context, /* referenceVideoPath= */ uriString, outputVideoFile.getPath());
context,
/* referenceVideoPath= */ checkNotNull(mediaItem.localConfiguration).uri.toString(),
outputVideoFile.getPath());
resultBuilder.setSsim(ssim);
}
} catch (InterruptedException interruptedException) {

33
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -16,9 +16,12 @@
package androidx.media3.transformer;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_URI_STRING;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.MediaItem;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import org.junit.Test;
@ -50,7 +53,7 @@ public class TransformerEndToEndTest {
.build()
.run(
/* testId= */ "videoEditing_completesWithConsistentFrameCount",
MP4_ASSET_URI_STRING);
MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)));
assertThat(result.transformationResult.videoFrameCount).isEqualTo(expectedFrameCount);
}
@ -71,8 +74,34 @@ public class TransformerEndToEndTest {
TransformationTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(/* testId= */ "videoOnly_completesWithConsistentDuration", MP4_ASSET_URI_STRING);
.run(
/* testId= */ "videoOnly_completesWithConsistentDuration",
MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)));
assertThat(result.transformationResult.durationMs).isEqualTo(expectedDurationMs);
}
@Test
public void clippedMedia_completesWithClippedDuration() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
Transformer transformer = new Transformer.Builder(context).build();
long clippingStartMs = 10_000;
long clippingEndMs = 11_000;
MediaItem mediaItem =
new MediaItem.Builder()
.setUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING))
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(clippingStartMs)
.setEndPositionMs(clippingEndMs)
.build())
.build();
TransformationTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(/* testId= */ "clippedMedia_completesWithClippedDuration", mediaItem);
assertThat(result.transformationResult.durationMs).isAtMost(clippingEndMs - clippingStartMs);
}
}

8
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/RepeatedTranscodeTransformationTest.java
View File

@ -19,6 +19,8 @@ import static androidx.media3.common.util.Assertions.checkNotNull;
import static com.google.common.truth.Truth.assertWithMessage;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.MediaItem;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.TransformationRequest;
import androidx.media3.transformer.TransformationTestResult;
@ -56,7 +58,7 @@ public final class RepeatedTranscodeTransformationTest {
TransformationTestResult testResult =
transformerRunner.run(
/* testId= */ "repeatedTranscode_givesConsistentLengthOutput_" + i,
AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING);
MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING)));
differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes));
}
@ -86,7 +88,7 @@ public final class RepeatedTranscodeTransformationTest {
TransformationTestResult testResult =
transformerRunner.run(
/* testId= */ "repeatedTranscodeNoAudio_givesConsistentLengthOutput_" + i,
AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING);
MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING)));
differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes));
}
@ -115,7 +117,7 @@ public final class RepeatedTranscodeTransformationTest {
TransformationTestResult testResult =
transformerRunner.run(
/* testId= */ "repeatedTranscodeNoVideo_givesConsistentLengthOutput_" + i,
AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING);
MediaItem.fromUri(Uri.parse(AndroidTestUtil.MP4_REMOTE_10_SECONDS_URI_STRING)));
differentOutputSizesBytes.add(checkNotNull(testResult.transformationResult.fileSizeBytes));
}

5
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/SetFrameEditTransformationTest.java
View File

@ -18,6 +18,8 @@ package androidx.media3.transformer.mh;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.MediaItem;
import androidx.media3.transformer.TransformationRequest;
import androidx.media3.transformer.Transformer;
import androidx.media3.transformer.TransformerAndroidTestRunner;
@ -41,6 +43,7 @@ public class SetFrameEditTransformationTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(
/* testId= */ "SetFrameEditTransform", MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
/* testId= */ "SetFrameEditTransform",
MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
}
}

17
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TranscodeQualityTest.java
View File

@ -19,6 +19,8 @@ package androidx.media3.transformer.mh;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.MediaItem;
import androidx.media3.common.MimeTypes;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.TransformationRequest;
@ -58,7 +60,10 @@ public final class TranscodeQualityTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.build()
.run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
.run(
testId,
MediaItem.fromUri(
Uri.parse(AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
if (result.ssim != TransformationTestResult.SSIM_UNSET) {
assertThat(result.ssim).isGreaterThan(0.90);
@ -92,7 +97,10 @@ public final class TranscodeQualityTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.build()
.run(testId, AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
.run(
testId,
MediaItem.fromUri(
Uri.parse(AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
if (result.ssim != TransformationTestResult.SSIM_UNSET) {
assertThat(result.ssim).isGreaterThan(0.90);
@ -121,7 +129,10 @@ public final class TranscodeQualityTest {
.build()
.run(
testId,
AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING);
MediaItem.fromUri(
Uri.parse(
AndroidTestUtil
.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING)));
if (result.ssim != TransformationTestResult.SSIM_UNSET) {
assertThat(result.ssim).isGreaterThan(0.90);

16
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/TransformationTest.java
View File

@ -24,6 +24,8 @@ import static androidx.media3.transformer.AndroidTestUtil.MP4_REMOTE_4K60_PORTRA
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Util;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.DefaultEncoderFactory;
@ -54,7 +56,7 @@ public class TransformationTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.build()
.run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
}
@Test
@ -65,7 +67,7 @@ public class TransformationTest {
// No need to calculate SSIM because no decode/encoding, so input frames match output frames.
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
}
@Test
@ -84,7 +86,7 @@ public class TransformationTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.build()
.run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
}
@Test
@ -106,7 +108,7 @@ public class TransformationTest {
.setCalculateSsim(true)
.setTimeoutSeconds(180)
.build()
.run(testId, MP4_REMOTE_4K60_PORTRAIT_URI_STRING);
.run(testId, MediaItem.fromUri(Uri.parse(MP4_REMOTE_4K60_PORTRAIT_URI_STRING)));
}
@Test
@ -121,7 +123,7 @@ public class TransformationTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.setCalculateSsim(true)
.build()
.run(testId, MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING);
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_URI_STRING)));
}
@Test
@ -135,7 +137,7 @@ public class TransformationTest {
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, MP4_ASSET_URI_STRING);
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING)));
}
@Test
@ -159,6 +161,6 @@ public class TransformationTest {
.build();
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, MP4_ASSET_SEF_URI_STRING);
.run(testId, MediaItem.fromUri(Uri.parse(MP4_ASSET_SEF_URI_STRING)));
}
}

4
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/BitrateAnalysisTest.java
View File

@ -19,6 +19,8 @@ import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_CBR;
import static android.media.MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR;
import android.content.Context;
import android.net.Uri;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Assertions;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.DefaultEncoderFactory;
@ -116,6 +118,6 @@ public class BitrateAnalysisTest {
.setInputValues(inputValues)
.setCalculateSsim(true)
.build()
.run(testId, fileUri);
.run(testId, MediaItem.fromUri(Uri.parse(fileUri)));
}
}

3
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/analysis/EncoderPerformanceAnalysisTest.java
View File

@ -22,6 +22,7 @@ import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
import android.content.Context;
import android.media.MediaFormat;
import android.net.Uri;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Util;
import androidx.media3.transformer.AndroidTestUtil;
import androidx.media3.transformer.DefaultEncoderFactory;
@ -136,6 +137,6 @@ public class EncoderPerformanceAnalysisTest {
new TransformerAndroidTestRunner.Builder(context, transformer)
.setInputValues(inputValues)
.build()
.run(testId, fileUri);
.run(testId, MediaItem.fromUri(Uri.parse(fileUri)));
}
}

4
libraries/transformer/src/main/java/androidx/media3/transformer/PassthroughSamplePipeline.java
View File

@ -25,14 +25,17 @@ import androidx.media3.decoder.DecoderInputBuffer;
private final DecoderInputBuffer buffer;
private final Format format;
private final long outputPresentationTimeOffsetUs;
private boolean hasPendingBuffer;
public PassthroughSamplePipeline(
Format format,
long outputPresentationTimeOffsetUs,
TransformationRequest transformationRequest,
FallbackListener fallbackListener) {
this.format = format;
this.outputPresentationTimeOffsetUs = outputPresentationTimeOffsetUs;
buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT);
hasPendingBuffer = false;
fallbackListener.onTransformationRequestFinalized(transformationRequest);
@ -46,6 +49,7 @@ import androidx.media3.decoder.DecoderInputBuffer;
@Override
public void queueInputBuffer() {
buffer.timeUs -= outputPresentationTimeOffsetUs;
hasPendingBuffer = true;
}

3
libraries/transformer/src/main/java/androidx/media3/transformer/TransformationRequest.java
View File

@ -90,6 +90,9 @@ public final class TransformationRequest {
* Mp4Extractor#FLAG_READ_SEF_DATA} is set on the {@link Mp4Extractor} used. Otherwise, the slow
* motion metadata will be ignored and the input won't be flattened.
*
* <p>Using slow motion flattening together with {@link
* androidx.media3.common.MediaItem.ClippingConfiguration} is not supported yet.
*
* @param flattenForSlowMotion Whether to flatten for slow motion.
* @return This builder.
*/

7
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -61,6 +61,7 @@ import androidx.media3.extractor.DefaultExtractorsFactory;
import androidx.media3.extractor.mp4.Mp4Extractor;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
@ -685,6 +686,12 @@ public final class Transformer {
* @throws IOException If an error occurs opening the output file for writing.
*/
public void startTransformation(MediaItem mediaItem, String path) throws IOException {
if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET)
&& transformationRequest.flattenForSlowMotion) {
// TODO(b/233986762): Support clipping with SEF flattening.
throw new UnsupportedEncodingException(
"Clipping is not supported when slow motion flattening is requested");
}
startTransformation(mediaItem, muxerFactory.create(path, containerMimeType));
}

3
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java
View File

@ -70,7 +70,8 @@ import androidx.media3.extractor.metadata.mp4.SlowMotionData;
Format inputFormat = checkNotNull(formatHolder.format);
if (shouldPassthrough(inputFormat)) {
samplePipeline =
new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
new PassthroughSamplePipeline(
inputFormat, startPositionOffsetUs, transformationRequest, fallbackListener);
} else {
samplePipeline =
new AudioTranscodingSamplePipeline(

2
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerBaseRenderer.java
View File

@ -45,6 +45,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
protected boolean muxerWrapperTrackAdded;
protected boolean muxerWrapperTrackEnded;
protected long streamOffsetUs;
protected long startPositionOffsetUs;
protected @MonotonicNonNull SamplePipeline samplePipeline;
public TransformerBaseRenderer(
@ -109,6 +110,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@Override
protected final void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) {
this.streamOffsetUs = offsetUs;
this.startPositionOffsetUs = startPositionUs - offsetUs;
}
@Override

8
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
View File

@ -86,12 +86,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
Format inputFormat = checkNotNull(formatHolder.format);
if (shouldPassthrough(inputFormat)) {
samplePipeline =
new PassthroughSamplePipeline(inputFormat, transformationRequest, fallbackListener);
new PassthroughSamplePipeline(
inputFormat, startPositionOffsetUs, transformationRequest, fallbackListener);
} else {
samplePipeline =
new VideoTranscodingSamplePipeline(
context,
inputFormat,
startPositionOffsetUs,
transformationRequest,
effects,
decoderFactory,
@ -108,6 +110,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
private boolean shouldPassthrough(Format inputFormat) {
// TODO(b/233988291): Use passthrough pipeline if the clipping start is a key-frame.
if (startPositionOffsetUs != 0) {
return false;
}
if (encoderFactory.videoNeedsEncoding()) {
return false;
}

35
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -27,6 +27,7 @@ import androidx.media3.common.Format;
import androidx.media3.common.util.Util;
import androidx.media3.decoder.DecoderInputBuffer;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
import org.checkerframework.dataflow.qual.Pure;
@ -35,9 +36,12 @@ import org.checkerframework.dataflow.qual.Pure;
*/
/* package */ final class VideoTranscodingSamplePipeline implements SamplePipeline {
private final int outputRotationDegrees;
private final long outputPresentationTimeOffsetUs;
private final int maxPendingFrameCount;
private final DecoderInputBuffer decoderInputBuffer;
private final Codec decoder;
private final int maxPendingFrameCount;
private final ArrayList<Long> decodeOnlyPresentationTimestamps;
private final FrameProcessorChain frameProcessorChain;
@ -49,6 +53,7 @@ import org.checkerframework.dataflow.qual.Pure;
public VideoTranscodingSamplePipeline(
Context context,
Format inputFormat,
long outputPresentationTimeOffsetUs,
TransformationRequest transformationRequest,
ImmutableList<GlEffect> effects,
Codec.DecoderFactory decoderFactory,
@ -58,10 +63,12 @@ import org.checkerframework.dataflow.qual.Pure;
FrameProcessorChain.Listener frameProcessorChainListener,
Transformer.DebugViewProvider debugViewProvider)
throws TransformationException {
this.outputPresentationTimeOffsetUs = outputPresentationTimeOffsetUs;
decoderInputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
encoderOutputBuffer =
new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED);
decodeOnlyPresentationTimestamps = new ArrayList<>();
// The decoder rotates encoded frames for display by inputFormat.rotationDegrees.
int decodedWidth =
@ -148,6 +155,9 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public void queueInputBuffer() throws TransformationException {
if (decoderInputBuffer.isDecodeOnly()) {
decodeOnlyPresentationTimestamps.add(decoderInputBuffer.timeUs);
}
decoder.queueInputBuffer(decoderInputBuffer);
}
@ -192,7 +202,7 @@ import org.checkerframework.dataflow.qual.Pure;
return null;
}
MediaCodec.BufferInfo bufferInfo = checkNotNull(encoder.getOutputBufferInfo());
encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs;
encoderOutputBuffer.timeUs = bufferInfo.presentationTimeUs - outputPresentationTimeOffsetUs;
encoderOutputBuffer.setFlags(bufferInfo.flags);
return encoderOutputBuffer;
}
@ -251,10 +261,16 @@ import org.checkerframework.dataflow.qual.Pure;
* @throws TransformationException If a problem occurs while processing the frame.
*/
private boolean maybeProcessDecoderOutput() throws TransformationException {
if (decoder.getOutputBufferInfo() == null) {
@Nullable MediaCodec.BufferInfo decoderOutputBufferInfo = decoder.getOutputBufferInfo();
if (decoderOutputBufferInfo == null) {
return false;
}
if (isDecodeOnlyBuffer(decoderOutputBufferInfo.presentationTimeUs)) {
decoder.releaseOutputBuffer(/* render= */ false);
return true;
}
if (maxPendingFrameCount != Codec.UNLIMITED_PENDING_FRAME_COUNT
&& frameProcessorChain.getPendingFrameCount() == maxPendingFrameCount) {
return false;
@ -264,4 +280,17 @@ import org.checkerframework.dataflow.qual.Pure;
decoder.releaseOutputBuffer(/* render= */ true);
return true;
}
private boolean isDecodeOnlyBuffer(long presentationTimeUs) {
// We avoid using decodeOnlyPresentationTimestamps.remove(presentationTimeUs) because it would
// box presentationTimeUs, creating a Long object that would need to be garbage collected.
int size = decodeOnlyPresentationTimestamps.size();
for (int i = 0; i < size; i++) {
if (decodeOnlyPresentationTimestamps.get(i) == presentationTimeUs) {
decodeOnlyPresentationTimestamps.remove(i);
return true;
}
}
return false;
}
}