Add rough progress updates in trim optimization

PiperOrigin-RevId: 593116025
This commit is contained in:
tofunmi 2023-12-22 07:35:06 -08:00 committed by Copybara-Service
parent 1632f37d70
commit 7e12b9e15f
3 changed files with 432 additions and 23 deletions

View File

@ -0,0 +1,243 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Util.isRunningOnEmulator;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_AVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_NOT_STARTED;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_UNAVAILABLE;
import static androidx.media3.transformer.Transformer.PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assume.assumeFalse;
import static org.junit.Assume.assumeTrue;
import android.content.Context;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.NullableType;
import androidx.media3.common.util.Util;
import androidx.media3.effect.DebugTraceUtil;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.platform.app.InstrumentationRegistry;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.SettableFuture;
import java.io.File;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
/**
* End-to-end instrumentation test for {@link Transformer#getProgress} when {@link
* Transformer.Builder#experimentalSetTrimOptimizationEnabled} is enabled.
*/
@RunWith(AndroidJUnit4.class)
public class TransformerTrimOptimizationProgressTest {
@Rule public final TestName testName = new TestName();
private final Context context = ApplicationProvider.getApplicationContext();
private @MonotonicNonNull String testId;
@Before
@EnsuresNonNull({"testId"})
public void setUp() {
testId = testName.getMethodName();
}
@Test
@RequiresNonNull("testId")
public void getProgress_trimOptimizationEnabledAndApplied_givesIncreasingPercentages()
throws Exception {
// The trim optimization is only guaranteed to work on emulator for this file.
assumeTrue(isRunningOnEmulator());
// MediaCodec returns a segmentation fault fails at this SDK level on emulators.
assumeFalse(Util.SDK_INT == 26);
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
MediaItem mediaItem =
new MediaItem.Builder()
.setUri("asset:///media/mp4/internal_emulator_transformer_output.mp4")
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(500)
.setEndPositionMs(2500)
.build())
.build();
// Written to on main sync, read on test thread.
Queue<Integer> progresses = new ConcurrentLinkedDeque<>();
SettableFuture<@NullableType Exception> transformerExceptionFuture = SettableFuture.create();
DebugTraceUtil.enableTracing = true;
// Created on test thread, only used on main sync.
Transformer testTransformer =
transformer
.buildUpon()
.addListener(
new Transformer.Listener() {
@Override
public void onCompleted(Composition composition, ExportResult exportResult) {
transformerExceptionFuture.set(null);
}
@Override
public void onError(
Composition composition,
ExportResult exportResult,
ExportException exportException) {
transformerExceptionFuture.set(exportException);
}
})
.build();
File outputVideoFile =
AndroidTestUtil.createExternalCacheFile(context, /* fileName= */ testId + "-output.mp4");
InstrumentationRegistry.getInstrumentation()
.runOnMainSync(
() -> {
try {
testTransformer.start(mediaItem, outputVideoFile.getAbsolutePath());
// Catch all exceptions to report. Exceptions thrown here that are not caught will
// NOT propagate.
} catch (RuntimeException e) {
transformerExceptionFuture.set(e);
}
});
while (!transformerExceptionFuture.isDone()) {
InstrumentationRegistry.getInstrumentation()
.runOnMainSync(
() -> {
ProgressHolder progressHolder = new ProgressHolder();
if (testTransformer.getProgress(progressHolder) == PROGRESS_STATE_AVAILABLE
&& (progresses.isEmpty()
|| Iterables.getLast(progresses) != progressHolder.progress)) {
progresses.add(progressHolder.progress);
}
});
Thread.sleep(/* millis= */ 200);
}
assertThat(transformerExceptionFuture.get()).isNull();
assertThat(progresses).isInOrder();
if (!progresses.isEmpty()) {
// The progress list could be empty if the export ends before any progress can be retrieved.
assertThat(Iterables.getFirst(progresses, /* defaultValue= */ -1)).isAtLeast(0);
assertThat(Iterables.getLast(progresses)).isAtMost(100);
}
}
@Test
@RequiresNonNull("testId")
public void getProgress_trimOptimizationEnabledAndActive_returnsConsistentStates()
throws Exception {
// The trim optimization is only guaranteed to work on emulator for this file.
assumeTrue(isRunningOnEmulator());
// MediaCodec returns a segmentation fault fails at this SDK level on emulators.
assumeFalse(Util.SDK_INT == 26);
Transformer transformer =
new Transformer.Builder(context).experimentalSetTrimOptimizationEnabled(true).build();
MediaItem mediaItem =
new MediaItem.Builder()
.setUri("asset:///media/mp4/internal_emulator_transformer_output.mp4")
.setClippingConfiguration(
new MediaItem.ClippingConfiguration.Builder()
.setStartPositionMs(500)
.setEndPositionMs(2500)
.build())
.build();
AtomicInteger previousProgressState =
new AtomicInteger(PROGRESS_STATE_WAITING_FOR_AVAILABILITY);
AtomicBoolean foundInconsistentState = new AtomicBoolean();
SettableFuture<@NullableType Exception> transformerExceptionFuture = SettableFuture.create();
DebugTraceUtil.enableTracing = true;
// Created on test thread, only used on main sync.
Transformer testTransformer =
transformer
.buildUpon()
.addListener(
new Transformer.Listener() {
@Override
public void onCompleted(Composition composition, ExportResult exportResult) {
transformerExceptionFuture.set(null);
}
@Override
public void onError(
Composition composition,
ExportResult exportResult,
ExportException exportException) {
transformerExceptionFuture.set(exportException);
}
})
.build();
File outputVideoFile =
AndroidTestUtil.createExternalCacheFile(context, /* fileName= */ testId + "-output.mp4");
InstrumentationRegistry.getInstrumentation()
.runOnMainSync(
() -> {
try {
testTransformer.start(mediaItem, outputVideoFile.getAbsolutePath());
// Catch all exceptions to report. Exceptions thrown here that are not caught will
// NOT propagate.
} catch (RuntimeException e) {
transformerExceptionFuture.set(e);
}
});
while (!transformerExceptionFuture.isDone()) {
InstrumentationRegistry.getInstrumentation()
.runOnMainSync(
() -> {
@Transformer.ProgressState
int progressState = transformer.getProgress(new ProgressHolder());
if (progressState == PROGRESS_STATE_UNAVAILABLE) {
foundInconsistentState.set(true);
return;
}
switch (previousProgressState.get()) {
case PROGRESS_STATE_WAITING_FOR_AVAILABILITY:
break;
case PROGRESS_STATE_AVAILABLE:
if (progressState == PROGRESS_STATE_WAITING_FOR_AVAILABILITY) {
foundInconsistentState.set(true);
return;
}
break;
case PROGRESS_STATE_NOT_STARTED:
if (progressState != PROGRESS_STATE_NOT_STARTED) {
foundInconsistentState.set(true);
return;
}
break;
default:
throw new IllegalStateException();
}
previousProgressState.set(progressState);
});
Thread.sleep(/* millis= */ 200);
}
assertThat(transformerExceptionFuture.get()).isNull();
assertThat(foundInconsistentState.get()).isFalse();
}
}

View File

@ -28,6 +28,7 @@ import static androidx.media3.transformer.ExportResult.OPTIMIZATION_FAILED_FORMA
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeAudio;
import static androidx.media3.transformer.TransformerUtil.shouldTranscodeVideo;
import static androidx.media3.transformer.TransmuxTranscodeHelper.buildNewCompositionWithClipTimes;
import static java.lang.Math.round;
import static java.lang.annotation.ElementType.TYPE_USE;
import android.content.Context;
@ -294,7 +295,6 @@ public final class Transformer {
return this;
}
// TODO: b/304476154 - Support progress updates in trim optimization.
/**
* Sets whether to attempt to optimize trims from the start of the {@link EditedMediaItem} by
* transcoding as little of the file as possible and transmuxing the rest.
@ -305,7 +305,6 @@ public final class Transformer {
* <li>Only supported for single-asset (i.e. only one {@link EditedMediaItem} in the whole
* {@link Composition}) exports of mp4 files.
* <li>Not guaranteed to work with any effects.
* <li>Progress updates will be unavailable.
* </ul>
*
* <p>This process relies on the given {@linkplain #setEncoderFactory EncoderFactory} providing
@ -781,7 +780,7 @@ public final class Transformer {
getResumeMetadataFuture;
private @MonotonicNonNull ListenableFuture<Void> copyOutputFuture;
private @MonotonicNonNull ListenableFuture<Mp4MetadataInfo> getMp4MetadataInfoFuture;
private @MonotonicNonNull Mp4MetadataInfo mp4MetadataInfo;
@Nullable private Mp4MetadataInfo mediaItemMetadataInfo;
private Transformer(
Context context,
@ -821,6 +820,7 @@ public final class Transformer {
this.looper = looper;
this.debugViewProvider = debugViewProvider;
this.clock = clock;
transformerState = TRANSFORMER_STATE_PROCESS_FULL_INPUT;
applicationHandler = clock.createHandler(looper, /* callback= */ null);
componentListener = new ComponentListener();
exportResultBuilder = new ExportResult.Builder();
@ -1069,13 +1069,66 @@ public final class Transformer {
*/
public @ProgressState int getProgress(ProgressHolder progressHolder) {
verifyApplicationThread();
if (transformerState != TRANSFORMER_STATE_PROCESS_FULL_INPUT) {
return PROGRESS_STATE_UNAVAILABLE;
}
if (transformerState == TRANSFORMER_STATE_PROCESS_FULL_INPUT) {
return transformerInternal == null
? PROGRESS_STATE_NOT_STARTED
: transformerInternal.getProgress(progressHolder);
}
if (isExportResumed()) {
return PROGRESS_STATE_UNAVAILABLE;
}
return getTrimOptimizationProgress(progressHolder);
}
private boolean isExportResumed() {
return transformerState == TRANSFORMER_STATE_REMUX_PROCESSED_VIDEO
|| transformerState == TRANSFORMER_STATE_PROCESS_REMAINING_VIDEO
|| transformerState == TRANSFORMER_STATE_PROCESS_AUDIO
|| transformerState == TRANSFORMER_STATE_COPY_OUTPUT;
}
private @ProgressState int getTrimOptimizationProgress(ProgressHolder progressHolder) {
if (mediaItemMetadataInfo == null) {
return PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
}
MediaItem firstMediaItem =
checkNotNull(composition).sequences.get(0).editedMediaItems.get(0).mediaItem;
long trimStartTimeUs = firstMediaItem.clippingConfiguration.startPositionUs;
long transcodeDuration =
mediaItemMetadataInfo.firstSyncSampleTimestampUsAfterTimeUs - trimStartTimeUs;
float transcodeWeighting = (float) transcodeDuration / mediaItemMetadataInfo.durationUs;
if (transformerState == TRANSFORMER_STATE_PROCESS_MEDIA_START) {
if (transformerInternal == null) {
return PROGRESS_STATE_WAITING_FOR_AVAILABILITY;
}
@ProgressState
int processMediaStartProgressState = transformerInternal.getProgress(progressHolder);
if (processMediaStartProgressState == PROGRESS_STATE_AVAILABLE) {
progressHolder.progress = round(progressHolder.progress * transcodeWeighting);
}
return processMediaStartProgressState;
}
float fullTranscodeProgress = 100 * transcodeWeighting;
if (transformerInternal == null) {
// Transformer has not started remuxing the remaining media yet.
progressHolder.progress = round(fullTranscodeProgress);
return PROGRESS_STATE_AVAILABLE;
}
@ProgressState
int remuxRemainingMediaProgressState = transformerInternal.getProgress(progressHolder);
if (remuxRemainingMediaProgressState == PROGRESS_STATE_NOT_STARTED
|| remuxRemainingMediaProgressState == PROGRESS_STATE_WAITING_FOR_AVAILABILITY) {
progressHolder.progress = round(fullTranscodeProgress);
return PROGRESS_STATE_AVAILABLE;
} else if (remuxRemainingMediaProgressState == PROGRESS_STATE_AVAILABLE) {
progressHolder.progress =
round(fullTranscodeProgress + (1 - transcodeWeighting) * progressHolder.progress);
}
return remuxRemainingMediaProgressState;
}
/**
* Cancels the export that is currently in progress, if any.
@ -1329,7 +1382,7 @@ public final class Transformer {
processFullInput();
return;
}
Transformer.this.mp4MetadataInfo = mp4MetadataInfo;
Transformer.this.mediaItemMetadataInfo = mp4MetadataInfo;
Composition trancodeComposition =
buildNewCompositionWithClipTimes(
composition,
@ -1355,6 +1408,7 @@ public final class Transformer {
}
private void remuxRemainingMedia() {
Mp4MetadataInfo mediaItemMetadataInfo = checkNotNull(this.mediaItemMetadataInfo);
transformerState = TRANSFORMER_STATE_REMUX_REMAINING_MEDIA;
if (!doesFormatsMatch()) {
remuxingMuxerWrapper = null;
@ -1367,13 +1421,12 @@ public final class Transformer {
checkNotNull(composition).sequences.get(0).editedMediaItems.get(0).mediaItem;
long trimStartTimeUs = firstMediaItem.clippingConfiguration.startPositionUs;
long trimEndTimeUs = firstMediaItem.clippingConfiguration.endPositionUs;
checkNotNull(mp4MetadataInfo);
Composition transmuxComposition =
buildNewCompositionWithClipTimes(
composition,
mp4MetadataInfo.firstSyncSampleTimestampUsAfterTimeUs,
mediaItemMetadataInfo.firstSyncSampleTimestampUsAfterTimeUs,
trimEndTimeUs,
mp4MetadataInfo.durationUs,
mediaItemMetadataInfo.durationUs,
/* startsAtKeyFrame= */ true);
checkNotNull(remuxingMuxerWrapper);
remuxingMuxerWrapper.changeToAppendMode();
@ -1381,19 +1434,19 @@ public final class Transformer {
transmuxComposition,
remuxingMuxerWrapper,
componentListener,
/* initialTimestampOffsetUs= */ mp4MetadataInfo.firstSyncSampleTimestampUsAfterTimeUs
/* initialTimestampOffsetUs= */ mediaItemMetadataInfo.firstSyncSampleTimestampUsAfterTimeUs
- trimStartTimeUs);
}
private boolean doesFormatsMatch() {
checkNotNull(mp4MetadataInfo);
Mp4MetadataInfo mediaItemMetadataInfo = checkNotNull(this.mediaItemMetadataInfo);
boolean videoFormatMatches =
checkNotNull(remuxingMuxerWrapper)
.getTrackFormat(C.TRACK_TYPE_VIDEO)
.initializationDataEquals(checkNotNull(mp4MetadataInfo.videoFormat));
.initializationDataEquals(checkNotNull(mediaItemMetadataInfo.videoFormat));
boolean audioFormatMatches =
mp4MetadataInfo.audioFormat == null
|| mp4MetadataInfo.audioFormat.initializationDataEquals(
mediaItemMetadataInfo.audioFormat == null
|| mediaItemMetadataInfo.audioFormat.initializationDataEquals(
checkNotNull(remuxingMuxerWrapper).getTrackFormat(C.TRACK_TYPE_AUDIO));
return videoFormatMatches && audioFormatMatches;
}
@ -1501,6 +1554,8 @@ public final class Transformer {
} else if (transformerState == TRANSFORMER_STATE_PROCESS_MEDIA_START) {
remuxRemainingMedia();
} else if (transformerState == TRANSFORMER_STATE_REMUX_REMAINING_MEDIA) {
transformerState = TRANSFORMER_STATE_PROCESS_FULL_INPUT;
mediaItemMetadataInfo = null;
exportResultBuilder.setOptimizationResult(ExportResult.OPTIMIZATION_SUCCEEDED);
onExportCompletedWithSuccess();
} else {

View File

@ -1065,6 +1065,51 @@ public final class MediaItemExportTest {
/* originalFileName= */ FILE_AUDIO_VIDEO, /* modifications...= */ "rotated"));
}
@Test
public void getProgress_unknownDuration_returnsConsistentStates() throws Exception {
Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_UNKNOWN_DURATION);
AtomicInteger previousProgressState =
new AtomicInteger(PROGRESS_STATE_WAITING_FOR_AVAILABILITY);
AtomicBoolean foundInconsistentState = new AtomicBoolean();
Handler progressHandler =
new Handler(Looper.myLooper()) {
@Override
public void handleMessage(Message msg) {
@Transformer.ProgressState
int progressState = transformer.getProgress(new ProgressHolder());
switch (previousProgressState.get()) {
case PROGRESS_STATE_WAITING_FOR_AVAILABILITY:
break;
case PROGRESS_STATE_UNAVAILABLE:
case PROGRESS_STATE_AVAILABLE: // See [Internal: b/176145097].
if (progressState == PROGRESS_STATE_WAITING_FOR_AVAILABILITY) {
foundInconsistentState.set(true);
return;
}
break;
case PROGRESS_STATE_NOT_STARTED:
if (progressState != PROGRESS_STATE_NOT_STARTED) {
foundInconsistentState.set(true);
return;
}
break;
default:
throw new IllegalStateException();
}
previousProgressState.set(progressState);
sendEmptyMessage(0);
}
};
transformer.start(mediaItem, outputDir.newFile().getPath());
progressHandler.sendEmptyMessage(0);
TransformerTestRunner.runLooper(transformer);
assertThat(foundInconsistentState.get()).isFalse();
}
@Test
public void getProgress_knownDuration_returnsConsistentStates() throws Exception {
Transformer transformer =
@ -1165,10 +1210,14 @@ public final class MediaItemExportTest {
}
@Test
public void getProgress_unknownDuration_returnsConsistentStates() throws Exception {
public void
getProgress_trimOptimizationEnabledButNotApplied_withClippingConfigurationUnset_returnsConsistentStates()
throws Exception {
Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_UNKNOWN_DURATION);
createTransformerBuilder(muxerFactory, /* enableFallback= */ false)
.experimentalSetTrimOptimizationEnabled(true)
.build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_VIDEO_ONLY);
AtomicInteger previousProgressState =
new AtomicInteger(PROGRESS_STATE_WAITING_FOR_AVAILABILITY);
AtomicBoolean foundInconsistentState = new AtomicBoolean();
@ -1178,11 +1227,14 @@ public final class MediaItemExportTest {
public void handleMessage(Message msg) {
@Transformer.ProgressState
int progressState = transformer.getProgress(new ProgressHolder());
if (progressState == PROGRESS_STATE_UNAVAILABLE) {
foundInconsistentState.set(true);
return;
}
switch (previousProgressState.get()) {
case PROGRESS_STATE_WAITING_FOR_AVAILABILITY:
break;
case PROGRESS_STATE_UNAVAILABLE:
case PROGRESS_STATE_AVAILABLE: // See [Internal: b/176145097].
case PROGRESS_STATE_AVAILABLE:
if (progressState == PROGRESS_STATE_WAITING_FOR_AVAILABILITY) {
foundInconsistentState.set(true);
return;
@ -1198,17 +1250,76 @@ public final class MediaItemExportTest {
throw new IllegalStateException();
}
previousProgressState.set(progressState);
sendEmptyMessage(0);
sendEmptyMessageDelayed(/* what= */ 0, /* delayMillis= */ 50);
}
};
transformer.start(mediaItem, outputDir.newFile().getPath());
progressHandler.sendEmptyMessage(0);
progressHandler.sendEmptyMessage(/* what= */ 0);
TransformerTestRunner.runLooper(transformer);
assertThat(foundInconsistentState.get()).isFalse();
}
@Test
public void
getProgress_trimOptimizationEnabledButNotApplied_withClippingConfigurationUnset_givesIncreasingPercentages()
throws Exception {
Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false)
.experimentalSetTrimOptimizationEnabled(true)
.build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_VIDEO_ONLY);
List<Integer> progresses = new ArrayList<>();
Handler progressHandler =
new Handler(Looper.myLooper()) {
@Override
public void handleMessage(Message msg) {
ProgressHolder progressHolder = new ProgressHolder();
@Transformer.ProgressState int progressState = transformer.getProgress(progressHolder);
if (progressState == PROGRESS_STATE_NOT_STARTED) {
return;
}
if (progressState == PROGRESS_STATE_AVAILABLE
&& (progresses.isEmpty()
|| Iterables.getLast(progresses) != progressHolder.progress)) {
progresses.add(progressHolder.progress);
}
sendEmptyMessageDelayed(/* what= */ 0, /* delayMillis= */ 50);
}
};
transformer.start(mediaItem, outputDir.newFile().getPath());
progressHandler.sendEmptyMessage(/* what= */ 0);
TransformerTestRunner.runLooper(transformer);
assertThat(progresses).isInOrder();
if (!progresses.isEmpty()) {
// The progress list could be empty if the export ends before any progress can be retrieved.
assertThat(progresses.get(0)).isAtLeast(0);
assertThat(Iterables.getLast(progresses)).isAtMost(100);
}
}
@Test
public void
getProgress_trimOptimizationEnabledButNotApplied_withClippingConfigurationUnset_noCurrentExport_returnsNotStarted()
throws Exception {
Transformer transformer =
createTransformerBuilder(muxerFactory, /* enableFallback= */ false)
.experimentalSetTrimOptimizationEnabled(true)
.build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_VIDEO_ONLY);
ProgressHolder progressHolder = new ProgressHolder();
@Transformer.ProgressState int stateBeforeTransform = transformer.getProgress(progressHolder);
transformer.start(mediaItem, outputDir.newFile().getPath());
TransformerTestRunner.runLooper(transformer);
@Transformer.ProgressState int stateAfterTransform = transformer.getProgress(progressHolder);
assertThat(stateBeforeTransform).isEqualTo(PROGRESS_STATE_NOT_STARTED);
assertThat(stateAfterTransform).isEqualTo(PROGRESS_STATE_NOT_STARTED);
}
@Test
public void getProgress_fromWrongThread_throwsError() throws Exception {
Transformer transformer =