diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index bd7fa5efed..ff1f84eff4 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -133,7 +133,8 @@ public final class TransformerActivity extends AppCompatActivity { @Nullable private ExoPlayer inputPlayer; @Nullable private ExoPlayer outputPlayer; @Nullable private Transformer transformer; - @Nullable private File externalCacheFile; + @Nullable private File outputFile; + @Nullable private File oldOutputFile; @Override protected void onCreate(@Nullable Bundle savedInstanceState) { @@ -153,7 +154,7 @@ public final class TransformerActivity extends AppCompatActivity { cancelButton = findViewById(R.id.cancel_button); cancelButton.setOnClickListener(this::cancelExport); resumeButton = findViewById(R.id.resume_button); - resumeButton.setOnClickListener(this::resumeExport); + resumeButton.setOnClickListener(view -> startExport()); debugFrame = findViewById(R.id.debug_aspect_ratio_frame_layout); displayInputButton = findViewById(R.id.display_input_button); displayInputButton.setOnClickListener(this::toggleInputVideoDisplay); @@ -195,8 +196,12 @@ public final class TransformerActivity extends AppCompatActivity { checkNotNull(outputPlayerView).onPause(); releasePlayer(); - checkNotNull(externalCacheFile).delete(); - externalCacheFile = null; + checkNotNull(outputFile).delete(); + outputFile = null; + if (oldOutputFile != null) { + oldOutputFile.delete(); + oldOutputFile = null; + } } private void startExport() { @@ -221,18 +226,23 @@ public final class TransformerActivity extends AppCompatActivity { Intent intent = getIntent(); Uri inputUri = checkNotNull(intent.getData()); try { - externalCacheFile = + outputFile = createExternalCacheFile("transformer-output-" + Clock.DEFAULT.elapsedRealtime() + ".mp4"); } catch (IOException e) { throw new IllegalStateException(e); } - String filePath = externalCacheFile.getAbsolutePath(); + String outputFilePath = outputFile.getAbsolutePath(); @Nullable Bundle bundle = intent.getExtras(); MediaItem mediaItem = createMediaItem(bundle, inputUri); - Transformer transformer = createTransformer(bundle, inputUri, filePath); + Transformer transformer = createTransformer(bundle, inputUri, outputFilePath); Composition composition = createComposition(mediaItem, bundle); + exportStopwatch.reset(); exportStopwatch.start(); - transformer.start(composition, filePath); + if (oldOutputFile == null) { + transformer.start(composition, outputFilePath); + } else { + transformer.resume(composition, outputFilePath, oldOutputFile.getAbsolutePath()); + } this.transformer = transformer; displayInputButton.setVisibility(View.GONE); inputCardView.setVisibility(View.GONE); @@ -243,6 +253,7 @@ public final class TransformerActivity extends AppCompatActivity { progressViewGroup.setVisibility(View.VISIBLE); cancelButton.setVisibility(View.VISIBLE); resumeButton.setVisibility(View.GONE); + progressIndicator.setProgress(0); Handler mainHandler = new Handler(getMainLooper()); ProgressHolder progressHolder = new ProgressHolder(); mainHandler.post( @@ -834,12 +845,10 @@ public final class TransformerActivity extends AppCompatActivity { exportStopwatch.stop(); cancelButton.setVisibility(View.GONE); resumeButton.setVisibility(View.VISIBLE); - } - - @RequiresNonNull({"exportStopwatch"}) - private void resumeExport(View view) { - exportStopwatch.reset(); - startExport(); + if (oldOutputFile != null) { + oldOutputFile.delete(); + } + oldOutputFile = outputFile; } private final class DemoDebugViewProvider implements DebugViewProvider { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java index c77c080712..36f9fea425 100644 --- a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java @@ -188,12 +188,28 @@ public class TransformerAndroidTestRunner { * @throws Exception The cause of the export not completing. */ public ExportTestResult run(String testId, Composition composition) throws Exception { + return run(testId, composition, /* oldFilePath= */ null); + } + + /** + * Exports the {@link Composition}, saving a summary of the export to the application cache. + * Resumes exporting if the {@code oldFilePath} is specified. + * + * @param testId A unique identifier for the transformer test run. + * @param composition The {@link Composition} to export. + * @param oldFilePath The old output file path to resume the export from. Passing {@code null} + * will restart the export from the beginning. + * @return The {@link ExportTestResult}. + * @throws Exception The cause of the export not completing. + */ + public ExportTestResult run(String testId, Composition composition, @Nullable String oldFilePath) + throws Exception { JSONObject resultJson = new JSONObject(); if (inputValues != null) { resultJson.put("inputValues", JSONObject.wrap(inputValues)); } try { - ExportTestResult exportTestResult = runInternal(testId, composition); + ExportTestResult exportTestResult = runInternal(testId, composition, oldFilePath); resultJson.put("exportResult", exportTestResult.asJsonObject()); if (exportTestResult.exportResult.exportException != null) { throw exportTestResult.exportResult.exportException; @@ -250,6 +266,8 @@ public class TransformerAndroidTestRunner { * * @param testId An identifier for the test. * @param composition The {@link Composition} to export. + * @param oldFilePath The old output file path to resume the export from. Passing {@code null} + * will restart the export from the beginning. * @return The {@link ExportTestResult}. * @throws IllegalStateException See {@link Transformer#start(Composition, String)}. * @throws InterruptedException If the thread is interrupted whilst waiting for transformer to @@ -258,7 +276,8 @@ public class TransformerAndroidTestRunner { * @throws TimeoutException If the export has not completed after {@linkplain * Builder#setTimeoutSeconds(int) the given timeout}. */ - private ExportTestResult runInternal(String testId, Composition composition) + private ExportTestResult runInternal( + String testId, Composition composition, @Nullable String oldFilePath) throws InterruptedException, IOException, TimeoutException { if (requestCalculateSsim) { checkArgument( @@ -347,7 +366,12 @@ public class TransformerAndroidTestRunner { .runOnMainSync( () -> { try { - testTransformer.start(composition, outputVideoFile.getAbsolutePath()); + if (oldFilePath == null) { + testTransformer.start(composition, outputVideoFile.getAbsolutePath()); + } else { + testTransformer.resume( + composition, outputVideoFile.getAbsolutePath(), oldFilePath); + } // Catch all exceptions to report. Exceptions thrown here and not caught will NOT // propagate. } catch (Exception e) { diff --git a/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerPauseResumeTest.java b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerPauseResumeTest.java new file mode 100644 index 0000000000..298a69a064 --- /dev/null +++ b/libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerPauseResumeTest.java @@ -0,0 +1,477 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT; +import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING; +import static androidx.test.core.app.ApplicationProvider.getApplicationContext; +import static com.google.common.truth.Truth.assertThat; +import static java.util.concurrent.TimeUnit.SECONDS; + +import android.content.Context; +import androidx.media3.common.C; +import androidx.media3.common.Effect; +import androidx.media3.common.Format; +import androidx.media3.common.MediaItem; +import androidx.media3.common.Metadata; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.audio.AudioProcessor; +import androidx.media3.common.audio.SonicAudioProcessor; +import androidx.media3.common.util.Util; +import androidx.media3.effect.RgbFilter; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; +import com.google.common.base.Ascii; +import com.google.common.collect.ImmutableList; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeoutException; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +/** End-to-end instrumentation tests for {@link Transformer} pause and resume scenarios. */ +@RunWith(AndroidJUnit4.class) +public class TransformerPauseResumeTest { + @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static final long DEFAULT_PRESENTATION_TIME_US_TO_BLOCK_FRAME = 5_000_000L; + private static final int DEFAULT_TIMEOUT_SECONDS = 120; + private static final int MP4_ASSET_FRAME_COUNT = 932; + + private final Context context = getApplicationContext(); + + @Test + public void resume_withSingleMediaItem_outputMatchesExpected() throws Exception { + String testId = "resume_withSingleMediaItem_outputMatchesExpected"; + if (shouldSkipDevice(testId)) { + return; + } + Composition composition = + buildSingleSequenceComposition( + /* clippingStartPositionMs= */ 0, + /* clippingEndPositionMs= */ C.TIME_END_OF_SOURCE, + /* mediaItemsInSequence= */ 1); + CountDownLatch countDownLatch = new CountDownLatch(1); + Transformer blockingTransformer = buildBlockingTransformer(countDownLatch::countDown); + String firstOutputPath = temporaryFolder.newFile("FirstOutput.mp4").getAbsolutePath(); + InstrumentationRegistry.getInstrumentation() + .runOnMainSync(() -> blockingTransformer.start(composition, firstOutputPath)); + // Block here until timeout reached or latch is counted down. + if (!countDownLatch.await(DEFAULT_TIMEOUT_SECONDS, SECONDS)) { + throw new TimeoutException( + "Transformer timed out after " + DEFAULT_TIMEOUT_SECONDS + " seconds."); + } + InstrumentationRegistry.getInstrumentation().runOnMainSync(blockingTransformer::cancel); + TransformerAndroidTestRunner testRunner = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build(); + + // Resume the export. + ExportResult exportResult = testRunner.run(testId, composition, firstOutputPath).exportResult; + + assertThat(exportResult.processedInputs).hasSize(4); + assertThat(exportResult.videoFrameCount) + .isEqualTo(MP4_ASSET_FRAME_COUNT - getDeviceSpecificMissingFrameCount()); + // The first processed media item corresponds to remuxing previous output video. + assertThat(exportResult.processedInputs.get(0).audioDecoderName).isNull(); + assertThat(exportResult.processedInputs.get(0).videoDecoderName).isNull(); + // The second processed media item corresponds to processing remaining video. + assertThat(exportResult.processedInputs.get(1).audioDecoderName).isNull(); + assertThat(exportResult.processedInputs.get(1).videoDecoderName).isNotEmpty(); + assertThat(exportResult.processedInputs.get(1).mediaItem.clippingConfiguration.startPositionMs) + .isGreaterThan(0); + // The third processed media item corresponds to processing audio. + assertThat(exportResult.processedInputs.get(2).audioDecoderName).isNotEmpty(); + assertThat(exportResult.processedInputs.get(2).videoDecoderName).isNull(); + // The fourth processed media item corresponds to transmuxing processed video. + assertThat(exportResult.processedInputs.get(3).audioDecoderName).isNull(); + assertThat(exportResult.processedInputs.get(3).videoDecoderName).isNull(); + } + + @Test + public void resume_withSingleMediaItemAfterImmediateCancellation_restartsExport() + throws Exception { + String testId = "resume_withSingleMediaItemAfterImmediateCancellation_restartsExport"; + if (shouldSkipDevice(testId)) { + return; + } + Composition composition = + buildSingleSequenceComposition( + /* clippingStartPositionMs= */ 0, + /* clippingEndPositionMs= */ C.TIME_END_OF_SOURCE, + /* mediaItemsInSequence= */ 1); + Transformer transformer = new Transformer.Builder(context).build(); + String firstOutputPath = temporaryFolder.newFile("FirstOutput.mp4").getAbsolutePath(); + InstrumentationRegistry.getInstrumentation() + .runOnMainSync( + () -> { + transformer.start(composition, firstOutputPath); + transformer.cancel(); + }); + + // Resume the export. + ExportResult exportResult = + new TransformerAndroidTestRunner.Builder(context, transformer) + .build() + .run(testId, composition, firstOutputPath) + .exportResult; + + // The first export did not progress because of the immediate cancellation hence resuming + // actually restarts the export. + assertThat(exportResult.processedInputs).hasSize(1); + } + + @Test + public void resume_withSingleMediaItem_outputMatchesWithoutResume() throws Exception { + String testId = "resume_withSingleMediaItem_outputMatchesWithoutResume"; + if (shouldSkipDevice(testId)) { + return; + } + Composition composition = + buildSingleSequenceComposition( + /* clippingStartPositionMs= */ 0, + /* clippingEndPositionMs= */ C.TIME_END_OF_SOURCE, + /* mediaItemsInSequence= */ 1); + // Export without resume. + ExportResult exportResultWithoutResume = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition) + .exportResult; + // Export with resume. + CountDownLatch countDownLatch = new CountDownLatch(1); + Transformer blockingTransformer = buildBlockingTransformer(countDownLatch::countDown); + String firstOutputPath = temporaryFolder.newFile("FirstOutput.mp4").getAbsolutePath(); + InstrumentationRegistry.getInstrumentation() + .runOnMainSync(() -> blockingTransformer.start(composition, firstOutputPath)); + // Block here until timeout reached or latch is counted down. + if (!countDownLatch.await(DEFAULT_TIMEOUT_SECONDS, SECONDS)) { + throw new TimeoutException( + "Transformer timed out after " + DEFAULT_TIMEOUT_SECONDS + " seconds."); + } + InstrumentationRegistry.getInstrumentation().runOnMainSync(blockingTransformer::cancel); + + // Resume the export. + ExportResult exportResultWithResume = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition, firstOutputPath) + .exportResult; + + assertThat(exportResultWithResume.processedInputs).hasSize(4); + assertThat(exportResultWithResume.audioEncoderName) + .isEqualTo(exportResultWithoutResume.audioEncoderName); + assertThat(exportResultWithResume.videoEncoderName) + .isEqualTo(exportResultWithoutResume.videoEncoderName); + assertThat(exportResultWithResume.videoFrameCount) + .isEqualTo( + exportResultWithoutResume.videoFrameCount - getDeviceSpecificMissingFrameCount()); + // TODO: b/306595508 - Remove this expected difference once inconsistent behaviour of audio + // encoder is fixed. + int maxDiffExpectedInDurationMs = 2; + assertThat(exportResultWithResume.durationMs - exportResultWithoutResume.durationMs) + .isLessThan(maxDiffExpectedInDurationMs); + } + + @Test + public void resume_withSingleMediaItemHavingClippingConfig_outputMatchesWithoutResume() + throws Exception { + String testId = "resume_withSingleMediaItemHavingClippingConfig_outputMatchesWithoutResume"; + if (shouldSkipDevice(testId)) { + return; + } + Composition composition = + buildSingleSequenceComposition( + /* clippingStartPositionMs= */ 2_000L, + /* clippingEndPositionMs= */ 13_000L, + /* mediaItemsInSequence= */ 1); + // Export without resume. + ExportResult exportResultWithoutResume = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition) + .exportResult; + // Export with resume. + CountDownLatch countDownLatch = new CountDownLatch(1); + Transformer blockingTransformer = buildBlockingTransformer(countDownLatch::countDown); + String firstOutputPath = temporaryFolder.newFile("FirstOutput.mp4").getAbsolutePath(); + InstrumentationRegistry.getInstrumentation() + .runOnMainSync(() -> blockingTransformer.start(composition, firstOutputPath)); + // Block here until timeout reached or latch is counted down. + if (!countDownLatch.await(DEFAULT_TIMEOUT_SECONDS, SECONDS)) { + throw new TimeoutException( + "Transformer timed out after " + DEFAULT_TIMEOUT_SECONDS + " seconds."); + } + InstrumentationRegistry.getInstrumentation().runOnMainSync(blockingTransformer::cancel); + + // Resume the export. + ExportResult exportResultWithResume = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition, firstOutputPath) + .exportResult; + + assertThat(exportResultWithResume.processedInputs).hasSize(4); + assertThat(exportResultWithResume.audioEncoderName) + .isEqualTo(exportResultWithoutResume.audioEncoderName); + assertThat(exportResultWithResume.videoEncoderName) + .isEqualTo(exportResultWithoutResume.videoEncoderName); + assertThat(exportResultWithResume.videoFrameCount) + .isEqualTo( + exportResultWithoutResume.videoFrameCount - getDeviceSpecificMissingFrameCount()); + int maxDiffExpectedInDurationMs = 2; + assertThat(exportResultWithResume.durationMs - exportResultWithoutResume.durationMs) + .isLessThan(maxDiffExpectedInDurationMs); + } + + @Test + public void resume_withTwoMediaItems_outputMatchesExpected() throws Exception { + String testId = "resume_withTwoMediaItems_outputMatchesExpected"; + if (shouldSkipDevice(testId)) { + return; + } + Composition composition = + buildSingleSequenceComposition( + /* clippingStartPositionMs= */ 0, + /* clippingEndPositionMs= */ C.TIME_END_OF_SOURCE, + /* mediaItemsInSequence= */ 2); + CountDownLatch countDownLatch = new CountDownLatch(1); + Transformer blockingTransformer = buildBlockingTransformer(countDownLatch::countDown); + String firstOutputPath = temporaryFolder.newFile("FirstOutput.mp4").getAbsolutePath(); + InstrumentationRegistry.getInstrumentation() + .runOnMainSync(() -> blockingTransformer.start(composition, firstOutputPath)); + // Block here until timeout reached or latch is counted down. + if (!countDownLatch.await(DEFAULT_TIMEOUT_SECONDS, SECONDS)) { + throw new TimeoutException( + "Transformer timed out after " + DEFAULT_TIMEOUT_SECONDS + " seconds."); + } + InstrumentationRegistry.getInstrumentation().runOnMainSync(blockingTransformer::cancel); + TransformerAndroidTestRunner testRunner = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build(); + + ExportResult exportResult = testRunner.run(testId, composition, firstOutputPath).exportResult; + + assertThat(exportResult.processedInputs).hasSize(6); + int expectedVideoFrameCount = MP4_ASSET_FRAME_COUNT * 2 - getDeviceSpecificMissingFrameCount(); + assertThat(exportResult.videoFrameCount).isEqualTo(expectedVideoFrameCount); + // The first processed media item corresponds to remuxing previous output video. + assertThat(exportResult.processedInputs.get(0).audioDecoderName).isNull(); + assertThat(exportResult.processedInputs.get(0).videoDecoderName).isNull(); + // The next two processed media item corresponds to processing remaining video. + assertThat(exportResult.processedInputs.get(1).audioDecoderName).isNull(); + assertThat(exportResult.processedInputs.get(1).videoDecoderName).isNotEmpty(); + assertThat(exportResult.processedInputs.get(1).mediaItem.clippingConfiguration.startPositionMs) + .isGreaterThan(0); + assertThat(exportResult.processedInputs.get(2).audioDecoderName).isNull(); + assertThat(exportResult.processedInputs.get(2).videoDecoderName).isNotEmpty(); + // The next two processed media item corresponds to processing audio. + assertThat(exportResult.processedInputs.get(3).audioDecoderName).isNotEmpty(); + assertThat(exportResult.processedInputs.get(3).videoDecoderName).isNull(); + assertThat(exportResult.processedInputs.get(4).audioDecoderName).isNotEmpty(); + assertThat(exportResult.processedInputs.get(4).videoDecoderName).isNull(); + // The last processed media item corresponds to transmuxing processed video. + assertThat(exportResult.processedInputs.get(5).audioDecoderName).isNull(); + assertThat(exportResult.processedInputs.get(5).videoDecoderName).isNull(); + } + + @Test + public void resume_withTwoMediaItems_outputMatchesWithoutResume() throws Exception { + String testId = "resume_withTwoMediaItems_outputMatchesWithoutResume"; + if (shouldSkipDevice(testId)) { + return; + } + Composition composition = + buildSingleSequenceComposition( + /* clippingStartPositionMs= */ 0, + /* clippingEndPositionMs= */ C.TIME_END_OF_SOURCE, + /* mediaItemsInSequence= */ 2); + // Export without resume. + ExportResult exportResultWithoutResume = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build() + .run(testId, composition) + .exportResult; + // Export with resume. + CountDownLatch countDownLatch = new CountDownLatch(1); + Transformer blockingTransformer = buildBlockingTransformer(countDownLatch::countDown); + String firstOutputPath = temporaryFolder.newFile("FirstOutput.mp4").getAbsolutePath(); + InstrumentationRegistry.getInstrumentation() + .runOnMainSync(() -> blockingTransformer.start(composition, firstOutputPath)); + // Block here until timeout reached or latch is counted down. + if (!countDownLatch.await(DEFAULT_TIMEOUT_SECONDS, SECONDS)) { + throw new TimeoutException( + "Transformer timed out after " + DEFAULT_TIMEOUT_SECONDS + " seconds."); + } + InstrumentationRegistry.getInstrumentation().runOnMainSync(blockingTransformer::cancel); + TransformerAndroidTestRunner testRunner = + new TransformerAndroidTestRunner.Builder(context, new Transformer.Builder(context).build()) + .build(); + + ExportResult exportResultWithResume = + testRunner.run(testId, composition, firstOutputPath).exportResult; + + assertThat(exportResultWithResume.processedInputs).hasSize(6); + assertThat(exportResultWithResume.audioEncoderName) + .isEqualTo(exportResultWithoutResume.audioEncoderName); + assertThat(exportResultWithResume.videoEncoderName) + .isEqualTo(exportResultWithoutResume.videoEncoderName); + assertThat(exportResultWithResume.videoFrameCount) + .isEqualTo( + exportResultWithoutResume.videoFrameCount - getDeviceSpecificMissingFrameCount()); + int maxDiffExpectedInDurationMs = 2; + assertThat(exportResultWithResume.durationMs - exportResultWithoutResume.durationMs) + .isLessThan(maxDiffExpectedInDurationMs); + } + + private static Composition buildSingleSequenceComposition( + long clippingStartPositionMs, long clippingEndPositionMs, int mediaItemsInSequence) { + SonicAudioProcessor sonic = new SonicAudioProcessor(); + sonic.setPitch(/* pitch= */ 2f); + ImmutableList audioEffects = ImmutableList.of(sonic); + + ImmutableList videoEffects = ImmutableList.of(RgbFilter.createInvertedFilter()); + + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + new MediaItem.Builder() + .setUri(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_URI_STRING) + .setClippingConfiguration( + new MediaItem.ClippingConfiguration.Builder() + .setStartPositionMs(clippingStartPositionMs) + .setEndPositionMs(clippingEndPositionMs) + .build()) + .build()) + .setEffects(new Effects(audioEffects, videoEffects)) + .build(); + + List editedMediaItemList = new ArrayList<>(); + while (mediaItemsInSequence-- > 0) { + editedMediaItemList.add(editedMediaItem); + } + + return new Composition.Builder(new EditedMediaItemSequence(editedMediaItemList)).build(); + } + + private static Transformer buildBlockingTransformer(FrameBlockingMuxer.Listener listener) { + return new Transformer.Builder(getApplicationContext()) + .setMuxerFactory(new FrameBlockingMuxerFactory(listener)) + .build(); + } + + private static boolean shouldSkipDevice(String testId) throws Exception { + // v26 emulators are not producing I-frames, due to which resuming export does not work as + // expected. + return AndroidTestUtil.skipAndLogIfFormatsUnsupported( + getApplicationContext(), + testId, + /* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT, + /* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S_FORMAT) + || (Util.SDK_INT == 26 + && (Ascii.toLowerCase(Util.DEVICE).contains("emulator") + || Ascii.toLowerCase(Util.DEVICE).contains("generic"))); + } + + private static int getDeviceSpecificMissingFrameCount() { + // TODO: b/307700189 - Remove this after investigating pause/resume behaviour with B-frames. + return (Util.SDK_INT == 27 + && (Ascii.equalsIgnoreCase(Util.MODEL, "asus_x00td") + || Ascii.equalsIgnoreCase(Util.MODEL, "tc77"))) + ? 1 + : 0; + } + + private static final class FrameBlockingMuxerFactory implements Muxer.Factory { + private final Muxer.Factory wrappedMuxerFactory; + private final FrameBlockingMuxer.Listener listener; + + public FrameBlockingMuxerFactory(FrameBlockingMuxer.Listener listener) { + this.wrappedMuxerFactory = new DefaultMuxer.Factory(); + this.listener = listener; + } + + @Override + public Muxer create(String path) throws Muxer.MuxerException { + return new FrameBlockingMuxer(wrappedMuxerFactory.create(path), listener); + } + + @Override + public ImmutableList getSupportedSampleMimeTypes(@C.TrackType int trackType) { + return wrappedMuxerFactory.getSupportedSampleMimeTypes(trackType); + } + } + + private static final class FrameBlockingMuxer implements Muxer { + interface Listener { + void onFrameBlocked(); + } + + private final Muxer wrappedMuxer; + private final FrameBlockingMuxer.Listener listener; + + private boolean notifiedListener; + private int videoTrackIndex; + + private FrameBlockingMuxer(Muxer wrappedMuxer, FrameBlockingMuxer.Listener listener) { + this.wrappedMuxer = wrappedMuxer; + this.listener = listener; + videoTrackIndex = C.INDEX_UNSET; + } + + @Override + public int addTrack(Format format) throws MuxerException { + int trackIndex = wrappedMuxer.addTrack(format); + if (MimeTypes.isVideo(format.sampleMimeType)) { + videoTrackIndex = trackIndex; + } + return trackIndex; + } + + @Override + public void writeSampleData( + int trackIndex, ByteBuffer data, long presentationTimeUs, @C.BufferFlags int flags) + throws MuxerException { + if (trackIndex == videoTrackIndex + && presentationTimeUs >= DEFAULT_PRESENTATION_TIME_US_TO_BLOCK_FRAME) { + if (!notifiedListener) { + listener.onFrameBlocked(); + notifiedListener = true; + } + return; + } + wrappedMuxer.writeSampleData(trackIndex, data, presentationTimeUs, flags); + } + + @Override + public void addMetadata(Metadata metadata) { + wrappedMuxer.addMetadata(metadata); + } + + @Override + public void release(boolean forCancellation) throws MuxerException { + wrappedMuxer.release(forCancellation); + } + + @Override + public long getMaxDelayBetweenSamplesMs() { + return wrappedMuxer.getMaxDelayBetweenSamplesMs(); + } + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java index 077ee90ea5..d793cd754c 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResult.java @@ -24,6 +24,7 @@ import androidx.media3.common.MediaItem; import androidx.media3.common.util.UnstableApi; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.List; import java.util.Objects; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -32,7 +33,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; public final class ExportResult { /** A builder for {@link ExportResult} instances. */ public static final class Builder { - private ImmutableList processedInputs; + private ImmutableList.Builder processedInputsBuilder; private long durationMs; private long fileSizeBytes; private int averageAudioBitrate; @@ -48,22 +49,15 @@ public final class ExportResult { @Nullable private ExportException exportException; /** Creates a builder. */ + @SuppressWarnings({"initialization.fields.uninitialized", "nullness:method.invocation"}) public Builder() { - processedInputs = ImmutableList.of(); - durationMs = C.TIME_UNSET; - fileSizeBytes = C.LENGTH_UNSET; - averageAudioBitrate = C.RATE_UNSET_INT; - channelCount = C.LENGTH_UNSET; - sampleRate = C.RATE_UNSET_INT; - averageVideoBitrate = C.RATE_UNSET_INT; - height = C.LENGTH_UNSET; - width = C.LENGTH_UNSET; + reset(); } - /** Sets the {@linkplain ProcessedInput processed inputs}. */ + /** Adds {@linkplain ProcessedInput processed inputs} to the {@link ProcessedInput} list. */ @CanIgnoreReturnValue - public Builder setProcessedInputs(ImmutableList processedInputs) { - this.processedInputs = processedInputs; + public Builder addProcessedInputs(List processedInputs) { + this.processedInputsBuilder.addAll(processedInputs); return this; } @@ -208,7 +202,7 @@ public final class ExportResult { /** Builds an {@link ExportResult} instance. */ public ExportResult build() { return new ExportResult( - processedInputs, + processedInputsBuilder.build(), durationMs, fileSizeBytes, averageAudioBitrate, @@ -223,6 +217,24 @@ public final class ExportResult { videoEncoderName, exportException); } + + /** Resets all the fields to their default values. */ + public void reset() { + processedInputsBuilder = new ImmutableList.Builder<>(); + durationMs = C.TIME_UNSET; + fileSizeBytes = C.LENGTH_UNSET; + averageAudioBitrate = C.RATE_UNSET_INT; + channelCount = C.LENGTH_UNSET; + sampleRate = C.RATE_UNSET_INT; + audioEncoderName = null; + averageVideoBitrate = C.RATE_UNSET_INT; + colorInfo = null; + height = C.LENGTH_UNSET; + width = C.LENGTH_UNSET; + videoFrameCount = 0; + videoEncoderName = null; + exportException = null; + } } /** An input entirely or partially processed. */ @@ -333,7 +345,7 @@ public final class ExportResult { public Builder buildUpon() { return new Builder() - .setProcessedInputs(processedInputs) + .addProcessedInputs(processedInputs) .setDurationMs(durationMs) .setFileSizeBytes(fileSizeBytes) .setAverageAudioBitrate(averageAudioBitrate) diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResumeHelper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResumeHelper.java new file mode 100644 index 0000000000..09e89089bb --- /dev/null +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/ExportResumeHelper.java @@ -0,0 +1,306 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.transformer; + +import static androidx.media3.common.util.Assertions.checkNotNull; + +import android.content.Context; +import android.util.Pair; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.MediaItem; +import androidx.media3.common.util.Util; +import com.google.common.collect.ImmutableList; +import com.google.common.io.ByteStreams; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +/** Utility methods for resuming an export. */ +/* package */ final class ExportResumeHelper { + + /** Provides metadata required to resume an export. */ + public static final class ResumeMetadata { + /** The last sync sample timestamp of the previous output file. */ + public final long lastSyncSampleTimestampUs; + + /** + * A {@link List} containing the index of the first {@link EditedMediaItem} to process and its + * {@linkplain androidx.media3.common.MediaItem.ClippingConfiguration#startPositionMs additional + * offset} for each {@link EditedMediaItemSequence} in a {@link Composition}. + */ + public final ImmutableList> firstMediaItemIndexAndOffsetInfo; + + public ResumeMetadata( + long lastSyncSampleTimestampUs, + ImmutableList> firstMediaItemIndexAndOffsetInfo) { + this.lastSyncSampleTimestampUs = lastSyncSampleTimestampUs; + this.firstMediaItemIndexAndOffsetInfo = firstMediaItemIndexAndOffsetInfo; + } + } + + private ExportResumeHelper() {} + + /** + * Returns a video only {@link Composition} from the given {@code filePath} and {@code + * clippingEndPositionUs}. + */ + public static Composition createVideoOnlyComposition( + String filePath, long clippingEndPositionUs) { + MediaItem.ClippingConfiguration clippingConfiguration = + new MediaItem.ClippingConfiguration.Builder() + .setEndPositionMs(Util.usToMs(clippingEndPositionUs)) + .build(); + EditedMediaItem editedMediaItem = + new EditedMediaItem.Builder( + new MediaItem.Builder() + .setUri(filePath) + .setClippingConfiguration(clippingConfiguration) + .build()) + .setRemoveAudio(true) + .build(); + EditedMediaItemSequence sequence = + new EditedMediaItemSequence(ImmutableList.of(editedMediaItem)); + return new Composition.Builder(ImmutableList.of(sequence)).build(); + } + + /** + * Returns a {@link Composition} for transcoding audio and transmuxing video. + * + * @param composition The {@link Composition} to transcode audio from. + * @param videoFilePath The video only file path to transmux video. + * @return The {@link Composition}. + */ + public static Composition createAudioTranscodeAndVideoTransmuxComposition( + Composition composition, String videoFilePath) { + Composition audioOnlyComposition = + ExportResumeHelper.buildUponComposition( + checkNotNull(composition), + /* removeAudio= */ false, + /* removeVideo= */ true, + /* resumeMetadata= */ null); + + Composition.Builder compositionBuilder = audioOnlyComposition.buildUpon(); + List sequences = new ArrayList<>(audioOnlyComposition.sequences); + + // Video stream sequence. + EditedMediaItem videoOnlyEditedMediaItem = + new EditedMediaItem.Builder(new MediaItem.Builder().setUri(videoFilePath).build()).build(); + EditedMediaItemSequence videoOnlySequence = + new EditedMediaItemSequence(ImmutableList.of(videoOnlyEditedMediaItem)); + + sequences.add(videoOnlySequence); + compositionBuilder.setSequences(sequences); + compositionBuilder.setTransmuxVideo(true); + return compositionBuilder.build(); + } + + /** + * Builds a new {@link Composition} from a given {@link Composition}. + * + *

The new {@link Composition} will be built based on {@link + * ResumeMetadata#firstMediaItemIndexAndOffsetInfo}. + */ + public static Composition buildUponComposition( + Composition composition, + boolean removeAudio, + boolean removeVideo, + @Nullable ResumeMetadata resumeMetadata) { + Composition.Builder compositionBuilder = composition.buildUpon(); + ImmutableList editedMediaItemSequenceList = composition.sequences; + List newEditedMediaItemSequenceList = new ArrayList<>(); + @Nullable + List> firstMediaItemIndexAndOffsetInfo = + resumeMetadata != null ? resumeMetadata.firstMediaItemIndexAndOffsetInfo : null; + + for (int sequenceIndex = 0; + sequenceIndex < editedMediaItemSequenceList.size(); + sequenceIndex++) { + EditedMediaItemSequence currentEditedMediaItemSequence = + editedMediaItemSequenceList.get(sequenceIndex); + ImmutableList editedMediaItemList = + currentEditedMediaItemSequence.editedMediaItems; + List newEditedMediaItemList = new ArrayList<>(); + + int firstMediaItemIndex = 0; + long firstMediaItemOffsetUs = 0L; + + if (firstMediaItemIndexAndOffsetInfo != null) { + firstMediaItemIndex = firstMediaItemIndexAndOffsetInfo.get(sequenceIndex).first; + firstMediaItemOffsetUs = firstMediaItemIndexAndOffsetInfo.get(sequenceIndex).second; + } + + for (int mediaItemIndex = firstMediaItemIndex; + mediaItemIndex < editedMediaItemList.size(); + mediaItemIndex++) { + EditedMediaItem currentEditedMediaItem = editedMediaItemList.get(mediaItemIndex); + EditedMediaItem.Builder newEditedMediaItemBuilder = currentEditedMediaItem.buildUpon(); + + if (mediaItemIndex == firstMediaItemIndex) { + MediaItem.ClippingConfiguration clippingConfiguration = + currentEditedMediaItem + .mediaItem + .clippingConfiguration + .buildUpon() + .setStartPositionMs( + currentEditedMediaItem.mediaItem.clippingConfiguration.startPositionMs + + Util.usToMs(firstMediaItemOffsetUs)) + .build(); + newEditedMediaItemBuilder.setMediaItem( + currentEditedMediaItem + .mediaItem + .buildUpon() + .setClippingConfiguration(clippingConfiguration) + .build()); + } + + if (removeAudio) { + newEditedMediaItemBuilder.setRemoveAudio(true); + } + if (removeVideo) { + newEditedMediaItemBuilder.setRemoveVideo(true); + } + + newEditedMediaItemList.add(newEditedMediaItemBuilder.build()); + } + + newEditedMediaItemSequenceList.add( + new EditedMediaItemSequence( + newEditedMediaItemList, currentEditedMediaItemSequence.isLooping)); + } + compositionBuilder.setSequences(newEditedMediaItemSequenceList); + return compositionBuilder.build(); + } + + /** + * Returns a {@link ListenableFuture} that provides {@link ResumeMetadata} for given input. + * + * @param context The {@link Context}. + * @param filePath The old file path to resume the export from. + * @param composition The {@link Composition} to export. + * @return A {@link ListenableFuture} that provides {@link ResumeMetadata}. + */ + public static ListenableFuture getResumeMetadataAsync( + Context context, String filePath, Composition composition) { + SettableFuture resumeMetadataSettableFuture = SettableFuture.create(); + new Thread("ExportResumeHelper:ResumeMetadata") { + @Override + public void run() { + try { + if (resumeMetadataSettableFuture.isCancelled()) { + return; + } + long lastSyncSampleTimestampUs = + Mp4MetadataInfo.create(context, filePath).lastSyncSampleTimestampUs; + + ImmutableList.Builder> firstMediaItemIndexAndOffsetInfoBuilder = + new ImmutableList.Builder<>(); + if (lastSyncSampleTimestampUs != C.TIME_UNSET) { + for (int compositionSequenceIndex = 0; + compositionSequenceIndex < composition.sequences.size(); + compositionSequenceIndex++) { + ImmutableList editedMediaItemList = + composition.sequences.get(compositionSequenceIndex).editedMediaItems; + long remainingDurationUsToSkip = lastSyncSampleTimestampUs; + int editedMediaItemIndex = 0; + long mediaItemOffset = 0L; + while (editedMediaItemIndex < editedMediaItemList.size() + && remainingDurationUsToSkip > 0) { + long mediaItemDuration = + getMediaItemDurationUs( + context, editedMediaItemList.get(editedMediaItemIndex).mediaItem); + if (mediaItemDuration > remainingDurationUsToSkip) { + mediaItemOffset = remainingDurationUsToSkip; + break; + } + + remainingDurationUsToSkip -= mediaItemDuration; + editedMediaItemIndex++; + } + firstMediaItemIndexAndOffsetInfoBuilder.add( + new Pair<>(editedMediaItemIndex, mediaItemOffset)); + } + } + resumeMetadataSettableFuture.set( + new ResumeMetadata( + lastSyncSampleTimestampUs, firstMediaItemIndexAndOffsetInfoBuilder.build())); + } catch (Exception ex) { + resumeMetadataSettableFuture.setException(ex); + } + } + }.start(); + + return resumeMetadataSettableFuture; + } + + /** Copies {@link File} content from source to destination asynchronously. */ + public static ListenableFuture copyFileAsync(File source, File destination) { + SettableFuture copyFileSettableFuture = SettableFuture.create(); + new Thread("ExportResumeHelper:CopyFile") { + @Override + public void run() { + if (copyFileSettableFuture.isCancelled()) { + return; + } + InputStream input = null; + OutputStream output = null; + try { + input = new FileInputStream(source); + output = new FileOutputStream(destination); + ByteStreams.copy(input, output); + copyFileSettableFuture.set(null); + } catch (Exception ex) { + copyFileSettableFuture.setException(ex); + } finally { + try { + if (input != null) { + input.close(); + } + if (output != null) { + output.close(); + } + } catch (IOException exception) { + // If the file copy was successful then this exception can be ignored and if there + // was some other error during copy operation then that exception has already been + // propagated in the catch block. + } + } + } + }.start(); + return copyFileSettableFuture; + } + + private static long getMediaItemDurationUs(Context context, MediaItem mediaItem) + throws IOException { + String filePath = checkNotNull(mediaItem.localConfiguration).uri.toString(); + long startUs = Util.msToUs(mediaItem.clippingConfiguration.startPositionMs); + long endUs; + if (mediaItem.clippingConfiguration.endPositionMs != C.TIME_END_OF_SOURCE) { + endUs = Util.msToUs(mediaItem.clippingConfiguration.endPositionMs); + } else { + endUs = Mp4MetadataInfo.create(context, filePath).durationUs; + } + + return endUs - startUs; + } +} diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java index ee59edbbd1..5ff7bcb505 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MuxerWrapper.java @@ -238,12 +238,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; // Ensure that video formats are the same. Some fields like codecs, averageBitrate, framerate, // etc, don't match exactly in the Extractor output format and the Encoder output // format but these fields can be ignored. + // TODO: b/308180225 - Compare Format.colorInfo as well. Format existingFormat = videoTrackInfo.format; checkArgument(Util.areEqual(existingFormat.sampleMimeType, format.sampleMimeType)); checkArgument(existingFormat.width == format.width); checkArgument(existingFormat.height == format.height); checkArgument(existingFormat.initializationDataEquals(format)); - checkArgument(Util.areEqual(existingFormat.colorInfo, format.colorInfo)); checkNotNull(muxer); resetAbortTimer(); diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java index 3a8a1c6f15..b7b14b45bd 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java @@ -49,8 +49,13 @@ import androidx.media3.effect.DefaultVideoFrameProcessor; import androidx.media3.effect.Presentation; import androidx.media3.exoplayer.source.DefaultMediaSourceFactory; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.InlineMe; +import java.io.File; +import java.io.IOException; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -653,6 +658,24 @@ public final class Transformer { /** Indicates that the progress is permanently unavailable. */ public static final int PROGRESS_STATE_UNAVAILABLE = 3; + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target(TYPE_USE) + @IntDef({ + TRANSFORMER_STATE_PROCESS_FULL_INPUT, + TRANSFORMER_STATE_REMUX_PROCESSED_VIDEO, + TRANSFORMER_STATE_PROCESS_REMAINING_VIDEO, + TRANSFORMER_STATE_PROCESS_AUDIO, + TRANSFORMER_STATE_COPY_OUTPUT + }) + private @interface TransformerState {} + + private static final int TRANSFORMER_STATE_PROCESS_FULL_INPUT = 0; + private static final int TRANSFORMER_STATE_REMUX_PROCESSED_VIDEO = 1; + private static final int TRANSFORMER_STATE_PROCESS_REMAINING_VIDEO = 2; + private static final int TRANSFORMER_STATE_PROCESS_AUDIO = 3; + private static final int TRANSFORMER_STATE_COPY_OUTPUT = 4; + private final Context context; private final TransformationRequest transformationRequest; private final ImmutableList audioProcessors; @@ -669,8 +692,20 @@ public final class Transformer { private final Looper looper; private final DebugViewProvider debugViewProvider; private final Clock clock; + private final HandlerWrapper applicationHandler; + private final ComponentListener componentListener; + private final ExportResult.Builder exportResultBuilder; @Nullable private TransformerInternal transformerInternal; + @Nullable private MuxerWrapper remuxingMuxerWrapper; + private @MonotonicNonNull Composition composition; + private @MonotonicNonNull String outputFilePath; + private @MonotonicNonNull String oldFilePath; + private @TransformerState int transformerState; + private ExportResumeHelper.@MonotonicNonNull ResumeMetadata resumeMetadata; + private @MonotonicNonNull ListenableFuture + getResumeMetadataFuture; + private @MonotonicNonNull ListenableFuture copyOutputFuture; private Transformer( Context context, @@ -706,6 +741,9 @@ public final class Transformer { this.looper = looper; this.debugViewProvider = debugViewProvider; this.clock = clock; + applicationHandler = clock.createHandler(looper, /* callback= */ null); + componentListener = new ComponentListener(); + exportResultBuilder = new ExportResult.Builder(); } /** Returns a {@link Transformer.Builder} initialized with the values of this instance. */ @@ -828,11 +866,12 @@ public final class Transformer { * @throws IllegalStateException If an export is already in progress. */ public void start(Composition composition, String path) { - ComponentListener componentListener = new ComponentListener(composition); + initialize(composition, path); startInternal( composition, new MuxerWrapper(path, muxerFactory, componentListener, MuxerWrapper.MUXER_MODE_DEFAULT), - componentListener); + componentListener, + /* initialTimestampOffsetUs= */ 0); } /** @@ -927,6 +966,9 @@ public final class Transformer { * Returns the current {@link ProgressState} and updates {@code progressHolder} with the current * progress if it is {@link #PROGRESS_STATE_AVAILABLE available}. * + *

If the export is {@linkplain #resume(Composition, String, String) resumed}, this method + * returns {@link #PROGRESS_STATE_UNAVAILABLE}. + * *

After an export {@linkplain Listener#onCompleted(Composition, ExportResult) completes}, this * method returns {@link #PROGRESS_STATE_NOT_STARTED}. * @@ -937,6 +979,9 @@ public final class Transformer { */ public @ProgressState int getProgress(ProgressHolder progressHolder) { verifyApplicationThread(); + if (transformerState != TRANSFORMER_STATE_PROCESS_FULL_INPUT) { + return PROGRESS_STATE_UNAVAILABLE; + } return transformerInternal == null ? PROGRESS_STATE_NOT_STARTED : transformerInternal.getProgress(progressHolder); @@ -959,6 +1004,159 @@ public final class Transformer { } finally { transformerInternal = null; } + + if (getResumeMetadataFuture != null && !getResumeMetadataFuture.isDone()) { + getResumeMetadataFuture.cancel(/* mayInterruptIfRunning= */ false); + } + if (copyOutputFuture != null && !copyOutputFuture.isDone()) { + copyOutputFuture.cancel(/* mayInterruptIfRunning= */ false); + } + } + + /** + * Resumes a previously {@linkplain #cancel() cancelled} export. + * + *

An export can be resumed only when: + * + *

    + *
  • The {@link Composition} contains a single {@link EditedMediaItemSequence} having + * continuous audio and video tracks. + *
  • The output is an MP4 file. + *
+ * + * @param composition The {@link Composition} to resume export. + * @param outputFilePath The path to the output file. This must be different from the output path + * of the cancelled export. + * @param oldFilePath The output path of the the cancelled export. + */ + public void resume(Composition composition, String outputFilePath, String oldFilePath) { + verifyApplicationThread(); + initialize(composition, outputFilePath); + this.oldFilePath = oldFilePath; + remuxProcessedVideo(); + } + + private void initialize(Composition composition, String outputFilePath) { + this.composition = composition; + this.outputFilePath = outputFilePath; + exportResultBuilder.reset(); + } + + private void processFullInput() { + transformerState = TRANSFORMER_STATE_PROCESS_FULL_INPUT; + startInternal( + checkNotNull(composition), + new MuxerWrapper( + checkNotNull(outputFilePath), + muxerFactory, + componentListener, + MuxerWrapper.MUXER_MODE_DEFAULT), + componentListener, + /* initialTimestampOffsetUs= */ 0); + } + + private void remuxProcessedVideo() { + transformerState = TRANSFORMER_STATE_REMUX_PROCESSED_VIDEO; + getResumeMetadataFuture = + ExportResumeHelper.getResumeMetadataAsync( + context, checkNotNull(oldFilePath), checkNotNull(composition)); + Futures.addCallback( + getResumeMetadataFuture, + new FutureCallback() { + @Override + public void onSuccess(ExportResumeHelper.ResumeMetadata resumeMetadata) { + // If there is no video track to remux or the last sync sample is actually the first + // sample, then start the normal Export. + if (resumeMetadata.lastSyncSampleTimestampUs == C.TIME_UNSET + || resumeMetadata.lastSyncSampleTimestampUs == 0) { + processFullInput(); + return; + } + + Transformer.this.resumeMetadata = resumeMetadata; + + remuxingMuxerWrapper = + new MuxerWrapper( + checkNotNull(outputFilePath), + muxerFactory, + componentListener, + MuxerWrapper.MUXER_MODE_MUX_PARTIAL_VIDEO); + + startInternal( + ExportResumeHelper.createVideoOnlyComposition( + oldFilePath, + /* clippingEndPositionUs= */ resumeMetadata.lastSyncSampleTimestampUs), + checkNotNull(remuxingMuxerWrapper), + componentListener, + /* initialTimestampOffsetUs= */ 0); + } + + @Override + public void onFailure(Throwable t) { + // In case of error fallback to normal Export. + processFullInput(); + } + }, + applicationHandler::post); + } + + private void processRemainingVideo() { + transformerState = TRANSFORMER_STATE_PROCESS_REMAINING_VIDEO; + Composition videoOnlyComposition = + ExportResumeHelper.buildUponComposition( + checkNotNull(composition), + /* removeAudio= */ true, + /* removeVideo= */ false, + resumeMetadata); + + checkNotNull(remuxingMuxerWrapper); + remuxingMuxerWrapper.changeToAppendVideoMode(); + + startInternal( + videoOnlyComposition, + remuxingMuxerWrapper, + componentListener, + /* initialTimestampOffsetUs= */ checkNotNull(resumeMetadata).lastSyncSampleTimestampUs); + } + + private void processAudio() { + transformerState = TRANSFORMER_STATE_PROCESS_AUDIO; + + startInternal( + ExportResumeHelper.createAudioTranscodeAndVideoTransmuxComposition( + checkNotNull(composition), checkNotNull(outputFilePath)), + new MuxerWrapper( + checkNotNull(oldFilePath), + muxerFactory, + componentListener, + MuxerWrapper.MUXER_MODE_DEFAULT), + componentListener, + /* initialTimestampOffsetUs= */ 0); + } + + // TODO: b/308253384 - Move copy output logic into MuxerWrapper. + private void copyOutput() { + transformerState = TRANSFORMER_STATE_COPY_OUTPUT; + copyOutputFuture = + ExportResumeHelper.copyFileAsync( + new File(checkNotNull(oldFilePath)), new File(checkNotNull(outputFilePath))); + + Futures.addCallback( + copyOutputFuture, + new FutureCallback() { + @Override + public void onSuccess(Void result) { + onExportCompletedWithSuccess(); + } + + @Override + public void onFailure(Throwable t) { + onExportCompletedWithError( + ExportException.createForUnexpected( + new IOException("Copy output task failed for the resumed export", t))); + } + }, + applicationHandler::post); } private void verifyApplicationThread() { @@ -968,11 +1166,13 @@ public final class Transformer { } private void startInternal( - Composition composition, MuxerWrapper muxerWrapper, ComponentListener componentListener) { + Composition composition, + MuxerWrapper muxerWrapper, + ComponentListener componentListener, + long initialTimestampOffsetUs) { checkArgument(composition.effects.audioProcessors.isEmpty()); verifyApplicationThread(); checkState(transformerInternal == null, "There is already an export in progress."); - HandlerWrapper applicationHandler = clock.createHandler(looper, /* callback= */ null); TransformationRequest transformationRequest = this.transformationRequest; if (composition.hdrMode != Composition.HDR_MODE_KEEP_HDR) { transformationRequest = @@ -1006,21 +1206,28 @@ public final class Transformer { applicationHandler, debugViewProvider, clock, - /* videoSampleTimestampOffsetUs= */ 0); + initialTimestampOffsetUs); transformerInternal.start(); } + private void onExportCompletedWithSuccess() { + listeners.queueEvent( + /* eventFlag= */ C.INDEX_UNSET, + listener -> listener.onCompleted(checkNotNull(composition), exportResultBuilder.build())); + listeners.flushEvents(); + } + + private void onExportCompletedWithError(ExportException exception) { + listeners.queueEvent( + /* eventFlag= */ C.INDEX_UNSET, + listener -> + listener.onError(checkNotNull(composition), exportResultBuilder.build(), exception)); + listeners.flushEvents(); + } + private final class ComponentListener implements TransformerInternal.Listener, MuxerWrapper.Listener { - private final Composition composition; - private final ExportResult.Builder exportResultBuilder; - - public ComponentListener(Composition composition) { - this.composition = composition; - this.exportResultBuilder = new ExportResult.Builder(); - } - // TransformerInternal.Listener implementation @Override @@ -1028,17 +1235,29 @@ public final class Transformer { ImmutableList processedInputs, @Nullable String audioEncoderName, @Nullable String videoEncoderName) { - exportResultBuilder - .setProcessedInputs(processedInputs) - .setAudioEncoderName(audioEncoderName) - .setVideoEncoderName(videoEncoderName); + exportResultBuilder.addProcessedInputs(processedInputs); + + // When an export is resumed, the audio and video encoder name (if any) can comes from + // different intermittent exports, so set encoder names only when they are available. + if (audioEncoderName != null) { + exportResultBuilder.setAudioEncoderName(audioEncoderName); + } + if (videoEncoderName != null) { + exportResultBuilder.setVideoEncoderName(videoEncoderName); + } // TODO(b/213341814): Add event flags for Transformer events. transformerInternal = null; - listeners.queueEvent( - /* eventFlag= */ C.INDEX_UNSET, - listener -> listener.onCompleted(composition, exportResultBuilder.build())); - listeners.flushEvents(); + if (transformerState == TRANSFORMER_STATE_REMUX_PROCESSED_VIDEO) { + processRemainingVideo(); + } else if (transformerState == TRANSFORMER_STATE_PROCESS_REMAINING_VIDEO) { + remuxingMuxerWrapper = null; + processAudio(); + } else if (transformerState == TRANSFORMER_STATE_PROCESS_AUDIO) { + copyOutput(); + } else { + onExportCompletedWithSuccess(); + } } @Override @@ -1048,17 +1267,20 @@ public final class Transformer { @Nullable String audioEncoderName, @Nullable String videoEncoderName, ExportException exportException) { - exportResultBuilder - .setProcessedInputs(processedInputs) - .setAudioEncoderName(audioEncoderName) - .setVideoEncoderName(videoEncoderName) - .setExportException(exportException); + exportResultBuilder.addProcessedInputs(processedInputs); + // When an export is resumed, the audio and video encoder name (if any) can comes from + // different intermittent exports, so set encoder names only when they are available. + if (audioEncoderName != null) { + exportResultBuilder.setAudioEncoderName(audioEncoderName); + } + if (videoEncoderName != null) { + exportResultBuilder.setVideoEncoderName(videoEncoderName); + } + + exportResultBuilder.setExportException(exportException); transformerInternal = null; - listeners.queueEvent( - /* eventFlag= */ C.INDEX_UNSET, - listener -> listener.onError(composition, exportResultBuilder.build(), exportException)); - listeners.flushEvents(); + onExportCompletedWithError(exportException); } // MuxerWrapper.Listener implementation