loliball/media
1
0
Fork 0
mirror of https://github.com/androidx/media.git synced 2025-05-18 13:09:56 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add test for audio and video from different sources

Browse Source 
PiperOrigin-RevId: 515379858
This commit is contained in:
kimvde 2023-03-09 18:43:39 +00:00 committed by tonihei
parent 7031d2c6f4
commit 1865e38108
6 changed files with 221 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerAndroidTestRunner.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static java.util.concurrent.TimeUnit.SECONDS;
@ -31,6 +32,7 @@ import androidx.media3.common.util.SystemClock;
import androidx.media3.common.util.Util;
import androidx.test.platform.app.InstrumentationRegistry;
import com.google.common.base.Ascii;
import com.google.common.collect.ImmutableList;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.io.File;
import java.io.IOException;
@ -174,20 +176,20 @@ public class TransformerAndroidTestRunner {
}
/**
* Exports the {@link EditedMediaItem}, saving a summary of the export to the application cache.
* Exports the {@link Composition}, saving a summary of the export to the application cache.
*
* @param testId A unique identifier for the transformer test run.
* @param editedMediaItem The {@link EditedMediaItem} to export.
* @param composition The {@link Composition} to export.
* @return The {@link ExportTestResult}.
* @throws Exception The cause of the export not completing.
*/
public ExportTestResult run(String testId, EditedMediaItem editedMediaItem) throws Exception {
public ExportTestResult run(String testId, Composition composition) throws Exception {
JSONObject resultJson = new JSONObject();
if (inputValues != null) {
resultJson.put("inputValues", JSONObject.wrap(inputValues));
}
try {
ExportTestResult exportTestResult = runInternal(testId, editedMediaItem);
ExportTestResult exportTestResult = runInternal(testId, composition);
resultJson.put("exportResult", exportTestResult.asJsonObject());
if (exportTestResult.exportResult.exportException != null) {
throw exportTestResult.exportResult.exportException;
@ -209,6 +211,21 @@ public class TransformerAndroidTestRunner {
}
}
/**
* Exports the {@link EditedMediaItem}, saving a summary of the export to the application cache.
*
* @param testId A unique identifier for the transformer test run.
* @param editedMediaItem The {@link EditedMediaItem} to export.
* @return The {@link ExportTestResult}.
* @throws Exception The cause of the export not completing.
*/
public ExportTestResult run(String testId, EditedMediaItem editedMediaItem) throws Exception {
EditedMediaItemSequence sequence =
new EditedMediaItemSequence(ImmutableList.of(editedMediaItem));
Composition composition = new Composition.Builder(ImmutableList.of(sequence)).build();
return run(testId, composition);
}
/**
* Exports the {@link MediaItem}, saving a summary of the export to the application cache.
*
@ -223,33 +240,49 @@ public class TransformerAndroidTestRunner {
}
/**
* Exports the {@link EditedMediaItem}.
* Exports the {@link Composition}.
*
* @param testId An identifier for the test.
* @param editedMediaItem The {@link EditedMediaItem} to export.
* @param composition The {@link Composition} to export.
* @return The {@link ExportTestResult}.
* @throws IllegalStateException See {@link Transformer#start(EditedMediaItem, String)}.
* @throws IllegalStateException See {@link Transformer#start(Composition, String)}.
* @throws InterruptedException If the thread is interrupted whilst waiting for transformer to
* complete.
* @throws IOException If an error occurs opening the output file for writing.
* @throws TimeoutException If the export has not completed after {@linkplain
* Builder#setTimeoutSeconds(int) the given timeout}.
*/
private ExportTestResult runInternal(String testId, EditedMediaItem editedMediaItem)
private ExportTestResult runInternal(String testId, Composition composition)
throws InterruptedException, IOException, TimeoutException {
MediaItem mediaItem = editedMediaItem.mediaItem;
if (!mediaItem.clippingConfiguration.equals(MediaItem.ClippingConfiguration.UNSET)
&& requestCalculateSsim) {
throw new UnsupportedOperationException(
if (requestCalculateSsim) {
checkArgument(
composition.sequences.size() == 1
&& composition.sequences.get(0).editedMediaItems.size() == 1,
"SSIM is only relevant for single MediaItem compositions");
checkArgument(
composition
.sequences
.get(0)
.editedMediaItems
.get(0)
.mediaItem
.clippingConfiguration
.equals(MediaItem.ClippingConfiguration.UNSET),
"SSIM calculation is not supported for clipped inputs.");
}
Uri mediaItemUri = checkNotNull(mediaItem.localConfiguration).uri;
String scheme = checkNotNull(mediaItemUri.getScheme());
if ((scheme.equals("http") || scheme.equals("https")) && !hasNetworkConnection(context)) {
throw new UnsupportedOperationException(
"Input network file requested on device with no network connection. Input file name: "
+ mediaItemUri);
if (!hasNetworkConnection(context)) {
for (EditedMediaItemSequence sequence : composition.sequences) {
for (EditedMediaItem editedMediaItem : sequence.editedMediaItems) {
Uri mediaItemUri = checkNotNull(editedMediaItem.mediaItem.localConfiguration).uri;
String scheme = checkNotNull(mediaItemUri.getScheme());
if ((scheme.equals("http") || scheme.equals("https"))) {
throw new IllegalArgumentException(
"Input network file requested on device with no network connection. Input file"
+ " name: "
+ mediaItemUri);
}
}
}
}
AtomicReference<@NullableType FallbackDetails> fallbackDetailsReference =
@ -307,7 +340,7 @@ public class TransformerAndroidTestRunner {
.runOnMainSync(
() -> {
try {
testTransformer.start(editedMediaItem, outputVideoFile.getAbsolutePath());
testTransformer.start(composition, outputVideoFile.getAbsolutePath());
// Catch all exceptions to report. Exceptions thrown here and not caught will NOT
// propagate.
} catch (Exception e) {
@ -359,6 +392,7 @@ public class TransformerAndroidTestRunner {
return testResultBuilder.build();
}
try {
MediaItem mediaItem = composition.sequences.get(0).editedMediaItems.get(0).mediaItem;
double ssim =
SsimHelper.calculate(
context,

48
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -27,7 +27,10 @@ import androidx.media3.common.C;
import androidx.media3.common.Effect;
import androidx.media3.common.Format;
import androidx.media3.common.MediaItem;
import androidx.media3.common.audio.AudioProcessor;
import androidx.media3.common.audio.SonicAudioProcessor;
import androidx.media3.effect.Presentation;
import androidx.media3.effect.RgbFilter;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
@ -185,6 +188,51 @@ public class TransformerEndToEndTest {
assertThat(exception).hasMessageThat().contains("video");
}
@Test
public void start_audioVideoTranscodedFromDifferentSequences_producesExpectedResult()
throws Exception {
Transformer transformer = new Transformer.Builder(context).build();
String testId = "start_audioVideoTranscodedFromDifferentSequences_producesExpectedResult";
ImmutableList<AudioProcessor> audioProcessors = ImmutableList.of(new SonicAudioProcessor());
ImmutableList<Effect> videoEffects = ImmutableList.of(RgbFilter.createGrayscaleFilter());
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET_URI_STRING));
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem)
.setEffects(new Effects(audioProcessors, videoEffects))
.build();
ExportTestResult expectedResult =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
EditedMediaItem audioEditedMediaItem =
new EditedMediaItem.Builder(mediaItem)
.setEffects(new Effects(audioProcessors, /* videoEffects= */ ImmutableList.of()))
.setRemoveVideo(true)
.build();
EditedMediaItemSequence audioSequence =
new EditedMediaItemSequence(ImmutableList.of(audioEditedMediaItem));
EditedMediaItem videoEditedMediaItem =
new EditedMediaItem.Builder(mediaItem)
.setEffects(new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects))
.setRemoveAudio(true)
.build();
EditedMediaItemSequence videoSequence =
new EditedMediaItemSequence(ImmutableList.of(videoEditedMediaItem));
Composition composition =
new Composition.Builder(ImmutableList.of(audioSequence, videoSequence)).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, composition);
assertThat(result.exportResult.channelCount)
.isEqualTo(expectedResult.exportResult.channelCount);
assertThat(result.exportResult.videoFrameCount)
.isEqualTo(expectedResult.exportResult.videoFrameCount);
assertThat(result.exportResult.durationMs).isEqualTo(expectedResult.exportResult.durationMs);
}
private static final class VideoUnsupportedEncoderFactory implements Codec.EncoderFactory {
private final Codec.EncoderFactory encoderFactory;

50
libraries/transformer/src/main/java/androidx/media3/transformer/ExoAssetLoaderBaseRenderer.java
View File

@ -17,7 +17,6 @@
package androidx.media3.transformer;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.decoder.DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DISABLED;
import static androidx.media3.exoplayer.source.SampleStream.FLAG_REQUIRE_FORMAT;
import static androidx.media3.transformer.AssetLoader.SUPPORTED_OUTPUT_TYPE_DECODED;
@ -54,6 +53,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private boolean isRunning;
private long streamStartPositionUs;
private boolean shouldInitDecoder;
public ExoAssetLoaderBaseRenderer(
@C.TrackType int trackType,
@ -97,7 +97,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
@Override
public void render(long positionUs, long elapsedRealtimeUs) {
try {
if (!isRunning || isEnded() || !hasReadInputFormat()) {
if (!isRunning || isEnded() || !readInputFormatAndInitDecoderIfNeeded()) {
return;
}
@ -195,35 +195,39 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* {@linkplain Codec decoder}.
*/
@EnsuresNonNullIf(expression = "inputFormat", result = true)
private boolean hasReadInputFormat() throws ExportException {
if (inputFormat != null) {
private boolean readInputFormatAndInitDecoderIfNeeded() throws ExportException {
if (inputFormat != null && !shouldInitDecoder) {
return true;
}
FormatHolder formatHolder = getFormatHolder();
@ReadDataResult
int result = readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT);
if (result != C.RESULT_FORMAT_READ) {
return false;
if (inputFormat == null) {
FormatHolder formatHolder = getFormatHolder();
@ReadDataResult
int result =
readSource(formatHolder, decoderInputBuffer, /* readFlags= */ FLAG_REQUIRE_FORMAT);
if (result != C.RESULT_FORMAT_READ) {
return false;
}
inputFormat = overrideFormat(checkNotNull(formatHolder.format));
onInputFormatRead(inputFormat);
shouldInitDecoder =
assetLoaderListener.onTrackAdded(
inputFormat,
SUPPORTED_OUTPUT_TYPE_DECODED | SUPPORTED_OUTPUT_TYPE_ENCODED,
streamStartPositionUs,
streamOffsetUs);
}
inputFormat = overrideFormat(checkNotNull(formatHolder.format));
onInputFormatRead(inputFormat);
boolean decodeOutput =
assetLoaderListener.onTrackAdded(
inputFormat,
SUPPORTED_OUTPUT_TYPE_DECODED | SUPPORTED_OUTPUT_TYPE_ENCODED,
streamStartPositionUs,
streamOffsetUs);
if (decodeOutput) {
if (getProcessedTrackType(inputFormat.sampleMimeType) == C.TRACK_TYPE_AUDIO) {
initDecoder(inputFormat);
} else {
if (shouldInitDecoder) {
if (getProcessedTrackType(inputFormat.sampleMimeType) == C.TRACK_TYPE_VIDEO) {
// TODO(b/237674316): Move surface creation out of video sampleConsumer. Init decoder and
// get decoder output Format before init sampleConsumer.
checkState(ensureSampleConsumerInitialized());
initDecoder(inputFormat);
if (!ensureSampleConsumerInitialized()) {
return false;
}
}
initDecoder(inputFormat);
shouldInitDecoder = false;
}
return true;

1
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -702,7 +702,6 @@ public final class Transformer {
* @throws IllegalStateException If an export is already in progress.
*/
public void start(Composition composition, String path) {
checkArgument(composition.sequences.size() == 1);
checkArgument(composition.effects.audioProcessors.isEmpty());
// Only supports Presentation in video effects.
ImmutableList<Effect> videoEffects = composition.effects.videoEffects;

2
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -465,7 +465,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} else {
outputHasVideo.set(true);
}
if (trackCountsToReport.get() == 0 && tracksToAdd.decrementAndGet() == 0) {
if (tracksToAdd.decrementAndGet() == 0 && trackCountsToReport.get() == 0) {
int outputTrackCount = (outputHasAudio.get() ? 1 : 0) + (outputHasVideo.get() ? 1 : 0);
muxerWrapper.setTrackCount(outputTrackCount);
fallbackListener.setTrackCount(outputTrackCount);

91
libraries/transformer/src/test/java/androidx/media3/transformer/CompositionExportTest.java Normal file
View File

@ -0,0 +1,91 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import static androidx.media3.transformer.TestUtil.ASSET_URI_PREFIX;
import static androidx.media3.transformer.TestUtil.FILE_AUDIO_VIDEO;
import static androidx.media3.transformer.TestUtil.createTransformerBuilder;
import static com.google.common.truth.Truth.assertThat;
import android.content.Context;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Util;
import androidx.media3.transformer.TestUtil.TestMuxerFactory.TestMuxerHolder;
import androidx.test.core.app.ApplicationProvider;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import com.google.common.collect.ImmutableList;
import java.nio.file.Files;
import java.nio.file.Paths;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
/**
* End-to-end test for exporting a {@link Composition} containing multiple {@link
* EditedMediaItemSequence} instances with {@link Transformer}.
*/
@RunWith(AndroidJUnit4.class)
public class CompositionExportTest {
private String outputPath;
private TestMuxerHolder testMuxerHolder;
@Before
public void setUp() throws Exception {
Context context = ApplicationProvider.getApplicationContext();
outputPath = Util.createTempFile(context, "TransformerTest").getPath();
testMuxerHolder = new TestUtil.TestMuxerFactory.TestMuxerHolder();
}
@After
public void tearDown() throws Exception {
Files.delete(Paths.get(outputPath));
}
@Test
public void start_audioVideoTransmuxedFromDifferentSequences_producesExpectedResult()
throws Exception {
Transformer transformer =
createTransformerBuilder(testMuxerHolder, /* enableFallback= */ false).build();
MediaItem mediaItem = MediaItem.fromUri(ASSET_URI_PREFIX + FILE_AUDIO_VIDEO);
transformer.start(mediaItem, outputPath);
ExportResult expectedExportResult = TransformerTestRunner.runLooper(transformer);
EditedMediaItem audioEditedMediaItem =
new EditedMediaItem.Builder(mediaItem).setRemoveVideo(true).build();
EditedMediaItemSequence audioSequence =
new EditedMediaItemSequence(ImmutableList.of(audioEditedMediaItem));
EditedMediaItem videoEditedMediaItem =
new EditedMediaItem.Builder(mediaItem).setRemoveAudio(true).build();
EditedMediaItemSequence videoSequence =
new EditedMediaItemSequence(ImmutableList.of(videoEditedMediaItem));
Composition composition =
new Composition.Builder(ImmutableList.of(audioSequence, videoSequence))
.setTransmuxAudio(true)
.setTransmuxVideo(true)
.build();
transformer.start(composition, outputPath);
ExportResult exportResult = TransformerTestRunner.runLooper(transformer);
// We can't compare the muxer output against a dump file because the asset loaders in each
// sequence load samples from their own thread, independently of each other, which makes the
// output non-deterministic.
assertThat(exportResult.channelCount).isEqualTo(expectedExportResult.channelCount);
assertThat(exportResult.videoFrameCount).isEqualTo(expectedExportResult.videoFrameCount);
assertThat(exportResult.durationMs).isEqualTo(expectedExportResult.durationMs);
}
}