loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compositor: Add tests for 1, 3, and 5 inputs.

Browse Source 
With this, Compositor now handles an arbitrary number of inputs!

PiperOrigin-RevId: 558813361
This commit is contained in:
huangdarwin 2023-08-21 17:32:41 +01:00 committed by Julia Bibik
parent fddb09be20
commit 350b394596
7 changed files with 100 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoCompositor.java
View File

@ -61,8 +61,6 @@ public final class DefaultVideoCompositor implements VideoCompositor {
// * Use a lock to synchronize inputFrameInfos more narrowly, to reduce blocking.
// * If the primary stream ends, consider setting the secondary stream as the new primary stream,
// so that secondary stream frames aren't dropped.
// * Consider adding info about the timestamps for each input frame used to composite an output
// frame, to aid debugging and testing.
private static final String THREAD_NAME = "Effect:DefaultVideoCompositor:GlThread";
private static final String TAG = "DefaultVideoCompositor";
@ -133,6 +131,9 @@ public final class DefaultVideoCompositor implements VideoCompositor {
* <p>The input source must be able to have at least two {@linkplain #queueInputTexture queued
* textures} before one texture is {@linkplain
* DefaultVideoFrameProcessor.ReleaseOutputTextureCallback released}.
*
* <p>When composited, textures are drawn in the reverse order of their registration order, so
* that the first registered source is on the very top.
*/
@Override
public synchronized int registerInputSource() {
@ -269,9 +270,7 @@ public final class DefaultVideoCompositor implements VideoCompositor {
ensureGlProgramConfigured();
// TODO: b/262694346 -
// * Support an arbitrary number of inputs.
// * Allow different input frame dimensions.
// TODO: b/262694346 - Allow different input frame dimensions.
InputFrameInfo primaryInputFrame = framesToComposite.get(PRIMARY_INPUT_ID);
GlTextureInfo primaryInputTexture = primaryInputFrame.texture;
outputTexturePool.ensureConfigured(

BIN
libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_0s_1s_0s.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 KiB

BIN
libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_1s_1s_0s.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 KiB

BIN
libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_2s_1s_2s.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 KiB

BIN
libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_grayscale_opaque_1s.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 6.9 KiB

BIN
libraries/test_data/src/test/assets/media/bitmap/CompositorTestTimestamps/output_grayscale_opaque_2s.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 7.5 KiB

106
libraries/transformer/src/androidTest/java/androidx/media3/transformer/DefaultVideoCompositorPixelTest.java
View File

@ -82,6 +82,8 @@ import org.junit.runners.Parameterized;
/** Pixel test for {@link DefaultVideoCompositor} compositing 2 input frames into 1 output frame. */
@RunWith(Parameterized.class)
public final class DefaultVideoCompositorPixelTest {
// TODO: b/262694346 - Have CompositorTestRunner queueBitmapToInput queue bitmaps at specified
// timestamps instead of frame rates.
@Parameterized.Parameters(name = "useSharedExecutor={0}")
public static ImmutableList<Boolean> useSharedExecutor() {
return ImmutableList.of(true, false);
@ -89,7 +91,7 @@ public final class DefaultVideoCompositorPixelTest {
// Golden images were generated on an API 33 emulator. API 26 emulators have a different text
// rendering implementation that leads to a larger pixel difference.
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_WITH_OVERLAY =
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_WITH_TEXT_OVERLAY =
(Ascii.toLowerCase(Util.DEVICE).contains("emulator")
|| Ascii.toLowerCase(Util.DEVICE).contains("generic"))
&& SDK_INT <= 26
@ -214,7 +216,7 @@ public final class DefaultVideoCompositorPixelTest {
@Test
@RequiresNonNull("testId")
public void compositeTwoInputs_withSecondaryAlphaZero_differentTimestamp_matchesExpectedBitmap()
public void compositeTwoInputs_withSecondaryTransparent_differentTimestamp_matchesExpectedBitmap()
throws Exception {
ImmutableList<ImmutableList<Effect>> inputEffects =
ImmutableList.of(
@ -498,17 +500,101 @@ public final class DefaultVideoCompositorPixelTest {
assertThat(compositorTestRunner.getCompositedTimestamps()).hasSize(numberOfFramesToQueue);
}
@Test
@RequiresNonNull("testId")
public void compositeFiveInputs_withFiveFramesFromEach_matchesExpectedFrameCount()
throws Exception {
compositorTestRunner =
new VideoCompositorTestRunner(
testId,
useSharedExecutor,
/* inputEffectLists= */ ImmutableList.of(
ImmutableList.of(),
ImmutableList.of(),
ImmutableList.of(),
ImmutableList.of(),
ImmutableList.of()));
int numberOfFramesToQueue = 5;
compositorTestRunner.queueBitmapToAllInputs(/* durationSec= */ numberOfFramesToQueue);
compositorTestRunner.endCompositing();
assertThat(compositorTestRunner.getCompositedTimestamps()).hasSize(numberOfFramesToQueue);
}
@Test
@RequiresNonNull("testId")
public void compositeOneInput_matchesExpectedBitmap() throws Exception {
compositorTestRunner =
new VideoCompositorTestRunner(
testId,
useSharedExecutor,
ImmutableList.of(
ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(100f))));
compositorTestRunner.queueBitmapToAllInputs(/* durationSec= */ 3);
compositorTestRunner.endCompositing();
ImmutableList<Long> primaryTimestamps =
ImmutableList.of(0 * C.MICROS_PER_SECOND, 1 * C.MICROS_PER_SECOND, 2 * C.MICROS_PER_SECOND);
assertThat(compositorTestRunner.inputBitmapReaders.get(0).getOutputTimestamps())
.containsExactlyElementsIn(primaryTimestamps)
.inOrder();
compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
ImmutableList.of("grayscale_opaque_0s", "grayscale_opaque_1s", "grayscale_opaque_2s"));
}
@Test
@RequiresNonNull("testId")
public void compositeThreeInputs_matchesExpectedBitmap() throws Exception {
compositorTestRunner =
new VideoCompositorTestRunner(
testId,
useSharedExecutor,
ImmutableList.of(
ImmutableList.of(RgbFilter.createInvertedFilter(), new AlphaScale(0.4f)),
ImmutableList.of(RgbFilter.createGrayscaleFilter(), new AlphaScale(0.7f)),
ImmutableList.of(
new ScaleAndRotateTransformation.Builder().setRotationDegrees(180).build())));
compositorTestRunner.queueBitmapToInput(
/* inputId= */ 0, /* durationSec= */ 3, /* offsetToAddSec= */ 0, /* frameRate= */ 1);
compositorTestRunner.queueBitmapToInput(
/* inputId= */ 1, /* durationSec= */ 1, /* offsetToAddSec= */ 1, /* frameRate= */ 1);
compositorTestRunner.queueBitmapToInput(
/* inputId= */ 2, /* durationSec= */ 3, /* offsetToAddSec= */ 0, /* frameRate= */ 0.5f);
compositorTestRunner.endCompositing();
ImmutableList<Long> primaryTimestamps =
ImmutableList.of(0 * C.MICROS_PER_SECOND, 1 * C.MICROS_PER_SECOND, 2 * C.MICROS_PER_SECOND);
ImmutableList<Long> secondary1Timestamps = ImmutableList.of(1 * C.MICROS_PER_SECOND);
ImmutableList<Long> secondary2Timestamps =
ImmutableList.of(0 * C.MICROS_PER_SECOND, 2 * C.MICROS_PER_SECOND);
assertThat(compositorTestRunner.inputBitmapReaders.get(0).getOutputTimestamps())
.containsExactlyElementsIn(primaryTimestamps)
.inOrder();
assertThat(compositorTestRunner.inputBitmapReaders.get(1).getOutputTimestamps())
.containsExactlyElementsIn(secondary1Timestamps)
.inOrder();
assertThat(compositorTestRunner.inputBitmapReaders.get(2).getOutputTimestamps())
.containsExactlyElementsIn(secondary2Timestamps)
.inOrder();
assertThat(compositorTestRunner.getCompositedTimestamps())
.containsExactlyElementsIn(primaryTimestamps)
.inOrder();
compositorTestRunner.saveAndAssertCompositedBitmapsMatchExpected(
ImmutableList.of("0s_1s_0s", "1s_1s_0s", "2s_1s_2s"));
}
/**
* A test runner for {@link DefaultVideoCompositor} tests.
*
* <p>Composites input bitmaps from two input sources.
*/
private static final class VideoCompositorTestRunner {
// Compositor tests rely on 2 VideoFrameProcessor instances, plus the compositor.
private static final int COMPOSITOR_TIMEOUT_MS = 2 * VIDEO_FRAME_PROCESSING_WAIT_MS;
private static final int COMPOSITOR_INPUT_SIZE = 2;
public final List<TextureBitmapReader> inputBitmapReaders;
private final int timeoutMs;
private final LinkedHashMap<Long, Bitmap> outputTimestampsToBitmaps;
private final List<VideoFrameProcessorTestRunner> inputVideoFrameProcessorTestRunners;
private final VideoCompositor videoCompositor;
@ -534,6 +620,7 @@ public final class DefaultVideoCompositorPixelTest {
ImmutableList<ImmutableList<Effect>> inputEffectLists)
throws GlUtil.GlException, VideoFrameProcessingException {
this.testId = testId;
timeoutMs = inputEffectLists.size() * VIDEO_FRAME_PROCESSING_WAIT_MS;
sharedExecutorService =
useSharedExecutor ? Util.newSingleThreadExecutor("Effect:Shared:GlThread") : null;
EGLContext sharedEglContext = AndroidTestUtil.createOpenGlObjects();
@ -578,7 +665,6 @@ public final class DefaultVideoCompositorPixelTest {
/* textureOutputCapacity= */ 1);
inputBitmapReaders = new ArrayList<>();
inputVideoFrameProcessorTestRunners = new ArrayList<>();
assertThat(inputEffectLists).hasSize(COMPOSITOR_INPUT_SIZE);
for (int i = 0; i < inputEffectLists.size(); i++) {
TextureBitmapReader textureBitmapReader = new TextureBitmapReader();
inputBitmapReaders.add(textureBitmapReader);
@ -629,11 +715,11 @@ public final class DefaultVideoCompositorPixelTest {
inputVideoFrameProcessorTestRunners.get(i).signalEndOfInput();
}
for (int i = 0; i < inputVideoFrameProcessorTestRunners.size(); i++) {
inputVideoFrameProcessorTestRunners.get(i).awaitFrameProcessingEnd(COMPOSITOR_TIMEOUT_MS);
inputVideoFrameProcessorTestRunners.get(i).awaitFrameProcessingEnd(timeoutMs);
}
@Nullable Exception endCompositingException = null;
try {
if (!compositorEnded.await(COMPOSITOR_TIMEOUT_MS, MILLISECONDS)) {
if (!compositorEnded.await(timeoutMs, MILLISECONDS)) {
endCompositingException = new IllegalStateException("Compositing timed out.");
}
} catch (InterruptedException e) {
@ -685,7 +771,7 @@ public final class DefaultVideoCompositorPixelTest {
if (sharedExecutorService != null) {
try {
sharedExecutorService.shutdown();
if (!sharedExecutorService.awaitTermination(COMPOSITOR_TIMEOUT_MS, MILLISECONDS)) {
if (!sharedExecutorService.awaitTermination(timeoutMs, MILLISECONDS)) {
throw new IllegalStateException("Missed shutdown timeout.");
}
} catch (InterruptedException unexpected) {
@ -806,6 +892,6 @@ public final class DefaultVideoCompositorPixelTest {
readBitmapUnpremultipliedAlpha(expectedBitmapAssetPath), actualBitmap, testId);
assertWithMessage("Pixel difference for bitmapLabel = " + actualBitmapLabel)
.that(averagePixelAbsoluteDifference)
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_WITH_OVERLAY);
.isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE_WITH_TEXT_OVERLAY);
}
}