diff --git a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java index 704022ace8..9ac0154488 100644 --- a/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java +++ b/library/transformer/src/androidTest/java/com/google/android/exoplayer2/transformer/FrameEditorDataProcessingTest.java @@ -24,6 +24,7 @@ import static java.lang.Math.max; import android.content.res.AssetFileDescriptor; import android.graphics.Bitmap; import android.graphics.BitmapFactory; +import android.graphics.Color; import android.graphics.Matrix; import android.graphics.PixelFormat; import android.media.Image; @@ -34,11 +35,11 @@ import android.media.MediaFormat; import androidx.annotation.Nullable; import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.exoplayer2.util.MimeTypes; +import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; import org.junit.After; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; @@ -47,12 +48,34 @@ import org.junit.runner.RunWith; * from emulators, so tests on physical devices may fail. To test on physical devices, please modify * the MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE. */ +// TODO(b/214510265): Fix these tests on Pixel 4 emulators. @RunWith(AndroidJUnit4.class) public final class FrameEditorDataProcessingTest { + // Input MP4 file to transform. private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4"; + + /* Expected first frames after transformation. + * To generate new "expected" assets: + * 1. Insert this code into a test, to download some editedBitmap. + * + try (FileOutputStream fileOutputStream = new FileOutputStream("/sdcard/tmp.png")) { + * + // quality is ignored + * + editedBitmap.compress(Bitmap.CompressFormat.PNG, 100, fileOutputStream); + * + } + * 2. Run the test on a "Nexus 6P API 23" emulator. Emulators are preferred as the automated + * presubmit that will run this test will also be an emulator. API versions 29+ have storage + * restrictions that complicate file generation. + * 3. Open the "Device File Explorer", find "/sdcard/tmp.png", and "Save As..." the file. + */ private static final String NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING = "media/bitmap/sample_mp4_first_frame.png"; + private static final String TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_translate_right.png"; + private static final String SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_scale_narrow.png"; + private static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING = + "media/bitmap/sample_mp4_first_frame_rotate90.png"; + /** * Maximum allowed average pixel difference between the expected and actual edited images for the * test to pass. The value is chosen so that differences in decoder behavior across emulator @@ -77,8 +100,85 @@ public final class FrameEditorDataProcessingTest { private @MonotonicNonNull ImageReader frameEditorOutputImageReader; private @MonotonicNonNull MediaFormat mediaFormat; - @Before - public void setUp() throws Exception { + @After + public void release() { + if (frameEditor != null) { + frameEditor.release(); + } + } + + @Test + public void processData_noEdits_producesExpectedOutput() throws Exception { + Matrix identityMatrix = new Matrix(); + setUpAndPrepareFirstFrame(identityMatrix); + + Bitmap expectedBitmap = getBitmap(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING); + checkNotNull(frameEditor).processData(); + Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); + Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); + + // TODO(internal b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void processData_translateRight_producesExpectedOutput() throws Exception { + Matrix translateRightMatrix = new Matrix(); + translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0); + setUpAndPrepareFirstFrame(translateRightMatrix); + + Bitmap expectedBitmap = getBitmap(TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING); + checkNotNull(frameEditor).processData(); + Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); + Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); + + // TODO(internal b/207848601): switch to using proper tooling for testing against golden + // data.simple + float averagePixelAbsoluteDifference = + getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void processData_scaleNarrow_producesExpectedOutput() throws Exception { + Matrix scaleNarrowMatrix = new Matrix(); + scaleNarrowMatrix.postScale(.5f, 1.2f); + setUpAndPrepareFirstFrame(scaleNarrowMatrix); + Bitmap expectedBitmap = getBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING); + + checkNotNull(frameEditor).processData(); + Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); + Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); + + // TODO(internal b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + @Test + public void processData_rotate90_producesExpectedOutput() throws Exception { + // TODO(internal b/213190310): After creating a Presentation class, move VideoSamplePipeline + // resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can + // test that rotation doesn't distort the image. + Matrix rotate90Matrix = new Matrix(); + rotate90Matrix.postRotate(/* degrees= */ 90); + setUpAndPrepareFirstFrame(rotate90Matrix); + + Bitmap expectedBitmap = getBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING); + checkNotNull(frameEditor).processData(); + Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); + Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); + + // TODO(internal b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + + private void setUpAndPrepareFirstFrame(Matrix transformationMatrix) throws Exception { // Set up the extractor to read the first video frame and get its format. MediaExtractor mediaExtractor = new MediaExtractor(); @Nullable MediaCodec mediaCodec = null; @@ -97,14 +197,13 @@ public final class FrameEditorDataProcessingTest { int height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT); frameEditorOutputImageReader = ImageReader.newInstance(width, height, PixelFormat.RGBA_8888, /* maxImages= */ 1); - Matrix identityMatrix = new Matrix(); frameEditor = FrameEditor.create( getApplicationContext(), width, height, PIXEL_WIDTH_HEIGHT_RATIO, - identityMatrix, + transformationMatrix, frameEditorOutputImageReader.getSurface(), Transformer.DebugViewProvider.NONE); @@ -156,29 +255,12 @@ public final class FrameEditorDataProcessingTest { } } - @After - public void tearDown() { - if (frameEditor != null) { - frameEditor.release(); + private Bitmap getBitmap(String expectedAssetString) throws IOException { + Bitmap bitmap; + try (InputStream inputStream = getApplicationContext().getAssets().open(expectedAssetString)) { + bitmap = BitmapFactory.decodeStream(inputStream); } - } - - @Test - public void processData_noEdits_producesExpectedOutput() throws Exception { - Bitmap expectedBitmap; - try (InputStream inputStream = - getApplicationContext().getAssets().open(NO_EDITS_EXPECTED_OUTPUT_PNG_ASSET_STRING)) { - expectedBitmap = BitmapFactory.decodeStream(inputStream); - } - - checkNotNull(frameEditor).processData(); - Image editedImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage(); - Bitmap editedBitmap = getArgb8888BitmapForRgba8888Image(editedImage); - - // TODO(internal b/207848601): switch to using proper tooling for testing against golden data. - float averagePixelAbsoluteDifference = - getAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, editedBitmap); - assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + return bitmap; } /** @@ -221,23 +303,20 @@ public final class FrameEditorDataProcessingTest { long sumMaximumAbsoluteDifferences = 0; for (int y = 0; y < height; y++) { for (int x = 0; x < width; x++) { - int color = actual.getPixel(x, y); + int actualColor = actual.getPixel(x, y); int expectedColor = expected.getPixel(x, y); + + int alphaDifference = abs(Color.alpha(actualColor) - Color.alpha(expectedColor)); + int redDifference = abs(Color.red(actualColor) - Color.red(expectedColor)); + int blueDifference = abs(Color.blue(actualColor) - Color.blue(expectedColor)); + int greenDifference = abs(Color.green(actualColor) - Color.green(expectedColor)); + int maximumAbsoluteDifference = 0; - maximumAbsoluteDifference = - max( - maximumAbsoluteDifference, - abs(((color >> 24) & 0xFF) - ((expectedColor >> 24) & 0xFF))); - maximumAbsoluteDifference = - max( - maximumAbsoluteDifference, - abs(((color >> 16) & 0xFF) - ((expectedColor >> 16) & 0xFF))); - maximumAbsoluteDifference = - max( - maximumAbsoluteDifference, - abs(((color >> 8) & 0xFF) - ((expectedColor >> 8) & 0xFF))); - maximumAbsoluteDifference = - max(maximumAbsoluteDifference, abs((color & 0xFF) - (expectedColor & 0xFF))); + maximumAbsoluteDifference = max(maximumAbsoluteDifference, alphaDifference); + maximumAbsoluteDifference = max(maximumAbsoluteDifference, redDifference); + maximumAbsoluteDifference = max(maximumAbsoluteDifference, blueDifference); + maximumAbsoluteDifference = max(maximumAbsoluteDifference, greenDifference); + sumMaximumAbsoluteDifferences += maximumAbsoluteDifference; } } diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate90.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate90.png new file mode 100644 index 0000000000..ac721443c9 Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_rotate90.png differ diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_scale_narrow.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_scale_narrow.png new file mode 100644 index 0000000000..0c53022a09 Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_scale_narrow.png differ diff --git a/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_translate_right.png b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_translate_right.png new file mode 100644 index 0000000000..8ad95fe043 Binary files /dev/null and b/testdata/src/test/assets/media/bitmap/sample_mp4_first_frame_translate_right.png differ