mirror of
https://github.com/androidx/media.git
synced 2025-04-30 06:46:50 +08:00
Convert FrameEditor to a FrameProcessorChain.
The FrameProcessorChain manages a List<GlFrameProcessor>. FrameProcessorChainDataProcessingTest now tests chaining ScaleToFit- and AdvancedFrameProcessors. PiperOrigin-RevId: 436468037
This commit is contained in:
hschlueter
Ian Baker
parent
5b4abc31f3
commit
ecfbc65a0d
@ -345,6 +345,7 @@ public final class GlUtil {
|
||||
* @param height of the new texture in pixels
|
||||
*/
|
||||
public static int createTexture(int width, int height) {
|
||||
assertValidTextureSize(width, height);
|
||||
int texId = generateAndBindTexture(GLES20.GL_TEXTURE_2D);
|
||||
ByteBuffer byteBuffer = ByteBuffer.allocateDirect(width * height * 4);
|
||||
GLES20.glTexImage2D(
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 338 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 416 KiB |
@ -44,7 +44,7 @@ import org.junit.runner.RunWith;
|
||||
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
||||
* devices may fail. To test on other devices, please increase the {@link
|
||||
* BitmapTestUtil#MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE} and/or inspect the saved output bitmaps
|
||||
* as recommended in {@link FrameEditorDataProcessingTest}.
|
||||
* as recommended in {@link FrameProcessorChainPixelTest}.
|
||||
*/
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public final class AdvancedFrameProcessorPixelTest {
|
||||
|
||||
@ -39,8 +39,8 @@ import java.io.InputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
|
||||
/**
|
||||
* Utilities for instrumentation tests for the {@link FrameEditor} and {@link GlFrameProcessor
|
||||
* GlFrameProcessors}.
|
||||
* Utilities for instrumentation tests for the {@link FrameProcessorChain} and {@link
|
||||
* GlFrameProcessor GlFrameProcessors}.
|
||||
*/
|
||||
public class BitmapTestUtil {
|
||||
|
||||
@ -53,6 +53,10 @@ public class BitmapTestUtil {
|
||||
"media/bitmap/sample_mp4_first_frame_translate_right.png";
|
||||
public static final String SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING =
|
||||
"media/bitmap/sample_mp4_first_frame_scale_narrow.png";
|
||||
public static final String ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
|
||||
"media/bitmap/sample_mp4_first_frame_rotate_then_translate.png";
|
||||
public static final String TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING =
|
||||
"media/bitmap/sample_mp4_first_frame_translate_then_rotate.png";
|
||||
public static final String ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING =
|
||||
"media/bitmap/sample_mp4_first_frame_rotate90.png";
|
||||
public static final String REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING =
|
||||
@ -63,13 +67,15 @@ public class BitmapTestUtil {
|
||||
* Maximum allowed average pixel difference between the expected and actual edited images for the
|
||||
* test to pass. The value is chosen so that differences in decoder behavior across emulator
|
||||
* versions don't affect whether the test passes for most emulators, but substantial distortions
|
||||
* introduced by changes in the behavior of the frame editor will cause the test to fail.
|
||||
* introduced by changes in the behavior of the {@link GlFrameProcessor GlFrameProcessors} will
|
||||
* cause the test to fail.
|
||||
*
|
||||
* <p>To run this test on physical devices, please use a value of 5f, rather than 0.1f. This
|
||||
* higher value will ignore some very small errors, but will allow for some differences caused by
|
||||
* graphics implementations to be ignored. When the difference is close to the threshold, manually
|
||||
* inspect expected/actual bitmaps to confirm failure, as it's possible this is caused by a
|
||||
* difference in the codec or graphics implementation as opposed to a FrameEditor issue.
|
||||
* difference in the codec or graphics implementation as opposed to a {@link GlFrameProcessor}
|
||||
* issue.
|
||||
*/
|
||||
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.1f;
|
||||
|
||||
|
||||
@ -20,11 +20,12 @@ import static androidx.media3.transformer.BitmapTestUtil.FIRST_FRAME_PNG_ASSET_S
|
||||
import static androidx.media3.transformer.BitmapTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
|
||||
import static androidx.media3.transformer.BitmapTestUtil.REQUEST_OUTPUT_HEIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
|
||||
import static androidx.media3.transformer.BitmapTestUtil.ROTATE45_SCALE_TO_FIT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
|
||||
import static androidx.media3.transformer.BitmapTestUtil.ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING;
|
||||
import static androidx.media3.transformer.BitmapTestUtil.SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING;
|
||||
import static androidx.media3.transformer.BitmapTestUtil.ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING;
|
||||
import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_RIGHT_EXPECTED_OUTPUT_PNG_ASSET_STRING;
|
||||
import static androidx.media3.transformer.BitmapTestUtil.TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING;
|
||||
import static androidx.test.core.app.ApplicationProvider.getApplicationContext;
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
import static java.util.Arrays.asList;
|
||||
|
||||
import android.content.Context;
|
||||
import android.content.res.AssetFileDescriptor;
|
||||
@ -40,14 +41,16 @@ import android.util.Size;
|
||||
import androidx.annotation.Nullable;
|
||||
import androidx.media3.common.MimeTypes;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.collect.Iterables;
|
||||
import java.nio.ByteBuffer;
|
||||
import java.util.List;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
import org.junit.After;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/**
|
||||
* Pixel test for frame processing via {@link FrameEditor}.
|
||||
* Pixel test for frame processing via {@link FrameProcessorChain}.
|
||||
*
|
||||
* <p>Expected images are taken from an emulator, so tests on different emulators or physical
|
||||
* devices may fail. To test on other devices, please increase the {@link
|
||||
@ -55,41 +58,35 @@ import org.junit.runner.RunWith;
|
||||
* bitmaps.
|
||||
*/
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public final class FrameEditorDataProcessingTest {
|
||||
// TODO(b/214975934): Once FrameEditor is converted to a FrameProcessorChain, replace these tests
|
||||
// with a test for a few example combinations of GlFrameProcessors rather than testing all use
|
||||
// cases of TransformationFrameProcessor.
|
||||
public final class FrameProcessorChainPixelTest {
|
||||
|
||||
/** Input video of which we only use the first frame. */
|
||||
private static final String INPUT_MP4_ASSET_STRING = "media/mp4/sample.mp4";
|
||||
/** Timeout for dequeueing buffers from the codec, in microseconds. */
|
||||
private static final int DEQUEUE_TIMEOUT_US = 5_000_000;
|
||||
/**
|
||||
* Time to wait for the decoded frame to populate the frame editor's input surface and the frame
|
||||
* editor to finish processing the frame, in milliseconds.
|
||||
* Time to wait for the decoded frame to populate the {@link FrameProcessorChain}'s input surface
|
||||
* and the {@link FrameProcessorChain} to finish processing the frame, in milliseconds.
|
||||
*/
|
||||
private static final int FRAME_PROCESSING_WAIT_MS = 1000;
|
||||
private static final int FRAME_PROCESSING_WAIT_MS = 2000;
|
||||
/** The ratio of width over height, for each pixel in a frame. */
|
||||
private static final float PIXEL_WIDTH_HEIGHT_RATIO = 1;
|
||||
|
||||
private @MonotonicNonNull FrameEditor frameEditor;
|
||||
private @MonotonicNonNull ImageReader frameEditorOutputImageReader;
|
||||
private @MonotonicNonNull FrameProcessorChain frameProcessorChain;
|
||||
private @MonotonicNonNull ImageReader outputImageReader;
|
||||
private @MonotonicNonNull MediaFormat mediaFormat;
|
||||
|
||||
@After
|
||||
public void release() {
|
||||
if (frameEditor != null) {
|
||||
frameEditor.release();
|
||||
if (frameProcessorChain != null) {
|
||||
frameProcessorChain.release();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void processData_noEdits_producesExpectedOutput() throws Exception {
|
||||
final String testId = "processData_noEdits";
|
||||
Matrix identityMatrix = new Matrix();
|
||||
GlFrameProcessor glFrameProcessor =
|
||||
new AdvancedFrameProcessor(getApplicationContext(), identityMatrix);
|
||||
setUpAndPrepareFirstFrame(glFrameProcessor);
|
||||
setUpAndPrepareFirstFrame();
|
||||
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(FIRST_FRAME_PNG_ASSET_STRING);
|
||||
|
||||
Bitmap actualBitmap = processFirstFrameAndEnd();
|
||||
@ -104,8 +101,9 @@ public final class FrameEditorDataProcessingTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void processData_translateRight_producesExpectedOutput() throws Exception {
|
||||
final String testId = "processData_translateRight";
|
||||
public void processData_withAdvancedFrameProcessor_translateRight_producesExpectedOutput()
|
||||
throws Exception {
|
||||
final String testId = "processData_withAdvancedFrameProcessor_translateRight";
|
||||
Matrix translateRightMatrix = new Matrix();
|
||||
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
|
||||
GlFrameProcessor glFrameProcessor =
|
||||
@ -126,15 +124,20 @@ public final class FrameEditorDataProcessingTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void processData_scaleNarrow_producesExpectedOutput() throws Exception {
|
||||
final String testId = "processData_scaleNarrow";
|
||||
Matrix scaleNarrowMatrix = new Matrix();
|
||||
scaleNarrowMatrix.postScale(.5f, 1.2f);
|
||||
GlFrameProcessor glFrameProcessor =
|
||||
new AdvancedFrameProcessor(getApplicationContext(), scaleNarrowMatrix);
|
||||
setUpAndPrepareFirstFrame(glFrameProcessor);
|
||||
public void processData_withAdvancedAndScaleToFitFrameProcessors_producesExpectedOutput()
|
||||
throws Exception {
|
||||
final String testId = "processData_withAdvancedAndScaleToFitFrameProcessors";
|
||||
Matrix translateRightMatrix = new Matrix();
|
||||
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
|
||||
GlFrameProcessor translateRightFrameProcessor =
|
||||
new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix);
|
||||
GlFrameProcessor rotate45FrameProcessor =
|
||||
new ScaleToFitFrameProcessor.Builder(getApplicationContext())
|
||||
.setRotationDegrees(45)
|
||||
.build();
|
||||
setUpAndPrepareFirstFrame(translateRightFrameProcessor, rotate45FrameProcessor);
|
||||
Bitmap expectedBitmap =
|
||||
BitmapTestUtil.readBitmap(SCALE_NARROW_EXPECTED_OUTPUT_PNG_ASSET_STRING);
|
||||
BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
|
||||
|
||||
Bitmap actualBitmap = processFirstFrameAndEnd();
|
||||
|
||||
@ -148,14 +151,20 @@ public final class FrameEditorDataProcessingTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void processData_rotate90_producesExpectedOutput() throws Exception {
|
||||
final String testId = "processData_rotate90";
|
||||
Matrix rotate90Matrix = new Matrix();
|
||||
rotate90Matrix.postRotate(/* degrees= */ 90);
|
||||
GlFrameProcessor glFrameProcessor =
|
||||
new AdvancedFrameProcessor(getApplicationContext(), rotate90Matrix);
|
||||
setUpAndPrepareFirstFrame(glFrameProcessor);
|
||||
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_90_EXPECTED_OUTPUT_PNG_ASSET_STRING);
|
||||
public void processData_withScaleToFitAndAdvancedFrameProcessors_producesExpectedOutput()
|
||||
throws Exception {
|
||||
final String testId = "processData_withScaleToFitAndAdvancedFrameProcessors";
|
||||
GlFrameProcessor rotate45FrameProcessor =
|
||||
new ScaleToFitFrameProcessor.Builder(getApplicationContext())
|
||||
.setRotationDegrees(45)
|
||||
.build();
|
||||
Matrix translateRightMatrix = new Matrix();
|
||||
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
|
||||
GlFrameProcessor translateRightFrameProcessor =
|
||||
new AdvancedFrameProcessor(getApplicationContext(), translateRightMatrix);
|
||||
setUpAndPrepareFirstFrame(rotate45FrameProcessor, translateRightFrameProcessor);
|
||||
Bitmap expectedBitmap =
|
||||
BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_EXPECTED_OUTPUT_PNG_ASSET_STRING);
|
||||
|
||||
Bitmap actualBitmap = processFirstFrameAndEnd();
|
||||
|
||||
@ -169,8 +178,9 @@ public final class FrameEditorDataProcessingTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void processData_requestOutputHeight_producesExpectedOutput() throws Exception {
|
||||
final String testId = "processData_requestOutputHeight";
|
||||
public void processData_withScaleToFitFrameProcessor_requestOutputHeight_producesExpectedOutput()
|
||||
throws Exception {
|
||||
final String testId = "processData_withScaleToFitFrameProcessor_requestOutputHeight";
|
||||
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
|
||||
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
|
||||
// test that rotation doesn't distort the image.
|
||||
@ -192,8 +202,9 @@ public final class FrameEditorDataProcessingTest {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void processData_rotate45_scaleToFit_producesExpectedOutput() throws Exception {
|
||||
final String testId = "processData_rotate45_scaleToFit";
|
||||
public void processData_withScaleToFitFrameProcessor_rotate45_producesExpectedOutput()
|
||||
throws Exception {
|
||||
final String testId = "processData_withScaleToFitFrameProcessor_rotate45";
|
||||
// TODO(b/213190310): After creating a Presentation class, move VideoSamplePipeline
|
||||
// resolution-based adjustments (ex. in cl/419619743) to that Presentation class, so we can
|
||||
// test that rotation doesn't distort the image.
|
||||
@ -218,12 +229,13 @@ public final class FrameEditorDataProcessingTest {
|
||||
|
||||
/**
|
||||
* Set up and prepare the first frame from an input video, as well as relevant test
|
||||
* infrastructure. The frame will be sent towards the {@link FrameEditor}, and may be accessed on
|
||||
* the {@link FrameEditor}'s output {@code frameEditorOutputImageReader}.
|
||||
* infrastructure. The frame will be sent towards the {@link FrameProcessorChain}, and may be
|
||||
* accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}.
|
||||
*
|
||||
* @param glFrameProcessor The frame processor that will apply changes to the input frame.
|
||||
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes
|
||||
* to the input frame.
|
||||
*/
|
||||
private void setUpAndPrepareFirstFrame(GlFrameProcessor glFrameProcessor) throws Exception {
|
||||
private void setUpAndPrepareFirstFrame(GlFrameProcessor... frameProcessors) throws Exception {
|
||||
// Set up the extractor to read the first video frame and get its format.
|
||||
MediaExtractor mediaExtractor = new MediaExtractor();
|
||||
@Nullable MediaCodec mediaCodec = null;
|
||||
@ -240,31 +252,35 @@ public final class FrameEditorDataProcessingTest {
|
||||
|
||||
int inputWidth = checkNotNull(mediaFormat).getInteger(MediaFormat.KEY_WIDTH);
|
||||
int inputHeight = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
|
||||
Size outputSize = glFrameProcessor.configureOutputSize(inputWidth, inputHeight);
|
||||
int outputWidth = outputSize.getWidth();
|
||||
int outputHeight = outputSize.getHeight();
|
||||
frameEditorOutputImageReader =
|
||||
List<GlFrameProcessor> frameProcessorsList = asList(frameProcessors);
|
||||
List<Size> sizes =
|
||||
FrameProcessorChain.configureSizes(inputWidth, inputHeight, frameProcessorsList);
|
||||
assertThat(sizes).isNotEmpty();
|
||||
outputImageReader =
|
||||
ImageReader.newInstance(
|
||||
outputWidth, outputHeight, PixelFormat.RGBA_8888, /* maxImages= */ 1);
|
||||
frameEditor =
|
||||
FrameEditor.create(
|
||||
Iterables.getLast(sizes).getWidth(),
|
||||
Iterables.getLast(sizes).getHeight(),
|
||||
PixelFormat.RGBA_8888,
|
||||
/* maxImages= */ 1);
|
||||
frameProcessorChain =
|
||||
FrameProcessorChain.create(
|
||||
context,
|
||||
inputWidth,
|
||||
inputHeight,
|
||||
outputWidth,
|
||||
outputHeight,
|
||||
PIXEL_WIDTH_HEIGHT_RATIO,
|
||||
glFrameProcessor,
|
||||
frameEditorOutputImageReader.getSurface(),
|
||||
frameProcessorsList,
|
||||
sizes,
|
||||
outputImageReader.getSurface(),
|
||||
/* enableExperimentalHdrEditing= */ false,
|
||||
Transformer.DebugViewProvider.NONE);
|
||||
frameEditor.registerInputFrame();
|
||||
frameProcessorChain.registerInputFrame();
|
||||
|
||||
// Queue the first video frame from the extractor.
|
||||
String mimeType = checkNotNull(mediaFormat.getString(MediaFormat.KEY_MIME));
|
||||
mediaCodec = MediaCodec.createDecoderByType(mimeType);
|
||||
mediaCodec.configure(
|
||||
mediaFormat, frameEditor.createInputSurface(), /* crypto= */ null, /* flags= */ 0);
|
||||
mediaFormat,
|
||||
frameProcessorChain.createInputSurface(),
|
||||
/* crypto= */ null,
|
||||
/* flags= */ 0);
|
||||
mediaCodec.start();
|
||||
int inputBufferIndex = mediaCodec.dequeueInputBuffer(DEQUEUE_TIMEOUT_US);
|
||||
assertThat(inputBufferIndex).isNotEqualTo(MediaCodec.INFO_TRY_AGAIN_LATER);
|
||||
@ -305,14 +321,15 @@ public final class FrameEditorDataProcessingTest {
|
||||
}
|
||||
|
||||
private Bitmap processFirstFrameAndEnd() throws InterruptedException, TransformationException {
|
||||
checkNotNull(frameEditor).signalEndOfInputStream();
|
||||
checkNotNull(frameProcessorChain).signalEndOfInputStream();
|
||||
Thread.sleep(FRAME_PROCESSING_WAIT_MS);
|
||||
assertThat(frameEditor.isEnded()).isTrue();
|
||||
frameEditor.getAndRethrowBackgroundExceptions();
|
||||
assertThat(frameProcessorChain.isEnded()).isTrue();
|
||||
frameProcessorChain.getAndRethrowBackgroundExceptions();
|
||||
|
||||
Image editorOutputImage = checkNotNull(frameEditorOutputImageReader).acquireLatestImage();
|
||||
Bitmap actualBitmap = BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(editorOutputImage);
|
||||
editorOutputImage.close();
|
||||
Image frameProcessorChainOutputImage = checkNotNull(outputImageReader).acquireLatestImage();
|
||||
Bitmap actualBitmap =
|
||||
BitmapTestUtil.createArgb8888BitmapFromRgba8888Image(frameProcessorChainOutputImage);
|
||||
frameProcessorChainOutputImage.close();
|
||||
return actualBitmap;
|
||||
}
|
||||
}
|
||||
@ -20,19 +20,21 @@ import static com.google.common.truth.Truth.assertThat;
|
||||
import static org.junit.Assert.assertThrows;
|
||||
|
||||
import android.content.Context;
|
||||
import android.graphics.Matrix;
|
||||
import android.graphics.SurfaceTexture;
|
||||
import android.util.Size;
|
||||
import android.view.Surface;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.util.List;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/**
|
||||
* Test for {@link FrameEditor#create(Context, int, int, int, int, float, GlFrameProcessor, Surface,
|
||||
* boolean, Transformer.DebugViewProvider) creating} a {@link FrameEditor}.
|
||||
* Test for {@link FrameProcessorChain#create(Context, float, List, List, Surface, boolean,
|
||||
* Transformer.DebugViewProvider) creating} a {@link FrameProcessorChain}.
|
||||
*/
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public final class FrameEditorTest {
|
||||
public final class FrameProcessorChainTest {
|
||||
// TODO(b/212539951): Make this a robolectric test by e.g. updating shadows or adding a
|
||||
// wrapper around GlUtil to allow the usage of mocks or fakes which don't need (Shadow)GLES20.
|
||||
|
||||
@ -41,15 +43,12 @@ public final class FrameEditorTest {
|
||||
throws TransformationException {
|
||||
Context context = getApplicationContext();
|
||||
|
||||
FrameEditor.create(
|
||||
FrameProcessorChain.create(
|
||||
context,
|
||||
/* inputWidth= */ 200,
|
||||
/* inputHeight= */ 100,
|
||||
/* outputWidth= */ 200,
|
||||
/* outputHeight= */ 100,
|
||||
/* pixelWidthHeightRatio= */ 1,
|
||||
new AdvancedFrameProcessor(context, new Matrix()),
|
||||
new Surface(new SurfaceTexture(false)),
|
||||
/* frameProcessors= */ ImmutableList.of(),
|
||||
/* sizes= */ ImmutableList.of(new Size(200, 100)),
|
||||
/* outputSurface= */ new Surface(new SurfaceTexture(false)),
|
||||
/* enableExperimentalHdrEditing= */ false,
|
||||
Transformer.DebugViewProvider.NONE);
|
||||
}
|
||||
@ -62,15 +61,12 @@ public final class FrameEditorTest {
|
||||
assertThrows(
|
||||
TransformationException.class,
|
||||
() ->
|
||||
FrameEditor.create(
|
||||
FrameProcessorChain.create(
|
||||
context,
|
||||
/* inputWidth= */ 200,
|
||||
/* inputHeight= */ 100,
|
||||
/* outputWidth= */ 200,
|
||||
/* outputHeight= */ 100,
|
||||
/* pixelWidthHeightRatio= */ 2,
|
||||
new AdvancedFrameProcessor(context, new Matrix()),
|
||||
new Surface(new SurfaceTexture(false)),
|
||||
/* frameProcessors= */ ImmutableList.of(),
|
||||
/* sizes= */ ImmutableList.of(new Size(200, 100)),
|
||||
/* outputSurface= */ new Surface(new SurfaceTexture(false)),
|
||||
/* enableExperimentalHdrEditing= */ false,
|
||||
Transformer.DebugViewProvider.NONE));
|
||||
|
||||
@ -15,8 +15,10 @@
|
||||
*/
|
||||
package androidx.media3.transformer;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
import static androidx.media3.common.util.Assertions.checkNotNull;
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
import static com.google.common.collect.Iterables.getLast;
|
||||
|
||||
import android.content.Context;
|
||||
import android.graphics.SurfaceTexture;
|
||||
@ -26,6 +28,7 @@ import android.opengl.EGLDisplay;
|
||||
import android.opengl.EGLExt;
|
||||
import android.opengl.EGLSurface;
|
||||
import android.opengl.GLES20;
|
||||
import android.util.Size;
|
||||
import android.view.Surface;
|
||||
import android.view.SurfaceView;
|
||||
import androidx.annotation.Nullable;
|
||||
@ -33,6 +36,8 @@ import androidx.media3.common.C;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import androidx.media3.common.util.Util;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ConcurrentLinkedQueue;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
@ -43,69 +48,90 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
|
||||
/**
|
||||
* {@code FrameEditor} applies changes to individual video frames.
|
||||
* {@code FrameProcessorChain} applies changes to individual video frames.
|
||||
*
|
||||
* <p>Input becomes available on its {@link #createInputSurface() input surface} asynchronously and
|
||||
* is processed on a background thread as it becomes available. All input frames should be {@link
|
||||
* #registerInputFrame() registered} before they are rendered to the input surface. {@link
|
||||
* #hasPendingFrames()} can be used to check whether there are frames that have not been fully
|
||||
* processed yet. Output is written to its {@link #create(Context, int, int, int, int, float,
|
||||
* GlFrameProcessor, Surface, boolean, Transformer.DebugViewProvider) output surface}.
|
||||
* processed yet. The {@code FrameProcessorChain} writes output to the surface passed to {@link
|
||||
* #create(Context, float, List, List, Surface, boolean, Transformer.DebugViewProvider)}.
|
||||
*/
|
||||
/* package */ final class FrameEditor {
|
||||
/* package */ final class FrameProcessorChain {
|
||||
|
||||
static {
|
||||
GlUtil.glAssertionsEnabled = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new {@code FrameEditor} for applying changes to individual frames.
|
||||
* Configures the output {@link Size sizes} of a list of {@link GlFrameProcessor
|
||||
* GlFrameProcessors}.
|
||||
*
|
||||
* @param inputWidth The width of frames passed to the first {@link GlFrameProcessor}.
|
||||
* @param inputHeight The height of frames passed to the first {@link GlFrameProcessor}.
|
||||
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors}.
|
||||
* @return A mutable {@link List} containing the input {@link Size} as well as the output {@link
|
||||
* Size} of each {@link GlFrameProcessor}.
|
||||
*/
|
||||
// TODO(b/218488308): Return an immutable list once VideoTranscodingSamplePipeline no longer needs
|
||||
// to modify this list for encoder fallback.
|
||||
public static List<Size> configureSizes(
|
||||
int inputWidth, int inputHeight, List<GlFrameProcessor> frameProcessors) {
|
||||
|
||||
List<Size> sizes = new ArrayList<>(frameProcessors.size() + 1);
|
||||
sizes.add(new Size(inputWidth, inputHeight));
|
||||
for (int i = 0; i < frameProcessors.size(); i++) {
|
||||
sizes.add(
|
||||
frameProcessors
|
||||
.get(i)
|
||||
.configureOutputSize(getLast(sizes).getWidth(), getLast(sizes).getHeight()));
|
||||
}
|
||||
return sizes;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a new {@code FrameProcessorChain} for applying changes to individual frames.
|
||||
*
|
||||
* @param context A {@link Context}.
|
||||
* @param inputWidth The input width in pixels.
|
||||
* @param inputHeight The input height in pixels.
|
||||
* @param outputWidth The output width in pixels.
|
||||
* @param outputHeight The output height in pixels.
|
||||
* @param pixelWidthHeightRatio The ratio of width over height, for each pixel.
|
||||
* @param transformationFrameProcessor The {@link GlFrameProcessor} to apply to each frame.
|
||||
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
|
||||
* Their output sizes must be {@link GlFrameProcessor#configureOutputSize(int, int)}
|
||||
* configured}.
|
||||
* @param sizes The input {@link Size} as well as the output {@link Size} of each {@link
|
||||
* GlFrameProcessor}.
|
||||
* @param outputSurface The {@link Surface}.
|
||||
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
|
||||
* @param debugViewProvider Provider for optional debug views to show intermediate output.
|
||||
* @return A configured {@code FrameEditor}.
|
||||
* @return A configured {@code FrameProcessorChain}.
|
||||
* @throws TransformationException If the {@code pixelWidthHeightRatio} isn't 1, reading shader
|
||||
* files fails, or an OpenGL error occurs while creating and configuring the OpenGL
|
||||
* components.
|
||||
*/
|
||||
// TODO(b/214975934): Take a List<GlFrameProcessor> as input and rename FrameEditor to
|
||||
// FrameProcessorChain.
|
||||
// TODO(b/218488308): Remove the need to input outputWidth and outputHeight into FrameEditor, that
|
||||
// stems from encoder fallback resolution. This could maybe be input into the last
|
||||
// GlFrameProcessor in the FrameEditor instead of being input directly into the FrameEditor.
|
||||
public static FrameEditor create(
|
||||
public static FrameProcessorChain create(
|
||||
Context context,
|
||||
int inputWidth,
|
||||
int inputHeight,
|
||||
int outputWidth,
|
||||
int outputHeight,
|
||||
float pixelWidthHeightRatio,
|
||||
GlFrameProcessor transformationFrameProcessor,
|
||||
List<GlFrameProcessor> frameProcessors,
|
||||
List<Size> sizes,
|
||||
Surface outputSurface,
|
||||
boolean enableExperimentalHdrEditing,
|
||||
Transformer.DebugViewProvider debugViewProvider)
|
||||
throws TransformationException {
|
||||
checkArgument(frameProcessors.size() + 1 == sizes.size());
|
||||
|
||||
if (pixelWidthHeightRatio != 1.0f) {
|
||||
// TODO(b/211782176): Consider implementing support for non-square pixels.
|
||||
throw TransformationException.createForFrameEditor(
|
||||
throw TransformationException.createForFrameProcessorChain(
|
||||
new UnsupportedOperationException(
|
||||
"Transformer's frame editor currently does not support frame edits on non-square"
|
||||
+ " pixels. The pixelWidthHeightRatio is: "
|
||||
"Transformer's FrameProcessorChain currently does not support frame edits on"
|
||||
+ " non-square pixels. The pixelWidthHeightRatio is: "
|
||||
+ pixelWidthHeightRatio),
|
||||
TransformationException.ERROR_CODE_GL_INIT_FAILED);
|
||||
}
|
||||
|
||||
@Nullable
|
||||
SurfaceView debugSurfaceView =
|
||||
debugViewProvider.getDebugPreviewSurfaceView(outputWidth, outputHeight);
|
||||
debugViewProvider.getDebugPreviewSurfaceView(
|
||||
getLast(sizes).getWidth(), getLast(sizes).getHeight());
|
||||
int debugPreviewWidth;
|
||||
int debugPreviewHeight;
|
||||
if (debugSurfaceView != null) {
|
||||
@ -120,51 +146,45 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
new ExternalCopyFrameProcessor(context, enableExperimentalHdrEditing);
|
||||
|
||||
ExecutorService singleThreadExecutorService = Util.newSingleThreadExecutor(THREAD_NAME);
|
||||
Future<FrameEditor> frameEditorFuture =
|
||||
Future<FrameProcessorChain> frameProcessorChainFuture =
|
||||
singleThreadExecutorService.submit(
|
||||
() ->
|
||||
createOpenGlObjectsAndFrameEditor(
|
||||
createOpenGlObjectsAndFrameProcessorChain(
|
||||
singleThreadExecutorService,
|
||||
externalCopyFrameProcessor,
|
||||
transformationFrameProcessor,
|
||||
frameProcessors,
|
||||
sizes,
|
||||
outputSurface,
|
||||
inputWidth,
|
||||
inputHeight,
|
||||
outputWidth,
|
||||
outputHeight,
|
||||
enableExperimentalHdrEditing,
|
||||
debugSurfaceView,
|
||||
debugPreviewWidth,
|
||||
debugPreviewHeight));
|
||||
try {
|
||||
return frameEditorFuture.get();
|
||||
return frameProcessorChainFuture.get();
|
||||
} catch (ExecutionException e) {
|
||||
throw TransformationException.createForFrameEditor(
|
||||
throw TransformationException.createForFrameProcessorChain(
|
||||
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw TransformationException.createForFrameEditor(
|
||||
throw TransformationException.createForFrameProcessorChain(
|
||||
e, TransformationException.ERROR_CODE_GL_INIT_FAILED);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a {@code FrameEditor} and its OpenGL objects.
|
||||
* Creates a {@code FrameProcessorChain} and its OpenGL objects.
|
||||
*
|
||||
* <p>As the {@code FrameEditor} will call OpenGL commands on the {@code
|
||||
* <p>As the {@code FrameProcessorChain} will call OpenGL commands on the {@code
|
||||
* singleThreadExecutorService}'s thread, the OpenGL context and objects also need to be created
|
||||
* on that thread. So this method should only be called on the {@code
|
||||
* singleThreadExecutorService}'s thread.
|
||||
*/
|
||||
private static FrameEditor createOpenGlObjectsAndFrameEditor(
|
||||
private static FrameProcessorChain createOpenGlObjectsAndFrameProcessorChain(
|
||||
ExecutorService singleThreadExecutorService,
|
||||
ExternalCopyFrameProcessor externalCopyFrameProcessor,
|
||||
GlFrameProcessor transformationFrameProcessor,
|
||||
List<GlFrameProcessor> frameProcessors,
|
||||
List<Size> sizes,
|
||||
Surface outputSurface,
|
||||
int inputWidth,
|
||||
int inputHeight,
|
||||
int outputWidth,
|
||||
int outputHeight,
|
||||
boolean enableExperimentalHdrEditing,
|
||||
@Nullable SurfaceView debugSurfaceView,
|
||||
int debugPreviewWidth,
|
||||
@ -192,35 +212,37 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
}
|
||||
}
|
||||
|
||||
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
|
||||
GlUtil.assertValidTextureSize(outputWidth, outputHeight);
|
||||
int inputExternalTexId = GlUtil.createExternalTexture();
|
||||
// TODO(b/214975934): Propagate output sizes through the chain of frame processors.
|
||||
externalCopyFrameProcessor.configureOutputSize(inputWidth, inputHeight);
|
||||
externalCopyFrameProcessor.initialize(/* inputTexId= */ inputExternalTexId);
|
||||
int intermediateTexId = GlUtil.createTexture(inputWidth, inputHeight);
|
||||
int frameBuffer = GlUtil.createFboForTexture(intermediateTexId);
|
||||
transformationFrameProcessor.initialize(/* inputTexId= */ intermediateTexId);
|
||||
GlUtil.focusEglSurface(
|
||||
eglDisplay, eglContext, eglSurface, getLast(sizes).getWidth(), getLast(sizes).getHeight());
|
||||
|
||||
return new FrameEditor(
|
||||
int inputExternalTexId = GlUtil.createExternalTexture();
|
||||
externalCopyFrameProcessor.configureOutputSize(
|
||||
/* inputWidth= */ sizes.get(0).getWidth(), /* inputHeight= */ sizes.get(0).getHeight());
|
||||
externalCopyFrameProcessor.initialize(inputExternalTexId);
|
||||
|
||||
int[] framebuffers = new int[frameProcessors.size()];
|
||||
for (int i = 0; i < frameProcessors.size(); i++) {
|
||||
int inputTexId = GlUtil.createTexture(sizes.get(i).getWidth(), sizes.get(i).getHeight());
|
||||
framebuffers[i] = GlUtil.createFboForTexture(inputTexId);
|
||||
frameProcessors.get(i).initialize(inputTexId);
|
||||
}
|
||||
|
||||
return new FrameProcessorChain(
|
||||
singleThreadExecutorService,
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
eglSurface,
|
||||
externalCopyFrameProcessor,
|
||||
transformationFrameProcessor,
|
||||
inputWidth,
|
||||
inputHeight,
|
||||
frameProcessors,
|
||||
inputExternalTexId,
|
||||
frameBuffer,
|
||||
outputWidth,
|
||||
outputHeight,
|
||||
framebuffers,
|
||||
sizes,
|
||||
debugPreviewEglSurface,
|
||||
debugPreviewWidth,
|
||||
debugPreviewHeight);
|
||||
}
|
||||
|
||||
private static final String THREAD_NAME = "Transformer:FrameEditor";
|
||||
private static final String THREAD_NAME = "Transformer:FrameProcessorChain";
|
||||
|
||||
/** Some OpenGL commands may block, so all OpenGL commands are run on a background thread. */
|
||||
private final ExecutorService singleThreadExecutorService;
|
||||
@ -235,25 +257,28 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
private final EGLContext eglContext;
|
||||
private final EGLSurface eglSurface;
|
||||
private final ExternalCopyFrameProcessor externalCopyFrameProcessor;
|
||||
private final GlFrameProcessor transformationFrameProcessor;
|
||||
|
||||
/** Identifier of the external texture the {@code FrameEditor} reads its input from. */
|
||||
private final List<GlFrameProcessor> frameProcessors;
|
||||
/**
|
||||
* Identifiers of a framebuffer object associated with the intermediate textures that receive
|
||||
* output from the previous {@link GlFrameProcessor}, and provide input for the following {@link
|
||||
* GlFrameProcessor}.
|
||||
*
|
||||
* <p>The {@link ExternalCopyFrameProcessor} writes to the first framebuffer.
|
||||
*/
|
||||
private final int[] framebuffers;
|
||||
/**
|
||||
* The input {@link Size}, i.e., the output {@link Size} of the {@link
|
||||
* ExternalCopyFrameProcessor}), as well as the output {@link Size} of each of the {@code
|
||||
* frameProcessors}.
|
||||
*/
|
||||
private final List<Size> sizes;
|
||||
/**
|
||||
* Identifier of the external texture the {@link ExternalCopyFrameProcessor} reads its input from.
|
||||
*/
|
||||
private final int inputExternalTexId;
|
||||
/** Transformation matrix associated with the surface texture. */
|
||||
private final float[] textureTransformMatrix;
|
||||
|
||||
/**
|
||||
* Identifier of a framebuffer object associated with the intermediate texture that receives
|
||||
* output from the prior {@link ExternalCopyFrameProcessor}, and provides input for the following
|
||||
* {@link GlFrameProcessor}.
|
||||
*/
|
||||
private final int frameBuffer;
|
||||
|
||||
private final int inputWidth;
|
||||
private final int inputHeight;
|
||||
private final int outputWidth;
|
||||
private final int outputHeight;
|
||||
|
||||
@Nullable private final EGLSurface debugPreviewEglSurface;
|
||||
private final int debugPreviewWidth;
|
||||
private final int debugPreviewHeight;
|
||||
@ -263,19 +288,16 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
private boolean inputStreamEnded;
|
||||
private volatile boolean releaseRequested;
|
||||
|
||||
private FrameEditor(
|
||||
private FrameProcessorChain(
|
||||
ExecutorService singleThreadExecutorService,
|
||||
EGLDisplay eglDisplay,
|
||||
EGLContext eglContext,
|
||||
EGLSurface eglSurface,
|
||||
ExternalCopyFrameProcessor externalCopyFrameProcessor,
|
||||
GlFrameProcessor transformationFrameProcessor,
|
||||
int inputWidth,
|
||||
int inputHeight,
|
||||
List<GlFrameProcessor> frameProcessors,
|
||||
int inputExternalTexId,
|
||||
int frameBuffer,
|
||||
int outputWidth,
|
||||
int outputHeight,
|
||||
int[] framebuffers,
|
||||
List<Size> sizes,
|
||||
@Nullable EGLSurface debugPreviewEglSurface,
|
||||
int debugPreviewWidth,
|
||||
int debugPreviewHeight) {
|
||||
@ -284,13 +306,10 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
this.eglContext = eglContext;
|
||||
this.eglSurface = eglSurface;
|
||||
this.externalCopyFrameProcessor = externalCopyFrameProcessor;
|
||||
this.transformationFrameProcessor = transformationFrameProcessor;
|
||||
this.inputWidth = inputWidth;
|
||||
this.inputHeight = inputHeight;
|
||||
this.frameProcessors = frameProcessors;
|
||||
this.inputExternalTexId = inputExternalTexId;
|
||||
this.frameBuffer = frameBuffer;
|
||||
this.outputWidth = outputWidth;
|
||||
this.outputHeight = outputHeight;
|
||||
this.framebuffers = framebuffers;
|
||||
this.sizes = sizes;
|
||||
this.debugPreviewEglSurface = debugPreviewEglSurface;
|
||||
this.debugPreviewWidth = debugPreviewWidth;
|
||||
this.debugPreviewHeight = debugPreviewHeight;
|
||||
@ -331,9 +350,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
}
|
||||
|
||||
/**
|
||||
* Informs the frame editor that a frame will be queued to its input surface.
|
||||
* Informs the {@code FrameProcessorChain} that a frame will be queued to its input surface.
|
||||
*
|
||||
* <p>Should be called before rendering a frame to the frame editor's input surface.
|
||||
* <p>Should be called before rendering a frame to the frame processor chain's input surface.
|
||||
*
|
||||
* @throws IllegalStateException If called after {@link #signalEndOfInputStream()}.
|
||||
*/
|
||||
@ -353,11 +372,11 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
try {
|
||||
oldestGlProcessingFuture.get();
|
||||
} catch (ExecutionException e) {
|
||||
throw TransformationException.createForFrameEditor(
|
||||
throw TransformationException.createForFrameProcessorChain(
|
||||
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw TransformationException.createForFrameEditor(
|
||||
throw TransformationException.createForFrameProcessorChain(
|
||||
e, TransformationException.ERROR_CODE_GL_PROCESSING_FAILED);
|
||||
}
|
||||
oldestGlProcessingFuture = futures.peek();
|
||||
@ -377,7 +396,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
return inputStreamEnded && !hasPendingFrames();
|
||||
}
|
||||
|
||||
/** Informs the {@code FrameEditor} that no further input frames should be accepted. */
|
||||
/** Informs the {@code FrameProcessorChain} that no further input frames should be accepted. */
|
||||
public void signalEndOfInputStream() {
|
||||
inputStreamEnded = true;
|
||||
}
|
||||
@ -385,9 +404,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
/**
|
||||
* Releases all resources.
|
||||
*
|
||||
* <p>If the frame editor is released before it has {@link #isEnded() ended}, it will attempt to
|
||||
* cancel processing any input frames that have already become available. Input frames that become
|
||||
* available after release are ignored.
|
||||
* <p>If the frame processor chain is released before it has {@link #isEnded() ended}, it will
|
||||
* attempt to cancel processing any input frames that have already become available. Input frames
|
||||
* that become available after release are ignored.
|
||||
*/
|
||||
public void release() {
|
||||
releaseRequested = true;
|
||||
@ -398,7 +417,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
singleThreadExecutorService.submit(
|
||||
() -> {
|
||||
externalCopyFrameProcessor.release();
|
||||
transformationFrameProcessor.release();
|
||||
for (int i = 0; i < frameProcessors.size(); i++) {
|
||||
frameProcessors.get(i).release();
|
||||
}
|
||||
GlUtil.destroyEglContext(eglDisplay, eglContext);
|
||||
}));
|
||||
if (inputSurfaceTexture != null) {
|
||||
@ -417,13 +438,41 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
inputSurfaceTexture.getTransformMatrix(textureTransformMatrix);
|
||||
long presentationTimeNs = inputSurfaceTexture.getTimestamp();
|
||||
|
||||
GlUtil.focusFramebuffer(
|
||||
eglDisplay, eglContext, eglSurface, frameBuffer, inputWidth, inputHeight);
|
||||
if (frameProcessors.isEmpty()) {
|
||||
GlUtil.focusEglSurface(
|
||||
eglDisplay, eglContext, eglSurface, sizes.get(0).getWidth(), sizes.get(0).getHeight());
|
||||
} else {
|
||||
GlUtil.focusFramebuffer(
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
eglSurface,
|
||||
framebuffers[0],
|
||||
sizes.get(0).getWidth(),
|
||||
sizes.get(0).getHeight());
|
||||
}
|
||||
externalCopyFrameProcessor.setTextureTransformMatrix(textureTransformMatrix);
|
||||
externalCopyFrameProcessor.updateProgramAndDraw(presentationTimeNs);
|
||||
|
||||
GlUtil.focusEglSurface(eglDisplay, eglContext, eglSurface, outputWidth, outputHeight);
|
||||
transformationFrameProcessor.updateProgramAndDraw(presentationTimeNs);
|
||||
for (int i = 0; i < frameProcessors.size() - 1; i++) {
|
||||
GlUtil.focusFramebuffer(
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
eglSurface,
|
||||
framebuffers[i + 1],
|
||||
sizes.get(i + 1).getWidth(),
|
||||
sizes.get(i + 1).getHeight());
|
||||
frameProcessors.get(i).updateProgramAndDraw(presentationTimeNs);
|
||||
}
|
||||
if (!frameProcessors.isEmpty()) {
|
||||
GlUtil.focusEglSurface(
|
||||
eglDisplay,
|
||||
eglContext,
|
||||
eglSurface,
|
||||
getLast(sizes).getWidth(),
|
||||
getLast(sizes).getHeight());
|
||||
getLast(frameProcessors).updateProgramAndDraw(presentationTimeNs);
|
||||
}
|
||||
|
||||
EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, presentationTimeNs);
|
||||
EGL14.eglSwapBuffers(eglDisplay, eglSurface);
|
||||
|
||||
@ -32,6 +32,9 @@ import java.io.IOException;
|
||||
* </ol>
|
||||
*/
|
||||
/* package */ interface GlFrameProcessor {
|
||||
// TODO(b/214975934): Investigate whether all configuration can be moved to initialize by
|
||||
// using a placeholder surface until the encoder surface is known. If so, convert
|
||||
// configureOutputSize to a simple getter.
|
||||
|
||||
/**
|
||||
* Returns the output {@link Size} of frames processed through {@link
|
||||
|
||||
@ -27,9 +27,7 @@ import androidx.media3.common.C;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import java.io.IOException;
|
||||
import org.checkerframework.checker.nullness.qual.EnsuresNonNull;
|
||||
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
|
||||
import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
|
||||
/**
|
||||
* Applies a simple rotation and/or scale in the vertex shader. All input frames' pixels will be
|
||||
@ -173,15 +171,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
*
|
||||
* <p>This method can only be called after {@link #configureOutputSize(int, int)}.
|
||||
*/
|
||||
@RequiresNonNull("adjustedTransformationMatrix")
|
||||
public boolean shouldProcess() {
|
||||
checkStateNotNull(adjustedTransformationMatrix);
|
||||
return inputWidth != outputWidth
|
||||
|| inputHeight != outputHeight
|
||||
|| !adjustedTransformationMatrix.isIdentity();
|
||||
}
|
||||
|
||||
@Override
|
||||
@EnsuresNonNull("adjustedTransformationMatrix")
|
||||
public Size configureOutputSize(int inputWidth, int inputHeight) {
|
||||
this.inputWidth = inputWidth;
|
||||
this.inputHeight = inputHeight;
|
||||
@ -191,7 +188,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
|
||||
int displayHeight = inputHeight;
|
||||
if (!transformationMatrix.isIdentity()) {
|
||||
float inputAspectRatio = (float) inputWidth / inputHeight;
|
||||
// Scale frames by inputAspectRatio, to account for FrameEditor's normalized device
|
||||
// Scale frames by inputAspectRatio, to account for FrameProcessorChain's normalized device
|
||||
// coordinates (NDC) (a square from -1 to 1 for both x and y) and preserve rectangular
|
||||
// display of input pixels during transformations (ex. rotations). With scaling,
|
||||
// transformationMatrix operations operate on a rectangle for x from -inputAspectRatio to
|
||||
|
||||
@ -274,15 +274,15 @@ public final class TransformationException extends Exception {
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates an instance for a {@link FrameEditor} related exception.
|
||||
* Creates an instance for a {@link FrameProcessorChain} related exception.
|
||||
*
|
||||
* @param cause The cause of the failure.
|
||||
* @param errorCode See {@link #errorCode}.
|
||||
* @return The created instance.
|
||||
*/
|
||||
/* package */ static TransformationException createForFrameEditor(
|
||||
/* package */ static TransformationException createForFrameProcessorChain(
|
||||
Throwable cause, int errorCode) {
|
||||
return new TransformationException("FrameEditor error", cause, errorCode);
|
||||
return new TransformationException("FrameProcessorChain error", cause, errorCode);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -28,6 +28,8 @@ import androidx.annotation.RequiresApi;
|
||||
import androidx.media3.common.Format;
|
||||
import androidx.media3.common.util.Util;
|
||||
import androidx.media3.decoder.DecoderInputBuffer;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import com.google.common.collect.Iterables;
|
||||
import java.util.List;
|
||||
import org.checkerframework.dataflow.qual.Pure;
|
||||
|
||||
@ -40,7 +42,7 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
private final DecoderInputBuffer decoderInputBuffer;
|
||||
private final Codec decoder;
|
||||
|
||||
@Nullable private final FrameEditor frameEditor;
|
||||
@Nullable private final FrameProcessorChain frameProcessorChain;
|
||||
|
||||
private final Codec encoder;
|
||||
private final DecoderInputBuffer encoderOutputBuffer;
|
||||
@ -74,14 +76,18 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
.setRotationDegrees(transformationRequest.rotationDegrees)
|
||||
.setResolution(transformationRequest.outputHeight)
|
||||
.build();
|
||||
Size requestedEncoderDimensions =
|
||||
scaleToFitFrameProcessor.configureOutputSize(decodedWidth, decodedHeight);
|
||||
// TODO(b/214975934): Allow a list of frame processors to be passed into the sample pipeline.
|
||||
ImmutableList<GlFrameProcessor> frameProcessors = ImmutableList.of(scaleToFitFrameProcessor);
|
||||
List<Size> frameProcessorSizes =
|
||||
FrameProcessorChain.configureSizes(decodedWidth, decodedHeight, frameProcessors);
|
||||
Size requestedEncoderSize = Iterables.getLast(frameProcessorSizes);
|
||||
// TODO(b/213190310): Move output rotation configuration to PresentationFrameProcessor.
|
||||
outputRotationDegrees = scaleToFitFrameProcessor.getOutputRotationDegrees();
|
||||
|
||||
Format requestedEncoderFormat =
|
||||
new Format.Builder()
|
||||
.setWidth(requestedEncoderDimensions.getWidth())
|
||||
.setHeight(requestedEncoderDimensions.getHeight())
|
||||
.setWidth(requestedEncoderSize.getWidth())
|
||||
.setHeight(requestedEncoderSize.getHeight())
|
||||
.setRotationDegrees(0)
|
||||
.setFrameRate(inputFormat.frameRate)
|
||||
.setSampleMimeType(
|
||||
@ -104,26 +110,30 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
|| inputFormat.width != encoderSupportedFormat.width
|
||||
|| scaleToFitFrameProcessor.shouldProcess()
|
||||
|| shouldAlwaysUseFrameEditor()) {
|
||||
frameEditor =
|
||||
FrameEditor.create(
|
||||
// TODO(b/218488308): Allow the final GlFrameProcessor to be re-configured if its output size
|
||||
// has to change due to encoder fallback or append another GlFrameProcessor.
|
||||
frameProcessorSizes.set(
|
||||
frameProcessorSizes.size() - 1,
|
||||
new Size(encoderSupportedFormat.width, encoderSupportedFormat.height));
|
||||
frameProcessorChain =
|
||||
FrameProcessorChain.create(
|
||||
context,
|
||||
/* inputWidth= */ decodedWidth,
|
||||
/* inputHeight= */ decodedHeight,
|
||||
/* outputWidth= */ encoderSupportedFormat.width,
|
||||
/* outputHeight= */ encoderSupportedFormat.height,
|
||||
inputFormat.pixelWidthHeightRatio,
|
||||
scaleToFitFrameProcessor,
|
||||
frameProcessors,
|
||||
frameProcessorSizes,
|
||||
/* outputSurface= */ encoder.getInputSurface(),
|
||||
transformationRequest.enableHdrEditing,
|
||||
debugViewProvider);
|
||||
} else {
|
||||
frameEditor = null;
|
||||
frameProcessorChain = null;
|
||||
}
|
||||
|
||||
decoder =
|
||||
decoderFactory.createForVideoDecoding(
|
||||
inputFormat,
|
||||
frameEditor == null ? encoder.getInputSurface() : frameEditor.createInputSurface());
|
||||
frameProcessorChain == null
|
||||
? encoder.getInputSurface()
|
||||
: frameProcessorChain.createInputSurface());
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -139,9 +149,9 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
|
||||
@Override
|
||||
public boolean processData() throws TransformationException {
|
||||
if (frameEditor != null) {
|
||||
frameEditor.getAndRethrowBackgroundExceptions();
|
||||
if (frameEditor.isEnded()) {
|
||||
if (frameProcessorChain != null) {
|
||||
frameProcessorChain.getAndRethrowBackgroundExceptions();
|
||||
if (frameProcessorChain.isEnded()) {
|
||||
if (!signaledEndOfStreamToEncoder) {
|
||||
encoder.signalEndOfInputStream();
|
||||
signaledEndOfStreamToEncoder = true;
|
||||
@ -163,8 +173,8 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
canProcessMoreDataImmediately = processDataDefault();
|
||||
}
|
||||
if (decoder.isEnded()) {
|
||||
if (frameEditor != null) {
|
||||
frameEditor.signalEndOfInputStream();
|
||||
if (frameProcessorChain != null) {
|
||||
frameProcessorChain.signalEndOfInputStream();
|
||||
} else {
|
||||
encoder.signalEndOfInputStream();
|
||||
signaledEndOfStreamToEncoder = true;
|
||||
@ -179,8 +189,8 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
*
|
||||
* <p>In this method the decoder could decode multiple frames in one invocation; as compared to
|
||||
* {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently,
|
||||
* if {@link FrameEditor} processes frames slower than the decoder, decoded frames are queued up
|
||||
* in the decoder's output surface.
|
||||
* if {@link FrameProcessorChain} processes frames slower than the decoder, decoded frames are
|
||||
* queued up in the decoder's output surface.
|
||||
*
|
||||
* <p>Prior to API 29, decoders may drop frames to keep their output surface from growing out of
|
||||
* bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame
|
||||
@ -195,12 +205,12 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
/**
|
||||
* Processes at most one input frame and returns whether a frame was processed.
|
||||
*
|
||||
* <p>Only renders decoder output to the {@link FrameEditor}'s input surface if the {@link
|
||||
* FrameEditor} has finished processing the previous frame.
|
||||
* <p>Only renders decoder output to the {@link FrameProcessorChain}'s input surface if the {@link
|
||||
* FrameProcessorChain} has finished processing the previous frame.
|
||||
*/
|
||||
private boolean processDataDefault() throws TransformationException {
|
||||
// TODO(b/214975934): Check whether this can be converted to a while-loop like processDataV29.
|
||||
if (frameEditor != null && frameEditor.hasPendingFrames()) {
|
||||
if (frameProcessorChain != null && frameProcessorChain.hasPendingFrames()) {
|
||||
return false;
|
||||
}
|
||||
return maybeProcessDecoderOutput();
|
||||
@ -240,8 +250,8 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
|
||||
@Override
|
||||
public void release() {
|
||||
if (frameEditor != null) {
|
||||
frameEditor.release();
|
||||
if (frameProcessorChain != null) {
|
||||
frameProcessorChain.release();
|
||||
}
|
||||
decoder.release();
|
||||
encoder.release();
|
||||
@ -299,8 +309,8 @@ import org.checkerframework.dataflow.qual.Pure;
|
||||
return false;
|
||||
}
|
||||
|
||||
if (frameEditor != null) {
|
||||
frameEditor.registerInputFrame();
|
||||
if (frameProcessorChain != null) {
|
||||
frameProcessorChain.registerInputFrame();
|
||||
}
|
||||
decoder.releaseOutputBuffer(/* render= */ true);
|
||||
return true;
|
||||
|
||||
@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Copyright 2021 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.transformer;
|
||||
|
||||
import static com.google.common.truth.Truth.assertThat;
|
||||
|
||||
import android.util.Size;
|
||||
import androidx.test.ext.junit.runners.AndroidJUnit4;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
import java.util.List;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
/**
|
||||
* Robolectric tests for {@link FrameProcessorChain}.
|
||||
*
|
||||
* <p>See {@code FrameProcessorChainTest} and {@code FrameProcessorChainPixelTest} in the
|
||||
* androidTest directory for instrumentation tests.
|
||||
*/
|
||||
@RunWith(AndroidJUnit4.class)
|
||||
public final class FrameProcessorChainTest {
|
||||
|
||||
@Test
|
||||
public void configureOutputDimensions_withEmptyList_returnsInputSize() {
|
||||
Size inputSize = new Size(200, 100);
|
||||
|
||||
List<Size> sizes =
|
||||
FrameProcessorChain.configureSizes(
|
||||
inputSize.getWidth(), inputSize.getHeight(), /* frameProcessors= */ ImmutableList.of());
|
||||
|
||||
assertThat(sizes).containsExactly(inputSize);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void configureOutputDimensions_withOneFrameProcessor_returnsItsInputAndOutputDimensions() {
|
||||
Size inputSize = new Size(200, 100);
|
||||
Size outputSize = new Size(300, 250);
|
||||
GlFrameProcessor frameProcessor = new FakeFrameProcessor(outputSize);
|
||||
|
||||
List<Size> sizes =
|
||||
FrameProcessorChain.configureSizes(
|
||||
inputSize.getWidth(), inputSize.getHeight(), ImmutableList.of(frameProcessor));
|
||||
|
||||
assertThat(sizes).containsExactly(inputSize, outputSize).inOrder();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void configureOutputDimensions_withThreeFrameProcessors_propagatesOutputDimensions() {
|
||||
Size inputSize = new Size(200, 100);
|
||||
Size outputSize1 = new Size(300, 250);
|
||||
Size outputSize2 = new Size(400, 244);
|
||||
Size outputSize3 = new Size(150, 160);
|
||||
GlFrameProcessor frameProcessor1 = new FakeFrameProcessor(outputSize1);
|
||||
GlFrameProcessor frameProcessor2 = new FakeFrameProcessor(outputSize2);
|
||||
GlFrameProcessor frameProcessor3 = new FakeFrameProcessor(outputSize3);
|
||||
|
||||
List<Size> sizes =
|
||||
FrameProcessorChain.configureSizes(
|
||||
inputSize.getWidth(),
|
||||
inputSize.getHeight(),
|
||||
ImmutableList.of(frameProcessor1, frameProcessor2, frameProcessor3));
|
||||
|
||||
assertThat(sizes).containsExactly(inputSize, outputSize1, outputSize2, outputSize3).inOrder();
|
||||
}
|
||||
|
||||
private static class FakeFrameProcessor implements GlFrameProcessor {
|
||||
|
||||
private final Size outputSize;
|
||||
|
||||
private FakeFrameProcessor(Size outputSize) {
|
||||
this.outputSize = outputSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Size configureOutputSize(int inputWidth, int inputHeight) {
|
||||
return outputSize;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void initialize(int inputTexId) {}
|
||||
|
||||
@Override
|
||||
public void updateProgramAndDraw(long presentationTimeNs) {}
|
||||
|
||||
@Override
|
||||
public void release() {}
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user