diff --git a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java index c323e0c86b..ae5c56975b 100644 --- a/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java +++ b/libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java @@ -141,6 +141,21 @@ public final class GlEffectsFrameProcessorPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void processData_noEditsWithCache_leavesFrameUnchanged() throws Exception { + String testId = "processData_noEditsWithCache"; + setUpAndPrepareFirstFrame(new FrameCache(/* capacity= */ 5)); + Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH); + + Bitmap actualBitmap = processFirstFrameAndEnd(); + + maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap); + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + @Test public void processData_withPixelWidthHeightRatio_producesExpectedOutput() throws Exception { String testId = "processData_withPixelWidthHeightRatio"; @@ -384,6 +399,49 @@ public final class GlEffectsFrameProcessorPixelTest { assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); } + @Test + public void + processData_fullRotationIncreaseBrightnessAndCenterCropWithCache_leavesFrameUnchanged() + throws Exception { + String testId = "processData_fullRotationIncreaseBrightnessAndCenterCropWithCache"; + Crop centerCrop = + new Crop(/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f); + ImmutableList increaseBrightnessFullRotationCenterCrop = + ImmutableList.of( + new Rotation(/* degrees= */ 90), + new RgbAdjustment.Builder().setRedScale(5).build(), + new RgbAdjustment.Builder().setGreenScale(5).build(), + new Rotation(/* degrees= */ 90), + new Rotation(/* degrees= */ 90), + new RgbAdjustment.Builder().setBlueScale(5).build(), + new FrameCache(/* capacity= */ 2), + new Rotation(/* degrees= */ 90), + new FrameCache(/* capacity= */ 2), + centerCrop); + setUpAndPrepareFirstFrame( + ImmutableList.of( + new RgbAdjustment.Builder().setRedScale(5).setBlueScale(5).setGreenScale(5).build(), + centerCrop)); + Bitmap centerCropAndBrightnessIncreaseResultBitmap = processFirstFrameAndEnd(); + setUpAndPrepareFirstFrame(increaseBrightnessFullRotationCenterCrop); + + Bitmap fullRotationBrightnessIncreaseAndCenterCropResultBitmap = processFirstFrameAndEnd(); + + maybeSaveTestBitmapToCacheDirectory( + testId, /* bitmapLabel= */ "centerCrop", centerCropAndBrightnessIncreaseResultBitmap); + maybeSaveTestBitmapToCacheDirectory( + testId, + /* bitmapLabel= */ "full4StepRotationBrightnessIncreaseAndCenterCrop", + fullRotationBrightnessIncreaseAndCenterCropResultBitmap); + // TODO(b/207848601): switch to using proper tooling for testing against golden data. + float averagePixelAbsoluteDifference = + getBitmapAveragePixelAbsoluteDifferenceArgb8888( + centerCropAndBrightnessIncreaseResultBitmap, + fullRotationBrightnessIncreaseAndCenterCropResultBitmap, + testId); + assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE); + } + @Test public void drawFrame_grayscaleAndIncreaseRedChannel_producesGrayscaleAndRedImage() throws Exception { diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FrameCache.java b/libraries/effect/src/main/java/androidx/media3/effect/FrameCache.java new file mode 100644 index 0000000000..d87d53d7eb --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/FrameCache.java @@ -0,0 +1,57 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkArgument; + +import android.content.Context; +import androidx.annotation.IntRange; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.util.UnstableApi; + +/** + * Caches the input frames. + * + *

Example usage: cache the processed frames when presenting them on screen, to accommodate for + * the possible fluctuation in frame processing time between frames. + */ +@UnstableApi +public class FrameCache implements GlEffect { + /** The capacity of the frame cache. */ + public final int capacity; + + /** + * Creates a new instance. + * + *

The {@code capacity} should be chosen carefully. OpenGL could crash unexpectedly if the + * device is not capable of allocating the requested buffer. + * + *

Currently up to 8 frames can be cached in one {@code FrameCache} instance. + * + * @param capacity The capacity of the frame cache, must be greater than zero. + */ + public FrameCache(@IntRange(from = 1, to = 8) int capacity) { + // TODO(b/243033952) Consider adding a global limit across many FrameCache instances. + checkArgument(capacity > 0 && capacity < 9); + this.capacity = capacity; + } + + @Override + public GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr) + throws FrameProcessingException { + return new FrameCacheTextureProcessor(context, capacity, useHdr); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FrameCacheTextureProcessor.java b/libraries/effect/src/main/java/androidx/media3/effect/FrameCacheTextureProcessor.java new file mode 100644 index 0000000000..44fde62981 --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/FrameCacheTextureProcessor.java @@ -0,0 +1,206 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkState; + +import android.content.Context; +import android.opengl.GLES20; +import androidx.media3.common.FrameProcessingException; +import androidx.media3.common.util.GlProgram; +import androidx.media3.common.util.GlUtil; +import com.google.common.collect.Iterables; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.concurrent.Executor; + +/** + * Manages a pool of {@linkplain TextureInfo textures}, and caches the input frame. + * + *

Implements {@link FrameCache}. + */ +/* package */ class FrameCacheTextureProcessor implements GlTextureProcessor { + private static final String VERTEX_SHADER_TRANSFORMATION_ES2_PATH = + "shaders/vertex_shader_transformation_es2.glsl"; + private static final String FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH = + "shaders/fragment_shader_transformation_es2.glsl"; + + private final ArrayDeque freeOutputTextures; + private final ArrayDeque inUseOutputTextures; + private final GlProgram copyProgram; + private final int capacity; + private final boolean useHdr; + + private InputListener inputListener; + private OutputListener outputListener; + private ErrorListener errorListener; + private Executor errorListenerExecutor; + + /** Creates a new instance. */ + public FrameCacheTextureProcessor(Context context, int capacity, boolean useHdr) + throws FrameProcessingException { + freeOutputTextures = new ArrayDeque<>(); + inUseOutputTextures = new ArrayDeque<>(); + try { + this.copyProgram = + new GlProgram( + context, + VERTEX_SHADER_TRANSFORMATION_ES2_PATH, + FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH); + } catch (IOException | GlUtil.GlException e) { + throw FrameProcessingException.from(e); + } + this.capacity = capacity; + this.useHdr = useHdr; + + float[] identityMatrix = GlUtil.create4x4IdentityMatrix(); + copyProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix); + copyProgram.setFloatsUniform("uTransformationMatrix", identityMatrix); + copyProgram.setFloatsUniform("uRgbMatrix", identityMatrix); + copyProgram.setBufferAttribute( + "aFramePosition", + GlUtil.getNormalizedCoordinateBounds(), + GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + + inputListener = new InputListener() {}; + outputListener = new OutputListener() {}; + errorListener = frameProcessingException -> {}; + errorListenerExecutor = MoreExecutors.directExecutor(); + } + + @Override + public void setInputListener(InputListener inputListener) { + this.inputListener = inputListener; + int numberOfFreeFramesToNotify; + if (getIteratorToAllTextures().hasNext()) { + // The frame buffers have already been allocated. + numberOfFreeFramesToNotify = freeOutputTextures.size(); + } else { + // Defer frame buffer allocation to when queueing input frames. + numberOfFreeFramesToNotify = capacity; + } + for (int i = 0; i < numberOfFreeFramesToNotify; i++) { + inputListener.onReadyToAcceptInputFrame(); + } + } + + @Override + public void setOutputListener(OutputListener outputListener) { + this.outputListener = outputListener; + } + + @Override + public void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) { + this.errorListenerExecutor = errorListenerExecutor; + this.errorListener = errorListener; + } + + @Override + public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) { + try { + configureAllOutputTextures(inputTexture.width, inputTexture.height); + + // Focus on the next free buffer. + TextureInfo outputTexture = freeOutputTextures.remove(); + inUseOutputTextures.add(outputTexture); + + // Copy frame to fbo. + GlUtil.focusFramebufferUsingCurrentContext( + outputTexture.fboId, outputTexture.width, outputTexture.height); + GlUtil.clearOutputFrame(); + drawFrame(inputTexture.texId); + inputListener.onInputFrameProcessed(inputTexture); + outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs); + } catch (GlUtil.GlException | NoSuchElementException e) { + errorListenerExecutor.execute( + () -> errorListener.onFrameProcessingError(FrameProcessingException.from(e))); + } + } + + private void drawFrame(int inputTexId) throws GlUtil.GlException { + copyProgram.use(); + copyProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0); + copyProgram.bindAttributesAndUniforms(); + GLES20.glDrawArrays( + GLES20.GL_TRIANGLE_STRIP, + /* first= */ 0, + /* count= */ GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE); + } + + @Override + public void releaseOutputFrame(TextureInfo outputTexture) { + checkState(inUseOutputTextures.contains(outputTexture)); + inUseOutputTextures.remove(outputTexture); + freeOutputTextures.add(outputTexture); + inputListener.onReadyToAcceptInputFrame(); + } + + @Override + public void signalEndOfCurrentInputStream() { + outputListener.onCurrentOutputStreamEnded(); + } + + @Override + public void release() throws FrameProcessingException { + try { + deleteAllOutputTextures(); + } catch (GlUtil.GlException e) { + throw new FrameProcessingException(e); + } + } + + private void configureAllOutputTextures(int inputWidth, int inputHeight) + throws GlUtil.GlException { + Iterator allTextures = getIteratorToAllTextures(); + if (!allTextures.hasNext()) { + createAllOutputTextures(inputWidth, inputHeight); + return; + } + TextureInfo outputTextureInfo = allTextures.next(); + if (outputTextureInfo.width != inputWidth || outputTextureInfo.height != inputHeight) { + deleteAllOutputTextures(); + createAllOutputTextures(inputWidth, inputHeight); + } + } + + private void createAllOutputTextures(int width, int height) throws GlUtil.GlException { + checkState(freeOutputTextures.isEmpty()); + checkState(inUseOutputTextures.isEmpty()); + for (int i = 0; i < capacity; i++) { + int outputTexId = GlUtil.createTexture(width, height, useHdr); + int outputFboId = GlUtil.createFboForTexture(outputTexId); + TextureInfo outputTexture = new TextureInfo(outputTexId, outputFboId, width, height); + freeOutputTextures.add(outputTexture); + } + } + + private void deleteAllOutputTextures() throws GlUtil.GlException { + Iterator allTextures = getIteratorToAllTextures(); + while (allTextures.hasNext()) { + TextureInfo textureInfo = allTextures.next(); + GlUtil.deleteTexture(textureInfo.texId); + } + freeOutputTextures.clear(); + inUseOutputTextures.clear(); + } + + private Iterator getIteratorToAllTextures() { + return Iterables.concat(freeOutputTextures, inUseOutputTextures).iterator(); + } +} diff --git a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java index 849ad52b4c..f74fd392b5 100644 --- a/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java +++ b/libraries/test_utils/src/main/java/androidx/media3/test/utils/BitmapPixelTestUtil.java @@ -61,7 +61,7 @@ public class BitmapPixelTestUtil { * this is caused by a difference in the codec or graphics implementation as opposed to an issue * in the tested component. */ - public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.5f; + public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 1.f; /** * Reads a bitmap from the specified asset location.