mirror of
https://github.com/androidx/media.git
synced 2025-04-30 06:46:50 +08:00
Add a frame cache
Frame cache compensates for the fluctuation in frame processing times. Imagine a frame takes 10ms to process, and the interval between two frames is 33ms. The third frame took 40ms to process. If we don't have frame cache: - Process frame 1, ready after 10ms, starts playback, now t=0 ms - Start processing frame 2, ready at t=10ms, - Release frame 2 at t=33ms - We start processing the third frame at t=33ms - The third frame is due presentation at t=66ms - But frame 3 is available at t=73ms, late If we have a frame cache of say 3 frams, - Process frame 1, ready after 10ms, starts playback, now t=0 ms - Start processing frame 2, ready at t=10ms - Start processing frame 3, ready at t=50ms - Release frame 2 at t=33ms - Start frame 4, ready at t=60ms - Frame 3 is due presentation at t=66ms - Frame 3 isn't late PiperOrigin-RevId: 501869948
This commit is contained in:
claincly
Rohit Singh
parent
a82fcdefcb
commit
693600a444
@ -141,6 +141,21 @@ public final class GlEffectsFrameProcessorPixelTest {
|
||||
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void processData_noEditsWithCache_leavesFrameUnchanged() throws Exception {
|
||||
String testId = "processData_noEditsWithCache";
|
||||
setUpAndPrepareFirstFrame(new FrameCache(/* capacity= */ 5));
|
||||
Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
|
||||
|
||||
Bitmap actualBitmap = processFirstFrameAndEnd();
|
||||
|
||||
maybeSaveTestBitmapToCacheDirectory(testId, /* bitmapLabel= */ "actual", actualBitmap);
|
||||
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
|
||||
float averagePixelAbsoluteDifference =
|
||||
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
|
||||
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void processData_withPixelWidthHeightRatio_producesExpectedOutput() throws Exception {
|
||||
String testId = "processData_withPixelWidthHeightRatio";
|
||||
@ -384,6 +399,49 @@ public final class GlEffectsFrameProcessorPixelTest {
|
||||
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void
|
||||
processData_fullRotationIncreaseBrightnessAndCenterCropWithCache_leavesFrameUnchanged()
|
||||
throws Exception {
|
||||
String testId = "processData_fullRotationIncreaseBrightnessAndCenterCropWithCache";
|
||||
Crop centerCrop =
|
||||
new Crop(/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f);
|
||||
ImmutableList<Effect> increaseBrightnessFullRotationCenterCrop =
|
||||
ImmutableList.of(
|
||||
new Rotation(/* degrees= */ 90),
|
||||
new RgbAdjustment.Builder().setRedScale(5).build(),
|
||||
new RgbAdjustment.Builder().setGreenScale(5).build(),
|
||||
new Rotation(/* degrees= */ 90),
|
||||
new Rotation(/* degrees= */ 90),
|
||||
new RgbAdjustment.Builder().setBlueScale(5).build(),
|
||||
new FrameCache(/* capacity= */ 2),
|
||||
new Rotation(/* degrees= */ 90),
|
||||
new FrameCache(/* capacity= */ 2),
|
||||
centerCrop);
|
||||
setUpAndPrepareFirstFrame(
|
||||
ImmutableList.of(
|
||||
new RgbAdjustment.Builder().setRedScale(5).setBlueScale(5).setGreenScale(5).build(),
|
||||
centerCrop));
|
||||
Bitmap centerCropAndBrightnessIncreaseResultBitmap = processFirstFrameAndEnd();
|
||||
setUpAndPrepareFirstFrame(increaseBrightnessFullRotationCenterCrop);
|
||||
|
||||
Bitmap fullRotationBrightnessIncreaseAndCenterCropResultBitmap = processFirstFrameAndEnd();
|
||||
|
||||
maybeSaveTestBitmapToCacheDirectory(
|
||||
testId, /* bitmapLabel= */ "centerCrop", centerCropAndBrightnessIncreaseResultBitmap);
|
||||
maybeSaveTestBitmapToCacheDirectory(
|
||||
testId,
|
||||
/* bitmapLabel= */ "full4StepRotationBrightnessIncreaseAndCenterCrop",
|
||||
fullRotationBrightnessIncreaseAndCenterCropResultBitmap);
|
||||
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
|
||||
float averagePixelAbsoluteDifference =
|
||||
getBitmapAveragePixelAbsoluteDifferenceArgb8888(
|
||||
centerCropAndBrightnessIncreaseResultBitmap,
|
||||
fullRotationBrightnessIncreaseAndCenterCropResultBitmap,
|
||||
testId);
|
||||
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void drawFrame_grayscaleAndIncreaseRedChannel_producesGrayscaleAndRedImage()
|
||||
throws Exception {
|
||||
|
||||
@ -0,0 +1,57 @@
|
||||
/*
|
||||
* Copyright 2022 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkArgument;
|
||||
|
||||
import android.content.Context;
|
||||
import androidx.annotation.IntRange;
|
||||
import androidx.media3.common.FrameProcessingException;
|
||||
import androidx.media3.common.util.UnstableApi;
|
||||
|
||||
/**
|
||||
* Caches the input frames.
|
||||
*
|
||||
* <p>Example usage: cache the processed frames when presenting them on screen, to accommodate for
|
||||
* the possible fluctuation in frame processing time between frames.
|
||||
*/
|
||||
@UnstableApi
|
||||
public class FrameCache implements GlEffect {
|
||||
/** The capacity of the frame cache. */
|
||||
public final int capacity;
|
||||
|
||||
/**
|
||||
* Creates a new instance.
|
||||
*
|
||||
* <p>The {@code capacity} should be chosen carefully. OpenGL could crash unexpectedly if the
|
||||
* device is not capable of allocating the requested buffer.
|
||||
*
|
||||
* <p>Currently up to 8 frames can be cached in one {@code FrameCache} instance.
|
||||
*
|
||||
* @param capacity The capacity of the frame cache, must be greater than zero.
|
||||
*/
|
||||
public FrameCache(@IntRange(from = 1, to = 8) int capacity) {
|
||||
// TODO(b/243033952) Consider adding a global limit across many FrameCache instances.
|
||||
checkArgument(capacity > 0 && capacity < 9);
|
||||
this.capacity = capacity;
|
||||
}
|
||||
|
||||
@Override
|
||||
public GlTextureProcessor toGlTextureProcessor(Context context, boolean useHdr)
|
||||
throws FrameProcessingException {
|
||||
return new FrameCacheTextureProcessor(context, capacity, useHdr);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,206 @@
|
||||
/*
|
||||
* Copyright 2022 The Android Open Source Project
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package androidx.media3.effect;
|
||||
|
||||
import static androidx.media3.common.util.Assertions.checkState;
|
||||
|
||||
import android.content.Context;
|
||||
import android.opengl.GLES20;
|
||||
import androidx.media3.common.FrameProcessingException;
|
||||
import androidx.media3.common.util.GlProgram;
|
||||
import androidx.media3.common.util.GlUtil;
|
||||
import com.google.common.collect.Iterables;
|
||||
import com.google.common.util.concurrent.MoreExecutors;
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayDeque;
|
||||
import java.util.Iterator;
|
||||
import java.util.NoSuchElementException;
|
||||
import java.util.concurrent.Executor;
|
||||
|
||||
/**
|
||||
* Manages a pool of {@linkplain TextureInfo textures}, and caches the input frame.
|
||||
*
|
||||
* <p>Implements {@link FrameCache}.
|
||||
*/
|
||||
/* package */ class FrameCacheTextureProcessor implements GlTextureProcessor {
|
||||
private static final String VERTEX_SHADER_TRANSFORMATION_ES2_PATH =
|
||||
"shaders/vertex_shader_transformation_es2.glsl";
|
||||
private static final String FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH =
|
||||
"shaders/fragment_shader_transformation_es2.glsl";
|
||||
|
||||
private final ArrayDeque<TextureInfo> freeOutputTextures;
|
||||
private final ArrayDeque<TextureInfo> inUseOutputTextures;
|
||||
private final GlProgram copyProgram;
|
||||
private final int capacity;
|
||||
private final boolean useHdr;
|
||||
|
||||
private InputListener inputListener;
|
||||
private OutputListener outputListener;
|
||||
private ErrorListener errorListener;
|
||||
private Executor errorListenerExecutor;
|
||||
|
||||
/** Creates a new instance. */
|
||||
public FrameCacheTextureProcessor(Context context, int capacity, boolean useHdr)
|
||||
throws FrameProcessingException {
|
||||
freeOutputTextures = new ArrayDeque<>();
|
||||
inUseOutputTextures = new ArrayDeque<>();
|
||||
try {
|
||||
this.copyProgram =
|
||||
new GlProgram(
|
||||
context,
|
||||
VERTEX_SHADER_TRANSFORMATION_ES2_PATH,
|
||||
FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH);
|
||||
} catch (IOException | GlUtil.GlException e) {
|
||||
throw FrameProcessingException.from(e);
|
||||
}
|
||||
this.capacity = capacity;
|
||||
this.useHdr = useHdr;
|
||||
|
||||
float[] identityMatrix = GlUtil.create4x4IdentityMatrix();
|
||||
copyProgram.setFloatsUniform("uTexTransformationMatrix", identityMatrix);
|
||||
copyProgram.setFloatsUniform("uTransformationMatrix", identityMatrix);
|
||||
copyProgram.setFloatsUniform("uRgbMatrix", identityMatrix);
|
||||
copyProgram.setBufferAttribute(
|
||||
"aFramePosition",
|
||||
GlUtil.getNormalizedCoordinateBounds(),
|
||||
GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
|
||||
|
||||
inputListener = new InputListener() {};
|
||||
outputListener = new OutputListener() {};
|
||||
errorListener = frameProcessingException -> {};
|
||||
errorListenerExecutor = MoreExecutors.directExecutor();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setInputListener(InputListener inputListener) {
|
||||
this.inputListener = inputListener;
|
||||
int numberOfFreeFramesToNotify;
|
||||
if (getIteratorToAllTextures().hasNext()) {
|
||||
// The frame buffers have already been allocated.
|
||||
numberOfFreeFramesToNotify = freeOutputTextures.size();
|
||||
} else {
|
||||
// Defer frame buffer allocation to when queueing input frames.
|
||||
numberOfFreeFramesToNotify = capacity;
|
||||
}
|
||||
for (int i = 0; i < numberOfFreeFramesToNotify; i++) {
|
||||
inputListener.onReadyToAcceptInputFrame();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setOutputListener(OutputListener outputListener) {
|
||||
this.outputListener = outputListener;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setErrorListener(Executor errorListenerExecutor, ErrorListener errorListener) {
|
||||
this.errorListenerExecutor = errorListenerExecutor;
|
||||
this.errorListener = errorListener;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void queueInputFrame(TextureInfo inputTexture, long presentationTimeUs) {
|
||||
try {
|
||||
configureAllOutputTextures(inputTexture.width, inputTexture.height);
|
||||
|
||||
// Focus on the next free buffer.
|
||||
TextureInfo outputTexture = freeOutputTextures.remove();
|
||||
inUseOutputTextures.add(outputTexture);
|
||||
|
||||
// Copy frame to fbo.
|
||||
GlUtil.focusFramebufferUsingCurrentContext(
|
||||
outputTexture.fboId, outputTexture.width, outputTexture.height);
|
||||
GlUtil.clearOutputFrame();
|
||||
drawFrame(inputTexture.texId);
|
||||
inputListener.onInputFrameProcessed(inputTexture);
|
||||
outputListener.onOutputFrameAvailable(outputTexture, presentationTimeUs);
|
||||
} catch (GlUtil.GlException | NoSuchElementException e) {
|
||||
errorListenerExecutor.execute(
|
||||
() -> errorListener.onFrameProcessingError(FrameProcessingException.from(e)));
|
||||
}
|
||||
}
|
||||
|
||||
private void drawFrame(int inputTexId) throws GlUtil.GlException {
|
||||
copyProgram.use();
|
||||
copyProgram.setSamplerTexIdUniform("uTexSampler", inputTexId, /* texUnitIndex= */ 0);
|
||||
copyProgram.bindAttributesAndUniforms();
|
||||
GLES20.glDrawArrays(
|
||||
GLES20.GL_TRIANGLE_STRIP,
|
||||
/* first= */ 0,
|
||||
/* count= */ GlUtil.HOMOGENEOUS_COORDINATE_VECTOR_SIZE);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void releaseOutputFrame(TextureInfo outputTexture) {
|
||||
checkState(inUseOutputTextures.contains(outputTexture));
|
||||
inUseOutputTextures.remove(outputTexture);
|
||||
freeOutputTextures.add(outputTexture);
|
||||
inputListener.onReadyToAcceptInputFrame();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void signalEndOfCurrentInputStream() {
|
||||
outputListener.onCurrentOutputStreamEnded();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void release() throws FrameProcessingException {
|
||||
try {
|
||||
deleteAllOutputTextures();
|
||||
} catch (GlUtil.GlException e) {
|
||||
throw new FrameProcessingException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private void configureAllOutputTextures(int inputWidth, int inputHeight)
|
||||
throws GlUtil.GlException {
|
||||
Iterator<TextureInfo> allTextures = getIteratorToAllTextures();
|
||||
if (!allTextures.hasNext()) {
|
||||
createAllOutputTextures(inputWidth, inputHeight);
|
||||
return;
|
||||
}
|
||||
TextureInfo outputTextureInfo = allTextures.next();
|
||||
if (outputTextureInfo.width != inputWidth || outputTextureInfo.height != inputHeight) {
|
||||
deleteAllOutputTextures();
|
||||
createAllOutputTextures(inputWidth, inputHeight);
|
||||
}
|
||||
}
|
||||
|
||||
private void createAllOutputTextures(int width, int height) throws GlUtil.GlException {
|
||||
checkState(freeOutputTextures.isEmpty());
|
||||
checkState(inUseOutputTextures.isEmpty());
|
||||
for (int i = 0; i < capacity; i++) {
|
||||
int outputTexId = GlUtil.createTexture(width, height, useHdr);
|
||||
int outputFboId = GlUtil.createFboForTexture(outputTexId);
|
||||
TextureInfo outputTexture = new TextureInfo(outputTexId, outputFboId, width, height);
|
||||
freeOutputTextures.add(outputTexture);
|
||||
}
|
||||
}
|
||||
|
||||
private void deleteAllOutputTextures() throws GlUtil.GlException {
|
||||
Iterator<TextureInfo> allTextures = getIteratorToAllTextures();
|
||||
while (allTextures.hasNext()) {
|
||||
TextureInfo textureInfo = allTextures.next();
|
||||
GlUtil.deleteTexture(textureInfo.texId);
|
||||
}
|
||||
freeOutputTextures.clear();
|
||||
inUseOutputTextures.clear();
|
||||
}
|
||||
|
||||
private Iterator<TextureInfo> getIteratorToAllTextures() {
|
||||
return Iterables.concat(freeOutputTextures, inUseOutputTextures).iterator();
|
||||
}
|
||||
}
|
||||
@ -61,7 +61,7 @@ public class BitmapPixelTestUtil {
|
||||
* this is caused by a difference in the codec or graphics implementation as opposed to an issue
|
||||
* in the tested component.
|
||||
*/
|
||||
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 0.5f;
|
||||
public static final float MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE = 1.f;
|
||||
|
||||
/**
|
||||
* Reads a bitmap from the specified asset location.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user