loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update frame & texture processors to handle SDR image input

Browse Source 
GLEffectsFrameProcessor, MatrixShaderProgram and FinalMatrixShaderProgramWrapper are currently setup to handle the input frames coming from an external input (i.e. a video decoder). Image input is loaded into Bitmap objects at the start of the pipeline, so they are not produced externally. The changes provide a way for the frame processing pipeline to handle this "internal" (i.e. non-external) input.

PiperOrigin-RevId: 508645244
This commit is contained in:
tofunmi 2023-02-10 14:32:07 +00:00 committed by christosts
parent 706431059c
commit 574424f626
9 changed files with 554 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
libraries/common/src/main/java/androidx/media3/common/FrameProcessor.java
View File

@ -127,20 +127,25 @@ public interface FrameProcessor {
/**
* Provides an input {@link Bitmap} to the {@link FrameProcessor}.
*
* <p>This method should only be used for when the {@link FrameProcessor} was created with {@link
* C#TRACK_TYPE_IMAGE} as the {@code inputTrackType}.
*
* <p>Can be called on any thread.
*
* @param inputBitmap The {@link Bitmap} queued to the {@link FrameProcessor}.
* @param durationUs The duration for which to display the {@code inputBitmap}, in microseconds.
* @param frameRate The frame rate at which to display the {@code inputBitmap}, in frames per
* second.
*/
// TODO(b/262693274): Remove duration & frameRate parameters when EditedMediaItem can be signalled
// down to the processors.
void queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate);
// TODO(b/262693274): Remove duration and frameRate parameters when EditedMediaItem can be
// signalled down to the processors.
void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate);
/**
* Returns the input {@link Surface}, where {@link FrameProcessor} consumes input frames from.
*
* <p>This method should only be used for when the {@link FrameProcessor} was created with {@link
* C#TRACK_TYPE_VIDEO} as the {@code inputTrackType}.
*
* <p>Can be called on any thread.
*/
Surface getInputSurface();
@ -167,6 +172,9 @@ public interface FrameProcessor {
*
* <p>Must be called before rendering a frame to the frame processor's input surface.
*
* <p>This method should only be used for when the {@link FrameProcessor} was created with {@link
* C#TRACK_TYPE_VIDEO} as the {@code inputTrackType}.
*
* <p>Can be called on any thread.
*
* @throws IllegalStateException If called after {@link #signalEndOfInput()} or before {@link
@ -178,6 +186,9 @@ public interface FrameProcessor {
* Returns the number of input frames that have been {@linkplain #registerInputFrame() registered}
* but not processed off the {@linkplain #getInputSurface() input surface} yet.
*
* <p>This method should only be used for when the {@link FrameProcessor} was created with {@link
* C#TRACK_TYPE_VIDEO} as the {@code inputTrackType}.
*
* <p>Can be called on any thread.
*/
int getPendingInputFrameCount();
@ -235,6 +246,9 @@ public interface FrameProcessor {
* <p>All the frames that are {@linkplain #registerInputFrame() registered} prior to calling this
* method are no longer considered to be registered when this method returns.
*
* <p>This method should only be used for when the {@link FrameProcessor} was created with {@link
* C#TRACK_TYPE_VIDEO} as the {@code inputTrackType}.
*
* <p>{@link Listener} methods invoked prior to calling this method should be ignored.
*/
void flush();

52
libraries/effect/src/androidTest/java/androidx/media3/effect/GlEffectsFrameProcessorPixelTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.effect;
import static androidx.media3.common.C.TRACK_TYPE_IMAGE;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.effect.OverlayShaderProgramPixelTest.OVERLAY_PNG_ASSET_PATH;
@ -54,6 +55,8 @@ import org.junit.runner.RunWith;
public final class GlEffectsFrameProcessorPixelTest {
public static final String ORIGINAL_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/original.png";
public static final String WRAPPED_CROP_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/image_input_with_wrapped_crop.png";
// This file is generated on a Pixel 7, because the emulator isn't able to generate this file.
public static final String BITMAP_OVERLAY_PNG_ASSET_PATH =
"media/bitmap/sample_mp4_first_frame/electrical_colors/overlay_bitmap_FrameProcessor.png";
@ -102,6 +105,50 @@ public final class GlEffectsFrameProcessorPixelTest {
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void noEffects_withImageInput_matchesGoldenFile() throws Exception {
String testId = "noEffects_withImageInput_matchesGoldenFile";
frameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setInputTrackType(TRACK_TYPE_IMAGE)
.build();
Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
Bitmap actualBitmap = frameProcessorTestRunner.processImageFrameAndEnd(expectedBitmap);
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
@Test
public void wrappedCrop_withImageInput_matchesGoldenFile() throws Exception {
String testId = "wrappedCrop_withImageInput_matchesGoldenFile";
frameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setInputTrackType(TRACK_TYPE_IMAGE)
.setEffects(
new GlEffectWrapper(
new Crop(
/* left= */ -0.5f,
/* right= */ 0.5f,
/* bottom= */ -0.5f,
/* top= */ 0.5f)))
.build();
Bitmap originalBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
Bitmap expectedBitmap = readBitmap(WRAPPED_CROP_PNG_ASSET_PATH);
Bitmap actualBitmap = frameProcessorTestRunner.processImageFrameAndEnd(originalBitmap);
// TODO(b/207848601): switch to using proper tooling for testing against golden data.
float averagePixelAbsoluteDifference =
getBitmapAveragePixelAbsoluteDifferenceArgb8888(expectedBitmap, actualBitmap, testId);
assertThat(averagePixelAbsoluteDifference).isAtMost(MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE);
}
// TODO(b/262693274): Once texture deletion is added to InternalTextureManager.java, add a test
// queuing multiple input bitmaps to ensure successfully completion without errors.
@Test
public void noEffects_withFrameCache_matchesGoldenFile() throws Exception {
String testId = "noEffects_withFrameCache_matchesGoldenFile";
@ -238,9 +285,10 @@ public final class GlEffectsFrameProcessorPixelTest {
frameProcessorTestRunner =
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setEffects(
new Crop(/* left= */ -.5f, /* right= */ .5f, /* bottom= */ -.5f, /* top= */ .5f),
new Crop(
/* left= */ -0.5f, /* right= */ 0.5f, /* bottom= */ -0.5f, /* top= */ 0.5f),
Presentation.createForAspectRatio(
/* aspectRatio= */ .5f, Presentation.LAYOUT_SCALE_TO_FIT))
/* aspectRatio= */ 0.5f, Presentation.LAYOUT_SCALE_TO_FIT))
.build();
Bitmap expectedBitmap = readBitmap(CROP_THEN_ASPECT_RATIO_PNG_ASSET_PATH);

103
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl Normal file
View File

@ -0,0 +1,103 @@
#version 100
// Copyright 2023 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ES 2 fragment shader that:
// 1. Samples from an input texture created from an internal texture (e.g. a
// texture created from a bitmap), with uTexSampler copying from this texture
// to the current output.
// 2. Transforms the electrical colors to optical colors using the SMPTE 170M
// EOTF.
// 3. Applies a 4x4 RGB color matrix to change the pixel colors.
// 4. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR
// for outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to
// output electrical colors via an OETF (e.g. to an encoder).
precision mediump float;
uniform sampler2D uTexSampler;
uniform mat4 uRgbMatrix;
varying vec2 vTexSamplingCoord;
// C.java#ColorTransfer value.
// Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
uniform int uOutputColorTransfer;
const float inverseGamma = 0.4500;
const float gamma = 1.0 / inverseGamma;
// Transforms a single channel from electrical to optical SDR using the SMPTE
// 170M OETF.
float smpte170mEotfSingleChannel(float electricalChannel) {
// Specification:
// https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en
return electricalChannel < 0.0812
? electricalChannel / 4.500
: pow((electricalChannel + 0.099) / 1.099, gamma);
}
// Transforms electrical to optical SDR using the SMPTE 170M EOTF.
vec3 smpte170mEotf(vec3 electricalColor) {
return vec3(
smpte170mEotfSingleChannel(electricalColor.r),
smpte170mEotfSingleChannel(electricalColor.g),
smpte170mEotfSingleChannel(electricalColor.b));
}
// Transforms a single channel from optical to electrical SDR.
float smpte170mOetfSingleChannel(float opticalChannel) {
// Specification:
// https://www.itu.int/rec/R-REC-BT.1700-0-200502-I/en
return opticalChannel < 0.018
? opticalChannel * 4.500
: 1.099 * pow(opticalChannel, inverseGamma) - 0.099;
}
// Transforms optical SDR colors to electrical SDR using the SMPTE 170M OETF.
vec3 smpte170mOetf(vec3 opticalColor) {
return vec3(
smpte170mOetfSingleChannel(opticalColor.r),
smpte170mOetfSingleChannel(opticalColor.g),
smpte170mOetfSingleChannel(opticalColor.b));
}
// Applies the appropriate OETF to convert linear optical signals to nonlinear
// electrical signals. Input and output are both normalized to [0, 1].
highp vec3 applyOetf(highp vec3 linearColor) {
// LINT.IfChange(color_transfer)
const int COLOR_TRANSFER_LINEAR = 1;
const int COLOR_TRANSFER_SDR_VIDEO = 3;
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
return linearColor;
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mOetf(linearColor);
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
}
void main() {
vec2 vTexSamplingCoordFlipped =
vec2(vTexSamplingCoord.x, 1.0 - vTexSamplingCoord.y);
// Whereas the Android system uses the top-left corner as (0,0) of the
// coordinate system, OpenGL uses the bottom-left corner as (0,0), so the
// texture gets flipped. We flip the texture vertically to ensure the
// orientation of the output is correct.
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoordFlipped);
vec3 linearInputColor = smpte170mEotf(inputColor.rgb);
vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);
gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a);
}

25
libraries/effect/src/main/java/androidx/media3/effect/FinalMatrixShaderProgramWrapper.java
View File

@ -71,7 +71,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final DebugViewProvider debugViewProvider;
private final boolean sampleFromExternalTexture;
private final boolean sampleFromInputTexture;
private final boolean isInputExternal;
private final ColorInfo inputColorInfo;
private final ColorInfo outputColorInfo;
private final boolean releaseFramesAutomatically;
@ -106,9 +107,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
ImmutableList<GlMatrixTransformation> matrixTransformations,
ImmutableList<RgbMatrix> rgbMatrices,
DebugViewProvider debugViewProvider,
boolean sampleFromExternalTexture,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean sampleFromInputTexture,
boolean isInputExternal,
boolean releaseFramesAutomatically,
Executor frameProcessorListenerExecutor,
FrameProcessor.Listener frameProcessorListener) {
@ -118,7 +120,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglDisplay = eglDisplay;
this.eglContext = eglContext;
this.debugViewProvider = debugViewProvider;
this.sampleFromExternalTexture = sampleFromExternalTexture;
this.sampleFromInputTexture = sampleFromInputTexture;
this.isInputExternal = isInputExternal;
this.inputColorInfo = inputColorInfo;
this.outputColorInfo = outputColorInfo;
this.releaseFramesAutomatically = releaseFramesAutomatically;
@ -402,14 +405,24 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
MatrixShaderProgram matrixShaderProgram;
ImmutableList<GlMatrixTransformation> expandedMatrixTransformations =
matrixTransformationListBuilder.build();
if (sampleFromExternalTexture) {
if (sampleFromInputTexture) {
if (isInputExternal) {
matrixShaderProgram =
MatrixShaderProgram.createWithExternalSampler(
context,
expandedMatrixTransformations,
rgbMatrices,
/* inputColorInfo= */ inputColorInfo,
/* outputColorInfo= */ outputColorInfo);
inputColorInfo,
outputColorInfo);
} else {
matrixShaderProgram =
MatrixShaderProgram.createWithInternalSampler(
context,
expandedMatrixTransformations,
rgbMatrices,
inputColorInfo,
outputColorInfo);
}
} else {
matrixShaderProgram =
MatrixShaderProgram.createApplyingOetf(

89
libraries/effect/src/main/java/androidx/media3/effect/GlEffectsFrameProcessor.java
View File

@ -16,6 +16,7 @@
package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static com.google.common.collect.Iterables.getLast;
@ -108,6 +109,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
checkArgument(inputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
checkArgument(outputColorInfo.isValid());
checkArgument(outputColorInfo.colorTransfer != C.COLOR_TRANSFER_LINEAR);
checkArgument(inputTrackType == C.TRACK_TYPE_VIDEO || inputTrackType == C.TRACK_TYPE_IMAGE);
if (inputColorInfo.colorSpace != outputColorInfo.colorSpace
|| ColorInfo.isTransferHdr(inputColorInfo) != ColorInfo.isTransferHdr(outputColorInfo)) {
@ -134,6 +136,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
debugViewProvider,
inputColorInfo,
outputColorInfo,
/* isInputExternal= */ inputTrackType == C.TRACK_TYPE_VIDEO,
releaseFramesAutomatically,
singleThreadExecutorService,
listenerExecutor,
@ -167,6 +170,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean isInputExternal,
boolean releaseFramesAutomatically,
ExecutorService singleThreadExecutorService,
Executor executor,
@ -208,6 +212,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
debugViewProvider,
inputColorInfo,
outputColorInfo,
isInputExternal,
releaseFramesAutomatically,
executor,
listener);
@ -219,6 +224,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
return new GlEffectsFrameProcessor(
eglDisplay,
eglContext,
isInputExternal,
frameProcessingTaskExecutor,
shaderPrograms,
releaseFramesAutomatically);
@ -243,6 +249,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
DebugViewProvider debugViewProvider,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean isInputExternal,
boolean releaseFramesAutomatically,
Executor executor,
Listener listener)
@ -251,7 +258,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
ImmutableList.Builder<GlMatrixTransformation> matrixTransformationListBuilder =
new ImmutableList.Builder<>();
ImmutableList.Builder<RgbMatrix> rgbMatrixListBuilder = new ImmutableList.Builder<>();
boolean sampleFromExternalTexture = true;
boolean sampleFromInputTexture = true;
ColorInfo linearColorInfo =
new ColorInfo(
outputColorInfo.colorSpace, outputColorInfo.colorRange, C.COLOR_TRANSFER_LINEAR, null);
@ -275,16 +282,18 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
matrixTransformationListBuilder.build();
ImmutableList<RgbMatrix> rgbMatrices = rgbMatrixListBuilder.build();
boolean isOutputTransferHdr = ColorInfo.isTransferHdr(outputColorInfo);
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromExternalTexture) {
if (!matrixTransformations.isEmpty() || !rgbMatrices.isEmpty() || sampleFromInputTexture) {
MatrixShaderProgram matrixShaderProgram;
if (sampleFromExternalTexture) {
if (sampleFromInputTexture) {
if (isInputExternal) {
matrixShaderProgram =
MatrixShaderProgram.createWithExternalSampler(
context,
matrixTransformations,
rgbMatrices,
/* inputColorInfo= */ inputColorInfo,
/* outputColorInfo= */ linearColorInfo);
context, matrixTransformations, rgbMatrices, inputColorInfo, linearColorInfo);
} else {
matrixShaderProgram =
MatrixShaderProgram.createWithInternalSampler(
context, matrixTransformations, rgbMatrices, inputColorInfo, linearColorInfo);
}
} else {
matrixShaderProgram =
MatrixShaderProgram.create(
@ -293,7 +302,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
shaderProgramListBuilder.add(matrixShaderProgram);
matrixTransformationListBuilder = new ImmutableList.Builder<>();
rgbMatrixListBuilder = new ImmutableList.Builder<>();
sampleFromExternalTexture = false;
sampleFromInputTexture = false;
}
shaderProgramListBuilder.add(glEffect.toGlShaderProgram(context, isOutputTransferHdr));
}
@ -306,9 +315,10 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
matrixTransformationListBuilder.build(),
rgbMatrixListBuilder.build(),
debugViewProvider,
sampleFromExternalTexture,
/* inputColorInfo= */ sampleFromExternalTexture ? inputColorInfo : linearColorInfo,
/* inputColorInfo= */ sampleFromInputTexture ? inputColorInfo : linearColorInfo,
outputColorInfo,
sampleFromInputTexture,
isInputExternal,
releaseFramesAutomatically,
executor,
listener));
@ -343,8 +353,10 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
private final EGLDisplay eglDisplay;
private final EGLContext eglContext;
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
private final ExternalTextureManager inputExternalTextureManager;
private final Surface inputSurface;
private @MonotonicNonNull InternalTextureManager inputInternalTextureManager;
private @MonotonicNonNull ExternalTextureManager inputExternalTextureManager;
// TODO(262693274): Move this variable to ExternalTextureManager.
private @MonotonicNonNull Surface inputExternalSurface;
private final boolean releaseFramesAutomatically;
private final FinalMatrixShaderProgramWrapper finalShaderProgramWrapper;
private final ImmutableList<GlShaderProgram> allShaderPrograms;
@ -361,6 +373,7 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
private GlEffectsFrameProcessor(
EGLDisplay eglDisplay,
EGLContext eglContext,
boolean isInputExternal,
FrameProcessingTaskExecutor frameProcessingTaskExecutor,
ImmutableList<GlShaderProgram> shaderPrograms,
boolean releaseFramesAutomatically)
@ -372,14 +385,23 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
this.releaseFramesAutomatically = releaseFramesAutomatically;
checkState(!shaderPrograms.isEmpty());
checkState(shaderPrograms.get(0) instanceof ExternalShaderProgram);
checkState(getLast(shaderPrograms) instanceof FinalMatrixShaderProgramWrapper);
ExternalShaderProgram inputExternalShaderProgram =
(ExternalShaderProgram) shaderPrograms.get(0);
GlShaderProgram inputShaderProgram = shaderPrograms.get(0);
if (isInputExternal) {
checkState(inputShaderProgram instanceof ExternalShaderProgram);
inputExternalTextureManager =
new ExternalTextureManager(inputExternalShaderProgram, frameProcessingTaskExecutor);
inputExternalShaderProgram.setInputListener(inputExternalTextureManager);
inputSurface = new Surface(inputExternalTextureManager.getSurfaceTexture());
new ExternalTextureManager(
(ExternalShaderProgram) inputShaderProgram, frameProcessingTaskExecutor);
inputShaderProgram.setInputListener(inputExternalTextureManager);
inputExternalSurface = new Surface(inputExternalTextureManager.getSurfaceTexture());
} else {
inputInternalTextureManager =
new InternalTextureManager(inputShaderProgram, frameProcessingTaskExecutor);
inputShaderProgram.setInputListener(inputInternalTextureManager);
}
finalShaderProgramWrapper = (FinalMatrixShaderProgramWrapper) getLast(shaderPrograms);
allShaderPrograms = shaderPrograms;
previousStreamOffsetUs = C.TIME_UNSET;
@ -400,19 +422,27 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
* call this method after instantiation to ensure that buffers are handled at full resolution. See
* {@link SurfaceTexture#setDefaultBufferSize(int, int)} for more information.
*
* <p>This method should only be used for when the {@link FrameProcessor} was created with {@link
* C#TRACK_TYPE_VIDEO} as the {@code inputTrackType}.
*
* @param width The default width for input buffers, in pixels.
* @param height The default height for input buffers, in pixels.
*/
public void setInputDefaultBufferSize(int width, int height) {
inputExternalTextureManager.getSurfaceTexture().setDefaultBufferSize(width, height);
checkNotNull(inputExternalTextureManager)
.getSurfaceTexture()
.setDefaultBufferSize(width, height);
}
@Override
public void queueInputBitmap(Bitmap inputBitmap, long durationUs, int frameRate) {}
public void queueInputBitmap(Bitmap inputBitmap, long durationUs, float frameRate) {
checkNotNull(inputInternalTextureManager)
.queueInputBitmap(inputBitmap, durationUs, frameRate, /* useHdr= */ false);
}
@Override
public Surface getInputSurface() {
return inputSurface;
return checkNotNull(inputExternalSurface);
}
@Override
@ -431,12 +461,12 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
checkStateNotNull(
nextInputFrameInfo, "setInputFrameInfo must be called before registering input frames");
inputExternalTextureManager.registerInputFrame(nextInputFrameInfo);
checkNotNull(inputExternalTextureManager).registerInputFrame(nextInputFrameInfo);
}
@Override
public int getPendingInputFrameCount() {
return inputExternalTextureManager.getPendingFrameCount();
return checkNotNull(inputExternalTextureManager).getPendingFrameCount();
}
@Override
@ -457,15 +487,20 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
public void signalEndOfInput() {
checkState(!inputStreamEnded);
inputStreamEnded = true;
if (inputInternalTextureManager != null) {
frameProcessingTaskExecutor.submit(inputInternalTextureManager::signalEndOfInput);
}
if (inputExternalTextureManager != null) {
frameProcessingTaskExecutor.submit(inputExternalTextureManager::signalEndOfInput);
}
}
@Override
public void flush() {
try {
frameProcessingTaskExecutor.flush();
CountDownLatch latch = new CountDownLatch(1);
inputExternalTextureManager.setOnFlushCompleteListener(latch::countDown);
checkNotNull(inputExternalTextureManager).setOnFlushCompleteListener(latch::countDown);
frameProcessingTaskExecutor.submit(finalShaderProgramWrapper::flush);
latch.await();
inputExternalTextureManager.setOnFlushCompleteListener(null);
@ -483,8 +518,10 @@ public final class GlEffectsFrameProcessor implements FrameProcessor {
Thread.currentThread().interrupt();
throw new IllegalStateException(unexpected);
}
if (inputExternalTextureManager != null) {
inputExternalTextureManager.release();
inputSurface.release();
checkNotNull(inputExternalSurface).release();
}
}
/**

159
libraries/effect/src/main/java/androidx/media3/effect/InternalTextureManager.java Normal file
View File

@ -0,0 +1,159 @@
/*
* Copyright 2023 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static java.lang.Math.round;
import android.graphics.Bitmap;
import android.opengl.GLES20;
import android.opengl.GLUtils;
import androidx.annotation.WorkerThread;
import androidx.media3.common.C;
import androidx.media3.common.FrameProcessingException;
import androidx.media3.common.FrameProcessor;
import androidx.media3.common.util.GlUtil;
import androidx.media3.common.util.UnstableApi;
import java.util.Queue;
import java.util.concurrent.LinkedBlockingQueue;
/** Forwards a frame produced from a {@link Bitmap} to a {@link GlShaderProgram} for consumption. */
@UnstableApi
/* package */ class InternalTextureManager implements GlShaderProgram.InputListener {
private final GlShaderProgram shaderProgram;
private final FrameProcessingTaskExecutor frameProcessingTaskExecutor;
private final Queue<BitmapFrameSequenceInfo> pendingBitmaps;
private int downstreamShaderProgramCapacity;
private int availableFrameCount;
private long currentPresentationTimeUs;
private long totalDurationUs;
private boolean inputEnded;
public InternalTextureManager(
GlShaderProgram shaderProgram, FrameProcessingTaskExecutor frameProcessingTaskExecutor) {
this.shaderProgram = shaderProgram;
this.frameProcessingTaskExecutor = frameProcessingTaskExecutor;
pendingBitmaps = new LinkedBlockingQueue<>();
}
@Override
public void onReadyToAcceptInputFrame() {
frameProcessingTaskExecutor.submit(
() -> {
downstreamShaderProgramCapacity++;
maybeQueueToShaderProgram();
});
}
@Override
public void onInputFrameProcessed(TextureInfo inputTexture) {
// TODO(b/262693274): Delete texture when last duplicate of the frame comes back from the shader
// program and change to only allocate one texId at a time. A change to method signature to
// include presentationTimeUs will probably be needed to do this.
frameProcessingTaskExecutor.submit(
() -> {
if (availableFrameCount == 0) {
signalEndOfInput();
}
});
}
public void queueInputBitmap(
Bitmap inputBitmap, long durationUs, float frameRate, boolean useHdr) {
frameProcessingTaskExecutor.submit(
() -> setupBitmap(inputBitmap, durationUs, frameRate, useHdr));
}
@WorkerThread
private void setupBitmap(Bitmap bitmap, long durationUs, float frameRate, boolean useHdr)
throws FrameProcessingException {
if (inputEnded) {
return;
}
try {
int bitmapTexId =
GlUtil.createTexture(
bitmap.getWidth(), bitmap.getHeight(), /* useHighPrecisionColorComponents= */ useHdr);
GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, bitmapTexId);
GLUtils.texImage2D(GLES20.GL_TEXTURE_2D, /* level= */ 0, bitmap, /* border= */ 0);
GlUtil.checkGlError();
TextureInfo textureInfo =
new TextureInfo(
bitmapTexId, /* fboId= */ C.INDEX_UNSET, bitmap.getWidth(), bitmap.getHeight());
int timeIncrementUs = round(C.MICROS_PER_SECOND / frameRate);
availableFrameCount += round((frameRate * durationUs) / C.MICROS_PER_SECOND);
totalDurationUs += durationUs;
pendingBitmaps.add(
new BitmapFrameSequenceInfo(textureInfo, timeIncrementUs, totalDurationUs));
} catch (GlUtil.GlException e) {
throw FrameProcessingException.from(e);
}
maybeQueueToShaderProgram();
}
@WorkerThread
private void maybeQueueToShaderProgram() {
if (inputEnded || availableFrameCount == 0 || downstreamShaderProgramCapacity == 0) {
return;
}
availableFrameCount--;
downstreamShaderProgramCapacity--;
BitmapFrameSequenceInfo currentFrame = checkNotNull(pendingBitmaps.peek());
shaderProgram.queueInputFrame(currentFrame.textureInfo, currentPresentationTimeUs);
currentPresentationTimeUs += currentFrame.timeIncrementUs;
if (currentPresentationTimeUs >= currentFrame.endPresentationTimeUs) {
pendingBitmaps.remove();
}
}
/**
* Signals the end of the input.
*
* @see FrameProcessor#signalEndOfInput()
*/
public void signalEndOfInput() {
frameProcessingTaskExecutor.submit(
() -> {
if (inputEnded) {
return;
}
inputEnded = true;
shaderProgram.signalEndOfCurrentInputStream();
});
}
/**
* Value class specifying information to generate all the frames associated with a specific {@link
* Bitmap}.
*/
private static final class BitmapFrameSequenceInfo {
public final TextureInfo textureInfo;
public final long timeIncrementUs;
public final long endPresentationTimeUs;
public BitmapFrameSequenceInfo(
TextureInfo textureInfo, long timeIncrementUs, long endPresentationTimeUs) {
this.textureInfo = textureInfo;
this.timeIncrementUs = timeIncrementUs;
this.endPresentationTimeUs = endPresentationTimeUs;
}
}
}

146
libraries/effect/src/main/java/androidx/media3/effect/MatrixShaderProgram.java
View File

@ -69,6 +69,8 @@ import java.util.List;
"shaders/fragment_shader_transformation_external_yuv_es3.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH =
"shaders/fragment_shader_transformation_sdr_external_es2.glsl";
private static final String FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH =
"shaders/fragment_shader_transformation_sdr_internal_es2.glsl";
private static final ImmutableList<float[]> NDC_SQUARE =
ImmutableList.of(
new float[] {-1, -1, 0, 1},
@ -163,6 +165,48 @@ import java.util.List;
useHdr);
}
/**
* Creates a new instance.
*
* <p>Input will be sampled from an internal (i.e. regular) texture.
*
* <p>Applies the {@linkplain ColorInfo#colorTransfer inputColorInfo EOTF} to convert from
* electrical color input, to intermediate optical {@link GlShaderProgram} color output, before
* {@code matrixTransformations} and {@code rgbMatrices} are applied. Also applies the {@linkplain
* ColorInfo#colorTransfer outputColorInfo OETF}, if needed, to convert back to an electrical
* color output.
*
* @param context The {@link Context}.
* @param matrixTransformations The {@link GlMatrixTransformation GlMatrixTransformations} to
* apply to each frame in order. Can be empty to apply no vertex transformations.
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param inputColorInfo The input electrical (nonlinear) {@link ColorInfo}.
* @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
* If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
* @throws FrameProcessingException If a problem occurs while reading shader files or an OpenGL
* operation fails or is unsupported.
*/
public static MatrixShaderProgram createWithInternalSampler(
Context context,
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo)
throws FrameProcessingException {
checkState(
!ColorInfo.isTransferHdr(inputColorInfo),
"MatrixShaderProgram doesn't support HDR internal sampler input yet.");
GlProgram glProgram =
createGlProgram(
context,
VERTEX_SHADER_TRANSFORMATION_PATH,
FRAGMENT_SHADER_TRANSFORMATION_SDR_INTERNAL_PATH);
return createWithSampler(
glProgram, matrixTransformations, rgbMatrices, inputColorInfo, outputColorInfo);
}
/**
* Creates a new instance.
*
@ -204,50 +248,12 @@ import java.util.List;
isInputTransferHdr
? FRAGMENT_SHADER_TRANSFORMATION_EXTERNAL_YUV_ES3_PATH
: FRAGMENT_SHADER_TRANSFORMATION_SDR_EXTERNAL_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
@C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer;
if (isInputTransferHdr) {
checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing input.");
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
inputColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
checkArgument(ColorInfo.isTransferHdr(inputColorInfo));
glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer);
// TODO(b/239735341): Add a setBooleanUniform method to GlProgram.
glProgram.setIntUniform(
"uApplyHdrToSdrToneMapping",
/* value= */ (outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020) ? 1 : 0);
checkArgument(
outputColorTransfer != Format.NO_VALUE && outputColorTransfer != C.COLOR_TRANSFER_SDR);
glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
} else {
checkArgument(
outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020,
"Converting from SDR to HDR is not supported.");
checkArgument(inputColorInfo.colorSpace == outputColorInfo.colorSpace);
checkArgument(
outputColorTransfer == C.COLOR_TRANSFER_SDR
|| outputColorTransfer == C.COLOR_TRANSFER_LINEAR);
// The SDR shader automatically applies an COLOR_TRANSFER_SDR EOTF.
glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
}
return new MatrixShaderProgram(
glProgram,
ImmutableList.copyOf(matrixTransformations),
ImmutableList.copyOf(rgbMatrices),
outputColorInfo.colorTransfer,
isInputTransferHdr);
return createWithSampler(
createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath),
matrixTransformations,
rgbMatrices,
inputColorInfo,
outputColorInfo);
}
/**
@ -305,6 +311,58 @@ import java.util.List;
outputIsHdr);
}
private static MatrixShaderProgram createWithSampler(
GlProgram glProgram,
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo)
throws FrameProcessingException {
boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
@C.ColorTransfer int outputColorTransfer = outputColorInfo.colorTransfer;
if (isInputTransferHdr) {
checkArgument(inputColorInfo.colorSpace == C.COLOR_SPACE_BT2020);
// In HDR editing mode the decoder output is sampled in YUV.
if (!GlUtil.isYuvTargetExtensionSupported()) {
throw new FrameProcessingException(
"The EXT_YUV_target extension is required for HDR editing input.");
}
glProgram.setFloatsUniform(
"uYuvToRgbColorTransform",
inputColorInfo.colorRange == C.COLOR_RANGE_FULL
? BT2020_FULL_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX
: BT2020_LIMITED_RANGE_YUV_TO_RGB_COLOR_TRANSFORM_MATRIX);
checkArgument(ColorInfo.isTransferHdr(inputColorInfo));
glProgram.setIntUniform("uInputColorTransfer", inputColorInfo.colorTransfer);
// TODO(b/239735341): Add a setBooleanUniform method to GlProgram.
glProgram.setIntUniform(
"uApplyHdrToSdrToneMapping",
/* value= */ (outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020) ? 1 : 0);
checkArgument(
outputColorTransfer != Format.NO_VALUE && outputColorTransfer != C.COLOR_TRANSFER_SDR);
glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
} else {
checkArgument(
outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020,
"Converting from SDR to HDR is not supported.");
checkArgument(inputColorInfo.colorSpace == outputColorInfo.colorSpace);
checkArgument(
outputColorTransfer == C.COLOR_TRANSFER_SDR
|| outputColorTransfer == C.COLOR_TRANSFER_LINEAR);
// The SDR shader automatically applies an COLOR_TRANSFER_SDR EOTF.
glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
}
return new MatrixShaderProgram(
glProgram,
ImmutableList.copyOf(matrixTransformations),
ImmutableList.copyOf(rgbMatrices),
outputColorInfo.colorTransfer,
isInputTransferHdr);
}
/**
* Creates a new instance.
*

BIN
libraries/test_data/src/test/assets/media/bitmap/sample_mp4_first_frame/electrical_colors/image_input_with_wrapped_crop.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 192 KiB

33
libraries/test_utils/src/main/java/androidx/media3/test/utils/FrameProcessorTestRunner.java
View File

@ -62,10 +62,12 @@ public final class FrameProcessorTestRunner {
private float pixelWidthHeightRatio;
private @MonotonicNonNull ColorInfo inputColorInfo;
private @MonotonicNonNull ColorInfo outputColorInfo;
private @C.TrackType int inputTrackType;
/** Creates a new instance with default values. */
public Builder() {
pixelWidthHeightRatio = DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO;
inputTrackType = C.TRACK_TYPE_VIDEO;
}
/**
@ -168,6 +170,16 @@ public final class FrameProcessorTestRunner {
this.outputColorInfo = outputColorInfo;
return this;
}
/**
* Sets the input track type. See {@link FrameProcessor.Factory#create}.
*
* <p>The default value is {@link C#TRACK_TYPE_VIDEO}.
*/
@CanIgnoreReturnValue
public Builder setInputTrackType(@C.TrackType int inputTrackType) {
this.inputTrackType = inputTrackType;
return this;
}
public FrameProcessorTestRunner build() throws FrameProcessingException {
checkStateNotNull(testId, "testId must be set.");
@ -182,7 +194,8 @@ public final class FrameProcessorTestRunner {
effects == null ? ImmutableList.of() : effects,
pixelWidthHeightRatio,
inputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : inputColorInfo,
outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo);
outputColorInfo == null ? ColorInfo.SDR_BT709_LIMITED : outputColorInfo,
inputTrackType);
}
}
@ -211,7 +224,8 @@ public final class FrameProcessorTestRunner {
ImmutableList<Effect> effects,
float pixelWidthHeightRatio,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo)
ColorInfo outputColorInfo,
@C.TrackType int inputTrackType)
throws FrameProcessingException {
this.testId = testId;
this.videoAssetPath = videoAssetPath;
@ -226,7 +240,7 @@ public final class FrameProcessorTestRunner {
DebugViewProvider.NONE,
inputColorInfo,
outputColorInfo,
C.TRACK_TYPE_VIDEO,
inputTrackType,
/* releaseFramesAutomatically= */ true,
MoreExecutors.directExecutor(),
new FrameProcessor.Listener() {
@ -278,6 +292,19 @@ public final class FrameProcessorTestRunner {
}
},
frameProcessor.getInputSurface());
return endFrameProcessingAndGetImage();
}
public Bitmap processImageFrameAndEnd(Bitmap inputBitmap) throws Exception {
frameProcessor.setInputFrameInfo(
new FrameInfo.Builder(inputBitmap.getWidth(), inputBitmap.getHeight())
.setPixelWidthHeightRatio(pixelWidthHeightRatio)
.build());
frameProcessor.queueInputBitmap(inputBitmap, C.MICROS_PER_SECOND, /* frameRate= */ 1);
return endFrameProcessingAndGetImage();
}
private Bitmap endFrameProcessingAndGetImage() throws Exception {
frameProcessor.signalEndOfInput();
Thread.sleep(FRAME_PROCESSING_WAIT_MS);