loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Effect:migrate setEnableColorTransfers to setSdrWorkingColorSpace

Browse Source 
Part of a two stage change to remove the conversion to linear colors in the SDR effects pipeline by default. Changes the boolean to an intdef, introducing a third option that gets all sdr input into the same colorspace.

This is a planned API breaking change, but this change should not change the behavior of the pipeline.

PiperOrigin-RevId: 629013747
This commit is contained in:
tofunmi 2024-04-29 03:24:19 -07:00 committed by Copybara-Service
parent 43de3b67cf
commit 7089253bef
9 changed files with 222 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libraries/effect/src/androidTest/java/androidx/media3/effect/DefaultVideoFrameProcessorPixelTest.java
View File

@ -17,6 +17,7 @@ package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkStateNotNull;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL;
import static androidx.media3.test.utils.BitmapPixelTestUtil.MAXIMUM_AVERAGE_PIXEL_ABSOLUTE_DIFFERENCE;
import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
@ -145,7 +146,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setVideoFrameProcessorFactory(
new DefaultVideoFrameProcessor.Factory.Builder()
.setEnableColorTransfers(false)
.setSdrWorkingColorSpace(WORKING_COLOR_SPACE_ORIGINAL)
.build())
.build();
Bitmap expectedBitmap = readBitmap(ORIGINAL_PNG_ASSET_PATH);
@ -209,7 +210,7 @@ public final class DefaultVideoFrameProcessorPixelTest {
getDefaultFrameProcessorTestRunnerBuilder(testId)
.setVideoFrameProcessorFactory(
new DefaultVideoFrameProcessor.Factory.Builder()
.setEnableColorTransfers(false)
.setSdrWorkingColorSpace(WORKING_COLOR_SPACE_ORIGINAL)
.build())
.setEffects(NO_OP_EFFECT)
.build();

83
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_external_es2.glsl
View File

@ -16,8 +16,9 @@
// ES 2 fragment shader that:
// 1. Samples from an external texture with uTexSampler copying from this
// texture to the current output.
// 2. Transforms the electrical colors to optical colors using the SMPTE 170M
// EOTF.
// 2. Transforms the electrical colors to "working" colors which is the input
// colorspace with the colors transferred to either linear or SMPTE 170M as
// requested by uSdrWorkingColorSpace.
// 3. Applies a 4x4 RGB color matrix to change the pixel colors.
// 4. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR
// for outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to
@ -31,12 +32,16 @@ varying vec2 vTexSamplingCoord;
// C.java#ColorTransfer value.
// Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
uniform int uOutputColorTransfer;
uniform int uEnableColorTransfer;
uniform int uSdrWorkingColorSpace;
const float inverseGamma = 0.4500;
const float gamma = 1.0 / inverseGamma;
const int GL_FALSE = 0;
const int GL_TRUE = 1;
// LINT.IfChange(working_color_space)
const int WORKING_COLOR_SPACE_DEFAULT = 0;
const int WORKING_COLOR_SPACE_ORIGINAL = 1;
const int WORKING_COLOR_SPACE_LINEAR = 2;
// Transforms a single channel from electrical to optical SDR using the SMPTE
// 170M OETF.
@ -71,39 +76,57 @@ vec3 smpte170mOetf(vec3 opticalColor) {
smpte170mOetfSingleChannel(opticalColor.b));
}
// Applies the appropriate OETF to convert linear optical signals to nonlinear
// electrical signals. Input and output are both normalized to [0, 1].
highp vec3 applyOetf(highp vec3 linearColor) {
// LINT.IfChange(color_transfer)
const int COLOR_TRANSFER_LINEAR = 1;
const int COLOR_TRANSFER_SDR_VIDEO = 3;
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR ||
uEnableColorTransfer == GL_FALSE) {
return linearColor;
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mOetf(linearColor);
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
}
vec3 applyEotf(vec3 electricalColor) {
if (uEnableColorTransfer == GL_TRUE) {
return smpte170mEotf(electricalColor);
} else if (uEnableColorTransfer == GL_FALSE) {
return electricalColor;
// Optionally applies the appropriate EOTF to convert nonlinear electrical
// signals to linear optical signals. Input and output are both normalized to
// [0, 1].
vec3 convertToWorkingColors(vec3 inputColor) {
if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_DEFAULT ||
uSdrWorkingColorSpace == WORKING_COLOR_SPACE_ORIGINAL) {
return inputColor;
} else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) {
return smpte170mEotf(inputColor);
} else {
// Output blue as an obviously visible error.
return vec3(0.0, 0.0, 1.0);
}
}
// Optionally applies the appropriate OETF to convert linear optical signals to
// nonlinear electrical signals. Input and output are both normalized to [0, 1].
highp vec3 convertToOutputColors(highp vec3 workingColors) {
// LINT.IfChange(color_transfer)
const int COLOR_TRANSFER_LINEAR = 1;
const int COLOR_TRANSFER_SDR_VIDEO = 3;
if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_DEFAULT) {
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
return smpte170mEotf(workingColors);
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return workingColors;
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
} else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_ORIGINAL) {
return workingColors;
} else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) {
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
return workingColors;
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mOetf(workingColors);
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
}
void main() {
vec4 inputColor = texture2D(uTexSampler, vTexSamplingCoord);
vec3 linearInputColor = applyEotf(inputColor.rgb);
vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);
gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a);
vec3 workingColors = convertToWorkingColors(inputColor.rgb);
vec4 transformedColors = uRgbMatrix * vec4(workingColors, 1);
gl_FragColor =
vec4(convertToOutputColors(transformedColors.rgb), inputColor.a);
}

79
libraries/effect/src/main/assets/shaders/fragment_shader_transformation_sdr_internal_es2.glsl
View File

@ -17,8 +17,9 @@
// 1. Samples from an input texture created from an internal texture (e.g. a
// texture created from a bitmap), with uTexSampler copying from this texture
// to the current output.
// 2. Transforms the electrical colors to optical colors using the SMPTE 170M
// EOTF or the sRGB EOTF, as requested by uInputColorTransfer.
// 2. Transforms the electrical colors to "working" colors which is the input
// colorspace with the colors transferred to either linear or SMPTE 170M as
// requested by uSdrWorkingColorSpace.
// 3. Applies a 4x4 RGB color matrix to change the pixel colors.
// 4. Outputs as requested by uOutputColorTransfer. Use COLOR_TRANSFER_LINEAR
// for outputting to intermediate shaders, or COLOR_TRANSFER_SDR_VIDEO to
@ -34,7 +35,7 @@ uniform int uInputColorTransfer;
// C.java#ColorTransfer value.
// Only COLOR_TRANSFER_LINEAR and COLOR_TRANSFER_SDR_VIDEO are allowed.
uniform int uOutputColorTransfer;
uniform int uEnableColorTransfer;
uniform int uSdrWorkingColorSpace;
const float inverseGamma = 0.4500;
const float gamma = 1.0 / inverseGamma;
@ -44,6 +45,10 @@ const int GL_TRUE = 1;
const int COLOR_TRANSFER_LINEAR = 1;
const int COLOR_TRANSFER_SRGB = 2;
const int COLOR_TRANSFER_SDR_VIDEO = 3;
// LINT.IfChange(working_color_space)
const int WORKING_COLOR_SPACE_DEFAULT = 0;
const int WORKING_COLOR_SPACE_ORIGINAL = 1;
const int WORKING_COLOR_SPACE_LINEAR = 2;
// Transforms a single channel from electrical to optical SDR using the sRGB
// EOTF.
@ -56,7 +61,7 @@ float srgbEotfSingleChannel(float electricalChannel) {
}
// Transforms electrical to optical SDR using the sRGB EOTF.
vec3 srgbEotf(const vec3 electricalColor) {
vec3 srgbEotf(vec3 electricalColor) {
return vec3(srgbEotfSingleChannel(electricalColor.r),
srgbEotfSingleChannel(electricalColor.g),
srgbEotfSingleChannel(electricalColor.b));
@ -94,34 +99,60 @@ vec3 smpte170mOetf(vec3 opticalColor) {
smpte170mOetfSingleChannel(opticalColor.g),
smpte170mOetfSingleChannel(opticalColor.b));
}
// Applies the appropriate EOTF to convert nonlinear electrical signals to
// linear optical signals. Input and output are both normalized to [0, 1].
vec3 applyEotf(vec3 electricalColor) {
if (uEnableColorTransfer == GL_TRUE) {
// Optionally applies the appropriate EOTF to convert nonlinear electrical
// signals to linear optical signals. Input and output are both normalized to
// [0, 1].
vec3 convertToWorkingColors(vec3 inputColor) {
if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_DEFAULT) {
if (uInputColorTransfer == COLOR_TRANSFER_SRGB) {
return srgbEotf(electricalColor);
return smpte170mOetf(srgbEotf(inputColor));
} else if (uInputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mEotf(electricalColor);
return inputColor;
} else {
// Output blue as an obviously visible error.
return vec3(0.0, 0.0, 1.0);
}
} else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_ORIGINAL) {
return inputColor;
} else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) {
if (uInputColorTransfer == COLOR_TRANSFER_SRGB) {
return srgbEotf(inputColor);
} else if (uInputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mEotf(inputColor);
} else {
// Output blue as an obviously visible error.
return vec3(0.0, 0.0, 1.0);
}
} else if (uEnableColorTransfer == GL_FALSE) {
return electricalColor;
} else {
// Output blue as an obviously visible error.
return vec3(0.0, 0.0, 1.0);
}
}
// Applies the appropriate OETF to convert linear optical signals to nonlinear
// electrical signals. Input and output are both normalized to [0, 1].
highp vec3 applyOetf(highp vec3 linearColor) {
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR ||
uEnableColorTransfer == GL_FALSE) {
return linearColor;
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mOetf(linearColor);
// Optionally applies the appropriate OETF to convert linear optical signals to
// nonlinear electrical signals. Input and output are both normalized to [0, 1].
highp vec3 convertToOutputColors(highp vec3 workingColors) {
if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_DEFAULT) {
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
return smpte170mEotf(workingColors);
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return workingColors;
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
} else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_ORIGINAL) {
return workingColors;
} else if (uSdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) {
if (uOutputColorTransfer == COLOR_TRANSFER_LINEAR) {
return workingColors;
} else if (uOutputColorTransfer == COLOR_TRANSFER_SDR_VIDEO) {
return smpte170mOetf(workingColors);
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
}
} else {
// Output red as an obviously visible error.
return vec3(1.0, 0.0, 0.0);
@ -143,8 +174,8 @@ vec2 getAdjustedTexSamplingCoord(vec2 originalTexSamplingCoord) {
void main() {
vec4 inputColor =
texture2D(uTexSampler, getAdjustedTexSamplingCoord(vTexSamplingCoord));
vec3 linearInputColor = applyEotf(inputColor.rgb);
vec4 transformedColors = uRgbMatrix * vec4(linearInputColor, 1);
gl_FragColor = vec4(applyOetf(transformedColors.rgb), inputColor.a);
vec3 workingColors = convertToWorkingColors(inputColor.rgb);
vec4 transformedColors = uRgbMatrix * vec4(workingColors, 1);
gl_FragColor =
vec4(convertToOutputColors(transformedColors.rgb), inputColor.a);
}

31
libraries/effect/src/main/java/androidx/media3/effect/DefaultShaderProgram.java
View File

@ -20,6 +20,7 @@ import static android.opengl.GLES20.GL_TRUE;
import static androidx.media3.common.VideoFrameProcessor.INPUT_TYPE_BITMAP;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR;
import android.content.Context;
import android.graphics.Bitmap;
@ -38,6 +39,7 @@ import androidx.media3.common.util.GlUtil.GlException;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.UnstableApi;
import androidx.media3.common.util.Util;
import androidx.media3.effect.DefaultVideoFrameProcessor.WorkingColorSpace;
import com.google.common.collect.ImmutableList;
import java.io.IOException;
import java.util.Arrays;
@ -182,6 +184,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
createGlProgram(
context, VERTEX_SHADER_TRANSFORMATION_PATH, FRAGMENT_SHADER_TRANSFORMATION_PATH);
// TODO: b/263306471 - when default working color space changes to WORKING_COLOR_SPACE_DEFAULT,
// make sure no color transfers are applied in shader.
// No transfer functions needed, because input and output are both optical colors.
return new DefaultShaderProgram(
glProgram,
@ -206,8 +211,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
* If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
* @param enableColorTransfers Whether to transfer colors to an intermediate color space when
* applying effects. If the input or output is HDR, this must be {@code true}.
* @param sdrWorkingColorSpace The {@link WorkingColorSpace} to apply effects in.
* @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
* OpenGL operation fails or is unsupported.
*/
@ -215,7 +219,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Context context,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
@WorkingColorSpace int sdrWorkingColorSpace,
@InputType int inputType)
throws VideoFrameProcessingException {
checkState(
@ -242,7 +246,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
"uApplyHdrToSdrToneMapping",
outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020 ? GL_TRUE : GL_FALSE);
}
return createWithSampler(glProgram, inputColorInfo, outputColorInfo, enableColorTransfers);
return createWithSampler(glProgram, inputColorInfo, outputColorInfo, sdrWorkingColorSpace);
}
/**
@ -262,8 +266,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param outputColorInfo The output electrical (nonlinear) or optical (linear) {@link ColorInfo}.
* If this is an optical color, it must be BT.2020 if {@code inputColorInfo} is {@linkplain
* ColorInfo#isTransferHdr(ColorInfo) HDR}, and RGB BT.709 if not.
* @param enableColorTransfers Whether to transfer colors to an intermediate color space when
* applying effects. If the input or output is HDR, this must be {@code true}.
* @param sdrWorkingColorSpace The {@link WorkingColorSpace} to apply effects in.
* @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
* OpenGL operation fails or is unsupported.
*/
@ -271,7 +274,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Context context,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers)
@WorkingColorSpace int sdrWorkingColorSpace)
throws VideoFrameProcessingException {
boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
String vertexShaderFilePath =
@ -300,7 +303,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputColorInfo.colorSpace != C.COLOR_SPACE_BT2020 ? GL_TRUE : GL_FALSE);
}
return createWithSampler(glProgram, inputColorInfo, outputColorInfo, enableColorTransfers);
return createWithSampler(glProgram, inputColorInfo, outputColorInfo, sdrWorkingColorSpace);
}
/**
@ -319,6 +322,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
* @param rgbMatrices The {@link RgbMatrix RgbMatrices} to apply to each frame in order. Can be
* empty to apply no color transformations.
* @param outputColorInfo The electrical (non-linear) {@link ColorInfo} describing output colors.
* @param sdrWorkingColorSpace The {@link WorkingColorSpace} to apply effects in.
* @throws VideoFrameProcessingException If a problem occurs while reading shader files or an
* OpenGL operation fails or is unsupported.
*/
@ -327,15 +331,16 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
List<GlMatrixTransformation> matrixTransformations,
List<RgbMatrix> rgbMatrices,
ColorInfo outputColorInfo,
boolean enableColorTransfers)
@WorkingColorSpace int sdrWorkingColorSpace)
throws VideoFrameProcessingException {
boolean outputIsHdr = ColorInfo.isTransferHdr(outputColorInfo);
boolean shouldApplyOetf = sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR;
String vertexShaderFilePath =
outputIsHdr ? VERTEX_SHADER_TRANSFORMATION_ES3_PATH : VERTEX_SHADER_TRANSFORMATION_PATH;
String fragmentShaderFilePath =
outputIsHdr
? FRAGMENT_SHADER_OETF_ES3_PATH
: enableColorTransfers
: shouldApplyOetf
? FRAGMENT_SHADER_TRANSFORMATION_SDR_OETF_ES2_PATH
: FRAGMENT_SHADER_TRANSFORMATION_PATH;
GlProgram glProgram = createGlProgram(context, vertexShaderFilePath, fragmentShaderFilePath);
@ -346,7 +351,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
outputColorTransfer == C.COLOR_TRANSFER_HLG
|| outputColorTransfer == C.COLOR_TRANSFER_ST2084);
glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
} else if (enableColorTransfers) {
} else if (shouldApplyOetf) {
checkArgument(
outputColorTransfer == C.COLOR_TRANSFER_SDR
|| outputColorTransfer == C.COLOR_TRANSFER_GAMMA_2_2);
@ -365,7 +370,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlProgram glProgram,
ColorInfo inputColorInfo,
ColorInfo outputColorInfo,
boolean enableColorTransfers) {
@WorkingColorSpace int sdrWorkingColorSpace) {
boolean isInputTransferHdr = ColorInfo.isTransferHdr(inputColorInfo);
boolean isExpandingColorGamut =
(inputColorInfo.colorSpace == C.COLOR_SPACE_BT709
@ -384,7 +389,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
} else if (isExpandingColorGamut) {
glProgram.setIntUniform("uOutputColorTransfer", outputColorTransfer);
} else {
glProgram.setIntUniform("uEnableColorTransfer", enableColorTransfers ? GL_TRUE : GL_FALSE);
glProgram.setIntUniform("uSdrWorkingColorSpace", sdrWorkingColorSpace);
checkArgument(
outputColorTransfer == C.COLOR_TRANSFER_SDR
|| outputColorTransfer == C.COLOR_TRANSFER_LINEAR);

98
libraries/effect/src/main/java/androidx/media3/effect/DefaultVideoFrameProcessor.java
View File

@ -25,6 +25,7 @@ import static androidx.media3.effect.DebugTraceUtil.EVENT_VFP_REGISTER_NEW_INPUT
import static androidx.media3.effect.DebugTraceUtil.EVENT_VFP_SIGNAL_ENDED;
import static androidx.media3.effect.DebugTraceUtil.logEvent;
import static com.google.common.collect.Iterables.getFirst;
import static java.lang.annotation.ElementType.TYPE_USE;
import android.content.Context;
import android.graphics.Bitmap;
@ -35,6 +36,7 @@ import android.opengl.GLES20;
import android.opengl.GLES30;
import android.view.Surface;
import androidx.annotation.GuardedBy;
import androidx.annotation.IntDef;
import androidx.annotation.IntRange;
import androidx.annotation.Nullable;
import androidx.annotation.VisibleForTesting;
@ -57,6 +59,10 @@ import androidx.media3.common.util.Util;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.MoreExecutors;
import com.google.errorprone.annotations.CanIgnoreReturnValue;
import java.lang.annotation.Documented;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
@ -80,13 +86,52 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
void release(long presentationTimeUs);
}
// LINT.IfChange(working_color_space)
/**
* Specifies the color space that frames passed to intermediate {@link GlShaderProgram}s will be
* represented in.
*/
@Documented
@Retention(RetentionPolicy.SOURCE)
@Target(TYPE_USE)
@IntDef({WORKING_COLOR_SPACE_DEFAULT, WORKING_COLOR_SPACE_ORIGINAL, WORKING_COLOR_SPACE_LINEAR})
public @interface WorkingColorSpace {}
/**
* Use BT709 color primaries with the standard SDR transfer function (SMPTE 170m) as the working
* color space.
*
* <p>Any SDR content in a different color space will be transferred to this one.
*/
public static final int WORKING_COLOR_SPACE_DEFAULT = 0;
/**
* Use the original color space of the input as the working color space when the input is SDR.
*
* <p>Tonemapped HDR content will be represented with BT709 color primaries and the standard SDR
* transfer function (SMPTE 170m).
*
* <p>No color transfers will be applied when the input is SDR.
*/
public static final int WORKING_COLOR_SPACE_ORIGINAL = 1;
/**
* The working color space will have the same primaries as the input and a linear transfer
* function.
*
* <p>This option is not recommended for SDR content since it may lead to color banding since
* 8-bit colors are used in SDR processing. It may also cause effects that modify a frame's output
* colors (for example {@linkplain OverlayEffect overlays}) to have incorrect output colors.
*/
public static final int WORKING_COLOR_SPACE_LINEAR = 2;
/** A factory for {@link DefaultVideoFrameProcessor} instances. */
public static final class Factory implements VideoFrameProcessor.Factory {
private static final String THREAD_NAME = "Effect:DefaultVideoFrameProcessor:GlThread";
/** A builder for {@link DefaultVideoFrameProcessor.Factory} instances. */
public static final class Builder {
private boolean enableColorTransfers;
private @WorkingColorSpace int sdrWorkingColorSpace;
@Nullable private ExecutorService executorService;
private @MonotonicNonNull GlObjectsProvider glObjectsProvider;
private GlTextureProducer.@MonotonicNonNull Listener textureOutputListener;
@ -95,12 +140,12 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
/** Creates an instance. */
public Builder() {
enableColorTransfers = true;
sdrWorkingColorSpace = WORKING_COLOR_SPACE_LINEAR;
requireRegisteringAllInputFrames = true;
}
private Builder(Factory factory) {
enableColorTransfers = factory.enableColorTransfers;
sdrWorkingColorSpace = factory.sdrWorkingColorSpace;
executorService = factory.executorService;
glObjectsProvider = factory.glObjectsProvider;
textureOutputListener = factory.textureOutputListener;
@ -108,22 +153,19 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
requireRegisteringAllInputFrames = !factory.repeatLastRegisteredFrame;
}
// TODO: b/263306471 - Change default to WORKING_COLOR_SPACE_DEFAULT.
/**
* Sets whether to transfer colors to an intermediate color space when applying effects.
* Sets the {@link WorkingColorSpace} in which frames passed to intermediate effects will be
* represented.
*
* <p>The default value is {@code true}.
* <p>The default value is {@link #WORKING_COLOR_SPACE_LINEAR}.
*
* <p>If the output is HDR, this is ignored as the working color space must have a linear
* transfer function.
*
* <p>If all input and output content will be SDR, it's recommended to set this value to
* {@code false}. This is because 8-bit colors in SDR may result in color banding.
*
* <p>This doesn't currently work with overlay effects (ex. {@link TextureOverlay}).
* <p>This setter doesn't affect the working color space for HDR output, since the working
* color space must have a linear transfer function for HDR output.
*/
@CanIgnoreReturnValue
public Builder setEnableColorTransfers(boolean enableColorTransfers) {
this.enableColorTransfers = enableColorTransfers;
public Builder setSdrWorkingColorSpace(@WorkingColorSpace int sdrWorkingColorSpace) {
this.sdrWorkingColorSpace = sdrWorkingColorSpace;
return this;
}
@ -215,7 +257,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
/** Builds an {@link DefaultVideoFrameProcessor.Factory} instance. */
public DefaultVideoFrameProcessor.Factory build() {
return new DefaultVideoFrameProcessor.Factory(
enableColorTransfers,
sdrWorkingColorSpace,
/* repeatLastRegisteredFrame= */ !requireRegisteringAllInputFrames,
glObjectsProvider == null ? new DefaultGlObjectsProvider() : glObjectsProvider,
executorService,
@ -224,7 +266,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
}
}
private final boolean enableColorTransfers;
private final @WorkingColorSpace int sdrWorkingColorSpace;
private final boolean repeatLastRegisteredFrame;
private final GlObjectsProvider glObjectsProvider;
@Nullable private final ExecutorService executorService;
@ -232,13 +274,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
private final int textureOutputCapacity;
private Factory(
boolean enableColorTransfers,
@WorkingColorSpace int sdrWorkingColorSpace,
boolean repeatLastRegisteredFrame,
GlObjectsProvider glObjectsProvider,
@Nullable ExecutorService executorService,
@Nullable GlTextureProducer.Listener textureOutputListener,
int textureOutputCapacity) {
this.enableColorTransfers = enableColorTransfers;
this.sdrWorkingColorSpace = sdrWorkingColorSpace;
this.repeatLastRegisteredFrame = repeatLastRegisteredFrame;
this.glObjectsProvider = glObjectsProvider;
this.executorService = executorService;
@ -298,7 +340,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
context,
debugViewProvider,
outputColorInfo,
enableColorTransfers,
sdrWorkingColorSpace,
renderFramesAutomatically,
videoFrameProcessingTaskExecutor,
listenerExecutor,
@ -488,10 +530,6 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
* {@link C#COLOR_TRANSFER_GAMMA_2_2}, for consistency with other tone-mapping and color behavior
* in the Android ecosystem (for example, MediaFormat's COLOR_TRANSFER_SDR_VIDEO is defined as
* SMPTE 170M, but most OEMs process it as Gamma 2.2).
*
* <p>If either {@link FrameInfo#colorInfo} or {@code outputColorInfo} {@linkplain
* ColorInfo#isTransferHdr} are HDR}, color transfers must {@linkplain
* Factory.Builder#setEnableColorTransfers be enabled}.
*/
@Override
public void registerInputStream(
@ -666,7 +704,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
Context context,
DebugViewProvider debugViewProvider,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
@WorkingColorSpace int sdrWorkingColorSpace,
boolean renderFramesAutomatically,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor videoFrameProcessorListenerExecutor,
@ -693,7 +731,9 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
ColorInfo intermediateColorInfo =
ColorInfo.isTransferHdr(outputColorInfo)
? linearColorInfo
: enableColorTransfers ? linearColorInfo : outputColorInfo;
: sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR
? linearColorInfo
: outputColorInfo;
InputSwitcher inputSwitcher =
new InputSwitcher(
context,
@ -702,7 +742,7 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
videoFrameProcessingTaskExecutor,
/* errorListenerExecutor= */ videoFrameProcessorListenerExecutor,
/* samplingShaderProgramErrorListener= */ listener::onError,
enableColorTransfers,
sdrWorkingColorSpace,
repeatLastRegisteredFrame);
FinalShaderProgramWrapper finalShaderProgramWrapper =
@ -712,13 +752,13 @@ public final class DefaultVideoFrameProcessor implements VideoFrameProcessor {
eglContext,
debugViewProvider,
outputColorInfo,
enableColorTransfers,
renderFramesAutomatically,
videoFrameProcessingTaskExecutor,
videoFrameProcessorListenerExecutor,
listener,
textureOutputListener,
textureOutputCapacity);
textureOutputCapacity,
sdrWorkingColorSpace,
renderFramesAutomatically);
return new DefaultVideoFrameProcessor(
context,

22
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -17,6 +17,7 @@ package androidx.media3.effect;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR;
import android.content.Context;
import android.opengl.EGL14;
@ -44,6 +45,7 @@ import androidx.media3.common.util.Log;
import androidx.media3.common.util.LongArrayQueue;
import androidx.media3.common.util.Size;
import androidx.media3.common.util.Util;
import androidx.media3.effect.DefaultVideoFrameProcessor.WorkingColorSpace;
import com.google.common.collect.ImmutableList;
import java.util.ArrayList;
import java.util.List;
@ -81,8 +83,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final EGLContext eglContext;
private final DebugViewProvider debugViewProvider;
private final ColorInfo outputColorInfo;
private final boolean enableColorTransfers;
private final boolean renderFramesAutomatically;
private final VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor;
private final Executor videoFrameProcessorListenerExecutor;
private final VideoFrameProcessor.Listener videoFrameProcessorListener;
@ -91,6 +91,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final LongArrayQueue outputTextureTimestamps; // Synchronized with outputTexturePool.
private final LongArrayQueue syncObjects;
@Nullable private final GlTextureProducer.Listener textureOutputListener;
private final @WorkingColorSpace int sdrWorkingColorSpace;
private final boolean renderFramesAutomatically;
private int inputWidth;
private int inputHeight;
@ -122,13 +124,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
EGLContext eglContext,
DebugViewProvider debugViewProvider,
ColorInfo outputColorInfo,
boolean enableColorTransfers,
boolean renderFramesAutomatically,
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor videoFrameProcessorListenerExecutor,
VideoFrameProcessor.Listener videoFrameProcessorListener,
@Nullable GlTextureProducer.Listener textureOutputListener,
int textureOutputCapacity) {
@Nullable Listener textureOutputListener,
int textureOutputCapacity,
@WorkingColorSpace int sdrWorkingColorSpace,
boolean renderFramesAutomatically) {
this.context = context;
this.matrixTransformations = new ArrayList<>();
this.rgbMatrices = new ArrayList<>();
@ -136,12 +138,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.eglContext = eglContext;
this.debugViewProvider = debugViewProvider;
this.outputColorInfo = outputColorInfo;
this.enableColorTransfers = enableColorTransfers;
this.renderFramesAutomatically = renderFramesAutomatically;
this.videoFrameProcessingTaskExecutor = videoFrameProcessingTaskExecutor;
this.videoFrameProcessorListenerExecutor = videoFrameProcessorListenerExecutor;
this.videoFrameProcessorListener = videoFrameProcessorListener;
this.textureOutputListener = textureOutputListener;
this.sdrWorkingColorSpace = sdrWorkingColorSpace;
this.renderFramesAutomatically = renderFramesAutomatically;
inputListener = new InputListener() {};
availableFrames = new ConcurrentLinkedQueue<>();
@ -525,7 +527,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
expandedMatrixTransformations,
rgbMatrices,
outputColorInfo,
enableColorTransfers);
sdrWorkingColorSpace);
Size outputSize = defaultShaderProgram.configure(inputWidth, inputHeight);
if (outputSurfaceInfo != null) {
@ -545,7 +547,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
.maybeRenderToSurfaceView(
() -> {
GlUtil.clearFocusedBuffers();
if (enableColorTransfers) {
if (sdrWorkingColorSpace == WORKING_COLOR_SPACE_LINEAR) {
@C.ColorTransfer
int configuredColorTransfer = defaultShaderProgram.getOutputColorTransfer();
defaultShaderProgram.setOutputColorTransfer(

11
libraries/effect/src/main/java/androidx/media3/effect/InputSwitcher.java
View File

@ -35,6 +35,7 @@ import androidx.media3.common.GlTextureInfo;
import androidx.media3.common.OnInputFrameProcessedListener;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.effect.DefaultVideoFrameProcessor.WorkingColorSpace;
import java.util.concurrent.Executor;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.checker.nullness.qual.Nullable;
@ -51,7 +52,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
private final GlShaderProgram.ErrorListener samplingShaderProgramErrorListener;
private final Executor errorListenerExecutor;
private final SparseArray<Input> inputs;
private final boolean enableColorTransfers;
private final @WorkingColorSpace int sdrWorkingColorSpace;
private @MonotonicNonNull GlShaderProgram downstreamShaderProgram;
private @MonotonicNonNull TextureManager activeTextureManager;
@ -63,7 +64,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
VideoFrameProcessingTaskExecutor videoFrameProcessingTaskExecutor,
Executor errorListenerExecutor,
GlShaderProgram.ErrorListener samplingShaderProgramErrorListener,
boolean enableColorTransfers,
@WorkingColorSpace int sdrWorkingColorSpace,
boolean repeatLastRegisteredFrame)
throws VideoFrameProcessingException {
this.context = context;
@ -73,7 +74,7 @@ import org.checkerframework.checker.nullness.qual.Nullable;
this.errorListenerExecutor = errorListenerExecutor;
this.samplingShaderProgramErrorListener = samplingShaderProgramErrorListener;
this.inputs = new SparseArray<>();
this.enableColorTransfers = enableColorTransfers;
this.sdrWorkingColorSpace = sdrWorkingColorSpace;
// TODO(b/274109008): Investigate lazy instantiating the texture managers.
inputs.put(
@ -98,13 +99,13 @@ import org.checkerframework.checker.nullness.qual.Nullable;
case INPUT_TYPE_SURFACE:
samplingShaderProgram =
DefaultShaderProgram.createWithExternalSampler(
context, inputColorInfo, outputColorInfo, enableColorTransfers);
context, inputColorInfo, outputColorInfo, sdrWorkingColorSpace);
break;
case INPUT_TYPE_BITMAP:
case INPUT_TYPE_TEXTURE_ID:
samplingShaderProgram =
DefaultShaderProgram.createWithInternalSampler(
context, inputColorInfo, outputColorInfo, enableColorTransfers, inputType);
context, inputColorInfo, outputColorInfo, sdrWorkingColorSpace, inputType);
break;
default:
throw new VideoFrameProcessingException("Unsupported input type " + inputType);

5
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/HdrEditingTest.java
View File

@ -15,6 +15,7 @@
*/
package androidx.media3.transformer.mh;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL;
import static androidx.media3.test.utils.TestUtil.retrieveTrackFormat;
import static androidx.media3.transformer.AndroidTestUtil.FORCE_TRANSCODE_VIDEO_EFFECTS;
import static androidx.media3.transformer.AndroidTestUtil.MP4_ASSET_1080P_5_SECOND_HLG10;
@ -210,7 +211,7 @@ public final class HdrEditingTest {
}
@Test
public void exportAndTranscodeHdr_withDisabledColorTransfers_whenHdrEditingIsSupported()
public void exportAndTranscodeHdr_ignoringSdrWorkingColorSpace_whenHdrEditingIsSupported()
throws Exception {
Context context = ApplicationProvider.getApplicationContext();
Format format = MP4_ASSET_1080P_5_SECOND_HLG10_FORMAT;
@ -222,7 +223,7 @@ public final class HdrEditingTest {
new Transformer.Builder(context)
.setVideoFrameProcessorFactory(
new DefaultVideoFrameProcessor.Factory.Builder()
.setEnableColorTransfers(false)
.setSdrWorkingColorSpace(WORKING_COLOR_SPACE_ORIGINAL)
.build())
.build();
EditedMediaItem editedMediaItem =

5
libraries/transformer/src/androidTest/java/androidx/media3/transformer/mh/ToneMapHdrToSdrUsingOpenGlPixelTest.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer.mh;
import static androidx.media3.common.MimeTypes.VIDEO_H265;
import static androidx.media3.effect.DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_ORIGINAL;
import static androidx.media3.test.utils.BitmapPixelTestUtil.getBitmapAveragePixelAbsoluteDifferenceArgb8888;
import static androidx.media3.test.utils.BitmapPixelTestUtil.readBitmap;
import static androidx.media3.transformer.AndroidTestUtil.recordTestSkipped;
@ -280,14 +281,14 @@ public final class ToneMapHdrToSdrUsingOpenGlPixelTest {
}
@Test
public void toneMap_withDisabledColorTransfers_matchesGoldenFile() throws Exception {
public void toneMap_withWorkingColorSpaceSetToOriginal_matchesGoldenFile() throws Exception {
assumeDeviceSupportsOpenGlToneMapping(testId, HLG_ASSET_FORMAT);
videoFrameProcessorTestRunner =
new VideoFrameProcessorTestRunner.Builder()
.setTestId(testId)
.setVideoFrameProcessorFactory(
new DefaultVideoFrameProcessor.Factory.Builder()
.setEnableColorTransfers(false)
.setSdrWorkingColorSpace(WORKING_COLOR_SPACE_ORIGINAL)
.build())
.setVideoAssetPath(HLG_ASSET_STRING)
.setOutputColorInfo(TONE_MAP_SDR_COLOR)