loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Introduce GlEffect interface for effect specification.

Browse Source 
PiperOrigin-RevId: 446143537
This commit is contained in:
hschlueter 2022-05-03 10:21:10 +01:00 committed by Ian Baker
parent 90ce9a6787
commit b410a922fe
11 changed files with 159 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java
View File

@ -58,7 +58,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
public static final String ENABLE_FALLBACK = "enable_fallback";
public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping";
public static final String ENABLE_HDR_EDITING = "enable_hdr_editing";
public static final String DEMO_FRAME_PROCESSORS_SELECTIONS = "demo_frame_processors_selections";
public static final String DEMO_EFFECTS_SELECTIONS = "demo_effects_selections";
public static final String PERIODIC_VIGNETTE_CENTER_X = "periodic_vignette_center_x";
public static final String PERIODIC_VIGNETTE_CENTER_Y = "periodic_vignette_center_y";
public static final String PERIODIC_VIGNETTE_INNER_RADIUS = "periodic_vignette_inner_radius";
@ -91,7 +91,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"DASH stream with non-square pixels",
"MP4 with HDR (HDR10) H265 video (encoding may fail)",
};
private static final String[] DEMO_FRAME_PROCESSORS = {
private static final String[] DEMO_EFFECTS = {
"Dizzy crop", "Periodic vignette", "3D spin", "Overlay logo & timer", "Zoom in start"
};
private static final int PERIODIC_VIGNETTE_INDEX = 1;
@ -111,8 +111,8 @@ public final class ConfigurationActivity extends AppCompatActivity {
private @MonotonicNonNull CheckBox enableFallbackCheckBox;
private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox;
private @MonotonicNonNull CheckBox enableHdrEditingCheckBox;
private @MonotonicNonNull Button selectDemoFrameProcessorsButton;
private boolean @MonotonicNonNull [] demoFrameProcessorsSelections;
private @MonotonicNonNull Button selectDemoEffectsButton;
private boolean @MonotonicNonNull [] demoEffectsSelections;
private int inputUriPosition;
private float periodicVignetteCenterX;
private float periodicVignetteCenterY;
@ -187,9 +187,9 @@ public final class ConfigurationActivity extends AppCompatActivity {
findViewById(R.id.request_sdr_tone_mapping).setEnabled(isRequestSdrToneMappingSupported());
enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox);
demoFrameProcessorsSelections = new boolean[DEMO_FRAME_PROCESSORS.length];
selectDemoFrameProcessorsButton = findViewById(R.id.select_demo_frameprocessors_button);
selectDemoFrameProcessorsButton.setOnClickListener(this::selectFrameProcessors);
demoEffectsSelections = new boolean[DEMO_EFFECTS.length];
selectDemoEffectsButton = findViewById(R.id.select_demo_effects_button);
selectDemoEffectsButton.setOnClickListener(this::selectDemoEffects);
}
@Override
@ -220,7 +220,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"enableFallbackCheckBox",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
"demoFrameProcessorsSelections"
"demoEffectsSelections"
})
private void startTransformation(View view) {
Intent transformerIntent = new Intent(/* packageContext= */ this, TransformerActivity.class);
@ -255,7 +255,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
bundle.putBoolean(
ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked());
bundle.putBoolean(ENABLE_HDR_EDITING, enableHdrEditingCheckBox.isChecked());
bundle.putBooleanArray(DEMO_FRAME_PROCESSORS_SELECTIONS, demoFrameProcessorsSelections);
bundle.putBooleanArray(DEMO_EFFECTS_SELECTIONS, demoEffectsSelections);
bundle.putFloat(PERIODIC_VIGNETTE_CENTER_X, periodicVignetteCenterX);
bundle.putFloat(PERIODIC_VIGNETTE_CENTER_Y, periodicVignetteCenterY);
bundle.putFloat(PERIODIC_VIGNETTE_INNER_RADIUS, periodicVignetteInnerRadius);
@ -278,13 +278,11 @@ public final class ConfigurationActivity extends AppCompatActivity {
.show();
}
private void selectFrameProcessors(View view) {
private void selectDemoEffects(View view) {
new AlertDialog.Builder(/* context= */ this)
.setTitle(R.string.select_demo_frameprocessors)
.setTitle(R.string.select_demo_effects)
.setMultiChoiceItems(
DEMO_FRAME_PROCESSORS,
checkNotNull(demoFrameProcessorsSelections),
this::selectFrameProcessor)
DEMO_EFFECTS, checkNotNull(demoEffectsSelections), this::selectDemoEffect)
.setPositiveButton(android.R.string.ok, /* listener= */ null)
.create()
.show();
@ -296,9 +294,9 @@ public final class ConfigurationActivity extends AppCompatActivity {
selectedFileTextView.setText(URI_DESCRIPTIONS[inputUriPosition]);
}
@RequiresNonNull("demoFrameProcessorsSelections")
private void selectFrameProcessor(DialogInterface dialog, int which, boolean isChecked) {
demoFrameProcessorsSelections[which] = isChecked;
@RequiresNonNull("demoEffectsSelections")
private void selectDemoEffect(DialogInterface dialog, int which, boolean isChecked) {
demoEffectsSelections[which] = isChecked;
if (!isChecked || which != PERIODIC_VIGNETTE_INDEX) {
return;
}
@ -337,7 +335,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"rotateSpinner",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
"selectDemoFrameProcessorsButton"
"selectDemoEffectsButton"
})
private void onRemoveAudio(View view) {
if (((CheckBox) view).isChecked()) {
@ -357,7 +355,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"rotateSpinner",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
"selectDemoFrameProcessorsButton"
"selectDemoEffectsButton"
})
private void onRemoveVideo(View view) {
if (((CheckBox) view).isChecked()) {
@ -376,7 +374,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
"rotateSpinner",
"enableRequestSdrToneMappingCheckBox",
"enableHdrEditingCheckBox",
"selectDemoFrameProcessorsButton"
"selectDemoEffectsButton"
})
private void enableTrackSpecificOptions(boolean isAudioEnabled, boolean isVideoEnabled) {
audioMimeSpinner.setEnabled(isAudioEnabled);
@ -387,7 +385,7 @@ public final class ConfigurationActivity extends AppCompatActivity {
enableRequestSdrToneMappingCheckBox.setEnabled(
isRequestSdrToneMappingSupported() && isVideoEnabled);
enableHdrEditingCheckBox.setEnabled(isVideoEnabled);
selectDemoFrameProcessorsButton.setEnabled(isVideoEnabled);
selectDemoEffectsButton.setEnabled(isVideoEnabled);
findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled);
findViewById(R.id.video_mime_text_view).setEnabled(isVideoEnabled);

49
demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java
View File

@ -40,7 +40,7 @@ import androidx.media3.exoplayer.ExoPlayer;
import androidx.media3.exoplayer.util.DebugTextViewHelper;
import androidx.media3.transformer.DefaultEncoderFactory;
import androidx.media3.transformer.EncoderSelector;
import androidx.media3.transformer.GlFrameProcessor;
import androidx.media3.transformer.GlEffect;
import androidx.media3.transformer.ProgressHolder;
import androidx.media3.transformer.TransformationException;
import androidx.media3.transformer.TransformationRequest;
@ -240,35 +240,36 @@ public final class TransformerActivity extends AppCompatActivity {
EncoderSelector.DEFAULT,
/* enableFallback= */ bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK)));
ImmutableList.Builder<GlFrameProcessor> frameProcessors = new ImmutableList.Builder<>();
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
@Nullable
boolean[] selectedFrameProcessors =
bundle.getBooleanArray(ConfigurationActivity.DEMO_FRAME_PROCESSORS_SELECTIONS);
if (selectedFrameProcessors != null) {
if (selectedFrameProcessors[0]) {
frameProcessors.add(AdvancedFrameProcessorFactory.createDizzyCropFrameProcessor());
boolean[] selectedEffects =
bundle.getBooleanArray(ConfigurationActivity.DEMO_EFFECTS_SELECTIONS);
if (selectedEffects != null) {
if (selectedEffects[0]) {
effects.add(AdvancedFrameProcessorFactory::createDizzyCropFrameProcessor);
}
if (selectedFrameProcessors[1]) {
frameProcessors.add(
new PeriodicVignetteFrameProcessor(
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS),
/* maxInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS)));
if (selectedEffects[1]) {
effects.add(
() ->
new PeriodicVignetteFrameProcessor(
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_X),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_CENTER_Y),
/* minInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_INNER_RADIUS),
/* maxInnerRadius= */ bundle.getFloat(
ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS),
bundle.getFloat(ConfigurationActivity.PERIODIC_VIGNETTE_OUTER_RADIUS)));
}
if (selectedFrameProcessors[2]) {
frameProcessors.add(AdvancedFrameProcessorFactory.createSpin3dFrameProcessor());
if (selectedEffects[2]) {
effects.add(AdvancedFrameProcessorFactory::createSpin3dFrameProcessor);
}
if (selectedFrameProcessors[3]) {
frameProcessors.add(new BitmapOverlayFrameProcessor());
if (selectedEffects[3]) {
effects.add(BitmapOverlayFrameProcessor::new);
}
if (selectedFrameProcessors[4]) {
frameProcessors.add(AdvancedFrameProcessorFactory.createZoomInTransitionFrameProcessor());
if (selectedEffects[4]) {
effects.add(AdvancedFrameProcessorFactory::createZoomInTransitionFrameProcessor);
}
transformerBuilder.setFrameProcessors(frameProcessors.build());
transformerBuilder.setVideoFrameEffects(effects.build());
}
}
return transformerBuilder

6
demos/transformer/src/main/res/layout/configuration_activity.xml
View File

@ -64,7 +64,7 @@
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent"
app:layout_constraintTop_toBottomOf="@+id/selected_file_text_view"
app:layout_constraintBottom_toTopOf="@+id/select_demo_frameprocessors_button">
app:layout_constraintBottom_toTopOf="@+id/select_demo_effects_button">
<TableLayout
android:layout_width="fill_parent"
android:layout_height="wrap_content"
@ -192,13 +192,13 @@
</TableLayout>
</androidx.core.widget.NestedScrollView>
<Button
android:id="@+id/select_demo_frameprocessors_button"
android:id="@+id/select_demo_effects_button"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_marginTop="32dp"
android:layout_marginStart="32dp"
android:layout_marginEnd="32dp"
android:text="@string/select_demo_frameprocessors"
android:text="@string/select_demo_effects"
app:layout_constraintBottom_toTopOf="@+id/transform_button"
app:layout_constraintEnd_toEndOf="parent"
app:layout_constraintStart_toStartOf="parent" />

2
demos/transformer/src/main/res/values/strings.xml
View File

@ -29,7 +29,7 @@
<string name="enable_fallback" translatable="false">Enable fallback</string>
<string name="request_sdr_tone_mapping" translatable="false">Request SDR tone-mapping (API 31+)</string>
<string name="hdr_editing" translatable="false">[Experimental] HDR editing</string>
<string name="select_demo_frameprocessors" translatable="false">Add demo effects</string>
<string name="select_demo_effects" translatable="false">Add demo effects</string>
<string name="periodic_vignette_options" translatable="false">Periodic vignette options</string>
<string name="transform" translatable="false">Transform</string>
<string name="debug_preview" translatable="false">Debug preview:</string>

23
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainPixelTest.java
View File

@ -130,7 +130,7 @@ public final class FrameProcessorChainPixelTest {
Matrix translateRightMatrix = new Matrix();
translateRightMatrix.postTranslate(/* dx= */ 1, /* dy= */ 0);
GlFrameProcessor glFrameProcessor = new AdvancedFrameProcessor(translateRightMatrix);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_RIGHT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -155,7 +155,9 @@ public final class FrameProcessorChainPixelTest {
GlFrameProcessor rotate45FrameProcessor =
new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build();
setUpAndPrepareFirstFrame(
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, translateRightFrameProcessor, rotate45FrameProcessor);
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO,
() -> translateRightFrameProcessor,
() -> rotate45FrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(TRANSLATE_THEN_ROTATE_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -180,7 +182,9 @@ public final class FrameProcessorChainPixelTest {
GlFrameProcessor translateRightFrameProcessor =
new AdvancedFrameProcessor(translateRightMatrix);
setUpAndPrepareFirstFrame(
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, rotate45FrameProcessor, translateRightFrameProcessor);
DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO,
() -> rotate45FrameProcessor,
() -> translateRightFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE_THEN_TRANSLATE_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -200,7 +204,7 @@ public final class FrameProcessorChainPixelTest {
String testId = "processData_withPresentationFrameProcessor_setResolution";
GlFrameProcessor glFrameProcessor =
new PresentationFrameProcessor.Builder().setResolution(480).build();
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(REQUEST_OUTPUT_HEIGHT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -220,7 +224,7 @@ public final class FrameProcessorChainPixelTest {
String testId = "processData_withScaleToFitFrameProcessor_rotate45";
GlFrameProcessor glFrameProcessor =
new ScaleToFitFrameProcessor.Builder().setRotationDegrees(45).build();
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, glFrameProcessor);
setUpAndPrepareFirstFrame(DEFAULT_PIXEL_WIDTH_HEIGHT_RATIO, () -> glFrameProcessor);
Bitmap expectedBitmap = BitmapTestUtil.readBitmap(ROTATE45_SCALE_TO_FIT_PNG_ASSET_PATH);
Bitmap actualBitmap = processFirstFrameAndEnd();
@ -240,11 +244,10 @@ public final class FrameProcessorChainPixelTest {
* accessed on the {@link FrameProcessorChain}'s output {@code outputImageReader}.
*
* @param pixelWidthHeightRatio The ratio of width over height for each pixel.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} that will apply changes
* to the input frame.
* @param effects The {@link GlEffect GlEffects} to apply to the input frame.
*/
private void setUpAndPrepareFirstFrame(
float pixelWidthHeightRatio, GlFrameProcessor... frameProcessors) throws Exception {
private void setUpAndPrepareFirstFrame(float pixelWidthHeightRatio, GlEffect... effects)
throws Exception {
// Set up the extractor to read the first video frame and get its format.
MediaExtractor mediaExtractor = new MediaExtractor();
@Nullable MediaCodec mediaCodec = null;
@ -267,7 +270,7 @@ public final class FrameProcessorChainPixelTest {
pixelWidthHeightRatio,
inputWidth,
inputHeight,
asList(frameProcessors),
asList(effects),
/* enableExperimentalHdrEditing= */ false);
Size outputSize = frameProcessorChain.getOutputSize();
outputImageReader =

6
libraries/transformer/src/androidTest/java/androidx/media3/transformer/FrameProcessorChainTest.java
View File

@ -112,16 +112,16 @@ public final class FrameProcessorChainTest {
private static FrameProcessorChain createFrameProcessorChainWithFakeFrameProcessors(
float pixelWidthHeightRatio, Size inputSize, List<Size> frameProcessorOutputSizes)
throws TransformationException {
ImmutableList.Builder<GlFrameProcessor> frameProcessors = new ImmutableList.Builder<>();
ImmutableList.Builder<GlEffect> effects = new ImmutableList.Builder<>();
for (Size element : frameProcessorOutputSizes) {
frameProcessors.add(new FakeFrameProcessor(element));
effects.add(() -> new FakeFrameProcessor(element));
}
return FrameProcessorChain.create(
getApplicationContext(),
pixelWidthHeightRatio,
inputSize.getWidth(),
inputSize.getHeight(),
frameProcessors.build(),
effects.build(),
/* enableExperimentalHdrEditing= */ false);
}

47
libraries/transformer/src/main/java/androidx/media3/transformer/FrameProcessorChain.java
View File

@ -75,7 +75,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
* by this ratio so that the output frame's pixels have a ratio of 1.
* @param inputWidth The input frame width, in pixels.
* @param inputHeight The input frame height, in pixels.
* @param frameProcessors The {@link GlFrameProcessor GlFrameProcessors} to apply to each frame.
* @param effects The {@link GlEffect GlEffects} to apply to each frame.
* @param enableExperimentalHdrEditing Whether to attempt to process the input as an HDR signal.
* @return A new instance.
* @throws TransformationException If reading shader files fails, or an OpenGL error occurs while
@ -86,7 +86,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
List<GlFrameProcessor> frameProcessors,
List<GlEffect> effects,
boolean enableExperimentalHdrEditing)
throws TransformationException {
checkArgument(inputWidth > 0, "inputWidth must be positive");
@ -103,7 +103,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
pixelWidthHeightRatio,
inputWidth,
inputHeight,
frameProcessors,
effects,
enableExperimentalHdrEditing,
singleThreadExecutorService))
.get();
@ -118,8 +118,9 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
}
/**
* Creates the OpenGL textures, framebuffers, initializes the {@link GlFrameProcessor
* GlFrameProcessors} and returns a new {@code FrameProcessorChain}.
* Creates the OpenGL textures and framebuffers, initializes the {@link GlFrameProcessor
* GlFrameProcessors} corresponding to the {@link GlEffect GlEffects}, and returns a new {@code
* FrameProcessorChain}.
*
* <p>This method must be executed using the {@code singleThreadExecutorService}.
*/
@ -129,15 +130,13 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
float pixelWidthHeightRatio,
int inputWidth,
int inputHeight,
List<GlFrameProcessor> frameProcessors,
List<GlEffect> effects,
boolean enableExperimentalHdrEditing,
ExecutorService singleThreadExecutorService)
throws IOException {
checkState(Thread.currentThread().getName().equals(THREAD_NAME));
EGLDisplay eglDisplay = GlUtil.createEglDisplay();
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(enableExperimentalHdrEditing);
EGLContext eglContext =
enableExperimentalHdrEditing
? GlUtil.createEglContextEs3Rgba1010102(eglDisplay)
@ -153,20 +152,21 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
GlUtil.focusPlaceholderEglSurface(eglContext, eglDisplay);
}
ImmutableList<GlFrameProcessor> expandedFrameProcessors =
getExpandedFrameProcessors(
externalCopyFrameProcessor, pixelWidthHeightRatio, frameProcessors);
ExternalCopyFrameProcessor externalCopyFrameProcessor =
new ExternalCopyFrameProcessor(enableExperimentalHdrEditing);
ImmutableList<GlFrameProcessor> frameProcessors =
getFrameProcessors(externalCopyFrameProcessor, pixelWidthHeightRatio, effects);
// Initialize frame processors.
int inputExternalTexId = GlUtil.createExternalTexture();
externalCopyFrameProcessor.initialize(context, inputExternalTexId, inputWidth, inputHeight);
int[] framebuffers = new int[expandedFrameProcessors.size() - 1];
int[] framebuffers = new int[frameProcessors.size() - 1];
Size inputSize = externalCopyFrameProcessor.getOutputSize();
for (int i = 1; i < expandedFrameProcessors.size(); i++) {
for (int i = 1; i < frameProcessors.size(); i++) {
int inputTexId = GlUtil.createTexture(inputSize.getWidth(), inputSize.getHeight());
framebuffers[i - 1] = GlUtil.createFboForTexture(inputTexId);
GlFrameProcessor frameProcessor = expandedFrameProcessors.get(i);
GlFrameProcessor frameProcessor = frameProcessors.get(i);
frameProcessor.initialize(context, inputTexId, inputSize.getWidth(), inputSize.getHeight());
inputSize = frameProcessor.getOutputSize();
}
@ -176,31 +176,34 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
singleThreadExecutorService,
inputExternalTexId,
framebuffers,
expandedFrameProcessors,
frameProcessors,
enableExperimentalHdrEditing);
}
private static ImmutableList<GlFrameProcessor> getExpandedFrameProcessors(
private static ImmutableList<GlFrameProcessor> getFrameProcessors(
ExternalCopyFrameProcessor externalCopyFrameProcessor,
float pixelWidthHeightRatio,
List<GlFrameProcessor> frameProcessors) {
ImmutableList.Builder<GlFrameProcessor> frameProcessorListBuilder =
List<GlEffect> effects) {
ImmutableList.Builder<GlFrameProcessor> frameProcessors =
new ImmutableList.Builder<GlFrameProcessor>().add(externalCopyFrameProcessor);
// Scale to expand the frame to apply the pixelWidthHeightRatio.
if (pixelWidthHeightRatio > 1f) {
frameProcessorListBuilder.add(
frameProcessors.add(
new ScaleToFitFrameProcessor.Builder()
.setScale(/* scaleX= */ pixelWidthHeightRatio, /* scaleY= */ 1f)
.build());
} else if (pixelWidthHeightRatio < 1f) {
frameProcessorListBuilder.add(
frameProcessors.add(
new ScaleToFitFrameProcessor.Builder()
.setScale(/* scaleX= */ 1f, /* scaleY= */ 1f / pixelWidthHeightRatio)
.build());
}
frameProcessorListBuilder.addAll(frameProcessors);
return frameProcessorListBuilder.build();
for (int i = 0; i < effects.size(); i++) {
frameProcessors.add(effects.get(i).toGlFrameProcessor());
}
return frameProcessors.build();
}
private static final String TAG = "FrameProcessorChain";

31
libraries/transformer/src/main/java/androidx/media3/transformer/GlEffect.java Normal file
View File

@ -0,0 +1,31 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.transformer;
import androidx.media3.common.util.UnstableApi;
/**
* Interface for a video frame effect with a {@link GlFrameProcessor} implementation.
*
* <p>Implementations contain information specifying the effect and can be {@linkplain
* #toGlFrameProcessor() converted} to a {@link GlFrameProcessor} which applies the effect.
*/
@UnstableApi
public interface GlEffect {
/** Returns a {@link GlFrameProcessor} that applies the the effect. */
GlFrameProcessor toGlFrameProcessor();
}

36
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -104,7 +104,7 @@ public final class Transformer {
private boolean removeVideo;
private String containerMimeType;
private TransformationRequest transformationRequest;
private ImmutableList<GlFrameProcessor> frameProcessors;
private ImmutableList<GlEffect> videoFrameEffects;
private ListenerSet<Transformer.Listener> listeners;
private DebugViewProvider debugViewProvider;
private Looper looper;
@ -124,7 +124,7 @@ public final class Transformer {
debugViewProvider = DebugViewProvider.NONE;
containerMimeType = MimeTypes.VIDEO_MP4;
transformationRequest = new TransformationRequest.Builder().build();
frameProcessors = ImmutableList.of();
videoFrameEffects = ImmutableList.of();
}
/**
@ -142,7 +142,7 @@ public final class Transformer {
debugViewProvider = DebugViewProvider.NONE;
containerMimeType = MimeTypes.VIDEO_MP4;
transformationRequest = new TransformationRequest.Builder().build();
frameProcessors = ImmutableList.of();
videoFrameEffects = ImmutableList.of();
}
/** Creates a builder with the values of the provided {@link Transformer}. */
@ -154,7 +154,7 @@ public final class Transformer {
this.removeVideo = transformer.removeVideo;
this.containerMimeType = transformer.containerMimeType;
this.transformationRequest = transformer.transformationRequest;
this.frameProcessors = transformer.frameProcessors;
this.videoFrameEffects = transformer.videoFrameEffects;
this.listeners = transformer.listeners;
this.looper = transformer.looper;
this.encoderFactory = transformer.encoderFactory;
@ -187,20 +187,20 @@ public final class Transformer {
}
/**
* Sets the {@linkplain GlFrameProcessor frame processors} to apply to each frame.
* Sets the {@linkplain GlEffect effects} to apply to each video frame.
*
* <p>The {@linkplain GlFrameProcessor frame processors} are applied before any {@linkplain
* <p>The {@linkplain GlEffect effects} are applied before any {@linkplain
* TransformationRequest.Builder#setScale(float, float) scale}, {@linkplain
* TransformationRequest.Builder#setRotationDegrees(float) rotation}, or {@linkplain
* TransformationRequest.Builder#setResolution(int) resolution} changes specified in the {@link
* #setTransformationRequest(TransformationRequest) TransformationRequest} but after {@linkplain
* TransformationRequest.Builder#setFlattenForSlowMotion(boolean) slow-motion flattening}.
*
* @param frameProcessors The {@linkplain GlFrameProcessor frame processors}.
* @param effects The {@linkplain GlEffect effects} to apply to each video frame.
* @return This builder.
*/
public Builder setFrameProcessors(List<GlFrameProcessor> frameProcessors) {
this.frameProcessors = ImmutableList.copyOf(frameProcessors);
public Builder setVideoFrameEffects(List<GlEffect> effects) {
this.videoFrameEffects = ImmutableList.copyOf(effects);
return this;
}
@ -432,7 +432,7 @@ public final class Transformer {
removeVideo,
containerMimeType,
transformationRequest,
frameProcessors,
videoFrameEffects,
listeners,
looper,
clock,
@ -554,7 +554,7 @@ public final class Transformer {
private final boolean removeVideo;
private final String containerMimeType;
private final TransformationRequest transformationRequest;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final ImmutableList<GlEffect> videoFrameEffects;
private final Looper looper;
private final Clock clock;
private final Codec.EncoderFactory encoderFactory;
@ -575,7 +575,7 @@ public final class Transformer {
boolean removeVideo,
String containerMimeType,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ImmutableList<GlEffect> videoFrameEffects,
ListenerSet<Transformer.Listener> listeners,
Looper looper,
Clock clock,
@ -590,7 +590,7 @@ public final class Transformer {
this.removeVideo = removeVideo;
this.containerMimeType = containerMimeType;
this.transformationRequest = transformationRequest;
this.frameProcessors = frameProcessors;
this.videoFrameEffects = videoFrameEffects;
this.listeners = listeners;
this.looper = looper;
this.clock = clock;
@ -730,7 +730,7 @@ public final class Transformer {
removeAudio,
removeVideo,
transformationRequest,
frameProcessors,
videoFrameEffects,
encoderFactory,
decoderFactory,
new FallbackListener(mediaItem, listeners, transformationRequest),
@ -842,7 +842,7 @@ public final class Transformer {
private final boolean removeAudio;
private final boolean removeVideo;
private final TransformationRequest transformationRequest;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final ImmutableList<GlEffect> videoFrameEffects;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final FallbackListener fallbackListener;
@ -854,7 +854,7 @@ public final class Transformer {
boolean removeAudio,
boolean removeVideo,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ImmutableList<GlEffect> videoFrameEffects,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
@ -864,7 +864,7 @@ public final class Transformer {
this.removeAudio = removeAudio;
this.removeVideo = removeVideo;
this.transformationRequest = transformationRequest;
this.frameProcessors = frameProcessors;
this.videoFrameEffects = videoFrameEffects;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.fallbackListener = fallbackListener;
@ -900,7 +900,7 @@ public final class Transformer {
muxerWrapper,
mediaClock,
transformationRequest,
frameProcessors,
videoFrameEffects,
encoderFactory,
decoderFactory,
fallbackListener,

10
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoRenderer.java
View File

@ -35,7 +35,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
private static final String TAG = "TVideoRenderer";
private final Context context;
private final ImmutableList<GlFrameProcessor> frameProcessors;
private final ImmutableList<GlEffect> effects;
private final Codec.EncoderFactory encoderFactory;
private final Codec.DecoderFactory decoderFactory;
private final Transformer.DebugViewProvider debugViewProvider;
@ -48,14 +48,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
MuxerWrapper muxerWrapper,
TransformerMediaClock mediaClock,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ImmutableList<GlEffect> effects,
Codec.EncoderFactory encoderFactory,
Codec.DecoderFactory decoderFactory,
FallbackListener fallbackListener,
Transformer.DebugViewProvider debugViewProvider) {
super(C.TRACK_TYPE_VIDEO, muxerWrapper, mediaClock, transformationRequest, fallbackListener);
this.context = context;
this.frameProcessors = frameProcessors;
this.effects = effects;
this.encoderFactory = encoderFactory;
this.decoderFactory = decoderFactory;
this.debugViewProvider = debugViewProvider;
@ -90,7 +90,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
context,
inputFormat,
transformationRequest,
frameProcessors,
effects,
decoderFactory,
encoderFactory,
muxerWrapper.getSupportedSampleMimeTypes(getTrackType()),
@ -137,7 +137,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull;
&& transformationRequest.outputHeight != inputFormat.height) {
return false;
}
if (!frameProcessors.isEmpty()) {
if (!effects.isEmpty()) {
return false;
}
return true;

30
libraries/transformer/src/main/java/androidx/media3/transformer/VideoTranscodingSamplePipeline.java
View File

@ -50,7 +50,7 @@ import org.checkerframework.dataflow.qual.Pure;
Context context,
Format inputFormat,
TransformationRequest transformationRequest,
ImmutableList<GlFrameProcessor> frameProcessors,
ImmutableList<GlEffect> effects,
Codec.DecoderFactory decoderFactory,
Codec.EncoderFactory encoderFactory,
List<String> allowedOutputMimeTypes,
@ -68,33 +68,35 @@ import org.checkerframework.dataflow.qual.Pure;
int decodedHeight =
(inputFormat.rotationDegrees % 180 == 0) ? inputFormat.height : inputFormat.width;
ImmutableList.Builder<GlFrameProcessor> frameProcessorsListBuilder =
new ImmutableList.Builder<GlFrameProcessor>().addAll(frameProcessors);
ImmutableList.Builder<GlEffect> effectsListBuilder =
new ImmutableList.Builder<GlEffect>().addAll(effects);
if (transformationRequest.scaleX != 1f
|| transformationRequest.scaleY != 1f
|| transformationRequest.rotationDegrees != 0f) {
frameProcessorsListBuilder.add(
new ScaleToFitFrameProcessor.Builder()
.setScale(transformationRequest.scaleX, transformationRequest.scaleY)
.setRotationDegrees(transformationRequest.rotationDegrees)
.build());
effectsListBuilder.add(
() ->
new ScaleToFitFrameProcessor.Builder()
.setScale(transformationRequest.scaleX, transformationRequest.scaleY)
.setRotationDegrees(transformationRequest.rotationDegrees)
.build());
}
if (transformationRequest.outputHeight != C.LENGTH_UNSET) {
frameProcessorsListBuilder.add(
new PresentationFrameProcessor.Builder()
.setResolution(transformationRequest.outputHeight)
.build());
effectsListBuilder.add(
() ->
new PresentationFrameProcessor.Builder()
.setResolution(transformationRequest.outputHeight)
.build());
}
EncoderCompatibilityFrameProcessor encoderCompatibilityFrameProcessor =
new EncoderCompatibilityFrameProcessor();
frameProcessorsListBuilder.add(encoderCompatibilityFrameProcessor);
effectsListBuilder.add(() -> encoderCompatibilityFrameProcessor);
frameProcessorChain =
FrameProcessorChain.create(
context,
inputFormat.pixelWidthHeightRatio,
/* inputWidth= */ decodedWidth,
/* inputHeight= */ decodedHeight,
frameProcessorsListBuilder.build(),
effectsListBuilder.build(),
transformationRequest.enableHdrEditing);
Size requestedEncoderSize = frameProcessorChain.getOutputSize();
outputRotationDegrees = encoderCompatibilityFrameProcessor.getOutputRotationDegrees();