From c235e4f4474af463b01f325afee26f12f89e02f2 Mon Sep 17 00:00:00 2001 From: hschlueter Date: Wed, 6 Apr 2022 11:28:55 +0100 Subject: [PATCH] Add matrix provider for AdvancedFrameProcessor and examples in demo. The matrix provider allows the transformation matrix to be updated for each frame based on the timestamp. The following example effects using this were added to the demo: * a zoom-in transition for the start of the video, * cropping a rotating rectangular frame portion, * rotating the frame around the y-axis in 3D. PiperOrigin-RevId: 439791592 --- .../AdvancedFrameProcessorFactory.java | 96 +++++++++++++++++++ .../transformer/ConfigurationActivity.java | 61 ++++++++---- .../demo/transformer/TransformerActivity.java | 23 +++++ .../res/layout/configuration_activity.xml | 23 +++-- .../src/main/res/values/strings.xml | 3 +- .../transformer/AdvancedFrameProcessor.java | 70 +++++++++++--- 6 files changed, 241 insertions(+), 35 deletions(-) create mode 100644 demos/transformer/src/main/java/androidx/media3/demo/transformer/AdvancedFrameProcessorFactory.java diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/AdvancedFrameProcessorFactory.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/AdvancedFrameProcessorFactory.java new file mode 100644 index 0000000000..51214ebc51 --- /dev/null +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/AdvancedFrameProcessorFactory.java @@ -0,0 +1,96 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.demo.transformer; + +import android.content.Context; +import android.graphics.Matrix; +import androidx.media3.common.C; +import androidx.media3.common.util.Util; +import androidx.media3.transformer.AdvancedFrameProcessor; +import androidx.media3.transformer.GlFrameProcessor; + +/** + * Factory for {@link GlFrameProcessor GlFrameProcessors} that create video effects by applying + * transformation matrices to the individual video frames using {@link AdvancedFrameProcessor}. + */ +/* package */ final class AdvancedFrameProcessorFactory { + /** + * Returns a {@link GlFrameProcessor} that rescales the frames over the first {@value + * #ZOOM_DURATION_SECONDS} seconds, such that the rectangle filled with the input frame increases + * linearly in size from a single point to filling the full output frame. + */ + public static GlFrameProcessor createZoomInTransitionFrameProcessor(Context context) { + return new AdvancedFrameProcessor( + context, + /* matrixProvider= */ AdvancedFrameProcessorFactory::calculateZoomInTransitionMatrix); + } + + /** + * Returns a {@link GlFrameProcessor} that crops frames to a rectangle that moves on an ellipse. + */ + public static GlFrameProcessor createDizzyCropFrameProcessor(Context context) { + return new AdvancedFrameProcessor( + context, /* matrixProvider= */ AdvancedFrameProcessorFactory::calculateDizzyCropMatrix); + } + + /** + * Returns a {@link GlFrameProcessor} that rotates a frame in 3D around the y-axis and applies + * perspective projection to 2D. + */ + public static GlFrameProcessor createSpin3dFrameProcessor(Context context) { + return new AdvancedFrameProcessor( + context, /* matrixProvider= */ AdvancedFrameProcessorFactory::calculate3dSpinMatrix); + } + + private static final float ZOOM_DURATION_SECONDS = 2f; + private static final float DIZZY_CROP_ROTATION_PERIOD_US = 1_500_000f; + + private static Matrix calculateZoomInTransitionMatrix(long presentationTimeUs) { + Matrix transformationMatrix = new Matrix(); + float scale = Math.min(1, presentationTimeUs / (C.MICROS_PER_SECOND * ZOOM_DURATION_SECONDS)); + transformationMatrix.postScale(/* sx= */ scale, /* sy= */ scale); + return transformationMatrix; + } + + private static android.graphics.Matrix calculateDizzyCropMatrix(long presentationTimeUs) { + double theta = presentationTimeUs * 2 * Math.PI / DIZZY_CROP_ROTATION_PERIOD_US; + float centerX = 0.5f * (float) Math.cos(theta); + float centerY = 0.5f * (float) Math.sin(theta); + android.graphics.Matrix transformationMatrix = new android.graphics.Matrix(); + transformationMatrix.postTranslate(/* dx= */ centerX, /* dy= */ centerY); + transformationMatrix.postScale(/* sx= */ 2f, /* sy= */ 2f); + return transformationMatrix; + } + + private static float[] calculate3dSpinMatrix(long presentationTimeUs) { + float[] transformationMatrix = new float[16]; + android.opengl.Matrix.frustumM( + transformationMatrix, + /* offset= */ 0, + /* left= */ -1f, + /* right= */ 1f, + /* bottom= */ -1f, + /* top= */ 1f, + /* near= */ 3f, + /* far= */ 5f); + android.opengl.Matrix.translateM( + transformationMatrix, /* mOffset= */ 0, /* x= */ 0f, /* y= */ 0f, /* z= */ -4f); + float theta = Util.usToMs(presentationTimeUs) / 10f; + android.opengl.Matrix.rotateM( + transformationMatrix, /* mOffset= */ 0, theta, /* x= */ 0f, /* y= */ 1f, /* z= */ 0f); + return transformationMatrix; + } +} diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java index 0245b9cc60..6182f3bdd1 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/ConfigurationActivity.java @@ -56,6 +56,7 @@ public final class ConfigurationActivity extends AppCompatActivity { public static final String ENABLE_FALLBACK = "enable_fallback"; public static final String ENABLE_REQUEST_SDR_TONE_MAPPING = "enable_request_sdr_tone_mapping"; public static final String ENABLE_HDR_EDITING = "enable_hdr_editing"; + public static final String FRAME_PROCESSOR_SELECTION = "frame_processor_selection"; private static final String[] INPUT_URIS = { "https://html5demos.com/assets/dizzy.mp4", "https://storage.googleapis.com/exoplayer-test-media-0/android-block-1080-hevc.mp4", @@ -80,10 +81,11 @@ public final class ConfigurationActivity extends AppCompatActivity { "SEF slow motion with 240 fps", "MP4 with HDR (HDR10) H265 video (encoding may fail)", }; + private static final String[] FRAME_PROCESSORS = {"Dizzy crop", "3D spin", "Zoom in start"}; private static final String SAME_AS_INPUT_OPTION = "same as input"; - private @MonotonicNonNull Button chooseFileButton; - private @MonotonicNonNull TextView chosenFileTextView; + private @MonotonicNonNull Button selectFileButton; + private @MonotonicNonNull TextView selectedFileTextView; private @MonotonicNonNull CheckBox removeAudioCheckbox; private @MonotonicNonNull CheckBox removeVideoCheckbox; private @MonotonicNonNull CheckBox flattenForSlowMotionCheckbox; @@ -95,6 +97,8 @@ public final class ConfigurationActivity extends AppCompatActivity { private @MonotonicNonNull CheckBox enableFallbackCheckBox; private @MonotonicNonNull CheckBox enableRequestSdrToneMappingCheckBox; private @MonotonicNonNull CheckBox enableHdrEditingCheckBox; + private @MonotonicNonNull Button selectFrameProcessorsButton; + private boolean @MonotonicNonNull [] selectedFrameProcessors; private int inputUriPosition; @Override @@ -104,11 +108,11 @@ public final class ConfigurationActivity extends AppCompatActivity { findViewById(R.id.transform_button).setOnClickListener(this::startTransformation); - chooseFileButton = findViewById(R.id.choose_file_button); - chooseFileButton.setOnClickListener(this::chooseFile); + selectFileButton = findViewById(R.id.select_file_button); + selectFileButton.setOnClickListener(this::selectFile); - chosenFileTextView = findViewById(R.id.chosen_file_text_view); - chosenFileTextView.setText(URI_DESCRIPTIONS[inputUriPosition]); + selectedFileTextView = findViewById(R.id.selected_file_text_view); + selectedFileTextView.setText(URI_DESCRIPTIONS[inputUriPosition]); removeAudioCheckbox = findViewById(R.id.remove_audio_checkbox); removeAudioCheckbox.setOnClickListener(this::onRemoveAudio); @@ -164,6 +168,10 @@ public final class ConfigurationActivity extends AppCompatActivity { enableRequestSdrToneMappingCheckBox.setEnabled(isRequestSdrToneMappingSupported()); findViewById(R.id.request_sdr_tone_mapping).setEnabled(isRequestSdrToneMappingSupported()); enableHdrEditingCheckBox = findViewById(R.id.hdr_editing_checkbox); + + selectedFrameProcessors = new boolean[FRAME_PROCESSORS.length]; + selectFrameProcessorsButton = findViewById(R.id.select_frameprocessors_button); + selectFrameProcessorsButton.setOnClickListener(this::selectFrameProcessors); } @Override @@ -171,8 +179,8 @@ public final class ConfigurationActivity extends AppCompatActivity { super.onResume(); @Nullable Uri intentUri = getIntent().getData(); if (intentUri != null) { - checkNotNull(chooseFileButton).setEnabled(false); - checkNotNull(chosenFileTextView).setText(intentUri.toString()); + checkNotNull(selectFileButton).setEnabled(false); + checkNotNull(selectedFileTextView).setText(intentUri.toString()); } } @@ -193,7 +201,8 @@ public final class ConfigurationActivity extends AppCompatActivity { "rotateSpinner", "enableFallbackCheckBox", "enableRequestSdrToneMappingCheckBox", - "enableHdrEditingCheckBox" + "enableHdrEditingCheckBox", + "selectedFrameProcessors" }) private void startTransformation(View view) { Intent transformerIntent = new Intent(this, TransformerActivity.class); @@ -228,6 +237,7 @@ public final class ConfigurationActivity extends AppCompatActivity { bundle.putBoolean( ENABLE_REQUEST_SDR_TONE_MAPPING, enableRequestSdrToneMappingCheckBox.isChecked()); bundle.putBoolean(ENABLE_HDR_EDITING, enableHdrEditingCheckBox.isChecked()); + bundle.putBooleanArray(FRAME_PROCESSOR_SELECTION, selectedFrameProcessors); transformerIntent.putExtras(bundle); @Nullable Uri intentUri = getIntent().getData(); @@ -237,19 +247,34 @@ public final class ConfigurationActivity extends AppCompatActivity { startActivity(transformerIntent); } - private void chooseFile(View view) { + private void selectFile(View view) { new AlertDialog.Builder(/* context= */ this) - .setTitle(R.string.choose_file_title) + .setTitle(R.string.select_file_title) .setSingleChoiceItems(URI_DESCRIPTIONS, inputUriPosition, this::selectFileInDialog) .setPositiveButton(android.R.string.ok, /* listener= */ null) .create() .show(); } - @RequiresNonNull("chosenFileTextView") + private void selectFrameProcessors(View view) { + new AlertDialog.Builder(/* context= */ this) + .setTitle(R.string.select_frameprocessors) + .setMultiChoiceItems( + FRAME_PROCESSORS, checkNotNull(selectedFrameProcessors), this::selectFrameProcessor) + .setPositiveButton(android.R.string.ok, /* listener= */ null) + .create() + .show(); + } + + @RequiresNonNull("selectedFileTextView") private void selectFileInDialog(DialogInterface dialog, int which) { inputUriPosition = which; - chosenFileTextView.setText(URI_DESCRIPTIONS[inputUriPosition]); + selectedFileTextView.setText(URI_DESCRIPTIONS[inputUriPosition]); + } + + @RequiresNonNull("selectedFrameProcessors") + private void selectFrameProcessor(DialogInterface dialog, int which, boolean isChecked) { + selectedFrameProcessors[which] = isChecked; } @RequiresNonNull({ @@ -260,7 +285,8 @@ public final class ConfigurationActivity extends AppCompatActivity { "scaleSpinner", "rotateSpinner", "enableRequestSdrToneMappingCheckBox", - "enableHdrEditingCheckBox" + "enableHdrEditingCheckBox", + "selectFrameProcessorsButton" }) private void onRemoveAudio(View view) { if (((CheckBox) view).isChecked()) { @@ -279,7 +305,8 @@ public final class ConfigurationActivity extends AppCompatActivity { "scaleSpinner", "rotateSpinner", "enableRequestSdrToneMappingCheckBox", - "enableHdrEditingCheckBox" + "enableHdrEditingCheckBox", + "selectFrameProcessorsButton" }) private void onRemoveVideo(View view) { if (((CheckBox) view).isChecked()) { @@ -297,7 +324,8 @@ public final class ConfigurationActivity extends AppCompatActivity { "scaleSpinner", "rotateSpinner", "enableRequestSdrToneMappingCheckBox", - "enableHdrEditingCheckBox" + "enableHdrEditingCheckBox", + "selectFrameProcessorsButton" }) private void enableTrackSpecificOptions(boolean isAudioEnabled, boolean isVideoEnabled) { audioMimeSpinner.setEnabled(isAudioEnabled); @@ -308,6 +336,7 @@ public final class ConfigurationActivity extends AppCompatActivity { enableRequestSdrToneMappingCheckBox.setEnabled( isRequestSdrToneMappingSupported() && isVideoEnabled); enableHdrEditingCheckBox.setEnabled(isVideoEnabled); + selectFrameProcessorsButton.setEnabled(isVideoEnabled); findViewById(R.id.audio_mime_text_view).setEnabled(isAudioEnabled); findViewById(R.id.video_mime_text_view).setEnabled(isVideoEnabled); diff --git a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java index 3a957cba3e..cb476adef1 100644 --- a/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java +++ b/demos/transformer/src/main/java/androidx/media3/demo/transformer/TransformerActivity.java @@ -40,6 +40,7 @@ import androidx.media3.exoplayer.ExoPlayer; import androidx.media3.exoplayer.util.DebugTextViewHelper; import androidx.media3.transformer.DefaultEncoderFactory; import androidx.media3.transformer.EncoderSelector; +import androidx.media3.transformer.GlFrameProcessor; import androidx.media3.transformer.ProgressHolder; import androidx.media3.transformer.TransformationException; import androidx.media3.transformer.TransformationRequest; @@ -50,6 +51,7 @@ import androidx.media3.ui.PlayerView; import com.google.android.material.progressindicator.LinearProgressIndicator; import com.google.common.base.Stopwatch; import com.google.common.base.Ticker; +import com.google.common.collect.ImmutableList; import java.io.File; import java.io.IOException; import java.util.concurrent.CountDownLatch; @@ -237,6 +239,27 @@ public final class TransformerActivity extends AppCompatActivity { new DefaultEncoderFactory( EncoderSelector.DEFAULT, /* enableFallback= */ bundle.getBoolean(ConfigurationActivity.ENABLE_FALLBACK))); + + ImmutableList.Builder frameProcessors = new ImmutableList.Builder<>(); + @Nullable + boolean[] selectedFrameProcessors = + bundle.getBooleanArray(ConfigurationActivity.FRAME_PROCESSOR_SELECTION); + if (selectedFrameProcessors != null) { + if (selectedFrameProcessors[0]) { + frameProcessors.add( + AdvancedFrameProcessorFactory.createDizzyCropFrameProcessor(/* context= */ this)); + } + if (selectedFrameProcessors[1]) { + frameProcessors.add( + AdvancedFrameProcessorFactory.createSpin3dFrameProcessor(/* context= */ this)); + } + if (selectedFrameProcessors[2]) { + frameProcessors.add( + AdvancedFrameProcessorFactory.createZoomInTransitionFrameProcessor( + /* context= */ this)); + } + transformerBuilder.setFrameProcessors(frameProcessors.build()); + } } return transformerBuilder .addListener( diff --git a/demos/transformer/src/main/res/layout/configuration_activity.xml b/demos/transformer/src/main/res/layout/configuration_activity.xml index c973bf4137..7464ce9f6b 100644 --- a/demos/transformer/src/main/res/layout/configuration_activity.xml +++ b/demos/transformer/src/main/res/layout/configuration_activity.xml @@ -34,18 +34,18 @@ app:layout_constraintStart_toStartOf="parent" app:layout_constraintTop_toTopOf="parent" />