Implements {@link FrameCache}. */ -/* package */ final class FrameCacheGlShaderProgram extends BaseGlShaderProgram { +/* package */ class FrameCacheGlShaderProgram extends BaseGlShaderProgram { private static final String VERTEX_SHADER_TRANSFORMATION_ES2_PATH = "shaders/vertex_shader_transformation_es2.glsl"; private static final String FRAGMENT_SHADER_TRANSFORMATION_ES2_PATH = diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FrameDropEffect.java b/libraries/effect/src/main/java/androidx/media3/effect/FrameDropEffect.java new file mode 100644 index 0000000000..fbee08f46f --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/FrameDropEffect.java @@ -0,0 +1,43 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import android.content.Context; +import androidx.media3.common.VideoFrameProcessingException; +import androidx.media3.common.util.UnstableApi; + +/** Drops frames to lower average frame rate to around {@code targetFrameRate}. */ +@UnstableApi +public class FrameDropEffect implements GlEffect { + + private final float targetFrameRate; + + /** + * Creates an instance. + * + * @param targetFrameRate The number of frames per second the output video should roughly have. + */ + public FrameDropEffect(float targetFrameRate) { + this.targetFrameRate = targetFrameRate; + } + + @Override + public GlShaderProgram toGlShaderProgram(Context context, boolean useHdr) + throws VideoFrameProcessingException { + return new FrameDroppingShaderProgram(context, useHdr, targetFrameRate); + } +} diff --git a/libraries/effect/src/main/java/androidx/media3/effect/FrameDroppingShaderProgram.java b/libraries/effect/src/main/java/androidx/media3/effect/FrameDroppingShaderProgram.java new file mode 100644 index 0000000000..e1d5326d5c --- /dev/null +++ b/libraries/effect/src/main/java/androidx/media3/effect/FrameDroppingShaderProgram.java @@ -0,0 +1,104 @@ +/* + * Copyright 2023 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package androidx.media3.effect; + +import static androidx.media3.common.util.Assertions.checkNotNull; +import static java.lang.Math.abs; + +import android.content.Context; +import androidx.annotation.Nullable; +import androidx.media3.common.C; +import androidx.media3.common.GlTextureInfo; +import androidx.media3.common.VideoFrameProcessingException; + +// TODO(b/227625363): Add tests for this file. +/** + * Drops frames by only queuing input frames that are chosen by the frame dropping strategy. + * + *
The strategy used is to queue the current frame, x, with timestamp T_x if and only if one of + * the following is true: + * + *
Where T_lastQueued is the timestamp of the last queued frame and T_(x+1) is the timestamp of + * the next frame. The target frame interval is determined from {@code targetFps}. + */ +/* package */ final class FrameDroppingShaderProgram extends FrameCacheGlShaderProgram { + private final long targetFrameDeltaUs; + + @Nullable private GlTextureInfo previousTexture; + private long previousPresentationTimeUs; + private long lastQueuedPresentationTimeUs; + private boolean isPreviousFrameFirstFrame; + + /** + * Creates a new instance. + * + * @param context The {@link Context}. + * @param useHdr Whether input textures come from an HDR source. If {@code true}, colors will be + * in linear RGB BT.2020. If {@code false}, colors will be in linear RGB BT.709. + * @param targetFps The number of frames per second the output video should roughly have. + */ + public FrameDroppingShaderProgram(Context context, boolean useHdr, float targetFps) + throws VideoFrameProcessingException { + super(context, /* capacity= */ 1, useHdr); + this.targetFrameDeltaUs = (long) (C.MICROS_PER_SECOND / targetFps); + lastQueuedPresentationTimeUs = C.TIME_UNSET; + previousPresentationTimeUs = C.TIME_UNSET; + } + + @Override + public void queueInputFrame(GlTextureInfo inputTexture, long presentationTimeUs) { + if (previousTexture == null) { + super.queueInputFrame(inputTexture, presentationTimeUs); + lastQueuedPresentationTimeUs = presentationTimeUs; + isPreviousFrameFirstFrame = true; + } else if (shouldQueuePreviousFrame(presentationTimeUs)) { + super.queueInputFrame(checkNotNull(previousTexture), previousPresentationTimeUs); + lastQueuedPresentationTimeUs = previousPresentationTimeUs; + } else { + inputListener.onInputFrameProcessed(checkNotNull(previousTexture)); + inputListener.onReadyToAcceptInputFrame(); + } + previousTexture = inputTexture; + previousPresentationTimeUs = presentationTimeUs; + } + + @Override + public void flush() { + super.flush(); + lastQueuedPresentationTimeUs = C.TIME_UNSET; + previousPresentationTimeUs = C.TIME_UNSET; + previousTexture = null; + } + + private boolean shouldQueuePreviousFrame(long currentPresentationTimeUs) { + if (isPreviousFrameFirstFrame) { + isPreviousFrameFirstFrame = false; + return false; + } + + long previousFrameTimeDeltaUs = previousPresentationTimeUs - lastQueuedPresentationTimeUs; + long currentFrameTimeDeltaUs = currentPresentationTimeUs - lastQueuedPresentationTimeUs; + + return abs(previousFrameTimeDeltaUs - targetFrameDeltaUs) + < abs(currentFrameTimeDeltaUs - targetFrameDeltaUs); + } +}