diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java index 51708ec0c6..27fc360fac 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/FrameEditor.java @@ -32,6 +32,7 @@ import androidx.annotation.Nullable; import androidx.media3.common.Format; import androidx.media3.common.util.GlUtil; import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; /** FrameEditor applies changes to individual video frames. */ /* package */ final class FrameEditor { @@ -178,6 +179,7 @@ import java.io.IOException; private final EGLContext eglContext; private final EGLSurface eglSurface; private final int textureId; + private final AtomicInteger pendingInputFrameCount; private final SurfaceTexture inputSurfaceTexture; private final Surface inputSurface; private final GlUtil.Program glProgram; @@ -187,8 +189,6 @@ import java.io.IOException; private final int debugPreviewWidth; private final int debugPreviewHeight; - private volatile boolean hasInputData; - private FrameEditor( EGLDisplay eglDisplay, EGLContext eglContext, @@ -205,6 +205,7 @@ import java.io.IOException; this.eglSurface = eglSurface; this.textureId = textureId; this.glProgram = glProgram; + this.pendingInputFrameCount = new AtomicInteger(); this.outputWidth = outputWidth; this.outputHeight = outputHeight; this.debugPreviewEglSurface = debugPreviewEglSurface; @@ -212,7 +213,8 @@ import java.io.IOException; this.debugPreviewHeight = debugPreviewHeight; textureTransformMatrix = new float[16]; inputSurfaceTexture = new SurfaceTexture(textureId); - inputSurfaceTexture.setOnFrameAvailableListener(surfaceTexture -> hasInputData = true); + inputSurfaceTexture.setOnFrameAvailableListener( + surfaceTexture -> pendingInputFrameCount.incrementAndGet()); inputSurface = new Surface(inputSurfaceTexture); } @@ -226,7 +228,7 @@ import java.io.IOException; * #processData()}. */ public boolean hasInputData() { - return hasInputData; + return pendingInputFrameCount.get() > 0; } /** Processes pending input frame. */ @@ -240,13 +242,12 @@ import java.io.IOException; long surfaceTextureTimestampNs = inputSurfaceTexture.getTimestamp(); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, surfaceTextureTimestampNs); EGL14.eglSwapBuffers(eglDisplay, eglSurface); + pendingInputFrameCount.decrementAndGet(); if (debugPreviewEglSurface != null) { focusAndDrawQuad(debugPreviewEglSurface, debugPreviewWidth, debugPreviewHeight); EGL14.eglSwapBuffers(eglDisplay, debugPreviewEglSurface); } - - hasInputData = false; } /** Releases all resources. */ diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/MediaCodecAdapterWrapper.java b/libraries/transformer/src/main/java/androidx/media3/transformer/MediaCodecAdapterWrapper.java index 6fcffa3a8b..eab805c806 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/MediaCodecAdapterWrapper.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/MediaCodecAdapterWrapper.java @@ -19,6 +19,7 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkArgument; import static androidx.media3.common.util.Assertions.checkNotNull; import static androidx.media3.common.util.Assertions.checkState; +import static androidx.media3.common.util.Util.SDK_INT; import android.annotation.SuppressLint; import android.media.MediaCodec; @@ -147,6 +148,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; MediaFormatUtil.maybeSetInteger( mediaFormat, MediaFormat.KEY_MAX_INPUT_SIZE, format.maxInputSize); MediaFormatUtil.setCsdBuffers(mediaFormat, format.initializationData); + + if (SDK_INT >= 29) { + // On API levels over 29, Transformer decodes as many frames as possible in one render + // cycle. This key ensures no frame dropping when the decoder's output surface is full. + mediaFormat.setInteger(MediaFormat.KEY_ALLOW_FRAME_DROP, 0); + } + adapter = new Factory() .createAdapter( diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java index 7e7c4e2ecd..ef589047fb 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/VideoSamplePipeline.java @@ -17,10 +17,13 @@ package androidx.media3.transformer; import static androidx.media3.common.util.Assertions.checkNotNull; +import static androidx.media3.common.util.Util.SDK_INT; import android.content.Context; import android.media.MediaCodec; +import android.media.MediaFormat; import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; import androidx.media3.common.C; import androidx.media3.common.Format; import androidx.media3.common.PlaybackException; @@ -145,6 +148,52 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return false; } + if (SDK_INT >= 29) { + return processDataV29(); + } else { + return processDataDefault(); + } + } + + /** + * Processes input data from API 29. + * + *
In this method the decoder could decode multiple frames in one invocation; as compared to + * {@link #processDataDefault()}, in which one frame is decoded in each invocation. Consequently, + * if {@link FrameEditor} processes frames slower than the decoder, decoded frames are queued up + * in the decoder's output surface. + * + *
Prior to API 29, decoders may drop frames to keep their output surface from growing out of + * bound; while after API 29, the {@link MediaFormat#KEY_ALLOW_FRAME_DROP} key prevents frame + * dropping even when the surface is full. As dropping random frames is not acceptable in {@code + * Transformer}, using this method requires API level 29 or higher. + */ + @RequiresApi(29) + private boolean processDataV29() { + if (frameEditor != null) { + while (frameEditor.hasInputData()) { + // Processes as much frames in one invocation: FrameEditor's output surface will block + // FrameEditor when it's full. There will be no frame drop, or FrameEditor's output surface + // growing out of bound. + frameEditor.processData(); + } + } + + while (decoder.getOutputBufferInfo() != null) { + decoder.releaseOutputBuffer(/* render= */ true); + } + + if (decoder.isEnded()) { + // TODO(internal b/208986865): Handle possible last frame drop. + encoder.signalEndOfInputStream(); + return false; + } + + return frameEditor != null && frameEditor.hasInputData(); + } + + /** Processes input data. */ + private boolean processDataDefault() { if (frameEditor != null) { if (frameEditor.hasInputData()) { waitingForFrameEditorInput = false;