loliball/media3
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Experimental flag to limit the number of frames in encoder

Browse Source 
Transformer.experimentalSetMaxFramesInEncoder controls max number
of frames in encoder.

VideoFrameProcessor now allows delayed releasing of frames to Surface,
while still using the original presentation time.

VideoSampleExporter can now configure video graphs to not render frames
automatically to output surface.

VideoSampleExporter.VideoGraphWrapper tracks how many frames are ready
to be rendered to Surface, and how many frames are already in-use by encoder.

PiperOrigin-RevId: 658429969
This commit is contained in:
dancho 2024-08-01 09:30:44 -07:00 committed by Copybara-Service
parent 766902634e
commit ddc86686b7
14 changed files with 298 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
libraries/common/src/main/java/androidx/media3/common/SurfaceInfo.java
View File

@ -41,6 +41,9 @@ public final class SurfaceInfo {
*/
public final int orientationDegrees;
/** Whether the {@link #surface} is an encoder input surface. */
public final boolean isEncoderInputSurface;
/** Creates a new instance. */
public SurfaceInfo(Surface surface, int width, int height) {
this(surface, width, height, /* orientationDegrees= */ 0);
@ -48,6 +51,16 @@ public final class SurfaceInfo {
/** Creates a new instance. */
public SurfaceInfo(Surface surface, int width, int height, int orientationDegrees) {
this(surface, width, height, orientationDegrees, /* isEncoderInputSurface= */ false);
}
/** Creates a new instance. */
public SurfaceInfo(
Surface surface,
int width,
int height,
int orientationDegrees,
boolean isEncoderInputSurface) {
checkArgument(
orientationDegrees == 0
|| orientationDegrees == 90
@ -58,6 +71,7 @@ public final class SurfaceInfo {
this.width = width;
this.height = height;
this.orientationDegrees = orientationDegrees;
this.isEncoderInputSurface = isEncoderInputSurface;
}
@Override
@ -72,6 +86,7 @@ public final class SurfaceInfo {
return width == that.width
&& height == that.height
&& orientationDegrees == that.orientationDegrees
&& isEncoderInputSurface == that.isEncoderInputSurface
&& surface.equals(that.surface);
}
@ -81,6 +96,7 @@ public final class SurfaceInfo {
result = 31 * result + width;
result = 31 * result + height;
result = 31 * result + orientationDegrees;
result = 31 * result + (isEncoderInputSurface ? 1 : 0);
return result;
}
}

12
libraries/common/src/main/java/androidx/media3/common/VideoFrameProcessor.java
View File

@ -186,6 +186,13 @@ public interface VideoFrameProcessor {
/** Indicates the frame should be dropped after {@link #renderOutputFrame(long)} is invoked. */
long DROP_OUTPUT_FRAME = -2;
/**
* Indicates the frame should preserve the input presentation time when {@link
* #renderOutputFrame(long)} is invoked.
*/
@SuppressWarnings("GoodTime-ApiWithNumericTimeUnit") // This is a named constant, not a time unit.
long RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME = -3;
/**
* Provides an input {@link Bitmap} to the {@link VideoFrameProcessor}.
*
@ -333,7 +340,10 @@ public interface VideoFrameProcessor {
*
* @param renderTimeNs The render time to use for the frame, in nanoseconds. The render time can
* be before or after the current system time. Use {@link #DROP_OUTPUT_FRAME} to drop the
* frame, or {@link #RENDER_OUTPUT_FRAME_IMMEDIATELY} to render the frame immediately.
* frame, or {@link #RENDER_OUTPUT_FRAME_IMMEDIATELY} to render the frame immediately, or
* {@link #RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME} to render the frame to the {@linkplain
* #setOutputSurfaceInfo output surface} with the presentation timestamp seen in {@link
* Listener#onOutputFrameAvailableForRendering(long)}.
*/
void renderOutputFrame(long renderTimeNs);

22
libraries/effect/src/main/java/androidx/media3/effect/FinalShaderProgramWrapper.java
View File

@ -15,6 +15,8 @@
*/
package androidx.media3.effect;
import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY;
import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.effect.DebugTraceUtil.COMPONENT_VFP;
@ -443,12 +445,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
GlUtil.clearFocusedBuffers();
defaultShaderProgram.drawFrame(inputTexture.texId, presentationTimeUs);
EGLExt.eglPresentationTimeANDROID(
eglDisplay,
outputEglSurface,
renderTimeNs == VideoFrameProcessor.RENDER_OUTPUT_FRAME_IMMEDIATELY
? System.nanoTime()
: renderTimeNs);
long eglPresentationTimeNs;
if (renderTimeNs == RENDER_OUTPUT_FRAME_IMMEDIATELY) {
eglPresentationTimeNs = System.nanoTime();
} else if (renderTimeNs == RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME) {
checkState(presentationTimeUs != C.TIME_UNSET);
eglPresentationTimeNs = presentationTimeUs * 1000;
} else {
eglPresentationTimeNs = renderTimeNs;
}
EGLExt.eglPresentationTimeANDROID(eglDisplay, outputEglSurface, eglPresentationTimeNs);
EGL14.eglSwapBuffers(eglDisplay, outputEglSurface);
DebugTraceUtil.logEvent(COMPONENT_VFP, EVENT_RENDERED_TO_OUTPUT_SURFACE, presentationTimeUs);
}
@ -524,8 +531,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
eglDisplay,
outputSurfaceInfo.surface,
outputColorInfo.colorTransfer,
// Frames are only rendered automatically when outputting to an encoder.
/* isEncoderInputSurface= */ renderFramesAutomatically);
outputSurfaceInfo.isEncoderInputSurface);
}
if (textureOutputListener != null) {
outputTexturePool.ensureConfigured(glObjectsProvider, outputWidth, outputHeight);

14
libraries/effect/src/main/java/androidx/media3/effect/MultipleInputVideoGraph.java
View File

@ -85,6 +85,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
private final SparseArray<CompositorOutputTextureRelease> compositorOutputTextureReleases;
private final long initialTimestampOffsetUs;
private final boolean renderFramesAutomatically;
@Nullable private VideoFrameProcessor compositionVideoFrameProcessor;
@Nullable private VideoCompositor videoCompositor;
@ -106,7 +107,8 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
long initialTimestampOffsetUs,
boolean renderFramesAutomatically) {
checkArgument(videoFrameProcessorFactory instanceof DefaultVideoFrameProcessor.Factory);
this.context = context;
this.outputColorInfo = outputColorInfo;
@ -116,6 +118,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
this.videoCompositorSettings = videoCompositorSettings;
this.compositionEffects = new ArrayList<>(compositionEffects);
this.initialTimestampOffsetUs = initialTimestampOffsetUs;
this.renderFramesAutomatically = renderFramesAutomatically;
lastRenderedPresentationTimeUs = C.TIME_UNSET;
preProcessors = new SparseArray<>();
sharedExecutorService = newSingleThreadScheduledExecutor(SHARED_EXECUTOR_NAME);
@ -150,7 +153,7 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
context,
debugViewProvider,
outputColorInfo,
/* renderFramesAutomatically= */ true,
renderFramesAutomatically,
/* listenerExecutor= */ MoreExecutors.directExecutor(),
new VideoFrameProcessor.Listener() {
// All of this listener's methods are called on the sharedExecutorService.
@ -174,6 +177,9 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
hasProducedFrameWithTimestampZero = true;
}
lastRenderedPresentationTimeUs = presentationTimeUs;
listenerExecutor.execute(
() -> listener.onOutputFrameAvailableForRendering(presentationTimeUs));
}
@Override
@ -312,6 +318,10 @@ public abstract class MultipleInputVideoGraph implements VideoGraph {
released = true;
}
protected VideoFrameProcessor getCompositionVideoFrameProcessor() {
return checkStateNotNull(compositionVideoFrameProcessor);
}
protected long getInitialTimestampOffsetUs() {
return initialTimestampOffsetUs;
}

2
libraries/transformer/src/androidTest/java/androidx/media3/transformer/AndroidTestUtil.java
View File

@ -276,6 +276,7 @@ public final class AndroidTestUtil {
.setCodecs("avc1.64001F")
.build())
.setVideoDurationUs(1_024_000L)
.setVideoFrameCount(30)
.setVideoTimestampsUs(
ImmutableList.of(
0L, 33_366L, 66_733L, 100_100L, 133_466L, 166_833L, 200_200L, 233_566L, 266_933L,
@ -390,6 +391,7 @@ public final class AndroidTestUtil {
.setFrameRate(30.00f)
.setCodecs("avc1.42C015")
.build())
.setVideoFrameCount(932)
.build();
public static final AssetInfo MP4_ASSET_WITH_SHORTER_AUDIO =

54
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerEndToEndTest.java
View File

@ -422,6 +422,60 @@ public class TransformerEndToEndTest {
assertThat(new File(result.filePath).length()).isGreaterThan(0);
}
@Test
public void videoEditing_withOneFrameInEncoder_completesWithConsistentFrameCount()
throws Exception {
assumeFormatsSupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat,
/* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat);
Transformer transformer =
new Transformer.Builder(context)
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context))
.experimentalSetMaxFramesInEncoder(1)
.build();
MediaItem mediaItem =
MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.uri));
EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.videoFrameCount)
.isEqualTo(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFrameCount);
assertThat(new File(result.filePath).length()).isGreaterThan(0);
}
@Test
public void videoEditing_withMaxFramesInEncoder_completesWithConsistentFrameCount()
throws Exception {
assumeFormatsSupported(
context,
testId,
/* inputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat,
/* outputFormat= */ MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFormat);
Transformer transformer =
new Transformer.Builder(context)
.setEncoderFactory(new AndroidTestUtil.ForceEncodeEncoderFactory(context))
.experimentalSetMaxFramesInEncoder(16)
.build();
MediaItem mediaItem =
MediaItem.fromUri(Uri.parse(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.uri));
EditedMediaItem editedMediaItem = new EditedMediaItem.Builder(mediaItem).build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, transformer)
.build()
.run(testId, editedMediaItem);
assertThat(result.exportResult.videoFrameCount)
.isEqualTo(MP4_ASSET_WITH_INCREASING_TIMESTAMPS_320W_240H_15S.videoFrameCount);
assertThat(new File(result.filePath).length()).isGreaterThan(0);
}
// TODO: b/345483531 - Migrate this test to a Parameterized ImageSequence test.
@Test
public void videoEditing_withShortAlternatingImages_completesWithCorrectFrameCountAndDuration()

50
libraries/transformer/src/androidTest/java/androidx/media3/transformer/TransformerMultiSequenceCompositionTest.java
View File

@ -31,6 +31,8 @@ import static com.google.common.truth.Truth.assertWithMessage;
import android.content.Context;
import android.graphics.Bitmap;
import android.net.Uri;
import androidx.media3.common.C;
import androidx.media3.common.Effect;
import androidx.media3.common.MediaItem;
import androidx.media3.common.util.Size;
@ -71,9 +73,15 @@ public final class TransformerMultiSequenceCompositionTest {
private static final int EXPORT_WIDTH = 360;
private static final int EXPORT_HEIGHT = 240;
@Parameters(name = "{0}")
public static ImmutableList<Boolean> workingColorSpaceLinear() {
return ImmutableList.of(false, true);
@Parameters(name = "{0},maxFramesInEncoder={1}")
public static ImmutableList<Object[]> parameters() {
ImmutableList.Builder<Object[]> listBuilder = new ImmutableList.Builder<>();
for (Boolean workingColorSpaceLinear : new boolean[] {false, true}) {
for (Integer maxFramesInEncoder : new int[] {C.INDEX_UNSET, 1, 16}) {
listBuilder.add(new Object[] {workingColorSpaceLinear, maxFramesInEncoder});
}
}
return listBuilder.build();
}
private final Context context = ApplicationProvider.getApplicationContext();
@ -81,7 +89,11 @@ public final class TransformerMultiSequenceCompositionTest {
private String testId;
@Parameter public boolean workingColorSpaceLinear;
@Parameter(0)
public boolean workingColorSpaceLinear;
@Parameter(1)
public int maxFramesInEncoder;
@Before
public void setUpTestId() {
@ -218,6 +230,33 @@ public final class TransformerMultiSequenceCompositionTest {
extractBitmapsFromVideo(context, checkNotNull(result.filePath)), testId);
}
@Test
public void export_completesWithConsistentFrameCount() throws Exception {
assumeFormatsSupported(
context,
testId,
/* inputFormat= */ MP4_ASSET.videoFormat,
/* outputFormat= */ MP4_ASSET.videoFormat);
MediaItem mediaItem = MediaItem.fromUri(Uri.parse(MP4_ASSET.uri));
ImmutableList<Effect> videoEffects = ImmutableList.of(Presentation.createForHeight(480));
Effects effects = new Effects(/* audioProcessors= */ ImmutableList.of(), videoEffects);
EditedMediaItem editedMediaItem =
new EditedMediaItem.Builder(mediaItem).setEffects(effects).build();
Composition composition =
new Composition.Builder(
new EditedMediaItemSequence(editedMediaItem),
new EditedMediaItemSequence(editedMediaItem))
.build();
ExportTestResult result =
new TransformerAndroidTestRunner.Builder(context, buildTransformer())
.build()
.run(testId, composition);
assertThat(result.exportResult.videoFrameCount).isEqualTo(MP4_ASSET.videoFrameCount);
assertThat(new File(result.filePath).length()).isGreaterThan(0);
}
private Transformer buildTransformer() {
// Use linear color space for grayscale effects.
Transformer.Builder builder = new Transformer.Builder(context);
@ -227,6 +266,7 @@ public final class TransformerMultiSequenceCompositionTest {
.setSdrWorkingColorSpace(DefaultVideoFrameProcessor.WORKING_COLOR_SPACE_LINEAR)
.build());
}
builder.experimentalSetMaxFramesInEncoder(maxFramesInEncoder);
return builder.build();
}
@ -278,7 +318,7 @@ public final class TransformerMultiSequenceCompositionTest {
Bitmap actualBitmap = actualBitmaps.get(i);
maybeSaveTestBitmap(
testId, /* bitmapLabel= */ String.valueOf(i), actualBitmap, /* path= */ null);
String subTestId = testId + "_" + i;
String subTestId = testId.replaceAll(",maxFramesInEncoder=-?\\d+", "") + "_" + i;
Bitmap expectedBitmap =
readBitmap(Util.formatInvariant("%s/%s.png", PNG_ASSET_BASE_PATH, subTestId));
float averagePixelAbsoluteDifference =

1
libraries/transformer/src/main/java/androidx/media3/transformer/ExperimentalAnalyzerModeFactory.java
View File

@ -74,6 +74,7 @@ public final class ExperimentalAnalyzerModeFactory {
return transformer
.buildUpon()
.experimentalSetTrimOptimizationEnabled(false)
.experimentalSetMaxFramesInEncoder(C.INDEX_UNSET)
.setEncoderFactory(new DroppingEncoder.Factory(context))
.setMaxDelayBetweenMuxerSamplesMs(C.TIME_UNSET)
.setMuxerFactory(

32
libraries/transformer/src/main/java/androidx/media3/transformer/Transformer.java
View File

@ -115,6 +115,7 @@ public final class Transformer {
private boolean trimOptimizationEnabled;
private boolean fileStartsOnVideoFrameEnabled;
private long maxDelayBetweenMuxerSamplesMs;
private int maxFramesInEncoder;
private ListenerSet<Transformer.Listener> listeners;
private AssetLoader.@MonotonicNonNull Factory assetLoaderFactory;
private AudioMixer.Factory audioMixerFactory;
@ -133,6 +134,7 @@ public final class Transformer {
public Builder(Context context) {
this.context = context.getApplicationContext();
maxDelayBetweenMuxerSamplesMs = DEFAULT_MAX_DELAY_BETWEEN_MUXER_SAMPLES_MS;
maxFramesInEncoder = C.INDEX_UNSET;
audioProcessors = ImmutableList.of();
videoEffects = ImmutableList.of();
audioMixerFactory = new DefaultAudioMixer.Factory();
@ -158,6 +160,7 @@ public final class Transformer {
this.trimOptimizationEnabled = transformer.trimOptimizationEnabled;
this.fileStartsOnVideoFrameEnabled = transformer.fileStartsOnVideoFrameEnabled;
this.maxDelayBetweenMuxerSamplesMs = transformer.maxDelayBetweenMuxerSamplesMs;
this.maxFramesInEncoder = transformer.maxFramesInEncoder;
this.listeners = transformer.listeners;
this.assetLoaderFactory = transformer.assetLoaderFactory;
this.audioMixerFactory = transformer.audioMixerFactory;
@ -333,6 +336,30 @@ public final class Transformer {
return this;
}
/**
* Limits how many video frames can be processed at any time by the {@linkplain Codec encoder}.
*
* <p>A video frame starts encoding when it enters the {@linkplain Codec#getInputSurface()
* encoder input surface}, and finishes encoding when the corresponding {@linkplain
* Codec#releaseOutputBuffer encoder output buffer is released}.
*
* <p>The default value is {@link C#INDEX_UNSET}, which means no limit is enforced.
*
* <p>This method is experimental and will be renamed or removed in a future release.
*
* @param maxFramesInEncoder The maximum number of frames that the video encoder is allowed to
* process at a time, or {@link C#INDEX_UNSET} if no limit is enforced.
* @return This builder.
* @throws IllegalArgumentException If {@code maxFramesInEncoder} is not equal to {@link
* C#INDEX_UNSET} and is non-positive.
*/
@CanIgnoreReturnValue
public Builder experimentalSetMaxFramesInEncoder(int maxFramesInEncoder) {
checkArgument(maxFramesInEncoder > 0 || maxFramesInEncoder == C.INDEX_UNSET);
this.maxFramesInEncoder = maxFramesInEncoder;
return this;
}
/**
* Sets whether to ensure that the output file starts on a video frame.
*
@ -592,6 +619,7 @@ public final class Transformer {
trimOptimizationEnabled,
fileStartsOnVideoFrameEnabled,
maxDelayBetweenMuxerSamplesMs,
maxFramesInEncoder,
listeners,
assetLoaderFactory,
audioMixerFactory,
@ -844,6 +872,7 @@ public final class Transformer {
private final boolean trimOptimizationEnabled;
private final boolean fileStartsOnVideoFrameEnabled;
private final long maxDelayBetweenMuxerSamplesMs;
private final int maxFramesInEncoder;
private final ListenerSet<Transformer.Listener> listeners;
@Nullable private final AssetLoader.Factory assetLoaderFactory;
@ -881,6 +910,7 @@ public final class Transformer {
boolean trimOptimizationEnabled,
boolean fileStartsOnVideoFrameEnabled,
long maxDelayBetweenMuxerSamplesMs,
int maxFramesInEncoder,
ListenerSet<Listener> listeners,
@Nullable AssetLoader.Factory assetLoaderFactory,
AudioMixer.Factory audioMixerFactory,
@ -901,6 +931,7 @@ public final class Transformer {
this.trimOptimizationEnabled = trimOptimizationEnabled;
this.fileStartsOnVideoFrameEnabled = fileStartsOnVideoFrameEnabled;
this.maxDelayBetweenMuxerSamplesMs = maxDelayBetweenMuxerSamplesMs;
this.maxFramesInEncoder = maxFramesInEncoder;
this.listeners = listeners;
this.assetLoaderFactory = assetLoaderFactory;
this.audioMixerFactory = audioMixerFactory;
@ -1611,6 +1642,7 @@ public final class Transformer {
audioMixerFactory,
videoFrameProcessorFactory,
encoderFactory,
maxFramesInEncoder,
muxerWrapper,
componentListener,
fallbackListener,

7
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerInternal.java
View File

@ -148,6 +148,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
private final Object setMaxSequenceDurationUsLock;
private final Object progressLock;
private final ProgressHolder internalProgressHolder;
private final int maxFramesInEncoder;
private boolean isDrainingExporters;
@ -192,6 +193,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
AudioMixer.Factory audioMixerFactory,
VideoFrameProcessor.Factory videoFrameProcessorFactory,
Codec.EncoderFactory encoderFactory,
int maxFramesInEncoder,
MuxerWrapper muxerWrapper,
Listener listener,
FallbackListener fallbackListener,
@ -202,6 +204,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
this.context = context;
this.composition = composition;
this.encoderFactory = new CapturingEncoderFactory(encoderFactory);
this.maxFramesInEncoder = maxFramesInEncoder;
this.listener = listener;
this.applicationHandler = applicationHandler;
this.clock = clock;
@ -738,8 +741,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
fallbackListener,
debugViewProvider,
videoSampleTimestampOffsetUs,
/* hasMultipleInputs= */ assetLoaderInputTracker
.hasMultipleConcurrentVideoTracks()));
/* hasMultipleInputs= */ assetLoaderInputTracker.hasMultipleConcurrentVideoTracks(),
maxFramesInEncoder));
}
}

20
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerMultipleInputVideoGraph.java
View File

@ -16,6 +16,8 @@
package androidx.media3.transformer;
import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME;
import android.content.Context;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
@ -53,7 +55,8 @@ import java.util.concurrent.Executor;
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
long initialTimestampOffsetUs,
boolean renderFramesAutomatically) {
return new TransformerMultipleInputVideoGraph(
context,
videoFrameProcessorFactory,
@ -63,7 +66,8 @@ import java.util.concurrent.Executor;
listenerExecutor,
videoCompositorSettings,
compositionEffects,
initialTimestampOffsetUs);
initialTimestampOffsetUs,
renderFramesAutomatically);
}
}
@ -76,7 +80,8 @@ import java.util.concurrent.Executor;
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
long initialTimestampOffsetUs,
boolean renderFramesAutomatically) {
super(
context,
videoFrameProcessorFactory,
@ -86,7 +91,8 @@ import java.util.concurrent.Executor;
listenerExecutor,
videoCompositorSettings,
compositionEffects,
initialTimestampOffsetUs);
initialTimestampOffsetUs,
renderFramesAutomatically);
}
@Override
@ -95,4 +101,10 @@ import java.util.concurrent.Executor;
return new VideoFrameProcessingWrapper(
getProcessor(inputIndex), /* presentation= */ null, getInitialTimestampOffsetUs());
}
@Override
public void renderOutputFrameWithMediaPresentationTime() {
getCompositionVideoFrameProcessor()
.renderOutputFrame(RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME);
}
}

11
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerSingleInputVideoGraph.java
View File

@ -16,6 +16,7 @@
package androidx.media3.transformer;
import static androidx.media3.common.VideoFrameProcessor.RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME;
import static androidx.media3.common.util.Assertions.checkState;
import android.content.Context;
@ -57,7 +58,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs) {
long initialTimestampOffsetUs,
boolean renderFramesAutomatically) {
@Nullable Presentation presentation = null;
for (int i = 0; i < compositionEffects.size(); i++) {
Effect effect = compositionEffects.get(i);
@ -73,7 +75,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
debugViewProvider,
listenerExecutor,
videoCompositorSettings,
/* renderFramesAutomatically= */ true,
renderFramesAutomatically,
presentation,
initialTimestampOffsetUs);
}
@ -114,4 +116,9 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
getProcessor(inputIndex), getPresentation(), getInitialTimestampOffsetUs());
return videoFrameProcessingWrapper;
}
@Override
public void renderOutputFrameWithMediaPresentationTime() {
getProcessor(getInputIndex()).renderOutputFrame(RENDER_OUTPUT_FRAME_WITH_PRESENTATION_TIME);
}
}

22
libraries/transformer/src/main/java/androidx/media3/transformer/TransformerVideoGraph.java
View File

@ -20,6 +20,7 @@ import android.content.Context;
import androidx.media3.common.ColorInfo;
import androidx.media3.common.DebugViewProvider;
import androidx.media3.common.Effect;
import androidx.media3.common.SurfaceInfo;
import androidx.media3.common.VideoFrameProcessingException;
import androidx.media3.common.VideoFrameProcessor;
import androidx.media3.common.VideoGraph;
@ -44,6 +45,10 @@ import java.util.concurrent.Executor;
* composition.
* @param compositionEffects A list of {@linkplain Effect effects} to apply to the composition.
* @param initialTimestampOffsetUs The timestamp offset for the first frame, in microseconds.
* @param renderFramesAutomatically If {@code true}, the instance will render output frames to
* the {@linkplain #setOutputSurfaceInfo(SurfaceInfo) output surface} automatically as the
* instance is done processing them. If {@code false}, the instance will block until {@link
* #renderOutputFrameWithMediaPresentationTime()} is called, to render the frame.
* @return A new instance.
* @throws VideoFrameProcessingException If a problem occurs while creating the {@link
* VideoFrameProcessor}.
@ -56,7 +61,8 @@ import java.util.concurrent.Executor;
Executor listenerExecutor,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects,
long initialTimestampOffsetUs)
long initialTimestampOffsetUs,
boolean renderFramesAutomatically)
throws VideoFrameProcessingException;
}
@ -73,4 +79,18 @@ import java.util.concurrent.Executor;
* @param inputIndex The index of the input, which could be used to order the inputs.
*/
GraphInput createInput(int inputIndex) throws VideoFrameProcessingException;
/**
* Renders the oldest unrendered output frame that has become {@linkplain
* Listener#onOutputFrameAvailableForRendering(long) available for rendering} to the output
* surface.
*
* <p>This method must only be called if {@code renderFramesAutomatically} was set to {@code
* false} using the {@link Factory} and should be called exactly once for each frame that becomes
* {@linkplain Listener#onOutputFrameAvailableForRendering(long) available for rendering}.
*
* <p>This will render the output frame to the {@linkplain #setOutputSurfaceInfo output surface}
* with the presentation seen in {@link Listener#onOutputFrameAvailableForRendering(long)}.
*/
void renderOutputFrameWithMediaPresentationTime();
}

67
libraries/transformer/src/main/java/androidx/media3/transformer/VideoSampleExporter.java
View File

@ -24,6 +24,7 @@ import static androidx.media3.common.ColorInfo.SRGB_BT709_FULL;
import static androidx.media3.common.ColorInfo.isTransferHdr;
import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkNotNull;
import static androidx.media3.common.util.Assertions.checkState;
import static androidx.media3.transformer.Composition.HDR_MODE_KEEP_HDR;
import static androidx.media3.transformer.Composition.HDR_MODE_TONE_MAP_HDR_TO_SDR_USING_OPEN_GL;
import static androidx.media3.transformer.TransformerUtil.getOutputMimeTypeAndHdrModeAfterFallback;
@ -54,13 +55,14 @@ import java.nio.ByteBuffer;
import java.util.List;
import java.util.Objects;
import org.checkerframework.checker.initialization.qual.Initialized;
import org.checkerframework.checker.lock.qual.GuardedBy;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
import org.checkerframework.dataflow.qual.Pure;
/** Processes, encodes and muxes raw video frames. */
/* package */ final class VideoSampleExporter extends SampleExporter {
private final TransformerVideoGraph videoGraph;
private final VideoGraphWrapper videoGraph;
private final EncoderWrapper encoderWrapper;
private final DecoderInputBuffer encoderOutputBuffer;
private final long initialTimestampOffsetUs;
@ -86,7 +88,8 @@ import org.checkerframework.dataflow.qual.Pure;
FallbackListener fallbackListener,
DebugViewProvider debugViewProvider,
long initialTimestampOffsetUs,
boolean hasMultipleInputs)
boolean hasMultipleInputs,
int maxFramesInEncoder)
throws ExportException {
// TODO(b/278259383) Consider delaying configuration of VideoSampleExporter to use the decoder
// output format instead of the extractor output format, to match AudioSampleExporter behavior.
@ -142,7 +145,8 @@ import org.checkerframework.dataflow.qual.Pure;
errorConsumer,
debugViewProvider,
videoCompositorSettings,
compositionEffects);
compositionEffects,
maxFramesInEncoder);
videoGraph.initialize();
} catch (VideoFrameProcessingException e) {
throw ExportException.createForVideoFrameProcessingException(e);
@ -199,6 +203,7 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
protected void releaseMuxerInputBuffer() throws ExportException {
encoderWrapper.releaseOutputBuffer(/* render= */ false);
videoGraph.onEncoderBufferReleased();
}
@Override
@ -340,7 +345,8 @@ import org.checkerframework.dataflow.qual.Pure;
encoder.getInputSurface(),
actualEncoderFormat.width,
actualEncoderFormat.height,
outputRotationDegrees);
outputRotationDegrees,
/* isEncoderInputSurface= */ true);
if (releaseEncoder) {
encoder.release();
@ -454,6 +460,12 @@ import org.checkerframework.dataflow.qual.Pure;
private final TransformerVideoGraph videoGraph;
private final Consumer<ExportException> errorConsumer;
private final int maxFramesInEncoder;
private final boolean renderFramesAutomatically;
private final Object lock;
private @GuardedBy("lock") int framesInEncoder;
private @GuardedBy("lock") int framesAvailableToRender;
public VideoGraphWrapper(
Context context,
@ -462,7 +474,8 @@ import org.checkerframework.dataflow.qual.Pure;
Consumer<ExportException> errorConsumer,
DebugViewProvider debugViewProvider,
VideoCompositorSettings videoCompositorSettings,
List<Effect> compositionEffects)
List<Effect> compositionEffects,
int maxFramesInEncoder)
throws VideoFrameProcessingException {
this.errorConsumer = errorConsumer;
// To satisfy the nullness checker by declaring an initialized this reference used in the
@ -470,6 +483,11 @@ import org.checkerframework.dataflow.qual.Pure;
@SuppressWarnings("nullness:assignment")
@Initialized
VideoGraphWrapper thisRef = this;
this.maxFramesInEncoder = maxFramesInEncoder;
// Automatically render frames if the sample exporter does not limit the number of frames in
// the encoder.
renderFramesAutomatically = maxFramesInEncoder < 1;
lock = new Object();
videoGraph =
videoGraphFactory.create(
context,
@ -479,7 +497,8 @@ import org.checkerframework.dataflow.qual.Pure;
/* listenerExecutor= */ MoreExecutors.directExecutor(),
videoCompositorSettings,
compositionEffects,
initialTimestampOffsetUs);
initialTimestampOffsetUs,
renderFramesAutomatically);
}
@Override
@ -495,7 +514,12 @@ import org.checkerframework.dataflow.qual.Pure;
@Override
public void onOutputFrameAvailableForRendering(long framePresentationTimeUs) {
// Do nothing.
if (!renderFramesAutomatically) {
synchronized (lock) {
framesAvailableToRender += 1;
}
maybeRenderEarliestOutputFrame();
}
}
@Override
@ -534,6 +558,11 @@ import org.checkerframework.dataflow.qual.Pure;
return videoGraph.createInput(inputIndex);
}
@Override
public void renderOutputFrameWithMediaPresentationTime() {
videoGraph.renderOutputFrameWithMediaPresentationTime();
}
@Override
public void setOutputSurfaceInfo(@Nullable SurfaceInfo outputSurfaceInfo) {
videoGraph.setOutputSurfaceInfo(outputSurfaceInfo);
@ -548,5 +577,29 @@ import org.checkerframework.dataflow.qual.Pure;
public void release() {
videoGraph.release();
}
public void onEncoderBufferReleased() {
if (!renderFramesAutomatically) {
synchronized (lock) {
checkState(framesInEncoder > 0);
framesInEncoder -= 1;
}
maybeRenderEarliestOutputFrame();
}
}
private void maybeRenderEarliestOutputFrame() {
boolean shouldRender = false;
synchronized (lock) {
if (framesAvailableToRender > 0 && framesInEncoder < maxFramesInEncoder) {
framesInEncoder += 1;
framesAvailableToRender -= 1;
shouldRender = true;
}
}
if (shouldRender) {
renderOutputFrameWithMediaPresentationTime();
}
}
}
}