diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java index e88299d8f8..d03ea15c7b 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerAudioRenderer.java @@ -36,6 +36,9 @@ import androidx.media3.exoplayer.audio.SonicAudioProcessor; import androidx.media3.exoplayer.source.SampleStream.ReadDataResult; import java.io.IOException; import java.nio.ByteBuffer; +import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; @RequiresApi(18) /* package */ final class TransformerAudioRenderer extends TransformerBaseRenderer { @@ -51,8 +54,8 @@ import java.nio.ByteBuffer; @Nullable private MediaCodecAdapterWrapper decoder; @Nullable private MediaCodecAdapterWrapper encoder; @Nullable private SpeedProvider speedProvider; - @Nullable private Format decoderInputFormat; - @Nullable private AudioFormat encoderInputAudioFormat; + private @MonotonicNonNull Format decoderInputFormat; + private @MonotonicNonNull AudioFormat encoderInputAudioFormat; private ByteBuffer sonicOutputBuffer; private long nextEncoderInputBufferTimeUs; @@ -100,8 +103,6 @@ import java.nio.ByteBuffer; encoder = null; } speedProvider = null; - decoderInputFormat = null; - encoderInputAudioFormat = null; sonicOutputBuffer = AudioProcessor.EMPTY_BUFFER; nextEncoderInputBufferTimeUs = 0; currentSpeed = SPEED_UNSET; @@ -117,16 +118,18 @@ import java.nio.ByteBuffer; } if (ensureDecoderConfigured()) { + MediaCodecAdapterWrapper decoder = this.decoder; if (ensureEncoderAndAudioProcessingConfigured()) { - while (feedMuxerFromEncoder()) {} + MediaCodecAdapterWrapper encoder = this.encoder; + while (feedMuxerFromEncoder(encoder)) {} if (sonicAudioProcessor.isActive()) { - while (feedEncoderFromSonic()) {} - while (feedSonicFromDecoder()) {} + while (feedEncoderFromSonic(decoder, encoder)) {} + while (feedSonicFromDecoder(decoder)) {} } else { - while (feedEncoderFromDecoder()) {} + while (feedEncoderFromDecoder(decoder, encoder)) {} } } - while (feedDecoderFromInput()) {} + while (feedDecoderFromInput(decoder)) {} } } @@ -134,8 +137,7 @@ import java.nio.ByteBuffer; * Attempts to write encoder output data to the muxer, and returns whether it may be possible to * write more data immediately by calling this method again. */ - private boolean feedMuxerFromEncoder() { - MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder); + private boolean feedMuxerFromEncoder(MediaCodecAdapterWrapper encoder) { if (!hasEncoderOutputFormat) { @Nullable Format encoderOutputFormat = encoder.getOutputFormat(); if (encoderOutputFormat == null) { @@ -170,15 +172,15 @@ import java.nio.ByteBuffer; * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to * pass more data immediately by calling this method again. */ - private boolean feedEncoderFromDecoder() { - MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder); - MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder); + @RequiresNonNull({"encoderInputAudioFormat"}) + private boolean feedEncoderFromDecoder( + MediaCodecAdapterWrapper decoder, MediaCodecAdapterWrapper encoder) { if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { return false; } if (decoder.isEnded()) { - queueEndOfStreamToEncoder(); + queueEndOfStreamToEncoder(encoder); return false; } @@ -190,7 +192,7 @@ import java.nio.ByteBuffer; flushSonicAndSetSpeed(currentSpeed); return false; } - feedEncoder(decoderOutputBuffer); + feedEncoder(encoder, decoderOutputBuffer); if (!decoderOutputBuffer.hasRemaining()) { decoder.releaseOutputBuffer(); } @@ -201,8 +203,9 @@ import java.nio.ByteBuffer; * Attempts to pass audio processor output data to the encoder, and returns whether it may be * possible to pass more data immediately by calling this method again. */ - private boolean feedEncoderFromSonic() { - MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder); + @RequiresNonNull({"encoderInputAudioFormat"}) + private boolean feedEncoderFromSonic( + MediaCodecAdapterWrapper decoder, MediaCodecAdapterWrapper encoder) { if (!encoder.maybeDequeueInputBuffer(encoderInputBuffer)) { return false; } @@ -210,14 +213,14 @@ import java.nio.ByteBuffer; if (!sonicOutputBuffer.hasRemaining()) { sonicOutputBuffer = sonicAudioProcessor.getOutput(); if (!sonicOutputBuffer.hasRemaining()) { - if (checkNotNull(decoder).isEnded() && sonicAudioProcessor.isEnded()) { - queueEndOfStreamToEncoder(); + if (decoder.isEnded() && sonicAudioProcessor.isEnded()) { + queueEndOfStreamToEncoder(encoder); } return false; } } - feedEncoder(sonicOutputBuffer); + feedEncoder(encoder, sonicOutputBuffer); return true; } @@ -225,9 +228,7 @@ import java.nio.ByteBuffer; * Attempts to process decoder output data, and returns whether it may be possible to process more * data immediately by calling this method again. */ - private boolean feedSonicFromDecoder() { - MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder); - + private boolean feedSonicFromDecoder(MediaCodecAdapterWrapper decoder) { if (drainingSonicForSpeedChange) { if (sonicAudioProcessor.isEnded() && !sonicOutputBuffer.hasRemaining()) { flushSonicAndSetSpeed(currentSpeed); @@ -268,8 +269,7 @@ import java.nio.ByteBuffer; * Attempts to pass input data to the decoder, and returns whether it may be possible to pass more * data immediately by calling this method again. */ - private boolean feedDecoderFromInput() { - MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder); + private boolean feedDecoderFromInput(MediaCodecAdapterWrapper decoder) { if (!decoder.maybeDequeueInputBuffer(decoderInputBuffer)) { return false; } @@ -295,9 +295,8 @@ import java.nio.ByteBuffer; * Feeds as much data as possible between the current position and limit of the specified {@link * ByteBuffer} to the encoder, and advances its position by the number of bytes fed. */ - private void feedEncoder(ByteBuffer inputBuffer) { - AudioFormat encoderInputAudioFormat = checkNotNull(this.encoderInputAudioFormat); - MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder); + @RequiresNonNull({"encoderInputAudioFormat"}) + private void feedEncoder(MediaCodecAdapterWrapper encoder, ByteBuffer inputBuffer) { ByteBuffer encoderInputBufferData = checkNotNull(encoderInputBuffer.data); int bufferLimit = inputBuffer.limit(); inputBuffer.limit(min(bufferLimit, inputBuffer.position() + encoderInputBufferData.capacity())); @@ -314,8 +313,7 @@ import java.nio.ByteBuffer; encoder.queueInputBuffer(encoderInputBuffer); } - private void queueEndOfStreamToEncoder() { - MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder); + private void queueEndOfStreamToEncoder(MediaCodecAdapterWrapper encoder) { checkState(checkNotNull(encoderInputBuffer.data).position() == 0); encoderInputBuffer.addFlag(C.BUFFER_FLAG_END_OF_STREAM); encoderInputBuffer.flip(); @@ -327,11 +325,15 @@ import java.nio.ByteBuffer; * Attempts to configure the {@link #encoder} and Sonic (if applicable), if they have not been * configured yet, and returns whether they have been configured. */ + @RequiresNonNull({"decoder", "decoderInputFormat"}) + @EnsuresNonNullIf( + expression = {"encoder", "encoderInputAudioFormat"}, + result = true) private boolean ensureEncoderAndAudioProcessingConfigured() throws ExoPlaybackException { - if (encoder != null) { + if (encoder != null && encoderInputAudioFormat != null) { return true; } - MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder); + MediaCodecAdapterWrapper decoder = this.decoder; @Nullable Format decoderOutputFormat = decoder.getOutputFormat(); if (decoderOutputFormat == null) { return false; @@ -352,7 +354,7 @@ import java.nio.ByteBuffer; } String audioMimeType = transformation.audioMimeType == null - ? checkNotNull(decoderInputFormat).sampleMimeType + ? decoderInputFormat.sampleMimeType : transformation.audioMimeType; try { encoder = @@ -375,8 +377,11 @@ import java.nio.ByteBuffer; * Attempts to configure the {@link #decoder} if it has not been configured yet, and returns * whether the decoder has been configured. */ + @EnsuresNonNullIf( + expression = {"decoderInputFormat", "decoder"}, + result = true) private boolean ensureDecoderConfigured() throws ExoPlaybackException { - if (decoder != null) { + if (decoder != null && decoderInputFormat != null) { return true; } @@ -386,6 +391,7 @@ import java.nio.ByteBuffer; return false; } decoderInputFormat = checkNotNull(formatHolder.format); + MediaCodecAdapterWrapper decoder; try { decoder = MediaCodecAdapterWrapper.createForAudioDecoding(decoderInputFormat); } catch (IOException e) { @@ -394,6 +400,7 @@ import java.nio.ByteBuffer; } speedProvider = new SegmentSpeedProvider(decoderInputFormat); currentSpeed = speedProvider.getSpeed(0); + this.decoder = decoder; return true; } diff --git a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java index aba840bc32..264248755a 100644 --- a/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java +++ b/libraries/transformer/src/main/java/androidx/media3/transformer/TransformerTranscodingVideoRenderer.java @@ -42,7 +42,10 @@ import androidx.media3.exoplayer.source.SampleStream; import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.nio.ByteBuffer; +import org.checkerframework.checker.nullness.qual.EnsuresNonNull; +import org.checkerframework.checker.nullness.qual.EnsuresNonNullIf; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; @RequiresApi(18) /* package */ final class TransformerTranscodingVideoRenderer extends TransformerBaseRenderer { @@ -101,14 +104,26 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return; } ensureEncoderConfigured(); + MediaCodecAdapterWrapper encoder = this.encoder; ensureOpenGlConfigured(); + EGLDisplay eglDisplay = this.eglDisplay; + EGLSurface eglSurface = this.eglSurface; + GlUtil.Uniform decoderTextureTransformUniform = this.decoderTextureTransformUniform; if (!ensureDecoderConfigured()) { return; } + MediaCodecAdapterWrapper decoder = this.decoder; + SurfaceTexture decoderSurfaceTexture = this.decoderSurfaceTexture; - while (feedMuxerFromEncoder()) {} - while (feedEncoderFromDecoder()) {} - while (feedDecoderFromInput()) {} + while (feedMuxerFromEncoder(encoder)) {} + while (feedEncoderFromDecoder( + decoder, + encoder, + decoderSurfaceTexture, + eglDisplay, + eglSurface, + decoderTextureTransformUniform)) {} + while (feedDecoderFromInput(decoder)) {} } @Override @@ -150,6 +165,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; muxerWrapperTrackEnded = false; } + @EnsuresNonNullIf(expression = "decoderInputFormat", result = true) private boolean ensureInputFormatRead() { if (decoderInputFormat != null) { return true; @@ -166,6 +182,8 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return true; } + @RequiresNonNull({"decoderInputFormat"}) + @EnsuresNonNull({"encoder"}) private void ensureEncoderConfigured() throws ExoPlaybackException { if (encoder != null) { return; @@ -175,7 +193,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; encoder = MediaCodecAdapterWrapper.createForVideoEncoding( new Format.Builder() - .setWidth(checkNotNull(decoderInputFormat).width) + .setWidth(decoderInputFormat.width) .setHeight(decoderInputFormat.height) .setSampleMimeType( transformation.videoMimeType != null @@ -186,18 +204,19 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } catch (IOException e) { throw createRendererException( // TODO(claincly): should be "ENCODER_INIT_FAILED" - e, - checkNotNull(this.decoder).getOutputFormat(), - PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); + e, decoderInputFormat, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); } } + @RequiresNonNull({"encoder", "decoderInputFormat"}) + @EnsuresNonNull({"eglDisplay", "eglSurface", "decoderTextureTransformUniform"}) private void ensureOpenGlConfigured() { - if (eglDisplay != null) { + if (eglDisplay != null && eglSurface != null && decoderTextureTransformUniform != null) { return; } - eglDisplay = GlUtil.createEglDisplay(); + MediaCodecAdapterWrapper encoder = this.encoder; + EGLDisplay eglDisplay = GlUtil.createEglDisplay(); EGLContext eglContext; try { eglContext = GlUtil.createEglContext(eglDisplay); @@ -205,14 +224,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } catch (GlUtil.UnsupportedEglVersionException e) { throw new IllegalStateException("EGL version is unsupported", e); } - eglSurface = - GlUtil.getEglSurface(eglDisplay, checkNotNull(checkNotNull(encoder).getInputSurface())); + EGLSurface eglSurface = + GlUtil.getEglSurface(eglDisplay, checkNotNull(encoder.getInputSurface())); GlUtil.focusSurface( - eglDisplay, - eglContext, - eglSurface, - checkNotNull(decoderInputFormat).width, - decoderInputFormat.height); + eglDisplay, eglContext, eglSurface, decoderInputFormat.width, decoderInputFormat.height); decoderTextureId = GlUtil.createExternalTexture(); String vertexShaderCode; String fragmentShaderCode; @@ -262,31 +277,36 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; throw new IllegalStateException("Unexpected uniform name."); } } + checkNotNull(decoderTextureTransformUniform); + this.eglDisplay = eglDisplay; + this.eglSurface = eglSurface; } + @RequiresNonNull({"decoderInputFormat"}) + @EnsuresNonNullIf( + expression = {"decoder", "decoderSurfaceTexture"}, + result = true) private boolean ensureDecoderConfigured() throws ExoPlaybackException { - if (decoder != null) { + if (decoder != null && decoderSurfaceTexture != null) { return true; } checkState(decoderTextureId != GlUtil.TEXTURE_ID_UNSET); - decoderSurfaceTexture = new SurfaceTexture(decoderTextureId); + SurfaceTexture decoderSurfaceTexture = new SurfaceTexture(decoderTextureId); decoderSurfaceTexture.setOnFrameAvailableListener( surfaceTexture -> isDecoderSurfacePopulated = true); decoderSurface = new Surface(decoderSurfaceTexture); try { - decoder = - MediaCodecAdapterWrapper.createForVideoDecoding( - checkNotNull(decoderInputFormat), decoderSurface); + decoder = MediaCodecAdapterWrapper.createForVideoDecoding(decoderInputFormat, decoderSurface); } catch (IOException e) { throw createRendererException( e, decoderInputFormat, PlaybackException.ERROR_CODE_DECODER_INIT_FAILED); } + this.decoderSurfaceTexture = decoderSurfaceTexture; return true; } - private boolean feedDecoderFromInput() { - MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder); + private boolean feedDecoderFromInput(MediaCodecAdapterWrapper decoder) { if (!decoder.maybeDequeueInputBuffer(decoderInputBuffer)) { return false; } @@ -294,7 +314,6 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; decoderInputBuffer.clear(); @SampleStream.ReadDataResult int result = readSource(getFormatHolder(), decoderInputBuffer, /* readFlags= */ 0); - switch (result) { case C.RESULT_FORMAT_READ: throw new IllegalStateException("Format changes are not supported."); @@ -310,8 +329,13 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } - private boolean feedEncoderFromDecoder() { - MediaCodecAdapterWrapper decoder = checkNotNull(this.decoder); + private boolean feedEncoderFromDecoder( + MediaCodecAdapterWrapper decoder, + MediaCodecAdapterWrapper encoder, + SurfaceTexture decoderSurfaceTexture, + EGLDisplay eglDisplay, + EGLSurface eglSurface, + GlUtil.Uniform decoderTextureTransformUniform) { if (decoder.isEnded()) { return false; } @@ -323,23 +347,18 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; waitingForPopulatedDecoderSurface = true; } if (decoder.isEnded()) { - checkNotNull(encoder).signalEndOfInputStream(); + encoder.signalEndOfInputStream(); } } return false; } waitingForPopulatedDecoderSurface = false; - SurfaceTexture decoderSurfaceTexture = checkNotNull(this.decoderSurfaceTexture); decoderSurfaceTexture.updateTexImage(); decoderSurfaceTexture.getTransformMatrix(decoderTextureTransformMatrix); - GlUtil.Uniform decoderTextureTransformUniform = - checkNotNull(this.decoderTextureTransformUniform); decoderTextureTransformUniform.setFloats(decoderTextureTransformMatrix); decoderTextureTransformUniform.bind(); GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); - EGLDisplay eglDisplay = checkNotNull(this.eglDisplay); - EGLSurface eglSurface = checkNotNull(this.eglSurface); long decoderSurfaceTextureTimestampNs = decoderSurfaceTexture.getTimestamp(); EGLExt.eglPresentationTimeANDROID(eglDisplay, eglSurface, decoderSurfaceTextureTimestampNs); EGL14.eglSwapBuffers(eglDisplay, eglSurface); @@ -347,8 +366,7 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return true; } - private boolean feedMuxerFromEncoder() { - MediaCodecAdapterWrapper encoder = checkNotNull(this.encoder); + private boolean feedMuxerFromEncoder(MediaCodecAdapterWrapper encoder) { if (!hasEncoderActualOutputFormat) { @Nullable Format encoderOutputFormat = encoder.getOutputFormat(); if (encoderOutputFormat == null) {