From c41f3ea75e7ed07c873f4eb086189cd2e77c56e8 Mon Sep 17 00:00:00 2001 From: anjalibh Date: Wed, 9 Dec 2015 14:23:36 -0800 Subject: [PATCH 1/5] Allow custom VPX output buffer renderers. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=109832096 --- .../exoplayer/demo/vp9opus/VideoPlayer.java | 4 +- .../ext/vp9/LibvpxVideoTrackRenderer.java | 77 +++++++----- .../android/exoplayer/ext/vp9/VpxDecoder.java | 6 +- .../exoplayer/ext/vp9/VpxDecoderWrapper.java | 113 ++++-------------- .../exoplayer/ext/vp9/VpxOutputBuffer.java | 85 +++++++++++++ .../ext/vp9/VpxOutputBufferRenderer.java | 28 +++++ .../exoplayer/ext/vp9/VpxRenderer.java | 61 +++++----- .../ext/vp9/VpxVideoSurfaceView.java | 7 +- extensions/vp9/src/main/jni/vpx_jni.cc | 4 +- 9 files changed, 225 insertions(+), 160 deletions(-) create mode 100644 extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBuffer.java create mode 100644 extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBufferRenderer.java diff --git a/demo_misc/vp9_opus_sw/src/main/java/com/google/android/exoplayer/demo/vp9opus/VideoPlayer.java b/demo_misc/vp9_opus_sw/src/main/java/com/google/android/exoplayer/demo/vp9opus/VideoPlayer.java index 4733fc0f90..d84618474a 100644 --- a/demo_misc/vp9_opus_sw/src/main/java/com/google/android/exoplayer/demo/vp9opus/VideoPlayer.java +++ b/demo_misc/vp9_opus_sw/src/main/java/com/google/android/exoplayer/demo/vp9opus/VideoPlayer.java @@ -173,7 +173,7 @@ public class VideoPlayer extends Activity implements OnClickListener, TrackRenderer videoRenderer = new LibvpxVideoTrackRenderer(sampleSource, true, handler, this, 50); if (useOpenGL) { - player.sendMessage(videoRenderer, LibvpxVideoTrackRenderer.MSG_SET_VPX_SURFACE_VIEW, + player.sendMessage(videoRenderer, LibvpxVideoTrackRenderer.MSG_SET_OUTPUT_BUFFER_RENDERER, vpxVideoSurfaceView); surfaceView.setVisibility(View.GONE); } else { @@ -202,7 +202,7 @@ public class VideoPlayer extends Activity implements OnClickListener, player.addListener(this); mediaController.setMediaPlayer(new PlayerControl(player)); mediaController.setEnabled(true); - player.sendMessage(renderers[0], LibvpxVideoTrackRenderer.MSG_SET_VPX_SURFACE_VIEW, + player.sendMessage(renderers[0], LibvpxVideoTrackRenderer.MSG_SET_OUTPUT_BUFFER_RENDERER, vpxVideoSurfaceView); player.prepare(renderers); player.setPlayWhenReady(true); diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/LibvpxVideoTrackRenderer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/LibvpxVideoTrackRenderer.java index fd2d0579e4..2b162eb3e6 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/LibvpxVideoTrackRenderer.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/LibvpxVideoTrackRenderer.java @@ -23,8 +23,7 @@ import com.google.android.exoplayer.MediaFormatHolder; import com.google.android.exoplayer.SampleSource; import com.google.android.exoplayer.SampleSourceTrackRenderer; import com.google.android.exoplayer.TrackRenderer; -import com.google.android.exoplayer.ext.vp9.VpxDecoderWrapper.InputBuffer; -import com.google.android.exoplayer.ext.vp9.VpxDecoderWrapper.OutputBuffer; +import com.google.android.exoplayer.ext.vp9.VpxDecoderWrapper.VpxInputBuffer; import com.google.android.exoplayer.util.MimeTypes; import android.graphics.Bitmap; @@ -88,7 +87,12 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { * should be the target {@link Surface}, or null. */ public static final int MSG_SET_SURFACE = 1; - public static final int MSG_SET_VPX_SURFACE_VIEW = 2; + /** + * The type of a message that can be passed to an instance of this class via + * {@link ExoPlayer#sendMessage} or {@link ExoPlayer#blockingSendMessage}. The message object + * should be the target {@link VpxOutputBufferRenderer}, or null. + */ + public static final int MSG_SET_OUTPUT_BUFFER_RENDERER = 2; public final CodecCounters codecCounters = new CodecCounters(); @@ -100,14 +104,15 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { private MediaFormat format; private VpxDecoderWrapper decoder; - private InputBuffer inputBuffer; - private OutputBuffer outputBuffer; + private VpxInputBuffer inputBuffer; + private VpxOutputBuffer outputBuffer; + private VpxOutputBuffer renderedOutputBuffer; private Bitmap bitmap; private boolean drawnToSurface; private boolean renderedFirstFrame; private Surface surface; - private VpxVideoSurfaceView vpxVideoSurfaceView; + private VpxOutputBufferRenderer outputBufferRenderer; private int outputMode; private boolean inputStreamEnded; @@ -176,7 +181,13 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { return; } sourceIsReady = continueBufferingSource(positionUs); - checkForDiscontinuity(positionUs); + try { + checkForDiscontinuity(positionUs); + } catch (VpxDecoderException e) { + notifyDecoderError(e); + throw new ExoPlaybackException(e); + } + // Try and read a format if we don't have one already. if (format == null && !readFormat(positionUs)) { @@ -215,7 +226,8 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { if (outputBuffer.flags == VpxDecoderWrapper.FLAG_END_OF_STREAM) { outputStreamEnded = true; - releaseOutputBuffer(); + releaseOutputBuffer(outputBuffer); + outputBuffer = null; return; } @@ -229,7 +241,8 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { if (droppedFrameCount == maxDroppedFrameCountToNotify) { notifyAndResetDroppedFrameCount(); } - releaseOutputBuffer(); + releaseOutputBuffer(outputBuffer); + outputBuffer = null; return; } @@ -266,18 +279,23 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { drawnToSurface = true; notifyDrawnToSurface(surface); } - } else if (outputBuffer.mode == VpxDecoder.OUTPUT_MODE_YUV && vpxVideoSurfaceView != null) { - vpxVideoSurfaceView.renderFrame(outputBuffer); + } else if (outputBuffer.mode == VpxDecoder.OUTPUT_MODE_YUV && outputBufferRenderer != null) { + outputBufferRenderer.setOutputBuffer(outputBuffer); } - releaseOutputBuffer(); - } - - private void releaseOutputBuffer() throws VpxDecoderException { - decoder.releaseOutputBuffer(outputBuffer); + // Release the output buffer we rendered during the previous cycle, now that we delivered a new + // buffer. + releaseOutputBuffer(renderedOutputBuffer); + renderedOutputBuffer = outputBuffer; outputBuffer = null; } - private void renderRgbFrame(OutputBuffer outputBuffer, boolean scale) { + private void releaseOutputBuffer(VpxOutputBuffer buffer) throws VpxDecoderException { + if (buffer != null) { + decoder.releaseOutputBuffer(buffer); + } + } + + private void renderRgbFrame(VpxOutputBuffer outputBuffer, boolean scale) { if (bitmap == null || bitmap.getWidth() != outputBuffer.width || bitmap.getHeight() != outputBuffer.height) { bitmap = Bitmap.createBitmap(outputBuffer.width, outputBuffer.height, Bitmap.Config.RGB_565); @@ -332,7 +350,7 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { return true; } - private void checkForDiscontinuity(long positionUs) { + private void checkForDiscontinuity(long positionUs) throws VpxDecoderException { if (decoder == null) { return; } @@ -342,9 +360,12 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { } } - private void flushDecoder() { + private void flushDecoder() throws VpxDecoderException { inputBuffer = null; + VpxOutputBuffer bufferToRelease = outputBuffer; + // Set this to null now because releaseOutputBuffer could throw an exception. outputBuffer = null; + releaseOutputBuffer(bufferToRelease); decoder.flush(); } @@ -417,8 +438,8 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { public void handleMessage(int messageType, Object message) throws ExoPlaybackException { if (messageType == MSG_SET_SURFACE) { setSurface((Surface) message); - } else if (messageType == MSG_SET_VPX_SURFACE_VIEW) { - setVpxVideoSurfaceView((VpxVideoSurfaceView) message); + } else if (messageType == MSG_SET_OUTPUT_BUFFER_RENDERER) { + setOutputBufferRenderer((VpxOutputBufferRenderer) message); } else { super.handleMessage(messageType, message); } @@ -429,7 +450,7 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { return; } this.surface = surface; - vpxVideoSurfaceView = null; + outputBufferRenderer = null; outputMode = (surface != null) ? VpxDecoder.OUTPUT_MODE_RGB : VpxDecoder.OUTPUT_MODE_UNKNOWN; if (decoder != null) { decoder.setOutputMode(outputMode); @@ -437,20 +458,20 @@ public final class LibvpxVideoTrackRenderer extends SampleSourceTrackRenderer { drawnToSurface = false; } - private void setVpxVideoSurfaceView(VpxVideoSurfaceView vpxVideoSurfaceView) { - if (this.vpxVideoSurfaceView == vpxVideoSurfaceView) { + private void setOutputBufferRenderer(VpxOutputBufferRenderer outputBufferRenderer) { + if (this.outputBufferRenderer == outputBufferRenderer) { return; } - this.vpxVideoSurfaceView = vpxVideoSurfaceView; + this.outputBufferRenderer = outputBufferRenderer; surface = null; - outputMode = - (vpxVideoSurfaceView != null) ? VpxDecoder.OUTPUT_MODE_YUV : VpxDecoder.OUTPUT_MODE_UNKNOWN; + outputMode = (outputBufferRenderer != null) + ? VpxDecoder.OUTPUT_MODE_YUV : VpxDecoder.OUTPUT_MODE_UNKNOWN; if (decoder != null) { decoder.setOutputMode(outputMode); } } - private void notifyIfVideoSizeChanged(final OutputBuffer outputBuffer) { + private void notifyIfVideoSizeChanged(final VpxOutputBuffer outputBuffer) { if (previousWidth == -1 || previousHeight == -1 || previousWidth != outputBuffer.width || previousHeight != outputBuffer.height) { previousWidth = outputBuffer.width; diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoder.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoder.java index 954304d0cd..2c0a18df6f 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoder.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoder.java @@ -15,8 +15,6 @@ */ package com.google.android.exoplayer.ext.vp9; -import com.google.android.exoplayer.ext.vp9.VpxDecoderWrapper.OutputBuffer; - import java.nio.ByteBuffer; /** @@ -64,7 +62,7 @@ import java.nio.ByteBuffer; * @return 0 on success with a frame to render. 1 on success without a frame to render. * @throws VpxDecoderException on decode failure. */ - public int decode(ByteBuffer encoded, int size, OutputBuffer outputBuffer) + public int decode(ByteBuffer encoded, int size, VpxOutputBuffer outputBuffer) throws VpxDecoderException { if (vpxDecode(vpxDecContext, encoded, size) != 0) { throw new VpxDecoderException("libvpx decode error: " + vpxGetErrorMessage(vpxDecContext)); @@ -94,7 +92,7 @@ import java.nio.ByteBuffer; private native long vpxInit(); private native long vpxClose(long context); private native long vpxDecode(long context, ByteBuffer encoded, int length); - private native int vpxGetFrame(long context, OutputBuffer outputBuffer); + private native int vpxGetFrame(long context, VpxOutputBuffer outputBuffer); private native String vpxGetErrorMessage(long context); } diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoderWrapper.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoderWrapper.java index 5999244da7..dffbed4727 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoderWrapper.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxDecoderWrapper.java @@ -28,16 +28,19 @@ import java.util.LinkedList; public static final int FLAG_END_OF_STREAM = 1; private static final int INPUT_BUFFER_SIZE = 768 * 1024; // Value based on cs/SoftVpx.cpp. + /** + * The total number of output buffers. {@link LibvpxVideoTrackRenderer} may hold on to 2 buffers + * at a time so this value should be high enough considering LibvpxVideoTrackRenderer requirement. + */ private static final int NUM_BUFFERS = 16; private final Object lock; - private final LinkedList dequeuedInputBuffers; - private final LinkedList queuedInputBuffers; - private final LinkedList queuedOutputBuffers; - private final LinkedList dequeuedOutputBuffers; - private final InputBuffer[] availableInputBuffers; - private final OutputBuffer[] availableOutputBuffers; + private final LinkedList dequeuedInputBuffers; + private final LinkedList queuedInputBuffers; + private final LinkedList queuedOutputBuffers; + private final VpxInputBuffer[] availableInputBuffers; + private final VpxOutputBuffer[] availableOutputBuffers; private int availableInputBufferCount; private int availableOutputBufferCount; @@ -57,14 +60,13 @@ import java.util.LinkedList; dequeuedInputBuffers = new LinkedList<>(); queuedInputBuffers = new LinkedList<>(); queuedOutputBuffers = new LinkedList<>(); - dequeuedOutputBuffers = new LinkedList<>(); - availableInputBuffers = new InputBuffer[NUM_BUFFERS]; - availableOutputBuffers = new OutputBuffer[NUM_BUFFERS]; + availableInputBuffers = new VpxInputBuffer[NUM_BUFFERS]; + availableOutputBuffers = new VpxOutputBuffer[NUM_BUFFERS]; availableInputBufferCount = NUM_BUFFERS; availableOutputBufferCount = NUM_BUFFERS; for (int i = 0; i < NUM_BUFFERS; i++) { - availableInputBuffers[i] = new InputBuffer(); - availableOutputBuffers[i] = new OutputBuffer(); + availableInputBuffers[i] = new VpxInputBuffer(); + availableOutputBuffers[i] = new VpxOutputBuffer(); } } @@ -72,13 +74,13 @@ import java.util.LinkedList; this.outputMode = outputMode; } - public InputBuffer dequeueInputBuffer() throws VpxDecoderException { + public VpxInputBuffer dequeueInputBuffer() throws VpxDecoderException { synchronized (lock) { maybeThrowDecoderError(); if (availableInputBufferCount == 0) { return null; } - InputBuffer inputBuffer = availableInputBuffers[--availableInputBufferCount]; + VpxInputBuffer inputBuffer = availableInputBuffers[--availableInputBufferCount]; inputBuffer.flags = 0; inputBuffer.sampleHolder.clearData(); dequeuedInputBuffers.addLast(inputBuffer); @@ -86,7 +88,7 @@ import java.util.LinkedList; } } - public void queueInputBuffer(InputBuffer inputBuffer) throws VpxDecoderException { + public void queueInputBuffer(VpxInputBuffer inputBuffer) throws VpxDecoderException { synchronized (lock) { maybeThrowDecoderError(); dequeuedInputBuffers.remove(inputBuffer); @@ -95,22 +97,20 @@ import java.util.LinkedList; } } - public OutputBuffer dequeueOutputBuffer() throws VpxDecoderException { + public VpxOutputBuffer dequeueOutputBuffer() throws VpxDecoderException { synchronized (lock) { maybeThrowDecoderError(); if (queuedOutputBuffers.isEmpty()) { return null; } - OutputBuffer outputBuffer = queuedOutputBuffers.removeFirst(); - dequeuedOutputBuffers.add(outputBuffer); + VpxOutputBuffer outputBuffer = queuedOutputBuffers.removeFirst(); return outputBuffer; } } - public void releaseOutputBuffer(OutputBuffer outputBuffer) throws VpxDecoderException { + public void releaseOutputBuffer(VpxOutputBuffer outputBuffer) throws VpxDecoderException { synchronized (lock) { maybeThrowDecoderError(); - dequeuedOutputBuffers.remove(outputBuffer); availableOutputBuffers[availableOutputBufferCount++] = outputBuffer; maybeNotifyDecodeLoop(); } @@ -128,9 +128,6 @@ import java.util.LinkedList; while (!queuedOutputBuffers.isEmpty()) { availableOutputBuffers[availableOutputBufferCount++] = queuedOutputBuffers.removeFirst(); } - while (!dequeuedOutputBuffers.isEmpty()) { - availableOutputBuffers[availableOutputBufferCount++] = dequeuedOutputBuffers.removeFirst(); - } } } @@ -187,8 +184,8 @@ import java.util.LinkedList; private boolean decodeBuffer(VpxDecoder decoder) throws InterruptedException, VpxDecoderException { - InputBuffer inputBuffer; - OutputBuffer outputBuffer; + VpxInputBuffer inputBuffer; + VpxOutputBuffer outputBuffer; // Wait until we have an input buffer to decode, and an output buffer to decode into. synchronized (lock) { @@ -238,7 +235,7 @@ import java.util.LinkedList; return true; } - /* package */ static final class InputBuffer { + /* package */ static final class VpxInputBuffer { public final SampleHolder sampleHolder; @@ -246,75 +243,11 @@ import java.util.LinkedList; public int height; public int flags; - public InputBuffer() { + public VpxInputBuffer() { sampleHolder = new SampleHolder(SampleHolder.BUFFER_REPLACEMENT_MODE_DIRECT); sampleHolder.data = ByteBuffer.allocateDirect(INPUT_BUFFER_SIZE); } } - /* package */ static final class OutputBuffer { - - public ByteBuffer data; - public long timestampUs; - public int width; - public int height; - public int flags; - public ByteBuffer[] yuvPlanes; - public int[] yuvStrides; - public int mode; - - /** - * This method is called from C++ through JNI after decoding is done. It will resize the - * buffer based on the given dimensions. - */ - public void initForRgbFrame(int width, int height) { - this.width = width; - this.height = height; - int minimumRgbSize = width * height * 2; - if (data == null || data.capacity() < minimumRgbSize) { - data = ByteBuffer.allocateDirect(minimumRgbSize); - yuvPlanes = null; - } - data.position(0); - data.limit(minimumRgbSize); - } - - /** - * This method is called from C++ through JNI after decoding is done. It will resize the - * buffer based on the given stride. - */ - public void initForYuvFrame(int width, int height, int yStride, int uvStride) { - this.width = width; - this.height = height; - int yLength = yStride * height; - int uvLength = uvStride * ((height + 1) / 2); - int minimumYuvSize = yLength + (uvLength * 2); - if (data == null || data.capacity() < minimumYuvSize) { - data = ByteBuffer.allocateDirect(minimumYuvSize); - } - data.limit(minimumYuvSize); - if (yuvPlanes == null) { - yuvPlanes = new ByteBuffer[3]; - } - // Rewrapping has to be done on every frame since the stride might have changed. - data.position(0); - yuvPlanes[0] = data.slice(); - yuvPlanes[0].limit(yLength); - data.position(yLength); - yuvPlanes[1] = data.slice(); - yuvPlanes[1].limit(uvLength); - data.position(yLength + uvLength); - yuvPlanes[2] = data.slice(); - yuvPlanes[2].limit(uvLength); - if (yuvStrides == null) { - yuvStrides = new int[3]; - } - yuvStrides[0] = yStride; - yuvStrides[1] = uvStride; - yuvStrides[2] = uvStride; - } - - } - } diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBuffer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBuffer.java new file mode 100644 index 0000000000..193de149b7 --- /dev/null +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBuffer.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer.ext.vp9; + +import java.nio.ByteBuffer; + +/** + * OutputBuffer for storing the video frame. + */ +public final class VpxOutputBuffer { + + public ByteBuffer data; + public long timestampUs; + public int width; + public int height; + public int flags; + public ByteBuffer[] yuvPlanes; + public int[] yuvStrides; + public int mode; + + /** + * This method is called from C++ through JNI after decoding is done. It will resize the + * buffer based on the given dimensions. + */ + public void initForRgbFrame(int width, int height) { + this.width = width; + this.height = height; + int minimumRgbSize = width * height * 2; + if (data == null || data.capacity() < minimumRgbSize) { + data = ByteBuffer.allocateDirect(minimumRgbSize); + yuvPlanes = null; + } + data.position(0); + data.limit(minimumRgbSize); + } + + /** + * This method is called from C++ through JNI after decoding is done. It will resize the + * buffer based on the given stride. + */ + public void initForYuvFrame(int width, int height, int yStride, int uvStride) { + this.width = width; + this.height = height; + int yLength = yStride * height; + int uvLength = uvStride * ((height + 1) / 2); + int minimumYuvSize = yLength + (uvLength * 2); + if (data == null || data.capacity() < minimumYuvSize) { + data = ByteBuffer.allocateDirect(minimumYuvSize); + } + data.limit(minimumYuvSize); + if (yuvPlanes == null) { + yuvPlanes = new ByteBuffer[3]; + } + // Rewrapping has to be done on every frame since the stride might have changed. + data.position(0); + yuvPlanes[0] = data.slice(); + yuvPlanes[0].limit(yLength); + data.position(yLength); + yuvPlanes[1] = data.slice(); + yuvPlanes[1].limit(uvLength); + data.position(yLength + uvLength); + yuvPlanes[2] = data.slice(); + yuvPlanes[2].limit(uvLength); + if (yuvStrides == null) { + yuvStrides = new int[3]; + } + yuvStrides[0] = yStride; + yuvStrides[1] = uvStride; + yuvStrides[2] = uvStride; + } + +} diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBufferRenderer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBufferRenderer.java new file mode 100644 index 0000000000..99855ce8df --- /dev/null +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBufferRenderer.java @@ -0,0 +1,28 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer.ext.vp9; + +/** + * Renders the {@link VpxOutputBuffer}. + */ +public interface VpxOutputBufferRenderer { + + /** + * Sets the output buffer to be rendered. + */ + void setOutputBuffer(VpxOutputBuffer outputBuffer); + +} diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxRenderer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxRenderer.java index c78619594f..68765968a1 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxRenderer.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxRenderer.java @@ -15,8 +15,6 @@ */ package com.google.android.exoplayer.ext.vp9; -import com.google.android.exoplayer.ext.vp9.VpxDecoderWrapper.OutputBuffer; - import android.opengl.GLES20; import android.opengl.GLSurfaceView; @@ -67,7 +65,7 @@ import javax.microedition.khronos.opengles.GL10; private int program; private int texLocation; private FloatBuffer textureCoords; - private volatile OutputBuffer outputBuffer; + private VpxOutputBuffer outputBuffer; private int previousWidth; private int previousStride; @@ -82,7 +80,7 @@ import javax.microedition.khronos.opengles.GL10; * * @param outputBuffer OutputBuffer containing the YUV Frame to be rendered */ - public void setFrame(OutputBuffer outputBuffer) { + public synchronized void setFrame(VpxOutputBuffer outputBuffer) { this.outputBuffer = outputBuffer; } @@ -121,33 +119,36 @@ import javax.microedition.khronos.opengles.GL10; @Override public void onDrawFrame(GL10 unused) { - OutputBuffer outputBuffer = this.outputBuffer; + synchronized (this) { + VpxOutputBuffer outputBuffer = this.outputBuffer; + if (outputBuffer == null) { + // Nothing to render yet. + return; + } + for (int i = 0; i < 3; i++) { + int h = (i == 0) ? outputBuffer.height : (outputBuffer.height + 1) / 2; + GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); + GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, + outputBuffer.yuvStrides[i], h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, + outputBuffer.yuvPlanes[i]); + } + // Set cropping of stride if either width or stride has changed. + if (previousWidth != outputBuffer.width || previousStride != outputBuffer.yuvStrides[0]) { + float crop = (float) outputBuffer.width / outputBuffer.yuvStrides[0]; + textureCoords = nativeFloatBuffer( + 0.0f, 0.0f, + 0.0f, 1.0f, + crop, 0.0f, + crop, 1.0f); + GLES20.glVertexAttribPointer( + texLocation, 2, GLES20.GL_FLOAT, false, 0, textureCoords); + previousWidth = outputBuffer.width; + previousStride = outputBuffer.yuvStrides[0]; + } + } GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); - if (outputBuffer == null) { - // Nothing to render yet. - return; - } - for (int i = 0; i < 3; i++) { - int h = (i == 0) ? outputBuffer.height : (outputBuffer.height + 1) / 2; - GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); - GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, yuvTextures[i]); - GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1); - GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_LUMINANCE, outputBuffer.yuvStrides[i], - h, 0, GLES20.GL_LUMINANCE, GLES20.GL_UNSIGNED_BYTE, outputBuffer.yuvPlanes[i]); - } - // Set cropping of stride if either width or stride has changed. - if (previousWidth != outputBuffer.width || previousStride != outputBuffer.yuvStrides[0]) { - float crop = (float) outputBuffer.width / outputBuffer.yuvStrides[0]; - textureCoords = nativeFloatBuffer( - 0.0f, 0.0f, - 0.0f, 1.0f, - crop, 0.0f, - crop, 1.0f); - GLES20.glVertexAttribPointer( - texLocation, 2, GLES20.GL_FLOAT, false, 0, textureCoords); - previousWidth = outputBuffer.width; - previousStride = outputBuffer.yuvStrides[0]; - } GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, 4); checkNoGLES2Error(); } diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxVideoSurfaceView.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxVideoSurfaceView.java index 1b8046a6d2..47d0770abe 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxVideoSurfaceView.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxVideoSurfaceView.java @@ -15,8 +15,6 @@ */ package com.google.android.exoplayer.ext.vp9; -import com.google.android.exoplayer.ext.vp9.VpxDecoderWrapper.OutputBuffer; - import android.annotation.TargetApi; import android.content.Context; import android.opengl.GLSurfaceView; @@ -26,7 +24,7 @@ import android.util.AttributeSet; * A GLSurfaceView extension that scales itself to the given aspect ratio. */ @TargetApi(11) -public class VpxVideoSurfaceView extends GLSurfaceView { +public class VpxVideoSurfaceView extends GLSurfaceView implements VpxOutputBufferRenderer { private final VpxRenderer renderer; @@ -43,7 +41,8 @@ public class VpxVideoSurfaceView extends GLSurfaceView { setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY); } - public void renderFrame(OutputBuffer outputBuffer) { + @Override + public void setOutputBuffer(VpxOutputBuffer outputBuffer) { renderer.setFrame(outputBuffer); requestRender(); } diff --git a/extensions/vp9/src/main/jni/vpx_jni.cc b/extensions/vp9/src/main/jni/vpx_jni.cc index 6d8fbd5298..ba19f299cb 100644 --- a/extensions/vp9/src/main/jni/vpx_jni.cc +++ b/extensions/vp9/src/main/jni/vpx_jni.cc @@ -44,7 +44,7 @@ Java_com_google_android_exoplayer_ext_vp9_VpxDecoder_ ## NAME \ (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ -// JNI references for OutputBuffer class. +// JNI references for VpxOutputBuffer class. static jmethodID initForRgbFrame; static jmethodID initForYuvFrame; static jfieldID dataField; @@ -69,7 +69,7 @@ FUNC(jlong, vpxInit) { // Populate JNI References. const jclass outputBufferClass = env->FindClass( - "com/google/android/exoplayer/ext/vp9/VpxDecoderWrapper$OutputBuffer"); + "com/google/android/exoplayer/ext/vp9/VpxOutputBuffer"); initForYuvFrame = env->GetMethodID(outputBufferClass, "initForYuvFrame", "(IIII)V"); initForRgbFrame = env->GetMethodID(outputBufferClass, "initForRgbFrame", From 3f5b475035fc99949ae3b6122c7b5b70f88b9678 Mon Sep 17 00:00:00 2001 From: anjalibh Date: Wed, 9 Dec 2015 17:46:07 -0800 Subject: [PATCH 2/5] Expose and support colorspaces in LibVpx. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=109851975 --- .../exoplayer/ext/vp9/VpxOutputBuffer.java | 14 +++++++- .../exoplayer/ext/vp9/VpxRenderer.java | 35 +++++++++++++++---- extensions/vp9/src/main/jni/vpx_jni.cc | 21 +++++++++-- 3 files changed, 60 insertions(+), 10 deletions(-) diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBuffer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBuffer.java index 193de149b7..a509e7bcb2 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBuffer.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxOutputBuffer.java @@ -22,14 +22,25 @@ import java.nio.ByteBuffer; */ public final class VpxOutputBuffer { + public static final int COLORSPACE_UNKNOWN = 0; + public static final int COLORSPACE_BT601 = 1; + public static final int COLORSPACE_BT709 = 2; + + /** + * RGB buffer for RGB mode. + */ public ByteBuffer data; public long timestampUs; public int width; public int height; public int flags; + /** + * YUV planes for YUV mode. + */ public ByteBuffer[] yuvPlanes; public int[] yuvStrides; public int mode; + public int colorspace; /** * This method is called from C++ through JNI after decoding is done. It will resize the @@ -51,9 +62,10 @@ public final class VpxOutputBuffer { * This method is called from C++ through JNI after decoding is done. It will resize the * buffer based on the given stride. */ - public void initForYuvFrame(int width, int height, int yStride, int uvStride) { + public void initForYuvFrame(int width, int height, int yStride, int uvStride, int colorspace) { this.width = width; this.height = height; + this.colorspace = colorspace; int yLength = yStride * height; int uvLength = uvStride * ((height + 1) / 2); int minimumYuvSize = yLength + (uvLength * 2); diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxRenderer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxRenderer.java index 68765968a1..6deeab5c3d 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxRenderer.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer/ext/vp9/VpxRenderer.java @@ -31,6 +31,18 @@ import javax.microedition.khronos.opengles.GL10; */ /* package */ class VpxRenderer implements GLSurfaceView.Renderer { + private static final float[] kColorConversion601 = { + 1.164f, 1.164f, 1.164f, + 0.0f, -0.392f, 2.017f, + 1.596f, -0.813f, 0.0f, + }; + + private static final float[] kColorConversion709 = { + 1.164f, 1.164f, 1.164f, + 0.0f, -0.213f, 2.112f, + 1.793f, -0.533f, 0.0f, + }; + private static final String VERTEX_SHADER = "varying vec2 interp_tc;\n" + "attribute vec4 in_pos;\n" @@ -46,14 +58,13 @@ import javax.microedition.khronos.opengles.GL10; + "uniform sampler2D y_tex;\n" + "uniform sampler2D u_tex;\n" + "uniform sampler2D v_tex;\n" + + "uniform mat3 mColorConversion;\n" + "void main() {\n" - + " float y = 1.164 * (texture2D(y_tex, interp_tc).r - 0.0625);\n" - + " float u = texture2D(u_tex, interp_tc).r - 0.5;\n" - + " float v = texture2D(v_tex, interp_tc).r - 0.5;\n" - + " gl_FragColor = vec4(y + 1.596 * v, " - + " y - 0.391 * u - 0.813 * v, " - + " y + 2.018 * u, " - + " 1.0);\n" + + " vec3 yuv;" + + " yuv.x = texture2D(y_tex, interp_tc).r - 0.0625;\n" + + " yuv.y = texture2D(u_tex, interp_tc).r - 0.5;\n" + + " yuv.z = texture2D(v_tex, interp_tc).r - 0.5;\n" + + " gl_FragColor = vec4(mColorConversion * yuv, 1.0);" + "}\n"; private static final FloatBuffer TEXTURE_VERTICES = nativeFloatBuffer( -1.0f, 1.0f, @@ -64,6 +75,7 @@ import javax.microedition.khronos.opengles.GL10; private int program; private int texLocation; + private int colorMatrixLocation; private FloatBuffer textureCoords; private VpxOutputBuffer outputBuffer; private int previousWidth; @@ -108,6 +120,9 @@ import javax.microedition.khronos.opengles.GL10; posLocation, 2, GLES20.GL_FLOAT, false, 0, TEXTURE_VERTICES); texLocation = GLES20.glGetAttribLocation(program, "in_tc"); GLES20.glEnableVertexAttribArray(texLocation); + checkNoGLES2Error(); + colorMatrixLocation = GLES20.glGetUniformLocation(program, "mColorConversion"); + checkNoGLES2Error(); setupTextures(); checkNoGLES2Error(); } @@ -125,6 +140,12 @@ import javax.microedition.khronos.opengles.GL10; // Nothing to render yet. return; } + + // Set color matrix. Assume BT709 if the color space is unknown. + float[] colorConversion = outputBuffer.colorspace == VpxOutputBuffer.COLORSPACE_BT601 + ? kColorConversion601 : kColorConversion709; + GLES20.glUniformMatrix3fv(colorMatrixLocation, 1, false, colorConversion, 0); + for (int i = 0; i < 3; i++) { int h = (i == 0) ? outputBuffer.height : (outputBuffer.height + 1) / 2; GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + i); diff --git a/extensions/vp9/src/main/jni/vpx_jni.cc b/extensions/vp9/src/main/jni/vpx_jni.cc index ba19f299cb..a3abe24399 100644 --- a/extensions/vp9/src/main/jni/vpx_jni.cc +++ b/extensions/vp9/src/main/jni/vpx_jni.cc @@ -71,7 +71,7 @@ FUNC(jlong, vpxInit) { const jclass outputBufferClass = env->FindClass( "com/google/android/exoplayer/ext/vp9/VpxOutputBuffer"); initForYuvFrame = env->GetMethodID(outputBufferClass, "initForYuvFrame", - "(IIII)V"); + "(IIIII)V"); initForRgbFrame = env->GetMethodID(outputBufferClass, "initForRgbFrame", "(II)V"); dataField = env->GetFieldID(outputBufferClass, "data", @@ -128,9 +128,26 @@ FUNC(jint, vpxGetFrame, jlong jContext, jobject jOutputBuffer) { img->planes[VPX_PLANE_V], img->stride[VPX_PLANE_V], dst, img->d_w * 2, img->d_w, img->d_h); } else if (outputMode == kOutputModeYuv) { + const int kColorspaceUnknown = 0; + const int kColorspaceBT601 = 1; + const int kColorspaceBT709 = 2; + + int colorspace = kColorspaceUnknown; + switch (img->cs) { + case VPX_CS_BT_601: + colorspace = kColorspaceBT601; + break; + case VPX_CS_BT_709: + colorspace = kColorspaceBT709; + break; + default: + break; + } + // resize buffer if required. env->CallVoidMethod(jOutputBuffer, initForYuvFrame, img->d_w, img->d_h, - img->stride[VPX_PLANE_Y], img->stride[VPX_PLANE_U]); + img->stride[VPX_PLANE_Y], img->stride[VPX_PLANE_U], + colorspace); // get pointer to the data buffer. const jobject dataObject = env->GetObjectField(jOutputBuffer, dataField); From 99b34a1287fe55fd6ddcc2302793ade6bfce6d30 Mon Sep 17 00:00:00 2001 From: olly Date: Thu, 10 Dec 2015 06:31:28 -0800 Subject: [PATCH 3/5] Tweak DashTest VP9 constants. This is a no-op change that just makes the definitions consistent with the H264 ones, for general neatness/consistency reasons. ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=109892547 --- .../exoplayer/playbacktests/gts/DashTest.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/playbacktests/src/main/java/com/google/android/exoplayer/playbacktests/gts/DashTest.java b/playbacktests/src/main/java/com/google/android/exoplayer/playbacktests/gts/DashTest.java index 36766ce130..731e08a596 100644 --- a/playbacktests/src/main/java/com/google/android/exoplayer/playbacktests/gts/DashTest.java +++ b/playbacktests/src/main/java/com/google/android/exoplayer/playbacktests/gts/DashTest.java @@ -101,6 +101,13 @@ public final class DashTest extends ActivityInstrumentationTestCase2 Date: Thu, 10 Dec 2015 09:33:48 -0800 Subject: [PATCH 4/5] Fix Seeker.getTimeUs for positions before the first frame. Issue: #1038 ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=109906628 --- .../android/exoplayer/extractor/mp3/ConstantBitrateSeeker.java | 3 ++- .../com/google/android/exoplayer/extractor/mp3/XingSeeker.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/library/src/main/java/com/google/android/exoplayer/extractor/mp3/ConstantBitrateSeeker.java b/library/src/main/java/com/google/android/exoplayer/extractor/mp3/ConstantBitrateSeeker.java index db134089b9..bf83902410 100644 --- a/library/src/main/java/com/google/android/exoplayer/extractor/mp3/ConstantBitrateSeeker.java +++ b/library/src/main/java/com/google/android/exoplayer/extractor/mp3/ConstantBitrateSeeker.java @@ -47,7 +47,8 @@ import com.google.android.exoplayer.C; @Override public long getTimeUs(long position) { - return ((position - firstFramePosition) * C.MICROS_PER_SECOND * BITS_PER_BYTE) / bitrate; + return (Math.max(0, position - firstFramePosition) * C.MICROS_PER_SECOND * BITS_PER_BYTE) + / bitrate; } @Override diff --git a/library/src/main/java/com/google/android/exoplayer/extractor/mp3/XingSeeker.java b/library/src/main/java/com/google/android/exoplayer/extractor/mp3/XingSeeker.java index bc548f676c..bef49dfa89 100644 --- a/library/src/main/java/com/google/android/exoplayer/extractor/mp3/XingSeeker.java +++ b/library/src/main/java/com/google/android/exoplayer/extractor/mp3/XingSeeker.java @@ -130,7 +130,7 @@ import com.google.android.exoplayer.util.Util; @Override public long getTimeUs(long position) { - if (!isSeekable()) { + if (!isSeekable() || position < firstFramePosition) { return 0L; } double offsetByte = 256.0 * (position - firstFramePosition) / sizeBytes; From d753378b2f5111bd715968530f2cf552015e2351 Mon Sep 17 00:00:00 2001 From: aptly-io Date: Sat, 12 Dec 2015 21:09:43 +0100 Subject: [PATCH 5/5] Allow multiple identical but intermittent spans Multiple identical TTML fontStyles or fontWeights or textDecorations on the content for a specific moment were rendered as if there's only one decoration (span). That's because SpannableStringBuilder.setSpan(span, start, end, flag) found an earlier set span (the static allocated span's reference is the same each time) and only refreshed this first span's start and end values instead of adding a new span at its (new) different range. This patch removes the static data members; this makes the newly allocated span objects distinguishable. A correct implementation is favoured over worries about memory consumption. --- .../exoplayer/text/ttml/TtmlRenderUtil.java | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/library/src/main/java/com/google/android/exoplayer/text/ttml/TtmlRenderUtil.java b/library/src/main/java/com/google/android/exoplayer/text/ttml/TtmlRenderUtil.java index 63b6cc8886..a2bff2ec1d 100644 --- a/library/src/main/java/com/google/android/exoplayer/text/ttml/TtmlRenderUtil.java +++ b/library/src/main/java/com/google/android/exoplayer/text/ttml/TtmlRenderUtil.java @@ -35,16 +35,6 @@ import java.util.Map; */ /* package */ final class TtmlRenderUtil { - /* spans which are always the same can be reused to avoid object creation */ - private static final StrikethroughSpan STRIKETHROUGH_SPAN = new StrikethroughSpan(); - private static final UnderlineSpan UNDERLINE_SPAN = new UnderlineSpan(); - private static final StyleSpan[] STYLE_SPANS = new StyleSpan[] { - new StyleSpan(TtmlStyle.STYLE_NORMAL), - new StyleSpan(TtmlStyle.STYLE_BOLD), - new StyleSpan(TtmlStyle.STYLE_ITALIC), - new StyleSpan(TtmlStyle.STYLE_BOLD_ITALIC), - }; - public static TtmlStyle resolveStyle(TtmlStyle style, String[] styleIds, Map globalStyles) { if (style == null && styleIds == null) { @@ -78,14 +68,14 @@ import java.util.Map; int start, int end, TtmlStyle style) { if (style.getStyle() != TtmlStyle.UNSPECIFIED) { - builder.setSpan(STYLE_SPANS[style.getStyle()], start, end, + builder.setSpan(new StyleSpan(style.getStyle()), start, end, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); } if (style.isLinethrough()) { - builder.setSpan(STRIKETHROUGH_SPAN, start, end, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); + builder.setSpan(new StrikethroughSpan(), start, end, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); } if (style.isUnderline()) { - builder.setSpan(UNDERLINE_SPAN, start, end, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); + builder.setSpan(new UnderlineSpan(), start, end, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE); } if (style.hasColorSpecified()) { builder.setSpan(new ForegroundColorSpan(style.getColor()), start, end,