From ee8a7f17ff9c81db1cbe665c99eaa9de78f14233 Mon Sep 17 00:00:00 2001 From: vigneshv Date: Wed, 9 Nov 2016 10:06:23 -0800 Subject: [PATCH] vp9_extension: Fix potential integer overflows ------------- Created by MOE: https://github.com/google/moe MOE_MIGRATED_REVID=138652561 --- .../exoplayer2/ext/vp9/VpxDecoder.java | 5 +++- .../exoplayer2/ext/vp9/VpxOutputBuffer.java | 30 +++++++++++++++---- extensions/vp9/src/main/jni/vpx_jni.cc | 19 ++++++++---- 3 files changed, 42 insertions(+), 12 deletions(-) diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxDecoder.java b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxDecoder.java index 9af997a58c..0d7547d125 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxDecoder.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxDecoder.java @@ -122,8 +122,11 @@ import java.nio.ByteBuffer; } outputBuffer.init(inputBuffer.timeUs, outputMode); - if (vpxGetFrame(vpxDecContext, outputBuffer) != 0) { + int getFrameResult = vpxGetFrame(vpxDecContext, outputBuffer); + if (getFrameResult == 1) { outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); + } else if (getFrameResult == -1) { + return new VpxDecoderException("Buffer initialization failed."); } return null; } diff --git a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer.java b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer.java index d07b1443fd..c76d0eda03 100644 --- a/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer.java +++ b/extensions/vp9/src/main/java/com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer.java @@ -66,28 +66,39 @@ import java.nio.ByteBuffer; /** * Resizes the buffer based on the given dimensions. Called via JNI after decoding completes. + * @return Whether the buffer was resized successfully. */ - public void initForRgbFrame(int width, int height) { + public boolean initForRgbFrame(int width, int height) { this.width = width; this.height = height; this.yuvPlanes = null; - + if (!isSafeToMultiply(width, height) || !isSafeToMultiply(width * height, 2)) { + return false; + } int minimumRgbSize = width * height * 2; initData(minimumRgbSize); + return true; } /** * Resizes the buffer based on the given stride. Called via JNI after decoding completes. + * @return Whether the buffer was resized successfully. */ - public void initForYuvFrame(int width, int height, int yStride, int uvStride, + public boolean initForYuvFrame(int width, int height, int yStride, int uvStride, int colorspace) { this.width = width; this.height = height; this.colorspace = colorspace; - + int uvHeight = (int) (((long) height + 1) / 2); + if (!isSafeToMultiply(yStride, height) || !isSafeToMultiply(uvStride, uvHeight)) { + return false; + } int yLength = yStride * height; - int uvLength = uvStride * ((height + 1) / 2); + int uvLength = uvStride * uvHeight; int minimumYuvSize = yLength + (uvLength * 2); + if (!isSafeToMultiply(uvLength, 2) || minimumYuvSize < yLength) { + return false; + } initData(minimumYuvSize); if (yuvPlanes == null) { @@ -108,6 +119,7 @@ import java.nio.ByteBuffer; yuvStrides[0] = yStride; yuvStrides[1] = uvStride; yuvStrides[2] = uvStride; + return true; } private void initData(int size) { @@ -119,4 +131,12 @@ import java.nio.ByteBuffer; } } + /** + * Ensures that the result of multiplying individual numbers can fit into the size limit of an + * integer. + */ + private boolean isSafeToMultiply(int a, int b) { + return a >= 0 && b >= 0 && !(b > 0 && a >= Integer.MAX_VALUE / b); + } + } diff --git a/extensions/vp9/src/main/jni/vpx_jni.cc b/extensions/vp9/src/main/jni/vpx_jni.cc index afaac1c8ae..137ff9ac21 100644 --- a/extensions/vp9/src/main/jni/vpx_jni.cc +++ b/extensions/vp9/src/main/jni/vpx_jni.cc @@ -83,9 +83,9 @@ DECODER_FUNC(jlong, vpxInit) { const jclass outputBufferClass = env->FindClass( "com/google/android/exoplayer2/ext/vp9/VpxOutputBuffer"); initForYuvFrame = env->GetMethodID(outputBufferClass, "initForYuvFrame", - "(IIIII)V"); + "(IIIII)Z"); initForRgbFrame = env->GetMethodID(outputBufferClass, "initForRgbFrame", - "(II)V"); + "(II)Z"); dataField = env->GetFieldID(outputBufferClass, "data", "Ljava/nio/ByteBuffer;"); outputModeField = env->GetFieldID(outputBufferClass, "mode", "I"); @@ -141,7 +141,11 @@ DECODER_FUNC(jint, vpxGetFrame, jlong jContext, jobject jOutputBuffer) { int outputMode = env->GetIntField(jOutputBuffer, outputModeField); if (outputMode == kOutputModeRgb) { // resize buffer if required. - env->CallVoidMethod(jOutputBuffer, initForRgbFrame, img->d_w, img->d_h); + jboolean initResult = env->CallBooleanMethod(jOutputBuffer, initForRgbFrame, + img->d_w, img->d_h); + if (initResult == JNI_FALSE) { + return -1; + } // get pointer to the data buffer. const jobject dataObject = env->GetObjectField(jOutputBuffer, dataField); @@ -170,9 +174,12 @@ DECODER_FUNC(jint, vpxGetFrame, jlong jContext, jobject jOutputBuffer) { } // resize buffer if required. - env->CallVoidMethod(jOutputBuffer, initForYuvFrame, img->d_w, img->d_h, - img->stride[VPX_PLANE_Y], img->stride[VPX_PLANE_U], - colorspace); + jboolean initResult = env->CallBooleanMethod( + jOutputBuffer, initForYuvFrame, img->d_w, img->d_h, + img->stride[VPX_PLANE_Y], img->stride[VPX_PLANE_U], colorspace); + if (initResult == JNI_FALSE) { + return -1; + } // get pointer to the data buffer. const jobject dataObject = env->GetObjectField(jOutputBuffer, dataField);