From a83e4a7ab83335d1230ffd213bfc084fa1b35fc9 Mon Sep 17 00:00:00 2001 From: andrewlewis Date: Thu, 3 Jun 2021 14:01:15 +0100 Subject: [PATCH] Fix texture transformation in gldemo `SurfaceTexture` provides a transform matrix with each buffer. Previously gldemo ignored this but it is important to apply it to have the video render properly. The transformation matrix from the surface texture includes flipping so this change removes the hard-coded flipping from `a_texcoord`. Issue: #8992 #minor-release PiperOrigin-RevId: 377271389 --- RELEASENOTES.md | 3 +++ .../bitmap_overlay_video_processor_vertex.glsl | 9 +++++---- .../gldemo/BitmapOverlayVideoProcessor.java | 9 ++++++--- .../gldemo/VideoProcessingGLSurfaceView.java | 12 +++++++++--- .../google/android/exoplayer2/util/GlUtil.java | 17 ++++++++++++++--- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 4c8f7b4ca2..b84943c257 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -41,6 +41,9 @@ ([#8985](https://github.com/google/ExoPlayer/issues/8985)). * RTSP: * Add support for RTSP basic and digest authentication. +* GL demo app: + * Fix texture transformation to avoid green bars shown on some videos + ([#8992](https://github.com/google/ExoPlayer/issues/8992)). ### 2.14.0 (2021-05-13) diff --git a/demos/gl/src/main/assets/bitmap_overlay_video_processor_vertex.glsl b/demos/gl/src/main/assets/bitmap_overlay_video_processor_vertex.glsl index 0c07c12a70..1cb01b8293 100644 --- a/demos/gl/src/main/assets/bitmap_overlay_video_processor_vertex.glsl +++ b/demos/gl/src/main/assets/bitmap_overlay_video_processor_vertex.glsl @@ -11,10 +11,11 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -attribute vec2 a_position; -attribute vec2 a_texcoord; +attribute vec4 a_position; +attribute vec4 a_texcoord; +uniform mat4 tex_transform; varying vec2 v_texcoord; void main() { - gl_Position = vec4(a_position.x, a_position.y, 0, 1); - v_texcoord = a_texcoord; + gl_Position = a_position; + v_texcoord = (tex_transform * a_texcoord).xy; } diff --git a/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java b/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java index 02399ba86c..38f5b7ff35 100644 --- a/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java +++ b/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/BitmapOverlayVideoProcessor.java @@ -88,9 +88,9 @@ import javax.microedition.khronos.opengles.GL10; GlUtil.Uniform[] uniforms = GlUtil.getUniforms(program); for (GlUtil.Attribute attribute : attributes) { if (attribute.name.equals("a_position")) { - attribute.setBuffer(new float[] {-1, -1, 1, -1, -1, 1, 1, 1}, 2); + attribute.setBuffer(new float[] {-1, -1, 0, 1, 1, -1, 0, 1, -1, 1, 0, 1, 1, 1, 0, 1}, 4); } else if (attribute.name.equals("a_texcoord")) { - attribute.setBuffer(new float[] {0, 1, 1, 1, 0, 0, 1, 0}, 2); + attribute.setBuffer(new float[] {0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1}, 4); } } this.attributes = attributes; @@ -111,7 +111,7 @@ import javax.microedition.khronos.opengles.GL10; } @Override - public void draw(int frameTexture, long frameTimestampUs) { + public void draw(int frameTexture, long frameTimestampUs, float[] transformMatrix) { // Draw to the canvas and store it in a texture. String text = String.format(Locale.US, "%.02f", frameTimestampUs / (float) C.MICROS_PER_SECOND); overlayBitmap.eraseColor(Color.TRANSPARENT); @@ -140,6 +140,9 @@ import javax.microedition.khronos.opengles.GL10; case "scaleY": uniform.setFloat(bitmapScaleY); break; + case "tex_transform": + uniform.setFloats(transformMatrix); + break; default: // fall out } } diff --git a/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/VideoProcessingGLSurfaceView.java b/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/VideoProcessingGLSurfaceView.java index d1202052fe..4c04ca4c70 100644 --- a/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/VideoProcessingGLSurfaceView.java +++ b/demos/gl/src/main/java/com/google/android/exoplayer2/gldemo/VideoProcessingGLSurfaceView.java @@ -25,6 +25,7 @@ import android.os.Handler; import android.view.Surface; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.ExoPlayer; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.util.Assertions; @@ -61,8 +62,9 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView { * * @param frameTexture The ID of a GL texture containing a video frame. * @param frameTimestampUs The presentation timestamp of the frame, in microseconds. + * @param transformMatrix The 4 * 4 transform matrix to be applied to the texture. */ - void draw(int frameTexture, long frameTimestampUs); + void draw(int frameTexture, long frameTimestampUs, float[] transformMatrix); } private static final int EGL_PROTECTED_CONTENT_EXT = 0x32C0; @@ -214,6 +216,7 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView { private final VideoProcessor videoProcessor; private final AtomicBoolean frameAvailable; private final TimedValueQueue sampleTimestampQueue; + private final float[] transformMatrix; private int texture; @Nullable private SurfaceTexture surfaceTexture; @@ -229,6 +232,8 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView { sampleTimestampQueue = new TimedValueQueue<>(); width = -1; height = -1; + frameTimestampUs = C.TIME_UNSET; + transformMatrix = new float[16]; } @Override @@ -271,13 +276,14 @@ public final class VideoProcessingGLSurfaceView extends GLSurfaceView { SurfaceTexture surfaceTexture = Assertions.checkNotNull(this.surfaceTexture); surfaceTexture.updateTexImage(); long lastFrameTimestampNs = surfaceTexture.getTimestamp(); - Long frameTimestampUs = sampleTimestampQueue.poll(lastFrameTimestampNs); + @Nullable Long frameTimestampUs = sampleTimestampQueue.poll(lastFrameTimestampNs); if (frameTimestampUs != null) { this.frameTimestampUs = frameTimestampUs; } + surfaceTexture.getTransformMatrix(transformMatrix); } - videoProcessor.draw(texture, frameTimestampUs); + videoProcessor.draw(texture, frameTimestampUs, transformMatrix); } @Override diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java index f38fd61caf..b41b6fd2b0 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java @@ -141,7 +141,7 @@ public final class GlUtil { location = GLES20.glGetUniformLocation(program, this.name); this.type = type[0]; - value = new float[1]; + value = new float[16]; } /** @@ -160,9 +160,14 @@ public final class GlUtil { this.value[0] = value; } + /** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */ + public void setFloats(float[] value) { + System.arraycopy(value, 0, this.value, 0, value.length); + } + /** - * Sets the uniform to whatever value was passed via {@link #setSamplerTexId(int, int)} or - * {@link #setFloat(float)}. + * Sets the uniform to whatever value was passed via {@link #setSamplerTexId(int, int)}, {@link + * #setFloat(float)} or {@link #setFloats(float[])}. * *

Should be called before each drawing call. */ @@ -173,6 +178,12 @@ public final class GlUtil { return; } + if (type == GLES20.GL_FLOAT_MAT4) { + GLES20.glUniformMatrix4fv(location, 1, false, value, 0); + checkGlError(); + return; + } + if (texId == 0) { throw new IllegalStateException("call setSamplerTexId before bind"); }