Use camera motion metadata to stabilize 360 videos

RELNOTES=true

-------------
Created by MOE: https://github.com/google/moe
MOE_MIGRATED_REVID=210537375

RELEASENOTES.md

@@ -98,7 +98,8 @@
 * Use default Deserializers if non given to DownloadManager.
 * 360:
   * Add monoscopic 360 surface type to PlayerView.
-  * Support VR180 videos.
+  * Support 
+  [VR180 video format](https://github.com/google/spatial-media/blob/master/docs/vr180.md).
 * Deprecate `Player.DefaultEventListener` as selective listener overrides can
   be directly made with the `Player.EventListener` interface.
 * Deprecate `DefaultAnalyticsListener` as selective listener overrides can be

library/core/src/main/java/com/google/android/exoplayer2/C.java

@@ -27,6 +27,7 @@ import com.google.android.exoplayer2.PlayerMessage.Target;
 import com.google.android.exoplayer2.audio.AuxEffectInfo;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
+import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.util.UUID;
@@ -595,34 +596,22 @@ public final class C {
    */
   public static final int DATA_TYPE_CUSTOM_BASE = 10000;
 
-  /**
+  /** A type constant for tracks of unknown type. */
-   * A type constant for tracks of unknown type.
-   */
   public static final int TRACK_TYPE_UNKNOWN = -1;
-  /**
+  /** A type constant for tracks of some default type, where the type itself is unknown. */
-   * A type constant for tracks of some default type, where the type itself is unknown.
-   */
   public static final int TRACK_TYPE_DEFAULT = 0;
-  /**
+  /** A type constant for audio tracks. */
-   * A type constant for audio tracks.
-   */
   public static final int TRACK_TYPE_AUDIO = 1;
-  /**
+  /** A type constant for video tracks. */
-   * A type constant for video tracks.
-   */
   public static final int TRACK_TYPE_VIDEO = 2;
-  /**
+  /** A type constant for text tracks. */
-   * A type constant for text tracks.
-   */
   public static final int TRACK_TYPE_TEXT = 3;
-  /**
+  /** A type constant for metadata tracks. */
-   * A type constant for metadata tracks.
-   */
   public static final int TRACK_TYPE_METADATA = 4;
-  /**
+  /** A type constant for camera motion tracks. */
-   * A type constant for a dummy or empty track.
+  public static final int TRACK_TYPE_CAMERA_MOTION = 5;
-   */
+  /** A type constant for a dummy or empty track. */
-  public static final int TRACK_TYPE_NONE = 5;
+  public static final int TRACK_TYPE_NONE = 6;
   /**
    * Applications or extensions may define custom {@code TRACK_TYPE_*} constants greater than or
    * equal to this value.
@@ -655,36 +644,27 @@ public final class C {
    */
   public static final int SELECTION_REASON_CUSTOM_BASE = 10000;
 
-  /**
+  /** A default size in bytes for an individual allocation that forms part of a larger buffer. */
-   * A default size in bytes for an individual allocation that forms part of a larger buffer.
-   */
   public static final int DEFAULT_BUFFER_SEGMENT_SIZE = 64 * 1024;
 
-  /**
+  /** A default size in bytes for a video buffer. */
-   * A default size in bytes for a video buffer.
-   */
   public static final int DEFAULT_VIDEO_BUFFER_SIZE = 200 * DEFAULT_BUFFER_SEGMENT_SIZE;
 
-  /**
+  /** A default size in bytes for an audio buffer. */
-   * A default size in bytes for an audio buffer.
-   */
   public static final int DEFAULT_AUDIO_BUFFER_SIZE = 54 * DEFAULT_BUFFER_SEGMENT_SIZE;
 
-  /**
+  /** A default size in bytes for a text buffer. */
-   * A default size in bytes for a text buffer.
-   */
   public static final int DEFAULT_TEXT_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE;
 
-  /**
+  /** A default size in bytes for a metadata buffer. */
-   * A default size in bytes for a metadata buffer.
-   */
   public static final int DEFAULT_METADATA_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE;
 
-  /**
+  /** A default size in bytes for a camera motion buffer. */
-   * A default size in bytes for a muxed buffer (e.g. containing video, audio and text).
+  public static final int DEFAULT_CAMERA_MOTION_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE;
-   */
+
-  public static final int DEFAULT_MUXED_BUFFER_SIZE = DEFAULT_VIDEO_BUFFER_SIZE
+  /** A default size in bytes for a muxed buffer (e.g. containing video, audio and text). */
-      + DEFAULT_AUDIO_BUFFER_SIZE + DEFAULT_TEXT_BUFFER_SIZE;
+  public static final int DEFAULT_MUXED_BUFFER_SIZE =
+      DEFAULT_VIDEO_BUFFER_SIZE + DEFAULT_AUDIO_BUFFER_SIZE + DEFAULT_TEXT_BUFFER_SIZE;
 
   /** "cenc" scheme type name as defined in ISO/IEC 23001-7:2016. */
   @SuppressWarnings("ConstantField")
@@ -798,6 +778,13 @@ public final class C {
    */
   public static final int MSG_SET_VIDEO_FRAME_METADATA_LISTENER = 6;
 
+  /**
+   * The type of a message that can be passed to a camera motion {@link Renderer} via {@link
+   * ExoPlayer#createMessage(Target)}. The message payload should be a {@link CameraMotionListener}
+   * instance, or null.
+   */
+  public static final int MSG_SET_CAMERA_MOTION_LISTENER = 7;
+
   /**
    * Applications or extensions may define custom {@code MSG_*} constants that can be passed to
    * {@link Renderer}s. These custom constants must be greater than or equal to this value.

library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java

@@ -35,6 +35,7 @@ import com.google.android.exoplayer2.text.TextRenderer;
 import com.google.android.exoplayer2.trackselection.TrackSelector;
 import com.google.android.exoplayer2.video.MediaCodecVideoRenderer;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
+import com.google.android.exoplayer2.video.spherical.CameraMotionRenderer;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 import java.lang.reflect.Constructor;
@@ -182,6 +183,7 @@ public class DefaultRenderersFactory implements RenderersFactory {
         extensionRendererMode, renderersList);
     buildMetadataRenderers(context, metadataRendererOutput, eventHandler.getLooper(),
         extensionRendererMode, renderersList);
+    buildCameraMotionRenderers(context, extensionRendererMode, renderersList);
     buildMiscellaneousRenderers(context, eventHandler, extensionRendererMode, renderersList);
     return renderersList.toArray(new Renderer[renderersList.size()]);
   }
@@ -360,12 +362,14 @@ public class DefaultRenderersFactory implements RenderersFactory {
    *
    * @param context The {@link Context} associated with the player.
    * @param output An output for the renderers.
-   * @param outputLooper The looper associated with the thread on which the output should be
+   * @param outputLooper The looper associated with the thread on which the output should be called.
-   *     called.
    * @param extensionRendererMode The extension renderer mode.
    * @param out An array to which the built renderers should be appended.
    */
-  protected void buildTextRenderers(Context context, TextOutput output, Looper outputLooper,
+  protected void buildTextRenderers(
+      Context context,
+      TextOutput output,
+      Looper outputLooper,
       @ExtensionRendererMode int extensionRendererMode,
       ArrayList<Renderer> out) {
     out.add(new TextRenderer(output, outputLooper));
@@ -376,16 +380,31 @@ public class DefaultRenderersFactory implements RenderersFactory {
    *
    * @param context The {@link Context} associated with the player.
    * @param output An output for the renderers.
-   * @param outputLooper The looper associated with the thread on which the output should be
+   * @param outputLooper The looper associated with the thread on which the output should be called.
-   *     called.
    * @param extensionRendererMode The extension renderer mode.
    * @param out An array to which the built renderers should be appended.
    */
-  protected void buildMetadataRenderers(Context context, MetadataOutput output, Looper outputLooper,
+  protected void buildMetadataRenderers(
-      @ExtensionRendererMode int extensionRendererMode, ArrayList<Renderer> out) {
+      Context context,
+      MetadataOutput output,
+      Looper outputLooper,
+      @ExtensionRendererMode int extensionRendererMode,
+      ArrayList<Renderer> out) {
     out.add(new MetadataRenderer(output, outputLooper));
   }
 
+  /**
+   * Builds camera motion renderers for use by the player.
+   *
+   * @param context The {@link Context} associated with the player.
+   * @param extensionRendererMode The extension renderer mode.
+   * @param out An array to which the built renderers should be appended.
+   */
+  protected void buildCameraMotionRenderers(
+      Context context, @ExtensionRendererMode int extensionRendererMode, ArrayList<Renderer> out) {
+    out.add(new CameraMotionRenderer());
+  }
+
   /**
    * Builds any miscellaneous renderers used by the player.
    *

library/core/src/main/java/com/google/android/exoplayer2/Player.java

@@ -31,6 +31,7 @@ import com.google.android.exoplayer2.trackselection.TrackSelectionArray;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoListener;
+import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
 import java.lang.annotation.Retention;
 import java.lang.annotation.RetentionPolicy;
 
@@ -185,6 +186,21 @@ public interface Player {
      */
     void clearVideoFrameMetadataListener(VideoFrameMetadataListener listener);
 
+    /**
+     * Sets a listener of camera motion events.
+     *
+     * @param listener The listener.
+     */
+    void setCameraMotionListener(CameraMotionListener listener);
+
+    /**
+     * Clears the listener which receives camera motion events if it matches the one passed. Else
+     * does nothing.
+     *
+     * @param listener The listener to clear.
+     */
+    void clearCameraMotionListener(CameraMotionListener listener);
+
     /**
      * Clears any {@link Surface}, {@link SurfaceHolder}, {@link SurfaceView} or {@link TextureView}
      * currently set on the player.

library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java

@@ -53,6 +53,7 @@ import com.google.android.exoplayer2.util.Clock;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
 import com.google.android.exoplayer2.video.VideoRendererEventListener;
+import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -107,6 +108,7 @@ public class SimpleExoPlayer
   private MediaSource mediaSource;
   private List<Cue> currentCues;
   private VideoFrameMetadataListener videoFrameMetadataListener;
+  private CameraMotionListener cameraMotionListener;
 
   /**
    * @param context A {@link Context}.
@@ -597,6 +599,36 @@ public class SimpleExoPlayer
     }
   }
 
+  @Override
+  public void setCameraMotionListener(CameraMotionListener listener) {
+    cameraMotionListener = listener;
+    for (Renderer renderer : renderers) {
+      if (renderer.getTrackType() == C.TRACK_TYPE_CAMERA_MOTION) {
+        player
+            .createMessage(renderer)
+            .setType(C.MSG_SET_CAMERA_MOTION_LISTENER)
+            .setPayload(listener)
+            .send();
+      }
+    }
+  }
+
+  @Override
+  public void clearCameraMotionListener(CameraMotionListener listener) {
+    if (cameraMotionListener != listener) {
+      return;
+    }
+    for (Renderer renderer : renderers) {
+      if (renderer.getTrackType() == C.TRACK_TYPE_CAMERA_MOTION) {
+        player
+            .createMessage(renderer)
+            .setType(C.MSG_SET_CAMERA_MOTION_LISTENER)
+            .setPayload(null)
+            .send();
+      }
+    }
+  }
+
   /**
    * Sets a listener to receive video events, removing all existing listeners.
    *

library/core/src/main/java/com/google/android/exoplayer2/util/EventLogger.java

@@ -599,6 +599,8 @@ public class EventLogger implements AnalyticsListener {
         return "default";
       case C.TRACK_TYPE_METADATA:
         return "metadata";
+      case C.TRACK_TYPE_CAMERA_MOTION:
+        return "camera motion";
       case C.TRACK_TYPE_NONE:
         return "none";
       case C.TRACK_TYPE_TEXT:

library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java

@@ -328,9 +328,10 @@ public final class MimeTypes {
       return C.TRACK_TYPE_TEXT;
     } else if (APPLICATION_ID3.equals(mimeType)
         || APPLICATION_EMSG.equals(mimeType)
-        || APPLICATION_SCTE35.equals(mimeType)
+        || APPLICATION_SCTE35.equals(mimeType)) {
-        || APPLICATION_CAMERA_MOTION.equals(mimeType)) {
       return C.TRACK_TYPE_METADATA;
+    } else if (APPLICATION_CAMERA_MOTION.equals(mimeType)) {
+      return C.TRACK_TYPE_CAMERA_MOTION;
     } else {
       return getTrackTypeForCustomMimeType(mimeType);
     }

library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java

@@ -64,7 +64,7 @@ public final class TimedValueQueue<V> {
 
   /**
    * Returns the value with the greatest timestamp which is less than or equal to the given
-   * timestamp. Removes all older values including the returned one from the buffer.
+   * timestamp. Removes all older values and the returned one from the buffer.
    *
    * @param timestamp The timestamp value.
    * @return The value with the greatest timestamp which is less than or equal to the given

library/core/src/main/java/com/google/android/exoplayer2/util/Util.java

@@ -1443,6 +1443,8 @@ public final class Util {
         return C.DEFAULT_TEXT_BUFFER_SIZE;
       case C.TRACK_TYPE_METADATA:
         return C.DEFAULT_METADATA_BUFFER_SIZE;
+      case C.TRACK_TYPE_CAMERA_MOTION:
+        return C.DEFAULT_CAMERA_MOTION_BUFFER_SIZE;
       default:
         throw new IllegalStateException();
     }

library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionListener.java

@@ -0,0 +1,32 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.video.spherical;
+
+/** Listens camera motion. */
+public interface CameraMotionListener {
+
+  /**
+   * Called when a new camera motion is read. This method is called on the playback thread.
+   *
+   * @param timeUs The presentation time of the data.
+   * @param rotation Angle axis orientation in radians representing the rotation from camera
+   *     coordinate system to world coordinate system.
+   */
+  void onCameraMotion(long timeUs, float[] rotation);
+
+  /** Called when the camera motion track position is reset. */
+  void onCameraMotionReset();
+}

library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionRenderer.java

@@ -0,0 +1,129 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.video.spherical;
+
+import android.support.annotation.Nullable;
+import com.google.android.exoplayer2.BaseRenderer;
+import com.google.android.exoplayer2.C;
+import com.google.android.exoplayer2.ExoPlaybackException;
+import com.google.android.exoplayer2.Format;
+import com.google.android.exoplayer2.FormatHolder;
+import com.google.android.exoplayer2.Renderer;
+import com.google.android.exoplayer2.decoder.DecoderInputBuffer;
+import com.google.android.exoplayer2.util.MimeTypes;
+import com.google.android.exoplayer2.util.ParsableByteArray;
+import com.google.android.exoplayer2.util.Util;
+import java.nio.ByteBuffer;
+
+/** A {@link Renderer} that parses the camera motion track. */
+public class CameraMotionRenderer extends BaseRenderer {
+
+  // The amount of time to read samples ahead of the current time.
+  private static final int SAMPLE_WINDOW_DURATION_US = 100000;
+
+  private final FormatHolder formatHolder;
+  private final DecoderInputBuffer buffer;
+  private final ParsableByteArray scratch;
+
+  private long offsetUs;
+  private @Nullable CameraMotionListener listener;
+  private long lastTimestampUs;
+
+  public CameraMotionRenderer() {
+    super(C.TRACK_TYPE_CAMERA_MOTION);
+    formatHolder = new FormatHolder();
+    buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL);
+    scratch = new ParsableByteArray();
+  }
+
+  @Override
+  public int supportsFormat(Format format) {
+    return MimeTypes.APPLICATION_CAMERA_MOTION.equals(format.sampleMimeType)
+        ? FORMAT_HANDLED
+        : FORMAT_UNSUPPORTED_TYPE;
+  }
+
+  @Override
+  public void handleMessage(int messageType, Object message) throws ExoPlaybackException {
+    if (messageType == C.MSG_SET_CAMERA_MOTION_LISTENER) {
+      listener = (CameraMotionListener) message;
+    } else {
+      super.handleMessage(messageType, message);
+    }
+  }
+
+  @Override
+  protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException {
+    this.offsetUs = offsetUs;
+  }
+
+  @Override
+  protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException {
+    lastTimestampUs = 0;
+    if (listener != null) {
+      listener.onCameraMotionReset();
+    }
+  }
+
+  @Override
+  protected void onDisabled() {
+    lastTimestampUs = 0;
+  }
+
+  @Override
+  public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException {
+    // Keep reading available samples as long as the sample time is not too far into the future.
+    while (!hasReadStreamToEnd() && lastTimestampUs < positionUs + SAMPLE_WINDOW_DURATION_US) {
+      buffer.clear();
+      int result = readSource(formatHolder, buffer, /* formatRequired= */ false);
+      if (result != C.RESULT_BUFFER_READ || buffer.isEndOfStream()) {
+        return;
+      }
+
+      buffer.flip();
+      lastTimestampUs = buffer.timeUs;
+      if (listener != null) {
+        float[] rotation = parseMetadata(buffer.data);
+        if (rotation != null) {
+          Util.castNonNull(listener).onCameraMotion(lastTimestampUs - offsetUs, rotation);
+        }
+      }
+    }
+  }
+
+  @Override
+  public boolean isEnded() {
+    return hasReadStreamToEnd();
+  }
+
+  @Override
+  public boolean isReady() {
+    return true;
+  }
+
+  private @Nullable float[] parseMetadata(ByteBuffer data) {
+    if (data.remaining() != 16) {
+      return null;
+    }
+    scratch.reset(data.array(), data.limit());
+    scratch.setPosition(data.arrayOffset() + 4); // skip reserved bytes too.
+    float[] result = new float[3];
+    for (int i = 0; i < 3; i++) {
+      result[i] = Float.intBitsToFloat(scratch.readLittleEndianInt());
+    }
+    return result;
+  }
+}

library/core/src/main/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueue.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.video.spherical;
+
+import android.opengl.Matrix;
+import com.google.android.exoplayer2.util.TimedValueQueue;
+
+/**
+ * This class serves multiple purposes:
+ *
+ * <ul>
+ *   <li>Queues the rotation metadata extracted from camera motion track.
+ *   <li>Converts the metadata to rotation matrices in OpenGl coordinate system.
+ *   <li>Recenters the rotations to componsate the yaw of the initial rotation.
+ * </ul>
+ */
+public final class FrameRotationQueue {
+  private final float[] recenterMatrix;
+  private final float[] rotationMatrix;
+  private final TimedValueQueue<float[]> rotations;
+  private boolean recenterMatrixComputed;
+
+  public FrameRotationQueue() {
+    recenterMatrix = new float[16];
+    rotationMatrix = new float[16];
+    rotations = new TimedValueQueue<>();
+  }
+
+  /**
+   * Sets a rotation for a given timestamp.
+   *
+   * @param timestampUs Timestamp of the rotation.
+   * @param angleAxis Angle axis orientation in radians representing the rotation from camera
+   *     coordinate system to world coordinate system.
+   */
+  public void setRotation(long timestampUs, float[] angleAxis) {
+    rotations.add(timestampUs, angleAxis);
+  }
+
+  /** Removes all of the rotations and forces rotations to be recentered. */
+  public void reset() {
+    rotations.clear();
+    recenterMatrixComputed = false;
+  }
+
+  /**
+   * Copies the rotation matrix with the greatest timestamp which is less than or equal to the given
+   * timestamp to {@code matrix}. Removes all older rotations and the returned one from the queue.
+   * Does nothing if there is no such rotation.
+   *
+   * @param matrix A float array to hold the rotation matrix.
+   * @param timestampUs The time in microseconds to query the rotation.
+   * @return Whether a rotation matrix is copied to {@code matrix}.
+   */
+  public boolean pollRotationMatrix(float[] matrix, long timestampUs) {
+    float[] rotation = rotations.pollFloor(timestampUs);
+    if (rotation == null) {
+      return false;
+    }
+    // TODO [Internal: b/113315546]: Slerp between the floor and ceil rotation.
+    getRotationMatrixFromAngleAxis(rotationMatrix, rotation);
+    if (!recenterMatrixComputed) {
+      computeRecenterMatrix(recenterMatrix, rotationMatrix);
+      recenterMatrixComputed = true;
+    }
+    Matrix.multiplyMM(matrix, 0, recenterMatrix, 0, rotationMatrix, 0);
+    return true;
+  }
+
+  /**
+   * Computes a recentering matrix from the given angle-axis rotation only accounting for yaw. Roll
+   * and tilt will not be compensated.
+   */
+  private static void computeRecenterMatrix(float[] recenterMatrix, float[] rotationMatrix) {
+    // The re-centering matrix is computed as follows:
+    // recenter.row(2) = temp.col(2).transpose();
+    // recenter.row(0) = recenter.row(1).cross(recenter.row(2)).normalized();
+    // recenter.row(2) = recenter.row(0).cross(recenter.row(1)).normalized();
+    //             | temp[10]  0   -temp[8]    0|
+    //             | 0         1    0          0|
+    // recenter =  | temp[8]   0    temp[10]   0|
+    //             | 0         0    0          1|
+    Matrix.setIdentityM(recenterMatrix, 0);
+    float normRowSqr =
+        rotationMatrix[10] * rotationMatrix[10] + rotationMatrix[8] * rotationMatrix[8];
+    float normRow = (float) Math.sqrt(normRowSqr);
+    recenterMatrix[0] = rotationMatrix[10] / normRow;
+    recenterMatrix[2] = rotationMatrix[8] / normRow;
+    recenterMatrix[8] = -rotationMatrix[8] / normRow;
+    recenterMatrix[10] = rotationMatrix[10] / normRow;
+  }
+
+  private static void getRotationMatrixFromAngleAxis(float[] matrix, float[] angleAxis) {
+    // Convert coordinates to OpenGL coordinates.
+    // CAMM motion metadata: +x right, +y down, and +z forward.
+    // OpenGL: +x right, +y up, -z forwards
+    float x = angleAxis[0];
+    float y = -angleAxis[1];
+    float z = -angleAxis[2];
+    float angleRad = Matrix.length(x, y, z);
+    if (angleRad != 0) {
+      float angleDeg = (float) Math.toDegrees(angleRad);
+      Matrix.setRotateM(matrix, 0, angleDeg, x / angleRad, y / angleRad, z / angleRad);
+    } else {
+      Matrix.setIdentityM(matrix, 0);
+    }
+  }
+}

library/core/src/test/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueueTest.java

@@ -0,0 +1,133 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.android.exoplayer2.video.spherical;
+
+import static com.google.common.truth.Truth.assertThat;
+
+import android.opengl.Matrix;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.robolectric.RobolectricTestRunner;
+
+/** Tests {@link FrameRotationQueue}. */
+@RunWith(RobolectricTestRunner.class)
+public class FrameRotationQueueTest {
+
+  private FrameRotationQueue frameRotationQueue;
+  private float[] rotationMatrix;
+
+  @Before
+  public void setUp() throws Exception {
+    frameRotationQueue = new FrameRotationQueue();
+    rotationMatrix = new float[16];
+  }
+
+  @Test
+  public void testGetRotationMatrixReturnsNull_whenEmpty() throws Exception {
+    assertThat(frameRotationQueue.pollRotationMatrix(rotationMatrix, 0)).isFalse();
+  }
+
+  @Test
+  public void testGetRotationMatrixReturnsNotNull_whenNotEmpty() throws Exception {
+    frameRotationQueue.setRotation(0, new float[] {1, 2, 3});
+    assertThat(frameRotationQueue.pollRotationMatrix(rotationMatrix, 0)).isTrue();
+    assertThat(rotationMatrix).hasLength(16);
+  }
+
+  @Test
+  public void testConvertsAngleAxisToRotationMatrix() throws Exception {
+    doTestAngleAxisToRotationMatrix(/* angleRadian= */ 0, /* x= */ 1, /* y= */ 0, /* z= */ 0);
+    frameRotationQueue.reset();
+    doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 1, /* y= */ 0, /* z= */ 0);
+    frameRotationQueue.reset();
+    doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 0, /* y= */ 0, /* z= */ 1);
+    // Don't reset frameRotationQueue as we use recenter matrix from previous calls.
+    doTestAngleAxisToRotationMatrix(/* angleRadian= */ -1, /* x= */ 0, /* y= */ 1, /* z= */ 0);
+    doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 1, /* y= */ 1, /* z= */ 1);
+  }
+
+  @Test
+  public void testRecentering_justYaw() throws Exception {
+    float[] actualMatrix =
+        getRotationMatrixFromAngleAxis(
+            /* angleRadian= */ (float) Math.PI, /* x= */ 0, /* y= */ 1, /* z= */ 0);
+    float[] expectedMatrix = new float[16];
+    Matrix.setIdentityM(expectedMatrix, 0);
+    assertEquals(actualMatrix, expectedMatrix);
+  }
+
+  @Test
+  public void testRecentering_yawAndPitch() throws Exception {
+    float[] matrix =
+        getRotationMatrixFromAngleAxis(
+            /* angleRadian= */ (float) Math.PI, /* x= */ 1, /* y= */ 1, /* z= */ 0);
+    assertMultiplication(
+        /* xr= */ 0, /* yr= */ 0, /* zr= */ 1, matrix, /* x= */ 0, /* y= */ 0, /* z= */ 1);
+  }
+
+  @Test
+  public void testRecentering_yawAndPitch2() throws Exception {
+    float[] matrix =
+        getRotationMatrixFromAngleAxis(
+            /* angleRadian= */ (float) Math.PI / 2, /* x= */ 1, /* y= */ 1, /* z= */ 0);
+    float sqrt2 = (float) Math.sqrt(2);
+    assertMultiplication(
+        /* xr= */ sqrt2, /* yr= */ 0, /* zr= */ 0, matrix, /* x= */ 1, /* y= */ -1, /* z= */ 0);
+  }
+
+  @Test
+  public void testRecentering_yawAndPitchAndRoll() throws Exception {
+    float[] matrix =
+        getRotationMatrixFromAngleAxis(
+            /* angleRadian= */ (float) Math.PI * 2 / 3, /* x= */ 1, /* y= */ 1, /* z= */ 1);
+    assertMultiplication(
+        /* xr= */ 0, /* yr= */ 0, /* zr= */ 1, matrix, /* x= */ 0, /* y= */ 0, /* z= */ 1);
+  }
+
+  private void doTestAngleAxisToRotationMatrix(float angleRadian, int x, int y, int z) {
+    float[] actualMatrix = getRotationMatrixFromAngleAxis(angleRadian, x, y, z);
+    float[] expectedMatrix = createRotationMatrix(angleRadian, x, y, z);
+    assertEquals(actualMatrix, expectedMatrix);
+  }
+
+  private float[] getRotationMatrixFromAngleAxis(float angleRadian, int x, int y, int z) {
+    float length = Matrix.length(x, y, z);
+    float factor = angleRadian / length;
+    // Negate y and z to revert OpenGL coordinate system conversion.
+    frameRotationQueue.setRotation(0, new float[] {x * factor, -y * factor, -z * factor});
+    frameRotationQueue.pollRotationMatrix(rotationMatrix, 0);
+    return rotationMatrix;
+  }
+
+  private static void assertMultiplication(
+      float xr, float yr, float zr, float[] actualMatrix, float x, float y, float z) {
+    float[] vector = new float[] {x, y, z, 0};
+    float[] resultVec = new float[4];
+    Matrix.multiplyMV(resultVec, 0, actualMatrix, 0, vector, 0);
+    assertEquals(resultVec, new float[] {xr, yr, zr, 0});
+  }
+
+  private static float[] createRotationMatrix(float angleRadian, int x, int y, int z) {
+    float[] expectedMatrix = new float[16];
+    Matrix.setRotateM(expectedMatrix, 0, (float) Math.toDegrees(angleRadian), x, y, z);
+    return expectedMatrix;
+  }
+
+  private static void assertEquals(float[] actual, float[] expected) {
+    assertThat(actual).usingTolerance(1.0e-5).containsExactly(expected).inOrder();
+  }
+}

library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java

@@ -515,6 +515,7 @@ public class PlayerView extends FrameLayout {
         } else if (surfaceView instanceof SphericalSurfaceView) {
           oldVideoComponent.clearVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
           oldVideoComponent.clearVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
+          oldVideoComponent.clearCameraMotionListener(((SphericalSurfaceView) surfaceView));
         } else if (surfaceView instanceof SurfaceView) {
           oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView);
         }
@@ -540,8 +541,9 @@ public class PlayerView extends FrameLayout {
         if (surfaceView instanceof TextureView) {
           newVideoComponent.setVideoTextureView((TextureView) surfaceView);
         } else if (surfaceView instanceof SphericalSurfaceView) {
-          newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
           newVideoComponent.setVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView));
+          newVideoComponent.setCameraMotionListener(((SphericalSurfaceView) surfaceView));
+          newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface());
         } else if (surfaceView instanceof SurfaceView) {
           newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView);
         }

library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java

@@ -19,9 +19,12 @@ import static com.google.android.exoplayer2.ui.spherical.GlUtil.checkGlError;
 
 import android.graphics.SurfaceTexture;
 import android.opengl.GLES20;
+import android.opengl.Matrix;
 import android.support.annotation.Nullable;
 import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
 import com.google.android.exoplayer2.util.Assertions;
+import com.google.android.exoplayer2.util.TimedValueQueue;
+import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
 import com.google.android.exoplayer2.video.spherical.Projection;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
@@ -35,17 +38,30 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
 
   private final AtomicBoolean frameAvailable;
   private final ProjectionRenderer projectionRenderer;
+  private final FrameRotationQueue frameRotationQueue;
+  private final TimedValueQueue<Long> sampleTimestampQueue;
+  private final float[] rotationMatrix;
+  private final float[] tempMatrix;
 
   private int textureId;
   private @MonotonicNonNull SurfaceTexture surfaceTexture;
   private @Nullable Projection pendingProjection;
   private long pendingProjectionTimeNs;
   private long lastFrameTimestamp;
+  private boolean resetRotationAtNextFrame;
 
-  public SceneRenderer(Projection projection) {
+  public SceneRenderer(
+      Projection projection,
+      FrameRotationQueue frameRotationQueue,
+      TimedValueQueue<Long> sampleTimestampQueue) {
+    this.frameRotationQueue = frameRotationQueue;
+    this.sampleTimestampQueue = sampleTimestampQueue;
     frameAvailable = new AtomicBoolean();
     projectionRenderer = new ProjectionRenderer();
     projectionRenderer.setProjection(projection);
+    rotationMatrix = new float[16];
+    tempMatrix = new float[16];
+    resetRotation();
   }
 
   /** Initializes the renderer. */
@@ -63,6 +79,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     return surfaceTexture;
   }
 
+  public void resetRotation() {
+    resetRotationAtNextFrame = true;
+  }
+
   /** Sets a {@link Projection} to be used to display video. */
   public void setProjection(Projection projection, long timeNs) {
     pendingProjection = projection;
@@ -84,13 +104,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
     if (frameAvailable.compareAndSet(true, false)) {
       Assertions.checkNotNull(surfaceTexture).updateTexImage();
       checkGlError();
+      if (resetRotationAtNextFrame) {
+        Matrix.setIdentityM(rotationMatrix, 0);
+      }
       lastFrameTimestamp = surfaceTexture.getTimestamp();
+      Long sampleTimestamp = sampleTimestampQueue.poll(lastFrameTimestamp);
+      if (sampleTimestamp != null) {
+        frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestamp);
+      }
     }
     if (pendingProjection != null && pendingProjectionTimeNs <= lastFrameTimestamp) {
       projectionRenderer.setProjection(pendingProjection);
       pendingProjection = null;
     }
-
+    Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0);
-    projectionRenderer.draw(textureId, viewProjectionMatrix, eyeType);
+    projectionRenderer.draw(textureId, tempMatrix, eyeType);
   }
 }

library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java

@@ -40,8 +40,11 @@ import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.Format;
 import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType;
 import com.google.android.exoplayer2.util.Assertions;
+import com.google.android.exoplayer2.util.TimedValueQueue;
 import com.google.android.exoplayer2.util.Util;
 import com.google.android.exoplayer2.video.VideoFrameMetadataListener;
+import com.google.android.exoplayer2.video.spherical.CameraMotionListener;
+import com.google.android.exoplayer2.video.spherical.FrameRotationQueue;
 import com.google.android.exoplayer2.video.spherical.Projection;
 import com.google.android.exoplayer2.video.spherical.ProjectionDecoder;
 import java.util.Arrays;
@@ -60,7 +63,7 @@ import javax.microedition.khronos.opengles.GL10;
  */
 @TargetApi(15)
 public final class SphericalSurfaceView extends GLSurfaceView
-    implements VideoFrameMetadataListener {
+    implements VideoFrameMetadataListener, CameraMotionListener {
 
   /**
    * This listener can be used to be notified when the {@link Surface} associated with this view is
@@ -91,6 +94,8 @@ public final class SphericalSurfaceView extends GLSurfaceView
   private final PhoneOrientationListener phoneOrientationListener;
   private final Renderer renderer;
   private final Handler mainHandler;
+  private final TimedValueQueue<Long> sampleTimestampQueue;
+  private final FrameRotationQueue frameRotationQueue;
   private final TouchTracker touchTracker;
   private @Nullable SurfaceListener surfaceListener;
   private @Nullable SurfaceTexture surfaceTexture;
@@ -105,9 +110,6 @@ public final class SphericalSurfaceView extends GLSurfaceView
 
   public SphericalSurfaceView(Context context, @Nullable AttributeSet attributeSet) {
     super(context, attributeSet);
-
-    defaultStereoMode = C.STEREO_MODE_MONO;
-    currentStereoMode = C.STEREO_MODE_MONO;
     mainHandler = new Handler(Looper.getMainLooper());
 
     // Configure sensors and touch.
@@ -120,7 +122,13 @@ public final class SphericalSurfaceView extends GLSurfaceView
     int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR;
     orientationSensor = sensorManager.getDefaultSensor(type);
 
-    renderer = new Renderer();
+    defaultStereoMode = C.STEREO_MODE_MONO;
+    currentStereoMode = defaultStereoMode;
+    Projection projection = Projection.createEquirectangular(defaultStereoMode);
+    frameRotationQueue = new FrameRotationQueue();
+    sampleTimestampQueue = new TimedValueQueue<>();
+    SceneRenderer scene = new SceneRenderer(projection, frameRotationQueue, sampleTimestampQueue);
+    renderer = new Renderer(scene);
 
     touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES);
     WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE);
@@ -161,12 +169,29 @@ public final class SphericalSurfaceView extends GLSurfaceView
     touchTracker.setSingleTapListener(listener);
   }
 
+  // VideoFrameMetadataListener implementation.
+
   @Override
   public void onVideoFrameAboutToBeRendered(
       long presentationTimeUs, long releaseTimeNs, Format format) {
+    sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs);
     setProjection(format.projectionData, format.stereoMode, releaseTimeNs);
   }
 
+  // CameraMotionListener implementation.
+
+  @Override
+  public void onCameraMotion(long timeUs, float[] rotation) {
+    frameRotationQueue.setRotation(timeUs, rotation);
+  }
+
+  @Override
+  public void onCameraMotionReset() {
+    sampleTimestampQueue.clear();
+    frameRotationQueue.reset();
+    queueEvent(renderer.scene::resetRotation);
+  }
+
   @Override
   public void onResume() {
     super.onResume();
@@ -354,8 +379,8 @@ public final class SphericalSurfaceView extends GLSurfaceView
     private final float[] viewMatrix = new float[16];
     private final float[] tempMatrix = new float[16];
 
-    public Renderer() {
+    public Renderer(SceneRenderer scene) {
-      scene = new SceneRenderer(Projection.createEquirectangular(C.STEREO_MODE_MONO));
+      this.scene = scene;
       Matrix.setIdentityM(deviceOrientationMatrix, 0);
       Matrix.setIdentityM(touchPitchMatrix, 0);
       Matrix.setIdentityM(touchYawMatrix, 0);