diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 9d2c06a918..3098bd9a76 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -98,7 +98,8 @@ * Use default Deserializers if non given to DownloadManager. * 360: * Add monoscopic 360 surface type to PlayerView. - * Support VR180 videos. + * Support + [VR180 video format](https://github.com/google/spatial-media/blob/master/docs/vr180.md). * Deprecate `Player.DefaultEventListener` as selective listener overrides can be directly made with the `Player.EventListener` interface. * Deprecate `DefaultAnalyticsListener` as selective listener overrides can be diff --git a/library/core/src/main/java/com/google/android/exoplayer2/C.java b/library/core/src/main/java/com/google/android/exoplayer2/C.java index 144fa76b33..0cbdc14b1c 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/C.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/C.java @@ -27,6 +27,7 @@ import com.google.android.exoplayer2.PlayerMessage.Target; import com.google.android.exoplayer2.audio.AuxEffectInfo; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.VideoFrameMetadataListener; +import com.google.android.exoplayer2.video.spherical.CameraMotionListener; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.util.UUID; @@ -595,34 +596,22 @@ public final class C { */ public static final int DATA_TYPE_CUSTOM_BASE = 10000; - /** - * A type constant for tracks of unknown type. - */ + /** A type constant for tracks of unknown type. */ public static final int TRACK_TYPE_UNKNOWN = -1; - /** - * A type constant for tracks of some default type, where the type itself is unknown. - */ + /** A type constant for tracks of some default type, where the type itself is unknown. */ public static final int TRACK_TYPE_DEFAULT = 0; - /** - * A type constant for audio tracks. - */ + /** A type constant for audio tracks. */ public static final int TRACK_TYPE_AUDIO = 1; - /** - * A type constant for video tracks. - */ + /** A type constant for video tracks. */ public static final int TRACK_TYPE_VIDEO = 2; - /** - * A type constant for text tracks. - */ + /** A type constant for text tracks. */ public static final int TRACK_TYPE_TEXT = 3; - /** - * A type constant for metadata tracks. - */ + /** A type constant for metadata tracks. */ public static final int TRACK_TYPE_METADATA = 4; - /** - * A type constant for a dummy or empty track. - */ - public static final int TRACK_TYPE_NONE = 5; + /** A type constant for camera motion tracks. */ + public static final int TRACK_TYPE_CAMERA_MOTION = 5; + /** A type constant for a dummy or empty track. */ + public static final int TRACK_TYPE_NONE = 6; /** * Applications or extensions may define custom {@code TRACK_TYPE_*} constants greater than or * equal to this value. @@ -655,36 +644,27 @@ public final class C { */ public static final int SELECTION_REASON_CUSTOM_BASE = 10000; - /** - * A default size in bytes for an individual allocation that forms part of a larger buffer. - */ + /** A default size in bytes for an individual allocation that forms part of a larger buffer. */ public static final int DEFAULT_BUFFER_SEGMENT_SIZE = 64 * 1024; - /** - * A default size in bytes for a video buffer. - */ + /** A default size in bytes for a video buffer. */ public static final int DEFAULT_VIDEO_BUFFER_SIZE = 200 * DEFAULT_BUFFER_SEGMENT_SIZE; - /** - * A default size in bytes for an audio buffer. - */ + /** A default size in bytes for an audio buffer. */ public static final int DEFAULT_AUDIO_BUFFER_SIZE = 54 * DEFAULT_BUFFER_SEGMENT_SIZE; - /** - * A default size in bytes for a text buffer. - */ + /** A default size in bytes for a text buffer. */ public static final int DEFAULT_TEXT_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE; - /** - * A default size in bytes for a metadata buffer. - */ + /** A default size in bytes for a metadata buffer. */ public static final int DEFAULT_METADATA_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE; - /** - * A default size in bytes for a muxed buffer (e.g. containing video, audio and text). - */ - public static final int DEFAULT_MUXED_BUFFER_SIZE = DEFAULT_VIDEO_BUFFER_SIZE - + DEFAULT_AUDIO_BUFFER_SIZE + DEFAULT_TEXT_BUFFER_SIZE; + /** A default size in bytes for a camera motion buffer. */ + public static final int DEFAULT_CAMERA_MOTION_BUFFER_SIZE = 2 * DEFAULT_BUFFER_SEGMENT_SIZE; + + /** A default size in bytes for a muxed buffer (e.g. containing video, audio and text). */ + public static final int DEFAULT_MUXED_BUFFER_SIZE = + DEFAULT_VIDEO_BUFFER_SIZE + DEFAULT_AUDIO_BUFFER_SIZE + DEFAULT_TEXT_BUFFER_SIZE; /** "cenc" scheme type name as defined in ISO/IEC 23001-7:2016. */ @SuppressWarnings("ConstantField") @@ -798,6 +778,13 @@ public final class C { */ public static final int MSG_SET_VIDEO_FRAME_METADATA_LISTENER = 6; + /** + * The type of a message that can be passed to a camera motion {@link Renderer} via {@link + * ExoPlayer#createMessage(Target)}. The message payload should be a {@link CameraMotionListener} + * instance, or null. + */ + public static final int MSG_SET_CAMERA_MOTION_LISTENER = 7; + /** * Applications or extensions may define custom {@code MSG_*} constants that can be passed to * {@link Renderer}s. These custom constants must be greater than or equal to this value. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java index ff7e953896..c0a117c241 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/DefaultRenderersFactory.java @@ -35,6 +35,7 @@ import com.google.android.exoplayer2.text.TextRenderer; import com.google.android.exoplayer2.trackselection.TrackSelector; import com.google.android.exoplayer2.video.MediaCodecVideoRenderer; import com.google.android.exoplayer2.video.VideoRendererEventListener; +import com.google.android.exoplayer2.video.spherical.CameraMotionRenderer; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.reflect.Constructor; @@ -182,6 +183,7 @@ public class DefaultRenderersFactory implements RenderersFactory { extensionRendererMode, renderersList); buildMetadataRenderers(context, metadataRendererOutput, eventHandler.getLooper(), extensionRendererMode, renderersList); + buildCameraMotionRenderers(context, extensionRendererMode, renderersList); buildMiscellaneousRenderers(context, eventHandler, extensionRendererMode, renderersList); return renderersList.toArray(new Renderer[renderersList.size()]); } @@ -360,12 +362,14 @@ public class DefaultRenderersFactory implements RenderersFactory { * * @param context The {@link Context} associated with the player. * @param output An output for the renderers. - * @param outputLooper The looper associated with the thread on which the output should be - * called. + * @param outputLooper The looper associated with the thread on which the output should be called. * @param extensionRendererMode The extension renderer mode. * @param out An array to which the built renderers should be appended. */ - protected void buildTextRenderers(Context context, TextOutput output, Looper outputLooper, + protected void buildTextRenderers( + Context context, + TextOutput output, + Looper outputLooper, @ExtensionRendererMode int extensionRendererMode, ArrayList out) { out.add(new TextRenderer(output, outputLooper)); @@ -376,16 +380,31 @@ public class DefaultRenderersFactory implements RenderersFactory { * * @param context The {@link Context} associated with the player. * @param output An output for the renderers. - * @param outputLooper The looper associated with the thread on which the output should be - * called. + * @param outputLooper The looper associated with the thread on which the output should be called. * @param extensionRendererMode The extension renderer mode. * @param out An array to which the built renderers should be appended. */ - protected void buildMetadataRenderers(Context context, MetadataOutput output, Looper outputLooper, - @ExtensionRendererMode int extensionRendererMode, ArrayList out) { + protected void buildMetadataRenderers( + Context context, + MetadataOutput output, + Looper outputLooper, + @ExtensionRendererMode int extensionRendererMode, + ArrayList out) { out.add(new MetadataRenderer(output, outputLooper)); } + /** + * Builds camera motion renderers for use by the player. + * + * @param context The {@link Context} associated with the player. + * @param extensionRendererMode The extension renderer mode. + * @param out An array to which the built renderers should be appended. + */ + protected void buildCameraMotionRenderers( + Context context, @ExtensionRendererMode int extensionRendererMode, ArrayList out) { + out.add(new CameraMotionRenderer()); + } + /** * Builds any miscellaneous renderers used by the player. * diff --git a/library/core/src/main/java/com/google/android/exoplayer2/Player.java b/library/core/src/main/java/com/google/android/exoplayer2/Player.java index 57c55b0070..e99d62a417 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/Player.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/Player.java @@ -31,6 +31,7 @@ import com.google.android.exoplayer2.trackselection.TrackSelectionArray; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoListener; +import com.google.android.exoplayer2.video.spherical.CameraMotionListener; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -185,6 +186,21 @@ public interface Player { */ void clearVideoFrameMetadataListener(VideoFrameMetadataListener listener); + /** + * Sets a listener of camera motion events. + * + * @param listener The listener. + */ + void setCameraMotionListener(CameraMotionListener listener); + + /** + * Clears the listener which receives camera motion events if it matches the one passed. Else + * does nothing. + * + * @param listener The listener to clear. + */ + void clearCameraMotionListener(CameraMotionListener listener); + /** * Clears any {@link Surface}, {@link SurfaceHolder}, {@link SurfaceView} or {@link TextureView} * currently set on the player. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java index a86644c370..c29055ddc7 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/SimpleExoPlayer.java @@ -53,6 +53,7 @@ import com.google.android.exoplayer2.util.Clock; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.VideoFrameMetadataListener; import com.google.android.exoplayer2.video.VideoRendererEventListener; +import com.google.android.exoplayer2.video.spherical.CameraMotionListener; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -107,6 +108,7 @@ public class SimpleExoPlayer private MediaSource mediaSource; private List currentCues; private VideoFrameMetadataListener videoFrameMetadataListener; + private CameraMotionListener cameraMotionListener; /** * @param context A {@link Context}. @@ -597,6 +599,36 @@ public class SimpleExoPlayer } } + @Override + public void setCameraMotionListener(CameraMotionListener listener) { + cameraMotionListener = listener; + for (Renderer renderer : renderers) { + if (renderer.getTrackType() == C.TRACK_TYPE_CAMERA_MOTION) { + player + .createMessage(renderer) + .setType(C.MSG_SET_CAMERA_MOTION_LISTENER) + .setPayload(listener) + .send(); + } + } + } + + @Override + public void clearCameraMotionListener(CameraMotionListener listener) { + if (cameraMotionListener != listener) { + return; + } + for (Renderer renderer : renderers) { + if (renderer.getTrackType() == C.TRACK_TYPE_CAMERA_MOTION) { + player + .createMessage(renderer) + .setType(C.MSG_SET_CAMERA_MOTION_LISTENER) + .setPayload(null) + .send(); + } + } + } + /** * Sets a listener to receive video events, removing all existing listeners. * diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/EventLogger.java b/library/core/src/main/java/com/google/android/exoplayer2/util/EventLogger.java index f9bd139298..626464ec69 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/util/EventLogger.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/EventLogger.java @@ -599,6 +599,8 @@ public class EventLogger implements AnalyticsListener { return "default"; case C.TRACK_TYPE_METADATA: return "metadata"; + case C.TRACK_TYPE_CAMERA_MOTION: + return "camera motion"; case C.TRACK_TYPE_NONE: return "none"; case C.TRACK_TYPE_TEXT: diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java b/library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java index e0b1df7739..f56aac7c70 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java @@ -328,9 +328,10 @@ public final class MimeTypes { return C.TRACK_TYPE_TEXT; } else if (APPLICATION_ID3.equals(mimeType) || APPLICATION_EMSG.equals(mimeType) - || APPLICATION_SCTE35.equals(mimeType) - || APPLICATION_CAMERA_MOTION.equals(mimeType)) { + || APPLICATION_SCTE35.equals(mimeType)) { return C.TRACK_TYPE_METADATA; + } else if (APPLICATION_CAMERA_MOTION.equals(mimeType)) { + return C.TRACK_TYPE_CAMERA_MOTION; } else { return getTrackTypeForCustomMimeType(mimeType); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java b/library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java index 160db74eda..3fe3c56c15 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/TimedValueQueue.java @@ -64,7 +64,7 @@ public final class TimedValueQueue { /** * Returns the value with the greatest timestamp which is less than or equal to the given - * timestamp. Removes all older values including the returned one from the buffer. + * timestamp. Removes all older values and the returned one from the buffer. * * @param timestamp The timestamp value. * @return The value with the greatest timestamp which is less than or equal to the given diff --git a/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java b/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java index ba17168d06..2f30612081 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/util/Util.java @@ -1443,6 +1443,8 @@ public final class Util { return C.DEFAULT_TEXT_BUFFER_SIZE; case C.TRACK_TYPE_METADATA: return C.DEFAULT_METADATA_BUFFER_SIZE; + case C.TRACK_TYPE_CAMERA_MOTION: + return C.DEFAULT_CAMERA_MOTION_BUFFER_SIZE; default: throw new IllegalStateException(); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionListener.java b/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionListener.java new file mode 100644 index 0000000000..33fc639412 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionListener.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.spherical; + +/** Listens camera motion. */ +public interface CameraMotionListener { + + /** + * Called when a new camera motion is read. This method is called on the playback thread. + * + * @param timeUs The presentation time of the data. + * @param rotation Angle axis orientation in radians representing the rotation from camera + * coordinate system to world coordinate system. + */ + void onCameraMotion(long timeUs, float[] rotation); + + /** Called when the camera motion track position is reset. */ + void onCameraMotionReset(); +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionRenderer.java new file mode 100644 index 0000000000..5fab84ed8d --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/CameraMotionRenderer.java @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.spherical; + +import android.support.annotation.Nullable; +import com.google.android.exoplayer2.BaseRenderer; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.ExoPlaybackException; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.FormatHolder; +import com.google.android.exoplayer2.Renderer; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.ParsableByteArray; +import com.google.android.exoplayer2.util.Util; +import java.nio.ByteBuffer; + +/** A {@link Renderer} that parses the camera motion track. */ +public class CameraMotionRenderer extends BaseRenderer { + + // The amount of time to read samples ahead of the current time. + private static final int SAMPLE_WINDOW_DURATION_US = 100000; + + private final FormatHolder formatHolder; + private final DecoderInputBuffer buffer; + private final ParsableByteArray scratch; + + private long offsetUs; + private @Nullable CameraMotionListener listener; + private long lastTimestampUs; + + public CameraMotionRenderer() { + super(C.TRACK_TYPE_CAMERA_MOTION); + formatHolder = new FormatHolder(); + buffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); + scratch = new ParsableByteArray(); + } + + @Override + public int supportsFormat(Format format) { + return MimeTypes.APPLICATION_CAMERA_MOTION.equals(format.sampleMimeType) + ? FORMAT_HANDLED + : FORMAT_UNSUPPORTED_TYPE; + } + + @Override + public void handleMessage(int messageType, Object message) throws ExoPlaybackException { + if (messageType == C.MSG_SET_CAMERA_MOTION_LISTENER) { + listener = (CameraMotionListener) message; + } else { + super.handleMessage(messageType, message); + } + } + + @Override + protected void onStreamChanged(Format[] formats, long offsetUs) throws ExoPlaybackException { + this.offsetUs = offsetUs; + } + + @Override + protected void onPositionReset(long positionUs, boolean joining) throws ExoPlaybackException { + lastTimestampUs = 0; + if (listener != null) { + listener.onCameraMotionReset(); + } + } + + @Override + protected void onDisabled() { + lastTimestampUs = 0; + } + + @Override + public void render(long positionUs, long elapsedRealtimeUs) throws ExoPlaybackException { + // Keep reading available samples as long as the sample time is not too far into the future. + while (!hasReadStreamToEnd() && lastTimestampUs < positionUs + SAMPLE_WINDOW_DURATION_US) { + buffer.clear(); + int result = readSource(formatHolder, buffer, /* formatRequired= */ false); + if (result != C.RESULT_BUFFER_READ || buffer.isEndOfStream()) { + return; + } + + buffer.flip(); + lastTimestampUs = buffer.timeUs; + if (listener != null) { + float[] rotation = parseMetadata(buffer.data); + if (rotation != null) { + Util.castNonNull(listener).onCameraMotion(lastTimestampUs - offsetUs, rotation); + } + } + } + } + + @Override + public boolean isEnded() { + return hasReadStreamToEnd(); + } + + @Override + public boolean isReady() { + return true; + } + + private @Nullable float[] parseMetadata(ByteBuffer data) { + if (data.remaining() != 16) { + return null; + } + scratch.reset(data.array(), data.limit()); + scratch.setPosition(data.arrayOffset() + 4); // skip reserved bytes too. + float[] result = new float[3]; + for (int i = 0; i < 3; i++) { + result[i] = Float.intBitsToFloat(scratch.readLittleEndianInt()); + } + return result; + } +} diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueue.java b/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueue.java new file mode 100644 index 0000000000..d7404cbce4 --- /dev/null +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueue.java @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.spherical; + +import android.opengl.Matrix; +import com.google.android.exoplayer2.util.TimedValueQueue; + +/** + * This class serves multiple purposes: + * + *
    + *
  • Queues the rotation metadata extracted from camera motion track. + *
  • Converts the metadata to rotation matrices in OpenGl coordinate system. + *
  • Recenters the rotations to componsate the yaw of the initial rotation. + *
+ */ +public final class FrameRotationQueue { + private final float[] recenterMatrix; + private final float[] rotationMatrix; + private final TimedValueQueue rotations; + private boolean recenterMatrixComputed; + + public FrameRotationQueue() { + recenterMatrix = new float[16]; + rotationMatrix = new float[16]; + rotations = new TimedValueQueue<>(); + } + + /** + * Sets a rotation for a given timestamp. + * + * @param timestampUs Timestamp of the rotation. + * @param angleAxis Angle axis orientation in radians representing the rotation from camera + * coordinate system to world coordinate system. + */ + public void setRotation(long timestampUs, float[] angleAxis) { + rotations.add(timestampUs, angleAxis); + } + + /** Removes all of the rotations and forces rotations to be recentered. */ + public void reset() { + rotations.clear(); + recenterMatrixComputed = false; + } + + /** + * Copies the rotation matrix with the greatest timestamp which is less than or equal to the given + * timestamp to {@code matrix}. Removes all older rotations and the returned one from the queue. + * Does nothing if there is no such rotation. + * + * @param matrix A float array to hold the rotation matrix. + * @param timestampUs The time in microseconds to query the rotation. + * @return Whether a rotation matrix is copied to {@code matrix}. + */ + public boolean pollRotationMatrix(float[] matrix, long timestampUs) { + float[] rotation = rotations.pollFloor(timestampUs); + if (rotation == null) { + return false; + } + // TODO [Internal: b/113315546]: Slerp between the floor and ceil rotation. + getRotationMatrixFromAngleAxis(rotationMatrix, rotation); + if (!recenterMatrixComputed) { + computeRecenterMatrix(recenterMatrix, rotationMatrix); + recenterMatrixComputed = true; + } + Matrix.multiplyMM(matrix, 0, recenterMatrix, 0, rotationMatrix, 0); + return true; + } + + /** + * Computes a recentering matrix from the given angle-axis rotation only accounting for yaw. Roll + * and tilt will not be compensated. + */ + private static void computeRecenterMatrix(float[] recenterMatrix, float[] rotationMatrix) { + // The re-centering matrix is computed as follows: + // recenter.row(2) = temp.col(2).transpose(); + // recenter.row(0) = recenter.row(1).cross(recenter.row(2)).normalized(); + // recenter.row(2) = recenter.row(0).cross(recenter.row(1)).normalized(); + // | temp[10] 0 -temp[8] 0| + // | 0 1 0 0| + // recenter = | temp[8] 0 temp[10] 0| + // | 0 0 0 1| + Matrix.setIdentityM(recenterMatrix, 0); + float normRowSqr = + rotationMatrix[10] * rotationMatrix[10] + rotationMatrix[8] * rotationMatrix[8]; + float normRow = (float) Math.sqrt(normRowSqr); + recenterMatrix[0] = rotationMatrix[10] / normRow; + recenterMatrix[2] = rotationMatrix[8] / normRow; + recenterMatrix[8] = -rotationMatrix[8] / normRow; + recenterMatrix[10] = rotationMatrix[10] / normRow; + } + + private static void getRotationMatrixFromAngleAxis(float[] matrix, float[] angleAxis) { + // Convert coordinates to OpenGL coordinates. + // CAMM motion metadata: +x right, +y down, and +z forward. + // OpenGL: +x right, +y up, -z forwards + float x = angleAxis[0]; + float y = -angleAxis[1]; + float z = -angleAxis[2]; + float angleRad = Matrix.length(x, y, z); + if (angleRad != 0) { + float angleDeg = (float) Math.toDegrees(angleRad); + Matrix.setRotateM(matrix, 0, angleDeg, x / angleRad, y / angleRad, z / angleRad); + } else { + Matrix.setIdentityM(matrix, 0); + } + } +} diff --git a/library/core/src/test/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueueTest.java b/library/core/src/test/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueueTest.java new file mode 100644 index 0000000000..071cd582d5 --- /dev/null +++ b/library/core/src/test/java/com/google/android/exoplayer2/video/spherical/FrameRotationQueueTest.java @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.video.spherical; + +import static com.google.common.truth.Truth.assertThat; + +import android.opengl.Matrix; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.robolectric.RobolectricTestRunner; + +/** Tests {@link FrameRotationQueue}. */ +@RunWith(RobolectricTestRunner.class) +public class FrameRotationQueueTest { + + private FrameRotationQueue frameRotationQueue; + private float[] rotationMatrix; + + @Before + public void setUp() throws Exception { + frameRotationQueue = new FrameRotationQueue(); + rotationMatrix = new float[16]; + } + + @Test + public void testGetRotationMatrixReturnsNull_whenEmpty() throws Exception { + assertThat(frameRotationQueue.pollRotationMatrix(rotationMatrix, 0)).isFalse(); + } + + @Test + public void testGetRotationMatrixReturnsNotNull_whenNotEmpty() throws Exception { + frameRotationQueue.setRotation(0, new float[] {1, 2, 3}); + assertThat(frameRotationQueue.pollRotationMatrix(rotationMatrix, 0)).isTrue(); + assertThat(rotationMatrix).hasLength(16); + } + + @Test + public void testConvertsAngleAxisToRotationMatrix() throws Exception { + doTestAngleAxisToRotationMatrix(/* angleRadian= */ 0, /* x= */ 1, /* y= */ 0, /* z= */ 0); + frameRotationQueue.reset(); + doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 1, /* y= */ 0, /* z= */ 0); + frameRotationQueue.reset(); + doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 0, /* y= */ 0, /* z= */ 1); + // Don't reset frameRotationQueue as we use recenter matrix from previous calls. + doTestAngleAxisToRotationMatrix(/* angleRadian= */ -1, /* x= */ 0, /* y= */ 1, /* z= */ 0); + doTestAngleAxisToRotationMatrix(/* angleRadian= */ 1, /* x= */ 1, /* y= */ 1, /* z= */ 1); + } + + @Test + public void testRecentering_justYaw() throws Exception { + float[] actualMatrix = + getRotationMatrixFromAngleAxis( + /* angleRadian= */ (float) Math.PI, /* x= */ 0, /* y= */ 1, /* z= */ 0); + float[] expectedMatrix = new float[16]; + Matrix.setIdentityM(expectedMatrix, 0); + assertEquals(actualMatrix, expectedMatrix); + } + + @Test + public void testRecentering_yawAndPitch() throws Exception { + float[] matrix = + getRotationMatrixFromAngleAxis( + /* angleRadian= */ (float) Math.PI, /* x= */ 1, /* y= */ 1, /* z= */ 0); + assertMultiplication( + /* xr= */ 0, /* yr= */ 0, /* zr= */ 1, matrix, /* x= */ 0, /* y= */ 0, /* z= */ 1); + } + + @Test + public void testRecentering_yawAndPitch2() throws Exception { + float[] matrix = + getRotationMatrixFromAngleAxis( + /* angleRadian= */ (float) Math.PI / 2, /* x= */ 1, /* y= */ 1, /* z= */ 0); + float sqrt2 = (float) Math.sqrt(2); + assertMultiplication( + /* xr= */ sqrt2, /* yr= */ 0, /* zr= */ 0, matrix, /* x= */ 1, /* y= */ -1, /* z= */ 0); + } + + @Test + public void testRecentering_yawAndPitchAndRoll() throws Exception { + float[] matrix = + getRotationMatrixFromAngleAxis( + /* angleRadian= */ (float) Math.PI * 2 / 3, /* x= */ 1, /* y= */ 1, /* z= */ 1); + assertMultiplication( + /* xr= */ 0, /* yr= */ 0, /* zr= */ 1, matrix, /* x= */ 0, /* y= */ 0, /* z= */ 1); + } + + private void doTestAngleAxisToRotationMatrix(float angleRadian, int x, int y, int z) { + float[] actualMatrix = getRotationMatrixFromAngleAxis(angleRadian, x, y, z); + float[] expectedMatrix = createRotationMatrix(angleRadian, x, y, z); + assertEquals(actualMatrix, expectedMatrix); + } + + private float[] getRotationMatrixFromAngleAxis(float angleRadian, int x, int y, int z) { + float length = Matrix.length(x, y, z); + float factor = angleRadian / length; + // Negate y and z to revert OpenGL coordinate system conversion. + frameRotationQueue.setRotation(0, new float[] {x * factor, -y * factor, -z * factor}); + frameRotationQueue.pollRotationMatrix(rotationMatrix, 0); + return rotationMatrix; + } + + private static void assertMultiplication( + float xr, float yr, float zr, float[] actualMatrix, float x, float y, float z) { + float[] vector = new float[] {x, y, z, 0}; + float[] resultVec = new float[4]; + Matrix.multiplyMV(resultVec, 0, actualMatrix, 0, vector, 0); + assertEquals(resultVec, new float[] {xr, yr, zr, 0}); + } + + private static float[] createRotationMatrix(float angleRadian, int x, int y, int z) { + float[] expectedMatrix = new float[16]; + Matrix.setRotateM(expectedMatrix, 0, (float) Math.toDegrees(angleRadian), x, y, z); + return expectedMatrix; + } + + private static void assertEquals(float[] actual, float[] expected) { + assertThat(actual).usingTolerance(1.0e-5).containsExactly(expected).inOrder(); + } +} diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java index 8ad8899216..142b251a61 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/PlayerView.java @@ -515,6 +515,7 @@ public class PlayerView extends FrameLayout { } else if (surfaceView instanceof SphericalSurfaceView) { oldVideoComponent.clearVideoSurface(((SphericalSurfaceView) surfaceView).getSurface()); oldVideoComponent.clearVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView)); + oldVideoComponent.clearCameraMotionListener(((SphericalSurfaceView) surfaceView)); } else if (surfaceView instanceof SurfaceView) { oldVideoComponent.clearVideoSurfaceView((SurfaceView) surfaceView); } @@ -540,8 +541,9 @@ public class PlayerView extends FrameLayout { if (surfaceView instanceof TextureView) { newVideoComponent.setVideoTextureView((TextureView) surfaceView); } else if (surfaceView instanceof SphericalSurfaceView) { - newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface()); newVideoComponent.setVideoFrameMetadataListener(((SphericalSurfaceView) surfaceView)); + newVideoComponent.setCameraMotionListener(((SphericalSurfaceView) surfaceView)); + newVideoComponent.setVideoSurface(((SphericalSurfaceView) surfaceView).getSurface()); } else if (surfaceView instanceof SurfaceView) { newVideoComponent.setVideoSurfaceView((SurfaceView) surfaceView); } diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java index d529de1ccb..023d68f988 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SceneRenderer.java @@ -19,9 +19,12 @@ import static com.google.android.exoplayer2.ui.spherical.GlUtil.checkGlError; import android.graphics.SurfaceTexture; import android.opengl.GLES20; +import android.opengl.Matrix; import android.support.annotation.Nullable; import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType; import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.TimedValueQueue; +import com.google.android.exoplayer2.video.spherical.FrameRotationQueue; import com.google.android.exoplayer2.video.spherical.Projection; import java.util.concurrent.atomic.AtomicBoolean; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; @@ -35,17 +38,30 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; private final AtomicBoolean frameAvailable; private final ProjectionRenderer projectionRenderer; + private final FrameRotationQueue frameRotationQueue; + private final TimedValueQueue sampleTimestampQueue; + private final float[] rotationMatrix; + private final float[] tempMatrix; private int textureId; private @MonotonicNonNull SurfaceTexture surfaceTexture; private @Nullable Projection pendingProjection; private long pendingProjectionTimeNs; private long lastFrameTimestamp; + private boolean resetRotationAtNextFrame; - public SceneRenderer(Projection projection) { + public SceneRenderer( + Projection projection, + FrameRotationQueue frameRotationQueue, + TimedValueQueue sampleTimestampQueue) { + this.frameRotationQueue = frameRotationQueue; + this.sampleTimestampQueue = sampleTimestampQueue; frameAvailable = new AtomicBoolean(); projectionRenderer = new ProjectionRenderer(); projectionRenderer.setProjection(projection); + rotationMatrix = new float[16]; + tempMatrix = new float[16]; + resetRotation(); } /** Initializes the renderer. */ @@ -63,6 +79,10 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return surfaceTexture; } + public void resetRotation() { + resetRotationAtNextFrame = true; + } + /** Sets a {@link Projection} to be used to display video. */ public void setProjection(Projection projection, long timeNs) { pendingProjection = projection; @@ -84,13 +104,20 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; if (frameAvailable.compareAndSet(true, false)) { Assertions.checkNotNull(surfaceTexture).updateTexImage(); checkGlError(); + if (resetRotationAtNextFrame) { + Matrix.setIdentityM(rotationMatrix, 0); + } lastFrameTimestamp = surfaceTexture.getTimestamp(); + Long sampleTimestamp = sampleTimestampQueue.poll(lastFrameTimestamp); + if (sampleTimestamp != null) { + frameRotationQueue.pollRotationMatrix(rotationMatrix, sampleTimestamp); + } } if (pendingProjection != null && pendingProjectionTimeNs <= lastFrameTimestamp) { projectionRenderer.setProjection(pendingProjection); pendingProjection = null; } - - projectionRenderer.draw(textureId, viewProjectionMatrix, eyeType); + Matrix.multiplyMM(tempMatrix, 0, viewProjectionMatrix, 0, rotationMatrix, 0); + projectionRenderer.draw(textureId, tempMatrix, eyeType); } } diff --git a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java index 483376dba4..30995aca5f 100644 --- a/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java +++ b/library/ui/src/main/java/com/google/android/exoplayer2/ui/spherical/SphericalSurfaceView.java @@ -40,8 +40,11 @@ import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.ui.spherical.ProjectionRenderer.EyeType; import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.TimedValueQueue; import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.VideoFrameMetadataListener; +import com.google.android.exoplayer2.video.spherical.CameraMotionListener; +import com.google.android.exoplayer2.video.spherical.FrameRotationQueue; import com.google.android.exoplayer2.video.spherical.Projection; import com.google.android.exoplayer2.video.spherical.ProjectionDecoder; import java.util.Arrays; @@ -60,7 +63,7 @@ import javax.microedition.khronos.opengles.GL10; */ @TargetApi(15) public final class SphericalSurfaceView extends GLSurfaceView - implements VideoFrameMetadataListener { + implements VideoFrameMetadataListener, CameraMotionListener { /** * This listener can be used to be notified when the {@link Surface} associated with this view is @@ -91,6 +94,8 @@ public final class SphericalSurfaceView extends GLSurfaceView private final PhoneOrientationListener phoneOrientationListener; private final Renderer renderer; private final Handler mainHandler; + private final TimedValueQueue sampleTimestampQueue; + private final FrameRotationQueue frameRotationQueue; private final TouchTracker touchTracker; private @Nullable SurfaceListener surfaceListener; private @Nullable SurfaceTexture surfaceTexture; @@ -105,9 +110,6 @@ public final class SphericalSurfaceView extends GLSurfaceView public SphericalSurfaceView(Context context, @Nullable AttributeSet attributeSet) { super(context, attributeSet); - - defaultStereoMode = C.STEREO_MODE_MONO; - currentStereoMode = C.STEREO_MODE_MONO; mainHandler = new Handler(Looper.getMainLooper()); // Configure sensors and touch. @@ -120,7 +122,13 @@ public final class SphericalSurfaceView extends GLSurfaceView int type = Util.SDK_INT >= 18 ? Sensor.TYPE_GAME_ROTATION_VECTOR : Sensor.TYPE_ROTATION_VECTOR; orientationSensor = sensorManager.getDefaultSensor(type); - renderer = new Renderer(); + defaultStereoMode = C.STEREO_MODE_MONO; + currentStereoMode = defaultStereoMode; + Projection projection = Projection.createEquirectangular(defaultStereoMode); + frameRotationQueue = new FrameRotationQueue(); + sampleTimestampQueue = new TimedValueQueue<>(); + SceneRenderer scene = new SceneRenderer(projection, frameRotationQueue, sampleTimestampQueue); + renderer = new Renderer(scene); touchTracker = new TouchTracker(context, renderer, PX_PER_DEGREES); WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); @@ -161,12 +169,29 @@ public final class SphericalSurfaceView extends GLSurfaceView touchTracker.setSingleTapListener(listener); } + // VideoFrameMetadataListener implementation. + @Override public void onVideoFrameAboutToBeRendered( long presentationTimeUs, long releaseTimeNs, Format format) { + sampleTimestampQueue.add(releaseTimeNs, presentationTimeUs); setProjection(format.projectionData, format.stereoMode, releaseTimeNs); } + // CameraMotionListener implementation. + + @Override + public void onCameraMotion(long timeUs, float[] rotation) { + frameRotationQueue.setRotation(timeUs, rotation); + } + + @Override + public void onCameraMotionReset() { + sampleTimestampQueue.clear(); + frameRotationQueue.reset(); + queueEvent(renderer.scene::resetRotation); + } + @Override public void onResume() { super.onResume(); @@ -354,8 +379,8 @@ public final class SphericalSurfaceView extends GLSurfaceView private final float[] viewMatrix = new float[16]; private final float[] tempMatrix = new float[16]; - public Renderer() { - scene = new SceneRenderer(Projection.createEquirectangular(C.STEREO_MODE_MONO)); + public Renderer(SceneRenderer scene) { + this.scene = scene; Matrix.setIdentityM(deviceOrientationMatrix, 0); Matrix.setIdentityM(touchPitchMatrix, 0); Matrix.setIdentityM(touchYawMatrix, 0);