From a40a71a534092e93bef18b79915fa8423cb2b8d5 Mon Sep 17 00:00:00 2001 From: christosts Date: Mon, 6 Dec 2021 12:37:06 +0000 Subject: [PATCH 01/27] Add AudioAttributes.spatializationBehavior The new field matches the platform's AudioAttributes.getSpatializationBehavior() API added in Sv2. At the moment, the platform API is called via reflection, until Sv2 is released and the compile SDK target can be increased to 32. PiperOrigin-RevId: 414406126 --- .../java/com/google/android/exoplayer2/C.java | 14 ++++ .../exoplayer2/audio/AudioAttributes.java | 74 +++++++++++++++++-- .../exoplayer2/audio/AudioAttributesTest.java | 1 + 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/C.java b/library/common/src/main/java/com/google/android/exoplayer2/C.java index 2c2548468b..b8d2def750 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/C.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/C.java @@ -272,6 +272,20 @@ public final class C { /** @see AudioFormat#ENCODING_DOLBY_TRUEHD */ public static final int ENCODING_DOLBY_TRUEHD = AudioFormat.ENCODING_DOLBY_TRUEHD; + /** Represents the behavior affecting whether spatialization will be used. */ + @Documented + @Retention(RetentionPolicy.SOURCE) + @Target({FIELD, METHOD, PARAMETER, LOCAL_VARIABLE, TYPE_USE}) + @IntDef({SPATIALIZATION_BEHAVIOR_AUTO, SPATIALIZATION_BEHAVIOR_NEVER}) + public @interface SpatializationBehavior {} + + // TODO[b/190759307]: Update constant values and javadoc to use SDK once compile SDK target is set + // to 32. + /** See AudioAttributes#SPATIALIZATION_BEHAVIOR_AUTO */ + public static final int SPATIALIZATION_BEHAVIOR_AUTO = 0; + /** See AudioAttributes#SPATIALIZATION_BEHAVIOR_NEVER */ + public static final int SPATIALIZATION_BEHAVIOR_NEVER = 1; + /** * Stream types for an {@link android.media.AudioTrack}. One of {@link #STREAM_TYPE_ALARM}, {@link * #STREAM_TYPE_DTMF}, {@link #STREAM_TYPE_MUSIC}, {@link #STREAM_TYPE_NOTIFICATION}, {@link diff --git a/library/common/src/main/java/com/google/android/exoplayer2/audio/AudioAttributes.java b/library/common/src/main/java/com/google/android/exoplayer2/audio/AudioAttributes.java index f503ab53c4..0ae911cf08 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/audio/AudioAttributes.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/audio/AudioAttributes.java @@ -16,6 +16,7 @@ package com.google.android.exoplayer2.audio; import android.os.Bundle; +import androidx.annotation.DoNotInline; import androidx.annotation.IntDef; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; @@ -25,6 +26,7 @@ import com.google.android.exoplayer2.util.Util; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; +import java.lang.reflect.Method; /** * Attributes for audio playback, which configure the underlying platform {@link @@ -48,6 +50,7 @@ public final class AudioAttributes implements Bundleable { private @C.AudioFlags int flags; private @C.AudioUsage int usage; private @C.AudioAllowedCapturePolicy int allowedCapturePolicy; + private @C.SpatializationBehavior int spatializationBehavior; /** * Creates a new builder for {@link AudioAttributes}. @@ -60,6 +63,7 @@ public final class AudioAttributes implements Bundleable { flags = 0; usage = C.USAGE_MEDIA; allowedCapturePolicy = C.ALLOW_CAPTURE_BY_ALL; + spatializationBehavior = C.SPATIALIZATION_BEHAVIOR_AUTO; } /** @see android.media.AudioAttributes.Builder#setContentType(int) */ @@ -86,9 +90,18 @@ public final class AudioAttributes implements Bundleable { return this; } + // TODO[b/190759307] Update javadoc to link to AudioAttributes.Builder#setSpatializationBehavior + // once compile SDK target is set to 32. + /** See AudioAttributes.Builder#setSpatializationBehavior(int). */ + public Builder setSpatializationBehavior(@C.SpatializationBehavior int spatializationBehavior) { + this.spatializationBehavior = spatializationBehavior; + return this; + } + /** Creates an {@link AudioAttributes} instance from this builder. */ public AudioAttributes build() { - return new AudioAttributes(contentType, flags, usage, allowedCapturePolicy); + return new AudioAttributes( + contentType, flags, usage, allowedCapturePolicy, spatializationBehavior); } } @@ -96,6 +109,7 @@ public final class AudioAttributes implements Bundleable { public final @C.AudioFlags int flags; public final @C.AudioUsage int usage; public final @C.AudioAllowedCapturePolicy int allowedCapturePolicy; + public final @C.SpatializationBehavior int spatializationBehavior; @Nullable private android.media.AudioAttributes audioAttributesV21; @@ -103,11 +117,13 @@ public final class AudioAttributes implements Bundleable { @C.AudioContentType int contentType, @C.AudioFlags int flags, @C.AudioUsage int usage, - @C.AudioAllowedCapturePolicy int allowedCapturePolicy) { + @C.AudioAllowedCapturePolicy int allowedCapturePolicy, + @C.SpatializationBehavior int spatializationBehavior) { this.contentType = contentType; this.flags = flags; this.usage = usage; this.allowedCapturePolicy = allowedCapturePolicy; + this.spatializationBehavior = spatializationBehavior; } /** @@ -124,7 +140,10 @@ public final class AudioAttributes implements Bundleable { .setFlags(flags) .setUsage(usage); if (Util.SDK_INT >= 29) { - builder.setAllowedCapturePolicy(allowedCapturePolicy); + Api29.setAllowedCapturePolicy(builder, allowedCapturePolicy); + } + if (Util.SDK_INT >= 32) { + Api32.setSpatializationBehavior(builder, spatializationBehavior); } audioAttributesV21 = builder.build(); } @@ -143,7 +162,8 @@ public final class AudioAttributes implements Bundleable { return this.contentType == other.contentType && this.flags == other.flags && this.usage == other.usage - && this.allowedCapturePolicy == other.allowedCapturePolicy; + && this.allowedCapturePolicy == other.allowedCapturePolicy + && this.spatializationBehavior == other.spatializationBehavior; } @Override @@ -153,6 +173,7 @@ public final class AudioAttributes implements Bundleable { result = 31 * result + flags; result = 31 * result + usage; result = 31 * result + allowedCapturePolicy; + result = 31 * result + spatializationBehavior; return result; } @@ -160,13 +181,20 @@ public final class AudioAttributes implements Bundleable { @Documented @Retention(RetentionPolicy.SOURCE) - @IntDef({FIELD_CONTENT_TYPE, FIELD_FLAGS, FIELD_USAGE, FIELD_ALLOWED_CAPTURE_POLICY}) + @IntDef({ + FIELD_CONTENT_TYPE, + FIELD_FLAGS, + FIELD_USAGE, + FIELD_ALLOWED_CAPTURE_POLICY, + FIELD_SPATIALIZATION_BEHAVIOR + }) private @interface FieldNumber {} private static final int FIELD_CONTENT_TYPE = 0; private static final int FIELD_FLAGS = 1; private static final int FIELD_USAGE = 2; private static final int FIELD_ALLOWED_CAPTURE_POLICY = 3; + private static final int FIELD_SPATIALIZATION_BEHAVIOR = 4; @Override public Bundle toBundle() { @@ -175,6 +203,7 @@ public final class AudioAttributes implements Bundleable { bundle.putInt(keyForField(FIELD_FLAGS), flags); bundle.putInt(keyForField(FIELD_USAGE), usage); bundle.putInt(keyForField(FIELD_ALLOWED_CAPTURE_POLICY), allowedCapturePolicy); + bundle.putInt(keyForField(FIELD_SPATIALIZATION_BEHAVIOR), spatializationBehavior); return bundle; } @@ -194,10 +223,45 @@ public final class AudioAttributes implements Bundleable { if (bundle.containsKey(keyForField(FIELD_ALLOWED_CAPTURE_POLICY))) { builder.setAllowedCapturePolicy(bundle.getInt(keyForField(FIELD_ALLOWED_CAPTURE_POLICY))); } + if (bundle.containsKey(keyForField(FIELD_SPATIALIZATION_BEHAVIOR))) { + builder.setSpatializationBehavior( + bundle.getInt(keyForField(FIELD_SPATIALIZATION_BEHAVIOR))); + } return builder.build(); }; private static String keyForField(@FieldNumber int field) { return Integer.toString(field, Character.MAX_RADIX); } + + @RequiresApi(29) + private static final class Api29 { + private Api29() {} + + @DoNotInline + public static void setAllowedCapturePolicy( + android.media.AudioAttributes.Builder builder, + @C.AudioAllowedCapturePolicy int allowedCapturePolicy) { + builder.setAllowedCapturePolicy(allowedCapturePolicy); + } + } + + @RequiresApi(32) + private static final class Api32 { + private Api32() {} + + @DoNotInline + public static void setSpatializationBehavior( + android.media.AudioAttributes.Builder builder, + @C.SpatializationBehavior int spatializationBehavior) { + try { + // TODO[b/190759307]: Remove reflection once compile SDK target is set to 32. + Method setSpatializationBehavior = + builder.getClass().getMethod("setSpatializationBehavior", Integer.TYPE); + setSpatializationBehavior.invoke(builder, spatializationBehavior); + } catch (Exception e) { + // Do nothing if reflection fails. + } + } + } } diff --git a/library/common/src/test/java/com/google/android/exoplayer2/audio/AudioAttributesTest.java b/library/common/src/test/java/com/google/android/exoplayer2/audio/AudioAttributesTest.java index 9296f60dd7..241633eb1b 100644 --- a/library/common/src/test/java/com/google/android/exoplayer2/audio/AudioAttributesTest.java +++ b/library/common/src/test/java/com/google/android/exoplayer2/audio/AudioAttributesTest.java @@ -34,6 +34,7 @@ public class AudioAttributesTest { .setFlags(C.FLAG_AUDIBILITY_ENFORCED) .setUsage(C.USAGE_ALARM) .setAllowedCapturePolicy(C.ALLOW_CAPTURE_BY_SYSTEM) + .setSpatializationBehavior(C.SPATIALIZATION_BEHAVIOR_NEVER) .build(); assertThat(AudioAttributes.CREATOR.fromBundle(audioAttributes.toBundle())) From daeea81e50d3dc10d58c76dce2db4b42302e4dfe Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Mon, 6 Dec 2021 12:41:11 +0000 Subject: [PATCH 02/27] Transformer GL: Create demo UI for changing resolution. Also, add 144p as an acceptable output resolution, to allow for a more obvious resolution difference when running the demo. PiperOrigin-RevId: 414406664 --- .../google/android/exoplayer2/transformer/Transformer.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java index 7a877e284b..1472506669 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/Transformer.java @@ -235,7 +235,7 @@ public final class Transformer { * Sets the output resolution using the output height. The default value is the same height as * the input. Output width will scale to preserve the input video's aspect ratio. * - *

For now, only "popular" heights like 240, 360, 480, 720, 1080, 1440, or 2160 are + *

For now, only "popular" heights like 144, 240, 360, 480, 720, 1080, 1440, or 2160 are * supported, to ensure compatibility on different devices. * *

For example, a 1920x1440 video can be scaled to 640x480 by calling setResolution(480). @@ -247,7 +247,8 @@ public final class Transformer { // TODO(Internal b/201293185): Restructure to input a Presentation class. // TODO(Internal b/201293185): Check encoder codec capabilities in order to allow arbitrary // resolutions and reasonable fallbacks. - if (outputHeight != 240 + if (outputHeight != 144 + && outputHeight != 240 && outputHeight != 360 && outputHeight != 480 && outputHeight != 720 @@ -255,7 +256,7 @@ public final class Transformer { && outputHeight != 1440 && outputHeight != 2160) { throw new IllegalArgumentException( - "Please use a height of 240, 360, 480, 720, 1080, 1440, or 2160."); + "Please use a height of 144, 240, 360, 480, 720, 1080, 1440, or 2160."); } this.outputHeight = outputHeight; return this; From 2e55643fbd22626e4749abf0767c8c18345a4f3a Mon Sep 17 00:00:00 2001 From: ibaker Date: Mon, 6 Dec 2021 13:24:26 +0000 Subject: [PATCH 03/27] Add MediaItem.SubtitleConfiguration#id field Issue: google/ExoPlayer#9673 #minor-release PiperOrigin-RevId: 414413320 --- RELEASENOTES.md | 5 ++++ .../google/android/exoplayer2/MediaItem.java | 28 +++++++++++++++---- .../android/exoplayer2/MediaItemTest.java | 3 ++ .../source/DefaultMediaSourceFactory.java | 1 + .../source/SingleSampleMediaSource.java | 1 + 5 files changed, 33 insertions(+), 5 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index eff30b74d2..6bb1b08cd7 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -24,6 +24,11 @@ * Extractors: * Fix inconsistency with spec in H.265 SPS nal units parsing ((#9719)[https://github.com/google/ExoPlayer/issues/9719]). +* Text: + * Add a `MediaItem.SubtitleConfiguration#id` field which is propagated to + the `Format#id` field of the subtitle track created from the + configuration + ((#9673)[https://github.com/google/ExoPlayer/issues/9673]). * DRM: * Remove `playbackLooper` from `DrmSessionManager.(pre)acquireSession`. When a `DrmSessionManager` is used by an app in a custom `MediaSource`, diff --git a/library/common/src/main/java/com/google/android/exoplayer2/MediaItem.java b/library/common/src/main/java/com/google/android/exoplayer2/MediaItem.java index b18c52018e..883b68f603 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/MediaItem.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/MediaItem.java @@ -1238,6 +1238,7 @@ public final class MediaItem implements Bundleable { private @C.SelectionFlags int selectionFlags; private @C.RoleFlags int roleFlags; @Nullable private String label; + @Nullable private String id; /** * Constructs an instance. @@ -1255,6 +1256,7 @@ public final class MediaItem implements Bundleable { this.selectionFlags = subtitleConfiguration.selectionFlags; this.roleFlags = subtitleConfiguration.roleFlags; this.label = subtitleConfiguration.label; + this.id = subtitleConfiguration.id; } /** Sets the {@link Uri} to the subtitle file. */ @@ -1293,6 +1295,12 @@ public final class MediaItem implements Bundleable { return this; } + /** Sets the optional ID for this subtitle track. */ + public Builder setId(@Nullable String id) { + this.id = id; + return this; + } + /** Creates a {@link SubtitleConfiguration} from the values of this builder. */ public SubtitleConfiguration build() { return new SubtitleConfiguration(this); @@ -1315,20 +1323,27 @@ public final class MediaItem implements Bundleable { public final @C.RoleFlags int roleFlags; /** The label. */ @Nullable public final String label; + /** + * The ID of the subtitles. This will be propagated to the {@link Format#id} of the subtitle + * track created from this configuration. + */ + @Nullable public final String id; private SubtitleConfiguration( Uri uri, String mimeType, @Nullable String language, - @C.SelectionFlags int selectionFlags, - @C.RoleFlags int roleFlags, - @Nullable String label) { + int selectionFlags, + int roleFlags, + @Nullable String label, + @Nullable String id) { this.uri = uri; this.mimeType = mimeType; this.language = language; this.selectionFlags = selectionFlags; this.roleFlags = roleFlags; this.label = label; + this.id = id; } private SubtitleConfiguration(Builder builder) { @@ -1338,6 +1353,7 @@ public final class MediaItem implements Bundleable { this.selectionFlags = builder.selectionFlags; this.roleFlags = builder.roleFlags; this.label = builder.label; + this.id = builder.id; } /** Returns a {@link Builder} initialized with the values of this instance. */ @@ -1361,7 +1377,8 @@ public final class MediaItem implements Bundleable { && Util.areEqual(language, other.language) && selectionFlags == other.selectionFlags && roleFlags == other.roleFlags - && Util.areEqual(label, other.label); + && Util.areEqual(label, other.label) + && Util.areEqual(id, other.id); } @Override @@ -1372,6 +1389,7 @@ public final class MediaItem implements Bundleable { result = 31 * result + selectionFlags; result = 31 * result + roleFlags; result = 31 * result + (label == null ? 0 : label.hashCode()); + result = 31 * result + (id == null ? 0 : id.hashCode()); return result; } } @@ -1402,7 +1420,7 @@ public final class MediaItem implements Bundleable { @C.SelectionFlags int selectionFlags, @C.RoleFlags int roleFlags, @Nullable String label) { - super(uri, mimeType, language, selectionFlags, roleFlags, label); + super(uri, mimeType, language, selectionFlags, roleFlags, label, /* id= */ null); } private Subtitle(Builder builder) { diff --git a/library/common/src/test/java/com/google/android/exoplayer2/MediaItemTest.java b/library/common/src/test/java/com/google/android/exoplayer2/MediaItemTest.java index 340f346a3e..457162d744 100644 --- a/library/common/src/test/java/com/google/android/exoplayer2/MediaItemTest.java +++ b/library/common/src/test/java/com/google/android/exoplayer2/MediaItemTest.java @@ -278,6 +278,7 @@ public class MediaItemTest { .setSelectionFlags(C.SELECTION_FLAG_FORCED) .setRoleFlags(C.ROLE_FLAG_ALTERNATE) .setLabel("label") + .setId("id") .build()); MediaItem mediaItem = @@ -619,6 +620,7 @@ public class MediaItemTest { .setSelectionFlags(C.SELECTION_FLAG_FORCED) .setRoleFlags(C.ROLE_FLAG_ALTERNATE) .setLabel("label") + .setId("id") .build())) .setTag(new Object()) .build(); @@ -675,6 +677,7 @@ public class MediaItemTest { .setSelectionFlags(C.SELECTION_FLAG_FORCED) .setRoleFlags(C.ROLE_FLAG_ALTERNATE) .setLabel("label") + .setId("id") .build())) .setTag(new Object()) .build(); diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/DefaultMediaSourceFactory.java b/library/core/src/main/java/com/google/android/exoplayer2/source/DefaultMediaSourceFactory.java index 548a252aba..a34f1023a0 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/source/DefaultMediaSourceFactory.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/DefaultMediaSourceFactory.java @@ -404,6 +404,7 @@ public final class DefaultMediaSourceFactory implements MediaSourceFactory { .setSelectionFlags(subtitleConfigurations.get(i).selectionFlags) .setRoleFlags(subtitleConfigurations.get(i).roleFlags) .setLabel(subtitleConfigurations.get(i).label) + .setId(subtitleConfigurations.get(i).id) .build(); ExtractorsFactory extractorsFactory = () -> diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/SingleSampleMediaSource.java b/library/core/src/main/java/com/google/android/exoplayer2/source/SingleSampleMediaSource.java index e5cb2bb976..af72496e5e 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/source/SingleSampleMediaSource.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/SingleSampleMediaSource.java @@ -170,6 +170,7 @@ public final class SingleSampleMediaSource extends BaseMediaSource { .setSelectionFlags(subtitleConfiguration.selectionFlags) .setRoleFlags(subtitleConfiguration.roleFlags) .setLabel(subtitleConfiguration.label) + .setId(subtitleConfiguration.id) .build(); dataSpec = new DataSpec.Builder() From 9e4d0db2171686932d1134f5a14ff6b76e96d705 Mon Sep 17 00:00:00 2001 From: kimvde Date: Mon, 6 Dec 2021 14:44:51 +0000 Subject: [PATCH 04/27] Various small improvements in Transformer PiperOrigin-RevId: 414428415 --- .../transformer/AudioSamplePipeline.java | 44 +++++++++---------- .../exoplayer2/transformer/FrameEditor.java | 4 +- .../transformer/SamplePipeline.java | 2 +- .../transformer/TransformerAudioRenderer.java | 18 +++++--- .../transformer/VideoSamplePipeline.java | 32 +++++++------- 5 files changed, 53 insertions(+), 47 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java index a3d7cc1bc0..fd9acd502d 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/AudioSamplePipeline.java @@ -98,12 +98,14 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } @Override - public void release() { - sonicAudioProcessor.reset(); - decoder.release(); - if (encoder != null) { - encoder.release(); - } + @Nullable + public DecoderInputBuffer dequeueInputBuffer() { + return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; + } + + @Override + public void queueInputBuffer() { + decoder.queueInputBuffer(decoderInputBuffer); } @Override @@ -118,28 +120,12 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; } } - @Override - @Nullable - public DecoderInputBuffer dequeueInputBuffer() { - return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; - } - - @Override - public void queueInputBuffer() { - decoder.queueInputBuffer(decoderInputBuffer); - } - @Override @Nullable public Format getOutputFormat() { return encoder != null ? encoder.getOutputFormat() : null; } - @Override - public boolean isEnded() { - return encoder != null && encoder.isEnded(); - } - @Override @Nullable public DecoderInputBuffer getOutputBuffer() { @@ -159,6 +145,20 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; checkStateNotNull(encoder).releaseOutputBuffer(); } + @Override + public boolean isEnded() { + return encoder != null && encoder.isEnded(); + } + + @Override + public void release() { + sonicAudioProcessor.reset(); + decoder.release(); + if (encoder != null) { + encoder.release(); + } + } + /** * Attempts to pass decoder output data to the encoder, and returns whether it may be possible to * pass more data immediately by calling this method again. diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java index 1e49815c40..b5e2a21649 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/FrameEditor.java @@ -100,7 +100,7 @@ import java.io.IOException; * Returns a 4x4, column-major Matrix float array, from an input {@link Matrix}. This is useful * for converting to the 4x4 column-major format commonly used in OpenGL. */ - private static final float[] getGlMatrixArray(Matrix matrix) { + private static float[] getGlMatrixArray(Matrix matrix) { float[] matrix3x3Array = new float[9]; matrix.getValues(matrix3x3Array); float[] matrix4x4Array = getMatrix4x4Array(matrix3x3Array); @@ -123,7 +123,7 @@ import java.io.IOException; * Input format: [a, b, c, d, e, f, g, h, i]
* Output format: [a, b, 0, c, d, e, 0, f, 0, 0, 1, 0, g, h, 0, i] */ - private static final float[] getMatrix4x4Array(float[] matrix3x3Array) { + private static float[] getMatrix4x4Array(float[] matrix3x3Array) { float[] matrix4x4Array = new float[16]; matrix4x4Array[10] = 1; for (int inputRow = 0; inputRow < 3; inputRow++) { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java index 6407b4f440..e2bf021dd2 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SamplePipeline.java @@ -41,7 +41,7 @@ import com.google.android.exoplayer2.decoder.DecoderInputBuffer; void queueInputBuffer(); /** - * Process the input data and returns whether more data can be processed by calling this method + * Processes the input data and returns whether more data can be processed by calling this method * again. */ boolean processData() throws ExoPlaybackException; diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java index ecd99f270c..1101ff067e 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/TransformerAudioRenderer.java @@ -60,12 +60,7 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; return false; } Format inputFormat = checkNotNull(formatHolder.format); - boolean shouldChangeMimeType = - transformation.audioMimeType != null - && !transformation.audioMimeType.equals(inputFormat.sampleMimeType); - boolean shouldFlattenForSlowMotion = - transformation.flattenForSlowMotion && isSlowMotion(inputFormat); - if (shouldChangeMimeType || shouldFlattenForSlowMotion) { + if (shouldTranscode(inputFormat)) { samplePipeline = new AudioSamplePipeline(inputFormat, transformation, getIndex()); } else { samplePipeline = new PassthroughSamplePipeline(inputFormat); @@ -73,6 +68,17 @@ import com.google.android.exoplayer2.source.SampleStream.ReadDataResult; return true; } + private boolean shouldTranscode(Format inputFormat) { + if (transformation.audioMimeType != null + && !transformation.audioMimeType.equals(inputFormat.sampleMimeType)) { + return true; + } + if (transformation.flattenForSlowMotion && isSlowMotion(inputFormat)) { + return true; + } + return false; + } + private static boolean isSlowMotion(Format format) { @Nullable Metadata metadata = format.metadata; if (metadata == null) { diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java index 3ca890e0d8..9d83af9f69 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/VideoSamplePipeline.java @@ -102,6 +102,17 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; } } + @Override + @Nullable + public DecoderInputBuffer dequeueInputBuffer() { + return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; + } + + @Override + public void queueInputBuffer() { + decoder.queueInputBuffer(decoderInputBuffer); + } + @Override public boolean processData() { if (decoder.isEnded()) { @@ -131,28 +142,12 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; return decoderHasOutputBuffer && !waitingForFrameEditorInput; } - @Override - @Nullable - public DecoderInputBuffer dequeueInputBuffer() { - return decoder.maybeDequeueInputBuffer(decoderInputBuffer) ? decoderInputBuffer : null; - } - - @Override - public void queueInputBuffer() { - decoder.queueInputBuffer(decoderInputBuffer); - } - @Override @Nullable public Format getOutputFormat() { return encoder.getOutputFormat(); } - @Override - public boolean isEnded() { - return encoder.isEnded(); - } - @Override @Nullable public DecoderInputBuffer getOutputBuffer() { @@ -171,6 +166,11 @@ import org.checkerframework.checker.nullness.qual.MonotonicNonNull; encoder.releaseOutputBuffer(); } + @Override + public boolean isEnded() { + return encoder.isEnded(); + } + @Override public void release() { if (frameEditor != null) { From 458e4b7397f7e38d620d352bca84cefe8f2423d7 Mon Sep 17 00:00:00 2001 From: kimvde Date: Mon, 6 Dec 2021 17:01:11 +0000 Subject: [PATCH 05/27] Fix re-encoding after flattening The decoder is using the SVC NAL unit prefix data on some Samsung devices. PiperOrigin-RevId: 414457181 --- .../transformer/SefSlowMotionFlattener.java | 22 +--- .../mp4/sample_sef_slow_motion.mp4.dump | 116 +++++++++--------- 2 files changed, 59 insertions(+), 79 deletions(-) diff --git a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SefSlowMotionFlattener.java b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SefSlowMotionFlattener.java index 1fcb4cc0a5..3fffad6105 100644 --- a/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SefSlowMotionFlattener.java +++ b/library/transformer/src/main/java/com/google/android/exoplayer2/transformer/SefSlowMotionFlattener.java @@ -33,7 +33,6 @@ import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; import com.google.common.collect.ImmutableList; import java.nio.ByteBuffer; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import org.checkerframework.checker.nullness.qual.RequiresNonNull; @@ -157,7 +156,7 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; // reused for the empty end-of-stream buffer. buffer.timeUs = getCurrentFrameOutputTimeUs(/* inputTimeUs= */ buffer.timeUs); if (shouldKeepFrame) { - skipToNextNalUnit(data); // Skip over prefix_nal_unit_svc. + data.position(originalPosition); return false; } return true; @@ -261,25 +260,6 @@ import org.checkerframework.checker.nullness.qual.RequiresNonNull; return Math.round(outputTimeUs * INPUT_FRAME_RATE / captureFrameRate); } - /** - * Advances the position of {@code data} to the start of the next NAL unit. - * - * @throws IllegalStateException If no NAL unit is found. - */ - private void skipToNextNalUnit(ByteBuffer data) { - int newPosition = data.position(); - while (data.remaining() >= NAL_START_CODE_LENGTH) { - data.get(scratch, 0, NAL_START_CODE_LENGTH); - if (Arrays.equals(scratch, NAL_START_CODE)) { - data.position(newPosition); - return; - } - newPosition++; - data.position(newPosition); - } - throw new IllegalStateException("Could not find NAL unit start code."); - } - /** Returns the {@link MetadataInfo} derived from the {@link Metadata} provided. */ private static MetadataInfo getMetadataInfo(@Nullable Metadata metadata) { MetadataInfo metadataInfo = new MetadataInfo(); diff --git a/testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump b/testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump index 816e26e384..a83228a55c 100644 --- a/testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump +++ b/testdata/src/test/assets/transformerdumps/mp4/sample_sef_slow_motion.mp4.dump @@ -18,110 +18,110 @@ format 1: data = length 10, hash 7A0D0F2B sample: trackIndex = 1 - dataHashCode = -549003117 - size = 5438 + dataHashCode = 1949079733 + size = 5446 isKeyFrame = true presentationTimeUs = 0 sample: trackIndex = 1 - dataHashCode = 593600631 - size = 117 + dataHashCode = -1397194508 + size = 125 isKeyFrame = false presentationTimeUs = 14000 sample: trackIndex = 1 - dataHashCode = -961321612 - size = 139 + dataHashCode = 1147159698 + size = 147 isKeyFrame = false presentationTimeUs = 47333 sample: trackIndex = 1 - dataHashCode = -386347143 - size = 141 + dataHashCode = 524634358 + size = 149 isKeyFrame = false presentationTimeUs = 80667 sample: trackIndex = 1 - dataHashCode = -1289764147 - size = 141 + dataHashCode = 2031178347 + size = 149 isKeyFrame = false presentationTimeUs = 114000 sample: trackIndex = 1 - dataHashCode = 1337088875 - size = 161 + dataHashCode = -625462168 + size = 169 isKeyFrame = false presentationTimeUs = 147333 sample: trackIndex = 1 - dataHashCode = -322406979 - size = 118 + dataHashCode = -973299745 + size = 126 isKeyFrame = false presentationTimeUs = 180667 sample: trackIndex = 1 - dataHashCode = -1688033783 - size = 112 + dataHashCode = -788426325 + size = 120 isKeyFrame = false presentationTimeUs = 228042 sample: trackIndex = 1 - dataHashCode = -700344608 - size = 118 + dataHashCode = 2009515523 + size = 126 isKeyFrame = false presentationTimeUs = 244708 sample: trackIndex = 1 - dataHashCode = -1441653629 - size = 1172 + dataHashCode = -874600600 + size = 1180 isKeyFrame = false presentationTimeUs = 334083 sample: trackIndex = 1 - dataHashCode = 1201357091 - size = 208 + dataHashCode = 984869991 + size = 216 isKeyFrame = false presentationTimeUs = 267416 sample: trackIndex = 1 - dataHashCode = -668484307 - size = 111 + dataHashCode = 2106811176 + size = 119 isKeyFrame = false presentationTimeUs = 234083 sample: trackIndex = 1 - dataHashCode = 653508165 - size = 137 + dataHashCode = -1981166365 + size = 145 isKeyFrame = false presentationTimeUs = 300750 sample: trackIndex = 1 - dataHashCode = -816848987 - size = 1266 + dataHashCode = 1234592714 + size = 1274 isKeyFrame = false presentationTimeUs = 467416 sample: trackIndex = 1 - dataHashCode = 1842436292 - size = 182 + dataHashCode = -13135608 + size = 190 isKeyFrame = false presentationTimeUs = 400750 sample: trackIndex = 1 - dataHashCode = -559603233 - size = 99 + dataHashCode = 1840621658 + size = 107 isKeyFrame = false presentationTimeUs = 367416 sample: trackIndex = 1 - dataHashCode = -666437886 - size = 117 + dataHashCode = 1637734271 + size = 125 isKeyFrame = false presentationTimeUs = 434083 sample: trackIndex = 1 - dataHashCode = 182521759 - size = 1101 + dataHashCode = 2112365658 + size = 1109 isKeyFrame = false presentationTimeUs = 600750 sample: @@ -276,68 +276,68 @@ sample: presentationTimeUs = 199105 sample: trackIndex = 1 - dataHashCode = 2139021989 - size = 242 + dataHashCode = -968901399 + size = 250 isKeyFrame = false presentationTimeUs = 534083 sample: trackIndex = 1 - dataHashCode = 2013165108 - size = 116 + dataHashCode = -1184738023 + size = 124 isKeyFrame = false presentationTimeUs = 500750 sample: trackIndex = 1 - dataHashCode = 405675195 - size = 126 + dataHashCode = 1756300509 + size = 134 isKeyFrame = false presentationTimeUs = 567416 sample: trackIndex = 1 - dataHashCode = -1893277090 - size = 1193 + dataHashCode = 823429273 + size = 1201 isKeyFrame = false presentationTimeUs = 734083 sample: trackIndex = 1 - dataHashCode = -1554795381 - size = 205 + dataHashCode = 651718599 + size = 213 isKeyFrame = false presentationTimeUs = 667416 sample: trackIndex = 1 - dataHashCode = 1197099206 - size = 117 + dataHashCode = 846349953 + size = 125 isKeyFrame = false presentationTimeUs = 634083 sample: trackIndex = 1 - dataHashCode = -674808173 - size = 106 + dataHashCode = 1331153462 + size = 114 isKeyFrame = false presentationTimeUs = 700750 sample: trackIndex = 1 - dataHashCode = -775517313 - size = 1002 + dataHashCode = -73358172 + size = 1010 isKeyFrame = false presentationTimeUs = 867416 sample: trackIndex = 1 - dataHashCode = -2045106113 - size = 201 + dataHashCode = -1395269253 + size = 209 isKeyFrame = false presentationTimeUs = 800750 sample: trackIndex = 1 - dataHashCode = 305167697 - size = 131 + dataHashCode = -1001367604 + size = 139 isKeyFrame = false presentationTimeUs = 767416 sample: trackIndex = 1 - dataHashCode = 554021920 - size = 130 + dataHashCode = -122569918 + size = 138 isKeyFrame = false presentationTimeUs = 834083 released = true From fcdb96f0f1481394693b61ba2959c540bc7c2483 Mon Sep 17 00:00:00 2001 From: huangdarwin Date: Mon, 6 Dec 2021 18:51:10 +0000 Subject: [PATCH 06/27] GL: Document ambiguous parameter names in comments. Also, made a few other refactoring changes for clarity. No functional changes intended. PiperOrigin-RevId: 414487729 --- .../android/exoplayer2/util/GlUtil.java | 97 +++++++++++++------ 1 file changed, 66 insertions(+), 31 deletions(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java b/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java index 6f9676becb..aace7d4f14 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/GlUtil.java @@ -104,7 +104,7 @@ public final class GlUtil { // Link and use the program, and enumerate attributes/uniforms. GLES20.glLinkProgram(programId); int[] linkStatus = new int[] {GLES20.GL_FALSE}; - GLES20.glGetProgramiv(programId, GLES20.GL_LINK_STATUS, linkStatus, 0); + GLES20.glGetProgramiv(programId, GLES20.GL_LINK_STATUS, linkStatus, /* offset= */ 0); if (linkStatus[0] != GLES20.GL_TRUE) { throwGlException( "Unable to link shader program: \n" + GLES20.glGetProgramInfoLog(programId)); @@ -112,7 +112,8 @@ public final class GlUtil { GLES20.glUseProgram(programId); attributeByName = new HashMap<>(); int[] attributeCount = new int[1]; - GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTES, attributeCount, 0); + GLES20.glGetProgramiv( + programId, GLES20.GL_ACTIVE_ATTRIBUTES, attributeCount, /* offset= */ 0); attributes = new Attribute[attributeCount[0]]; for (int i = 0; i < attributeCount[0]; i++) { Attribute attribute = Attribute.create(programId, i); @@ -121,7 +122,7 @@ public final class GlUtil { } uniformByName = new HashMap<>(); int[] uniformCount = new int[1]; - GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORMS, uniformCount, 0); + GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORMS, uniformCount, /* offset= */ 0); uniforms = new Uniform[uniformCount[0]]; for (int i = 0; i < uniformCount[0]; i++) { Uniform uniform = Uniform.create(programId, i); @@ -289,11 +290,11 @@ public final class GlUtil { int lastError = GLES20.GL_NO_ERROR; int error; while ((error = GLES20.glGetError()) != GLES20.GL_NO_ERROR) { - Log.e(TAG, "glError " + gluErrorString(error)); + Log.e(TAG, "glError: " + gluErrorString(error)); lastError = error; } if (lastError != GLES20.GL_NO_ERROR) { - throwGlException("glError " + gluErrorString(lastError)); + throwGlException("glError: " + gluErrorString(lastError)); } } @@ -314,7 +315,7 @@ public final class GlUtil { */ public static void deleteTexture(int textureId) { int[] textures = new int[] {textureId}; - GLES20.glDeleteTextures(1, textures, 0); + GLES20.glDeleteTextures(/* n= */ 1, textures, /* offset= */ 0); checkGlError(); } @@ -371,7 +372,7 @@ public final class GlUtil { */ public static int createExternalTexture() { int[] texId = new int[1]; - GLES20.glGenTextures(1, IntBuffer.wrap(texId)); + GLES20.glGenTextures(/* n= */ 1, IntBuffer.wrap(texId)); GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texId[0]); GLES20.glTexParameteri( GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); @@ -391,7 +392,7 @@ public final class GlUtil { GLES20.glCompileShader(shader); int[] result = new int[] {GLES20.GL_FALSE}; - GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, 0); + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, result, /* offset= */ 0); if (result[0] != GLES20.GL_TRUE) { throwGlException(GLES20.glGetShaderInfoLog(shader) + ", source: " + glsl); } @@ -440,7 +441,8 @@ public final class GlUtil { /* Returns the attribute at the given index in the program. */ public static Attribute create(int programId, int index) { int[] length = new int[1]; - GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_ATTRIBUTE_MAX_LENGTH, length, 0); + GLES20.glGetProgramiv( + programId, GLES20.GL_ACTIVE_ATTRIBUTE_MAX_LENGTH, length, /* offset= */ 0); int[] type = new int[1]; int[] size = new int[1]; @@ -448,8 +450,18 @@ public final class GlUtil { int[] ignore = new int[1]; GLES20.glGetActiveAttrib( - programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0); - String name = new String(nameBytes, 0, strlen(nameBytes)); + programId, + index, + length[0], + ignore, + /* lengthOffset= */ 0, + size, + /* sizeOffset= */ 0, + type, + /* typeOffset= */ 0, + nameBytes, + /* nameOffset= */ 0); + String name = new String(nameBytes, /* offset= */ 0, strlen(nameBytes)); int location = getAttributeLocation(programId, name); return new Attribute(name, index, location); @@ -489,14 +501,9 @@ public final class GlUtil { */ public void bind() { Buffer buffer = checkNotNull(this.buffer, "call setBuffer before bind"); - GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, 0); + GLES20.glBindBuffer(GLES20.GL_ARRAY_BUFFER, /* buffer= */ 0); GLES20.glVertexAttribPointer( - location, - size, // count - GLES20.GL_FLOAT, // type - false, // normalize - 0, // stride - buffer); + location, size, GLES20.GL_FLOAT, /* normalized= */ false, /* stride= */ 0, buffer); GLES20.glEnableVertexAttribArray(index); checkGlError(); } @@ -510,7 +517,8 @@ public final class GlUtil { /** Returns the uniform at the given index in the program. */ public static Uniform create(int programId, int index) { int[] length = new int[1]; - GLES20.glGetProgramiv(programId, GLES20.GL_ACTIVE_UNIFORM_MAX_LENGTH, length, 0); + GLES20.glGetProgramiv( + programId, GLES20.GL_ACTIVE_UNIFORM_MAX_LENGTH, length, /* offset= */ 0); int[] type = new int[1]; int[] size = new int[1]; @@ -518,8 +526,18 @@ public final class GlUtil { int[] ignore = new int[1]; GLES20.glGetActiveUniform( - programId, index, length[0], ignore, 0, size, 0, type, 0, nameBytes, 0); - String name = new String(nameBytes, 0, strlen(nameBytes)); + programId, + index, + length[0], + ignore, + /* lengthOffset= */ 0, + size, + /*sizeOffset= */ 0, + type, + /* typeOffset= */ 0, + nameBytes, + /* nameOffset= */ 0); + String name = new String(nameBytes, /* offset= */ 0, strlen(nameBytes)); int location = getUniformLocation(programId, name); return new Uniform(name, location, type[0]); @@ -560,7 +578,7 @@ public final class GlUtil { /** Configures {@link #bind()} to use the specified float[] {@code value} for this uniform. */ public void setFloats(float[] value) { - System.arraycopy(value, 0, this.value, 0, value.length); + System.arraycopy(value, /* srcPos= */ 0, this.value, /* destPos= */ 0, value.length); } /** @@ -571,19 +589,20 @@ public final class GlUtil { */ public void bind() { if (type == GLES20.GL_FLOAT) { - GLES20.glUniform1fv(location, 1, value, 0); + GLES20.glUniform1fv(location, /* count= */ 1, value, /* offset= */ 0); checkGlError(); return; } if (type == GLES20.GL_FLOAT_MAT4) { - GLES20.glUniformMatrix4fv(location, 1, false, value, 0); + GLES20.glUniformMatrix4fv( + location, /* count= */ 1, /* transpose= */ false, value, /* offset= */ 0); checkGlError(); return; } if (texId == 0) { - throw new IllegalStateException("Call setSamplerTexId before bind."); + throw new IllegalStateException("No call to setSamplerTexId() before bind."); } GLES20.glActiveTexture(GLES20.GL_TEXTURE0 + unit); if (type == GLES11Ext.GL_SAMPLER_EXTERNAL_OES) { @@ -614,7 +633,8 @@ public final class GlUtil { checkEglException(!eglDisplay.equals(EGL14.EGL_NO_DISPLAY), "No EGL display."); int[] major = new int[1]; int[] minor = new int[1]; - if (!EGL14.eglInitialize(eglDisplay, major, 0, minor, 0)) { + if (!EGL14.eglInitialize( + eglDisplay, major, /* majorOffset= */ 0, minor, /* minorOffset= */ 0)) { throwGlException("Error in eglInitialize."); } checkGlError(); @@ -627,7 +647,11 @@ public final class GlUtil { int[] contextAttributes = {EGL14.EGL_CONTEXT_CLIENT_VERSION, 2, EGL14.EGL_NONE}; EGLContext eglContext = EGL14.eglCreateContext( - eglDisplay, getEglConfig(eglDisplay), EGL14.EGL_NO_CONTEXT, contextAttributes, 0); + eglDisplay, + getEglConfig(eglDisplay), + EGL14.EGL_NO_CONTEXT, + contextAttributes, + /* offset= */ 0); if (eglContext == null) { EGL14.eglTerminate(eglDisplay); throw new UnsupportedEglVersionException(); @@ -639,20 +663,24 @@ public final class GlUtil { @DoNotInline public static EGLSurface getEglSurface(EGLDisplay eglDisplay, Object surface) { return EGL14.eglCreateWindowSurface( - eglDisplay, getEglConfig(eglDisplay), surface, new int[] {EGL14.EGL_NONE}, 0); + eglDisplay, + getEglConfig(eglDisplay), + surface, + new int[] {EGL14.EGL_NONE}, + /* offset= */ 0); } @DoNotInline public static void focusSurface( EGLDisplay eglDisplay, EGLContext eglContext, EGLSurface surface, int width, int height) { int[] fbos = new int[1]; - GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, fbos, 0); + GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, fbos, /* offset= */ 0); int noFbo = 0; if (fbos[0] != noFbo) { GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, noFbo); } EGL14.eglMakeCurrent(eglDisplay, surface, surface, eglContext); - GLES20.glViewport(0, 0, width, height); + GLES20.glViewport(/* x= */ 0, /* y= */ 0, width, height); } @DoNotInline @@ -700,7 +728,14 @@ public final class GlUtil { int[] configsCount = new int[1]; EGLConfig[] eglConfigs = new EGLConfig[1]; if (!EGL14.eglChooseConfig( - eglDisplay, defaultConfiguration, 0, eglConfigs, 0, 1, configsCount, 0)) { + eglDisplay, + defaultConfiguration, + /* attrib_listOffset= */ 0, + eglConfigs, + /* configsOffset= */ 0, + /* config_size= */ 1, + configsCount, + /* num_configOffset= */ 0)) { throwGlException("eglChooseConfig failed."); } return eglConfigs[0]; From 07352a4585cd939805bc2dafe53166f02c7e9025 Mon Sep 17 00:00:00 2001 From: ibaker Date: Tue, 7 Dec 2021 11:22:29 +0000 Subject: [PATCH 07/27] Retry creating a MediaCodec instance in MediaCodecRenderer It's been observed that some devices fail when releasing a secure codec attached to a surface and immediately trying to create a new codec (secure or insecure) attached to the same surface. This change catches all exceptions thrown during codec creation, sleeps for a short time, and then retries the codec creation. This is observed to fix the problem (we believe this is because it allows enough time for some background part of the previous codec release operation to complete). This change should have no effect on the control flow when codec creation succeeds first time. It will introduce a slight delay when creating the preferred codec fails (while we sleep and retry), which will either delay propagating a permanent error or attempting to initialize a fallback decoder. We can't avoid the extra delay to instantiating the fallback decoder because we can't know whether we expect the second attempt to create the preferred decoder to succeed or fail. The benefit to always retrying the preferred decoder creation (fixing playback failures) outweighs the unfortunate additional delay to instantiating fallback decoders. Issue: google/ExoPlayer#8696 #minor-release PiperOrigin-RevId: 414671743 --- RELEASENOTES.md | 4 ++++ .../mediacodec/MediaCodecRenderer.java | 24 ++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 6bb1b08cd7..0d27c53384 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -14,6 +14,10 @@ `TrackGroup` constructor. This fixes a crash when resuming playback after backgrounding the app with an active track override ((#9718)[https://github.com/google/ExoPlayer/issues/9718]). + * Sleep and retry when creating a `MediaCodec` instance fails. This works + around an issue that occurs on some devices when switching a surface + from a secure codec to another codec + (#8696)[https://github.com/google/ExoPlayer/issues/8696]. * Android 12 compatibility: * Upgrade the Cast extension to depend on `com.google.android.gms:play-services-cast-framework:20.1.0`. Earlier diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index cb2194bbc5..72497d2771 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -976,13 +976,27 @@ public abstract class MediaCodecRenderer extends BaseRenderer { DecoderInitializationException.NO_SUITABLE_DECODER_ERROR); } + MediaCodecInfo preferredCodecInfo = availableCodecInfos.peekFirst(); while (codec == null) { MediaCodecInfo codecInfo = availableCodecInfos.peekFirst(); if (!shouldInitCodec(codecInfo)) { return; } try { - initCodec(codecInfo, crypto); + try { + initCodec(codecInfo, crypto); + } catch (Exception e) { + if (codecInfo == preferredCodecInfo) { + // If creating the preferred decoder failed then sleep briefly before retrying. + // Workaround for [internal b/191966399]. + // See also https://github.com/google/ExoPlayer/issues/8696. + Log.w(TAG, "Preferred decoder instantiation failed. Sleeping for 50ms then retrying."); + Thread.sleep(/* millis= */ 50); + initCodec(codecInfo, crypto); + } else { + throw e; + } + } } catch (Exception e) { Log.w(TAG, "Failed to initialize decoder: " + codecInfo, e); // This codec failed to initialize, so fall back to the next codec in the list (if any). We @@ -1060,13 +1074,17 @@ public abstract class MediaCodecRenderer extends BaseRenderer { codecOperatingRate = CODEC_OPERATING_RATE_UNSET; } codecInitializingTimestamp = SystemClock.elapsedRealtime(); - TraceUtil.beginSection("createCodec:" + codecName); MediaCodecAdapter.Configuration configuration = getMediaCodecConfiguration(codecInfo, inputFormat, crypto, codecOperatingRate); if (Util.SDK_INT >= 31) { Api31.setLogSessionIdToMediaCodecFormat(configuration, getPlayerId()); } - codec = codecAdapterFactory.createAdapter(configuration); + try { + TraceUtil.beginSection("createCodec:" + codecName); + codec = codecAdapterFactory.createAdapter(configuration); + } finally { + TraceUtil.endSection(); + } codecInitializedTimestamp = SystemClock.elapsedRealtime(); this.codecInfo = codecInfo; From 5c2f618613a1ff32cefe4b88b8407b1822f54dbe Mon Sep 17 00:00:00 2001 From: ibaker Date: Tue, 7 Dec 2021 11:23:22 +0000 Subject: [PATCH 08/27] Migrate usages of Timeline#getPeriodPosition to getPeriodPositionUs #minor-release PiperOrigin-RevId: 414671861 --- .../exoplayer2/ext/ima/ImaAdsLoader.java | 2 +- .../google/android/exoplayer2/Timeline.java | 4 +- .../android/exoplayer2/ExoPlayerImpl.java | 42 ++++----- .../exoplayer2/ExoPlayerImplInternal.java | 86 +++++++++---------- .../android/exoplayer2/MediaPeriodQueue.java | 18 ++-- .../exoplayer2/source/MaskingMediaSource.java | 8 +- .../source/SinglePeriodTimelineTest.java | 59 +++++++++---- 7 files changed, 124 insertions(+), 95 deletions(-) diff --git a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java index 85a39c77e8..951053e048 100644 --- a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java +++ b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaAdsLoader.java @@ -700,7 +700,7 @@ public final class ImaAdsLoader implements Player.Listener, AdsLoader { return; } long periodPositionUs = - timeline.getPeriodPosition( + timeline.getPeriodPositionUs( window, period, period.windowIndex, /* windowPositionUs= */ C.TIME_UNSET) .second; nextAdTagLoader.maybePreloadAds(Util.usToMs(periodPositionUs), Util.usToMs(period.durationUs)); diff --git a/library/common/src/main/java/com/google/android/exoplayer2/Timeline.java b/library/common/src/main/java/com/google/android/exoplayer2/Timeline.java index cf77941844..ad5eef25c4 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/Timeline.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/Timeline.java @@ -1190,13 +1190,13 @@ public abstract class Timeline implements Bundleable { } /** - * Calls {@link #getPeriodPosition(Window, Period, int, long, long)} with a zero default position + * Calls {@link #getPeriodPositionUs(Window, Period, int, long)} with a zero default position * projection. */ public final Pair getPeriodPositionUs( Window window, Period period, int windowIndex, long windowPositionUs) { return Assertions.checkNotNull( - getPeriodPosition( + getPeriodPositionUs( window, period, windowIndex, windowPositionUs, /* defaultPositionProjectionUs= */ 0)); } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java index 41e31180f0..72e6ce60f8 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImpl.java @@ -499,7 +499,7 @@ import java.util.concurrent.CopyOnWriteArraySet; maskTimelineAndPosition( playbackInfo, newTimeline, - getPeriodPositionAfterTimelineChanged(oldTimeline, newTimeline)); + getPeriodPositionUsAfterTimelineChanged(oldTimeline, newTimeline)); internalPlayer.addMediaSources(index, holders, shuffleOrder); updatePlaybackInfo( newPlaybackInfo, @@ -543,7 +543,7 @@ import java.util.concurrent.CopyOnWriteArraySet; maskTimelineAndPosition( playbackInfo, newTimeline, - getPeriodPositionAfterTimelineChanged(oldTimeline, newTimeline)); + getPeriodPositionUsAfterTimelineChanged(oldTimeline, newTimeline)); internalPlayer.moveMediaSources(fromIndex, toIndex, newFromIndex, shuffleOrder); updatePlaybackInfo( newPlaybackInfo, @@ -562,7 +562,7 @@ import java.util.concurrent.CopyOnWriteArraySet; maskTimelineAndPosition( playbackInfo, timeline, - getPeriodPositionOrMaskWindowPosition( + maskWindowPositionMsOrGetPeriodPositionUs( timeline, getCurrentMediaItemIndex(), getCurrentPosition())); pendingOperationAcks++; this.shuffleOrder = shuffleOrder; @@ -680,7 +680,7 @@ import java.util.concurrent.CopyOnWriteArraySet; maskTimelineAndPosition( newPlaybackInfo, timeline, - getPeriodPositionOrMaskWindowPosition(timeline, mediaItemIndex, positionMs)); + maskWindowPositionMsOrGetPeriodPositionUs(timeline, mediaItemIndex, positionMs)); internalPlayer.seekTo(timeline, mediaItemIndex, Util.msToUs(positionMs)); updatePlaybackInfo( newPlaybackInfo, @@ -1441,7 +1441,7 @@ import java.util.concurrent.CopyOnWriteArraySet; maskTimelineAndPosition( playbackInfo, timeline, - getPeriodPositionOrMaskWindowPosition(timeline, startWindowIndex, startPositionMs)); + maskWindowPositionMsOrGetPeriodPositionUs(timeline, startWindowIndex, startPositionMs)); // Mask the playback state. int maskingPlaybackState = newPlaybackInfo.playbackState; if (startWindowIndex != C.INDEX_UNSET && newPlaybackInfo.playbackState != STATE_IDLE) { @@ -1499,7 +1499,7 @@ import java.util.concurrent.CopyOnWriteArraySet; maskTimelineAndPosition( playbackInfo, newTimeline, - getPeriodPositionAfterTimelineChanged(oldTimeline, newTimeline)); + getPeriodPositionUsAfterTimelineChanged(oldTimeline, newTimeline)); // Player transitions to STATE_ENDED if the current index is part of the removed tail. final boolean transitionsToEnded = newPlaybackInfo.playbackState != STATE_IDLE @@ -1526,8 +1526,8 @@ import java.util.concurrent.CopyOnWriteArraySet; } private PlaybackInfo maskTimelineAndPosition( - PlaybackInfo playbackInfo, Timeline timeline, @Nullable Pair periodPosition) { - Assertions.checkArgument(timeline.isEmpty() || periodPosition != null); + PlaybackInfo playbackInfo, Timeline timeline, @Nullable Pair periodPositionUs) { + Assertions.checkArgument(timeline.isEmpty() || periodPositionUs != null); Timeline oldTimeline = playbackInfo.timeline; // Mask the timeline. playbackInfo = playbackInfo.copyWithTimeline(timeline); @@ -1552,10 +1552,10 @@ import java.util.concurrent.CopyOnWriteArraySet; } Object oldPeriodUid = playbackInfo.periodId.periodUid; - boolean playingPeriodChanged = !oldPeriodUid.equals(castNonNull(periodPosition).first); + boolean playingPeriodChanged = !oldPeriodUid.equals(castNonNull(periodPositionUs).first); MediaPeriodId newPeriodId = - playingPeriodChanged ? new MediaPeriodId(periodPosition.first) : playbackInfo.periodId; - long newContentPositionUs = periodPosition.second; + playingPeriodChanged ? new MediaPeriodId(periodPositionUs.first) : playbackInfo.periodId; + long newContentPositionUs = periodPositionUs.second; long oldContentPositionUs = Util.msToUs(getContentPosition()); if (!oldTimeline.isEmpty()) { oldContentPositionUs -= @@ -1631,25 +1631,25 @@ import java.util.concurrent.CopyOnWriteArraySet; } @Nullable - private Pair getPeriodPositionAfterTimelineChanged( + private Pair getPeriodPositionUsAfterTimelineChanged( Timeline oldTimeline, Timeline newTimeline) { long currentPositionMs = getContentPosition(); if (oldTimeline.isEmpty() || newTimeline.isEmpty()) { boolean isCleared = !oldTimeline.isEmpty() && newTimeline.isEmpty(); - return getPeriodPositionOrMaskWindowPosition( + return maskWindowPositionMsOrGetPeriodPositionUs( newTimeline, isCleared ? C.INDEX_UNSET : getCurrentWindowIndexInternal(), isCleared ? C.TIME_UNSET : currentPositionMs); } int currentMediaItemIndex = getCurrentMediaItemIndex(); @Nullable - Pair oldPeriodPosition = - oldTimeline.getPeriodPosition( + Pair oldPeriodPositionUs = + oldTimeline.getPeriodPositionUs( window, period, currentMediaItemIndex, Util.msToUs(currentPositionMs)); - Object periodUid = castNonNull(oldPeriodPosition).first; + Object periodUid = castNonNull(oldPeriodPositionUs).first; if (newTimeline.getIndexOfPeriod(periodUid) != C.INDEX_UNSET) { // The old period position is still available in the new timeline. - return oldPeriodPosition; + return oldPeriodPositionUs; } // Period uid not found in new timeline. Try to get subsequent period. @Nullable @@ -1659,19 +1659,19 @@ import java.util.concurrent.CopyOnWriteArraySet; if (nextPeriodUid != null) { // Reset position to the default position of the window of the subsequent period. newTimeline.getPeriodByUid(nextPeriodUid, period); - return getPeriodPositionOrMaskWindowPosition( + return maskWindowPositionMsOrGetPeriodPositionUs( newTimeline, period.windowIndex, newTimeline.getWindow(period.windowIndex, window).getDefaultPositionMs()); } else { // No subsequent period found and the new timeline is not empty. Use the default position. - return getPeriodPositionOrMaskWindowPosition( + return maskWindowPositionMsOrGetPeriodPositionUs( newTimeline, /* windowIndex= */ C.INDEX_UNSET, /* windowPositionMs= */ C.TIME_UNSET); } } @Nullable - private Pair getPeriodPositionOrMaskWindowPosition( + private Pair maskWindowPositionMsOrGetPeriodPositionUs( Timeline timeline, int windowIndex, long windowPositionMs) { if (timeline.isEmpty()) { // If empty we store the initial seek in the masking variables. @@ -1686,7 +1686,7 @@ import java.util.concurrent.CopyOnWriteArraySet; windowIndex = timeline.getFirstWindowIndex(shuffleModeEnabled); windowPositionMs = timeline.getWindow(windowIndex, window).getDefaultPositionMs(); } - return timeline.getPeriodPosition(window, period, windowIndex, Util.msToUs(windowPositionMs)); + return timeline.getPeriodPositionUs(window, period, windowIndex, Util.msToUs(windowPositionMs)); } private long periodPositionUsToWindowPositionUs( diff --git a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java index ae8063516a..76471b5c7e 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/ExoPlayerImplInternal.java @@ -1123,7 +1123,7 @@ import java.util.concurrent.atomic.AtomicBoolean; boolean seekPositionAdjusted; @Nullable Pair resolvedSeekPosition = - resolveSeekPosition( + resolveSeekPositionUs( playbackInfo.timeline, seekPosition, /* trySubsequentPeriods= */ true, @@ -1134,10 +1134,10 @@ import java.util.concurrent.atomic.AtomicBoolean; if (resolvedSeekPosition == null) { // The seek position was valid for the timeline that it was performed into, but the // timeline has changed or is not ready and a suitable seek position could not be resolved. - Pair firstPeriodAndPosition = - getPlaceholderFirstMediaPeriodPosition(playbackInfo.timeline); - periodId = firstPeriodAndPosition.first; - periodPositionUs = firstPeriodAndPosition.second; + Pair firstPeriodAndPositionUs = + getPlaceholderFirstMediaPeriodPositionUs(playbackInfo.timeline); + periodId = firstPeriodAndPositionUs.first; + periodPositionUs = firstPeriodAndPositionUs.second; requestedContentPositionUs = C.TIME_UNSET; seekPositionAdjusted = !playbackInfo.timeline.isEmpty(); } else { @@ -1412,10 +1412,10 @@ import java.util.concurrent.atomic.AtomicBoolean; boolean resetTrackInfo = false; if (resetPosition) { pendingInitialSeekPosition = null; - Pair firstPeriodAndPosition = - getPlaceholderFirstMediaPeriodPosition(playbackInfo.timeline); - mediaPeriodId = firstPeriodAndPosition.first; - startPositionUs = firstPeriodAndPosition.second; + Pair firstPeriodAndPositionUs = + getPlaceholderFirstMediaPeriodPositionUs(playbackInfo.timeline); + mediaPeriodId = firstPeriodAndPositionUs.first; + startPositionUs = firstPeriodAndPositionUs.second; requestedContentPositionUs = C.TIME_UNSET; if (!mediaPeriodId.equals(playbackInfo.periodId)) { resetTrackInfo = true; @@ -1451,19 +1451,19 @@ import java.util.concurrent.atomic.AtomicBoolean; } } - private Pair getPlaceholderFirstMediaPeriodPosition(Timeline timeline) { + private Pair getPlaceholderFirstMediaPeriodPositionUs(Timeline timeline) { if (timeline.isEmpty()) { return Pair.create(PlaybackInfo.getDummyPeriodForEmptyTimeline(), 0L); } int firstWindowIndex = timeline.getFirstWindowIndex(shuffleModeEnabled); - Pair firstPeriodAndPosition = - timeline.getPeriodPosition( + Pair firstPeriodAndPositionUs = + timeline.getPeriodPositionUs( window, period, firstWindowIndex, /* windowPositionUs= */ C.TIME_UNSET); // Add ad metadata if any and propagate the window sequence number to new period id. MediaPeriodId firstPeriodId = queue.resolveMediaPeriodIdForAds( - timeline, firstPeriodAndPosition.first, /* positionUs= */ 0); - long positionUs = firstPeriodAndPosition.second; + timeline, firstPeriodAndPositionUs.first, /* positionUs= */ 0); + long positionUs = firstPeriodAndPositionUs.second; if (firstPeriodId.isAd()) { timeline.getPeriodByUid(firstPeriodId.periodUid, period); positionUs = @@ -2543,7 +2543,7 @@ import java.util.concurrent.atomic.AtomicBoolean; // Resolve initial seek position. @Nullable Pair periodPosition = - resolveSeekPosition( + resolveSeekPositionUs( timeline, pendingInitialSeekPosition, /* trySubsequentPeriods= */ true, @@ -2608,10 +2608,10 @@ import java.util.concurrent.atomic.AtomicBoolean; // at position 0 and don't need to be resolved. long windowPositionUs = oldContentPositionUs + period.getPositionInWindowUs(); int windowIndex = timeline.getPeriodByUid(newPeriodUid, period).windowIndex; - Pair periodPosition = - timeline.getPeriodPosition(window, period, windowIndex, windowPositionUs); - newPeriodUid = periodPosition.first; - newContentPositionUs = periodPosition.second; + Pair periodPositionUs = + timeline.getPeriodPositionUs(window, period, windowIndex, windowPositionUs); + newPeriodUid = periodPositionUs.first; + newContentPositionUs = periodPositionUs.second; } // Use an explicitly requested content position as new target live offset. setTargetLiveOffset = true; @@ -2620,14 +2620,14 @@ import java.util.concurrent.atomic.AtomicBoolean; // Set period uid for default positions and resolve position for ad resolution. long contentPositionForAdResolutionUs = newContentPositionUs; if (startAtDefaultPositionWindowIndex != C.INDEX_UNSET) { - Pair defaultPosition = - timeline.getPeriodPosition( + Pair defaultPositionUs = + timeline.getPeriodPositionUs( window, period, startAtDefaultPositionWindowIndex, /* windowPositionUs= */ C.TIME_UNSET); - newPeriodUid = defaultPosition.first; - contentPositionForAdResolutionUs = defaultPosition.second; + newPeriodUid = defaultPositionUs.first; + contentPositionForAdResolutionUs = defaultPositionUs.second; newContentPositionUs = C.TIME_UNSET; } @@ -2741,7 +2741,7 @@ import java.util.concurrent.atomic.AtomicBoolean; : Util.msToUs(pendingMessageInfo.message.getPositionMs()); @Nullable Pair periodPosition = - resolveSeekPosition( + resolveSeekPositionUs( newTimeline, new SeekPosition( pendingMessageInfo.message.getTimeline(), @@ -2786,12 +2786,12 @@ import java.util.concurrent.atomic.AtomicBoolean; pendingMessageInfo.resolvedPeriodTimeUs + period.getPositionInWindowUs(); int windowIndex = newTimeline.getPeriodByUid(pendingMessageInfo.resolvedPeriodUid, period).windowIndex; - Pair periodPosition = - newTimeline.getPeriodPosition(window, period, windowIndex, windowPositionUs); + Pair periodPositionUs = + newTimeline.getPeriodPositionUs(window, period, windowIndex, windowPositionUs); pendingMessageInfo.setResolvedPosition( - /* periodIndex= */ newTimeline.getIndexOfPeriod(periodPosition.first), - /* periodTimeUs= */ periodPosition.second, - /* periodUid= */ periodPosition.first); + /* periodIndex= */ newTimeline.getIndexOfPeriod(periodPositionUs.first), + /* periodTimeUs= */ periodPositionUs.second, + /* periodUid= */ periodPositionUs.first); } return true; } @@ -2820,7 +2820,7 @@ import java.util.concurrent.atomic.AtomicBoolean; * bounds of the timeline. */ @Nullable - private static Pair resolveSeekPosition( + private static Pair resolveSeekPositionUs( Timeline timeline, SeekPosition seekPosition, boolean trySubsequentPeriods, @@ -2839,10 +2839,10 @@ import java.util.concurrent.atomic.AtomicBoolean; seekTimeline = timeline; } // Map the SeekPosition to a position in the corresponding timeline. - Pair periodPosition; + Pair periodPositionUs; try { - periodPosition = - seekTimeline.getPeriodPosition( + periodPositionUs = + seekTimeline.getPeriodPositionUs( window, period, seekPosition.windowIndex, seekPosition.windowPositionUs); } catch (IndexOutOfBoundsException e) { // The window index of the seek position was outside the bounds of the timeline. @@ -2850,24 +2850,24 @@ import java.util.concurrent.atomic.AtomicBoolean; } if (timeline.equals(seekTimeline)) { // Our internal timeline is the seek timeline, so the mapped position is correct. - return periodPosition; + return periodPositionUs; } // Attempt to find the mapped period in the internal timeline. - int periodIndex = timeline.getIndexOfPeriod(periodPosition.first); + int periodIndex = timeline.getIndexOfPeriod(periodPositionUs.first); if (periodIndex != C.INDEX_UNSET) { // We successfully located the period in the internal timeline. - if (seekTimeline.getPeriodByUid(periodPosition.first, period).isPlaceholder + if (seekTimeline.getPeriodByUid(periodPositionUs.first, period).isPlaceholder && seekTimeline.getWindow(period.windowIndex, window).firstPeriodIndex - == seekTimeline.getIndexOfPeriod(periodPosition.first)) { + == seekTimeline.getIndexOfPeriod(periodPositionUs.first)) { // The seek timeline was using a placeholder, so we need to re-resolve using the updated // timeline in case the resolved position changed. Only resolve the first period in a window // because subsequent periods must start at position 0 and don't need to be resolved. - int newWindowIndex = timeline.getPeriodByUid(periodPosition.first, period).windowIndex; - periodPosition = - timeline.getPeriodPosition( + int newWindowIndex = timeline.getPeriodByUid(periodPositionUs.first, period).windowIndex; + periodPositionUs = + timeline.getPeriodPositionUs( window, period, newWindowIndex, seekPosition.windowPositionUs); } - return periodPosition; + return periodPositionUs; } if (trySubsequentPeriods) { // Try and find a subsequent period from the seek timeline in the internal timeline. @@ -2878,12 +2878,12 @@ import java.util.concurrent.atomic.AtomicBoolean; period, repeatMode, shuffleModeEnabled, - periodPosition.first, + periodPositionUs.first, seekTimeline, timeline); if (periodUid != null) { // We found one. Use the default position of the corresponding window. - return timeline.getPeriodPosition( + return timeline.getPeriodPositionUs( window, period, timeline.getPeriodByUid(periodUid, period).windowIndex, diff --git a/library/core/src/main/java/com/google/android/exoplayer2/MediaPeriodQueue.java b/library/core/src/main/java/com/google/android/exoplayer2/MediaPeriodQueue.java index f00ca859f1..e34dffd595 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/MediaPeriodQueue.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/MediaPeriodQueue.java @@ -661,18 +661,18 @@ import com.google.common.collect.ImmutableList; // forward by the duration of the buffer, and start buffering from this point. contentPositionUs = C.TIME_UNSET; @Nullable - Pair defaultPosition = - timeline.getPeriodPosition( + Pair defaultPositionUs = + timeline.getPeriodPositionUs( window, period, nextWindowIndex, /* windowPositionUs= */ C.TIME_UNSET, /* defaultPositionProjectionUs= */ max(0, bufferedDurationUs)); - if (defaultPosition == null) { + if (defaultPositionUs == null) { return null; } - nextPeriodUid = defaultPosition.first; - startPositionUs = defaultPosition.second; + nextPeriodUid = defaultPositionUs.first; + startPositionUs = defaultPositionUs.second; MediaPeriodHolder nextMediaPeriodHolder = mediaPeriodHolder.getNext(); if (nextMediaPeriodHolder != null && nextMediaPeriodHolder.uid.equals(nextPeriodUid)) { windowSequenceNumber = nextMediaPeriodHolder.info.id.windowSequenceNumber; @@ -716,17 +716,17 @@ import com.google.common.collect.ImmutableList; // If we're transitioning from an ad group to content starting from its default position, // project the start position forward as if this were a transition to a new window. @Nullable - Pair defaultPosition = - timeline.getPeriodPosition( + Pair defaultPositionUs = + timeline.getPeriodPositionUs( window, period, period.windowIndex, /* windowPositionUs= */ C.TIME_UNSET, /* defaultPositionProjectionUs= */ max(0, bufferedDurationUs)); - if (defaultPosition == null) { + if (defaultPositionUs == null) { return null; } - startPositionUs = defaultPosition.second; + startPositionUs = defaultPositionUs.second; } long minStartPositionUs = getMinStartPositionAfterAdGroupUs( diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/MaskingMediaSource.java b/library/core/src/main/java/com/google/android/exoplayer2/source/MaskingMediaSource.java index c1744ed57a..04304a87f4 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/source/MaskingMediaSource.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/MaskingMediaSource.java @@ -179,11 +179,11 @@ public final class MaskingMediaSource extends CompositeMediaSource { windowStartPositionUs = windowPreparePositionUs; } } - Pair periodPosition = - newTimeline.getPeriodPosition( + Pair periodUidAndPositionUs = + newTimeline.getPeriodPositionUs( window, period, /* windowIndex= */ 0, windowStartPositionUs); - Object periodUid = periodPosition.first; - long periodPositionUs = periodPosition.second; + Object periodUid = periodUidAndPositionUs.first; + long periodPositionUs = periodUidAndPositionUs.second; timeline = hasRealTimeline ? timeline.cloneWithUpdatedTimeline(newTimeline) diff --git a/library/core/src/test/java/com/google/android/exoplayer2/source/SinglePeriodTimelineTest.java b/library/core/src/test/java/com/google/android/exoplayer2/source/SinglePeriodTimelineTest.java index aec51e3cab..cfa5d8819f 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/source/SinglePeriodTimelineTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/source/SinglePeriodTimelineTest.java @@ -45,19 +45,31 @@ public final class SinglePeriodTimelineTest { public void getPeriodPositionDynamicWindowUnknownDuration() { SinglePeriodTimeline timeline = new SinglePeriodTimeline( - C.TIME_UNSET, + /* durationUs= */ C.TIME_UNSET, /* isSeekable= */ false, /* isDynamic= */ true, /* isLive= */ true, /* manifest= */ null, MediaItem.fromUri(Uri.EMPTY)); // Should return null with any positive position projection. - Pair position = timeline.getPeriodPosition(window, period, 0, C.TIME_UNSET, 1); - assertThat(position).isNull(); + Pair positionUs = + timeline.getPeriodPositionUs( + window, + period, + /* windowIndex= */ 0, + /* windowPositionUs= */ C.TIME_UNSET, + /* defaultPositionProjectionUs= */ 1); + assertThat(positionUs).isNull(); // Should return (0, 0) without a position projection. - position = timeline.getPeriodPosition(window, period, 0, C.TIME_UNSET, 0); - assertThat(position.first).isEqualTo(timeline.getUidOfPeriod(0)); - assertThat(position.second).isEqualTo(0); + positionUs = + timeline.getPeriodPositionUs( + window, + period, + /* windowIndex= */ 0, + /* windowPositionUs= */ C.TIME_UNSET, + /* defaultPositionProjectionUs= */ 0); + assertThat(positionUs.first).isEqualTo(timeline.getUidOfPeriod(0)); + assertThat(positionUs.second).isEqualTo(0); } @Test @@ -75,17 +87,34 @@ public final class SinglePeriodTimelineTest { /* manifest= */ null, MediaItem.fromUri(Uri.EMPTY)); // Should return null with a positive position projection beyond window duration. - Pair position = - timeline.getPeriodPosition(window, period, 0, C.TIME_UNSET, windowDurationUs + 1); - assertThat(position).isNull(); + Pair positionUs = + timeline.getPeriodPositionUs( + window, + period, + /* windowIndex= */ 0, + /* windowPositionUs= */ C.TIME_UNSET, + /* defaultPositionProjectionUs= */ windowDurationUs + 1); + assertThat(positionUs).isNull(); // Should return (0, duration) with a projection equal to window duration. - position = timeline.getPeriodPosition(window, period, 0, C.TIME_UNSET, windowDurationUs - 1); - assertThat(position.first).isEqualTo(timeline.getUidOfPeriod(0)); - assertThat(position.second).isEqualTo(windowDurationUs - 1); + positionUs = + timeline.getPeriodPositionUs( + window, + period, + /* windowIndex= */ 0, + /* windowPositionUs= */ C.TIME_UNSET, + /* defaultPositionProjectionUs= */ windowDurationUs - 1); + assertThat(positionUs.first).isEqualTo(timeline.getUidOfPeriod(0)); + assertThat(positionUs.second).isEqualTo(windowDurationUs - 1); // Should return (0, 0) without a position projection. - position = timeline.getPeriodPosition(window, period, 0, C.TIME_UNSET, 0); - assertThat(position.first).isEqualTo(timeline.getUidOfPeriod(0)); - assertThat(position.second).isEqualTo(0); + positionUs = + timeline.getPeriodPositionUs( + window, + period, + /* windowIndex= */ 0, + /* windowPositionUs= */ C.TIME_UNSET, + /* defaultPositionProjectionUs= */ 0); + assertThat(positionUs.first).isEqualTo(timeline.getUidOfPeriod(0)); + assertThat(positionUs.second).isEqualTo(0); } @Test From 0f48dfc93e4019ae4a6ead8b24205125f63209ff Mon Sep 17 00:00:00 2001 From: ibaker Date: Tue, 7 Dec 2021 11:25:22 +0000 Subject: [PATCH 09/27] Fix how drop-to-keyframe events are recorded in `DecoderCounters` The existing code creates an imbalance between `inputBufferCount` and `droppedBufferCount` by adding 'dropped source buffers' to `droppedBufferCount` but not to `inputBufferCount`. This results in assertion failures in `DashTestRunner`. PiperOrigin-RevId: 414672175 --- .../exoplayer2/decoder/DecoderCounters.java | 34 +++++++++++++++---- .../video/DecoderVideoRenderer.java | 25 +++++++++----- .../video/MediaCodecVideoRenderer.java | 31 ++++++++++------- .../playbacktests/gts/DashTestRunner.java | 14 ++------ .../testutil/DecoderCountersUtil.java | 18 +++++++++- 5 files changed, 81 insertions(+), 41 deletions(-) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java b/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java index 779513d412..8e3d8d9956 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java @@ -30,12 +30,12 @@ public final class DecoderCounters { public int decoderInitCount; /** The number of times a decoder has been released. */ public int decoderReleaseCount; - /** The number of queued input buffers. */ + /** The number of input buffers queued to the decoder. */ public int inputBufferCount; /** * The number of skipped input buffers. * - *

A skipped input buffer is an input buffer that was deliberately not sent to the decoder. + *

A skipped input buffer is an input buffer that was deliberately not queued to the decoder. */ public int skippedInputBufferCount; /** The number of rendered output buffers. */ @@ -43,16 +43,28 @@ public final class DecoderCounters { /** * The number of skipped output buffers. * - *

A skipped output buffer is an output buffer that was deliberately not rendered. + *

A skipped output buffer is an output buffer that was deliberately not rendered. This + * includes buffers that were never dequeued from the decoder and instead skipped while 'inside' + * the codec due to a flush. */ public int skippedOutputBufferCount; /** * The number of dropped buffers. * - *

A dropped buffer is an buffer that was supposed to be decoded/rendered, but was instead + *

A dropped buffer is a buffer that was supposed to be decoded/rendered, but was instead * dropped because it could not be rendered in time. + * + *

This includes all of {@link #droppedInputBufferCount} in addition to buffers dropped after + * being queued to the decoder. */ public int droppedBufferCount; + /** + * The number of input buffers dropped. + * + *

A dropped input buffer is a buffer that was not queued to the decoder because it would not + * be rendered in time. + */ + public int droppedInputBufferCount; /** * The maximum number of dropped buffers without an interleaving rendered output buffer. * @@ -62,9 +74,16 @@ public final class DecoderCounters { /** * The number of times all buffers to a keyframe were dropped. * - *

Each time buffers to a keyframe are dropped, this counter is increased by one, and the - * dropped buffer counters are increased by one (for the current output buffer) plus the number of - * buffers dropped from the source to advance to the keyframe. + *

Each time buffers to a keyframe are dropped: + * + *

*/ public int droppedToKeyframeCount; /** @@ -111,6 +130,7 @@ public final class DecoderCounters { renderedOutputBufferCount += other.renderedOutputBufferCount; skippedOutputBufferCount += other.skippedOutputBufferCount; droppedBufferCount += other.droppedBufferCount; + droppedInputBufferCount += other.droppedInputBufferCount; maxConsecutiveDroppedBufferCount = max(maxConsecutiveDroppedBufferCount, other.maxConsecutiveDroppedBufferCount); droppedToKeyframeCount += other.droppedToKeyframeCount; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java index b4af6cfb46..7ccc1ecbe6 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java @@ -485,7 +485,8 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { * @param outputBuffer The output buffer to drop. */ protected void dropOutputBuffer(VideoDecoderOutputBuffer outputBuffer) { - updateDroppedBufferCounters(1); + updateDroppedBufferCounters( + /* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1); outputBuffer.release(); } @@ -506,21 +507,27 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { decoderCounters.droppedToKeyframeCount++; // We dropped some buffers to catch up, so update the decoder counters and flush the decoder, // which releases all pending buffers buffers including the current output buffer. - updateDroppedBufferCounters(buffersInCodecCount + droppedSourceBufferCount); + updateDroppedBufferCounters( + droppedSourceBufferCount, /* droppedDecoderBufferCount= */ buffersInCodecCount); flushDecoder(); return true; } /** - * Updates decoder counters to reflect that {@code droppedBufferCount} additional buffers were - * dropped. + * Updates local counters and {@link #decoderCounters} to reflect that buffers were dropped. * - * @param droppedBufferCount The number of additional dropped buffers. + * @param droppedInputBufferCount The number of buffers dropped from the source before being + * passed to the decoder. + * @param droppedDecoderBufferCount The number of buffers dropped after being passed to the + * decoder. */ - protected void updateDroppedBufferCounters(int droppedBufferCount) { - decoderCounters.droppedBufferCount += droppedBufferCount; - droppedFrames += droppedBufferCount; - consecutiveDroppedFrameCount += droppedBufferCount; + protected void updateDroppedBufferCounters( + int droppedInputBufferCount, int droppedDecoderBufferCount) { + decoderCounters.droppedInputBufferCount += droppedInputBufferCount; + int totalDroppedBufferCount = droppedInputBufferCount + droppedDecoderBufferCount; + decoderCounters.droppedBufferCount += totalDroppedBufferCount; + droppedFrames += totalDroppedBufferCount; + consecutiveDroppedFrameCount += totalDroppedBufferCount; decoderCounters.maxConsecutiveDroppedBufferCount = max(consecutiveDroppedFrameCount, decoderCounters.maxConsecutiveDroppedBufferCount); if (maxDroppedFramesToNotify > 0 && droppedFrames >= maxDroppedFramesToNotify) { diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java index be506ae398..4c1d3d6b16 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/MediaCodecVideoRenderer.java @@ -1108,7 +1108,8 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { TraceUtil.beginSection("dropVideoBuffer"); codec.releaseOutputBuffer(index, false); TraceUtil.endSection(); - updateDroppedBufferCounters(1); + updateDroppedBufferCounters( + /* droppedInputBufferCount= */ 0, /* droppedDecoderBufferCount= */ 1); } /** @@ -1128,29 +1129,35 @@ public class MediaCodecVideoRenderer extends MediaCodecRenderer { if (droppedSourceBufferCount == 0) { return false; } - decoderCounters.droppedToKeyframeCount++; // We dropped some buffers to catch up, so update the decoder counters and flush the codec, // which releases all pending buffers buffers including the current output buffer. - int totalDroppedBufferCount = buffersInCodecCount + droppedSourceBufferCount; if (treatDroppedBuffersAsSkipped) { - decoderCounters.skippedOutputBufferCount += totalDroppedBufferCount; + decoderCounters.skippedInputBufferCount += droppedSourceBufferCount; + decoderCounters.skippedOutputBufferCount += buffersInCodecCount; } else { - updateDroppedBufferCounters(totalDroppedBufferCount); + decoderCounters.droppedToKeyframeCount++; + updateDroppedBufferCounters( + droppedSourceBufferCount, /* droppedDecoderBufferCount= */ buffersInCodecCount); } flushOrReinitializeCodec(); return true; } /** - * Updates local counters and {@link DecoderCounters} to reflect that {@code droppedBufferCount} - * additional buffers were dropped. + * Updates local counters and {@link #decoderCounters} to reflect that buffers were dropped. * - * @param droppedBufferCount The number of additional dropped buffers. + * @param droppedInputBufferCount The number of buffers dropped from the source before being + * passed to the decoder. + * @param droppedDecoderBufferCount The number of buffers dropped after being passed to the + * decoder. */ - protected void updateDroppedBufferCounters(int droppedBufferCount) { - decoderCounters.droppedBufferCount += droppedBufferCount; - droppedFrames += droppedBufferCount; - consecutiveDroppedFrameCount += droppedBufferCount; + protected void updateDroppedBufferCounters( + int droppedInputBufferCount, int droppedDecoderBufferCount) { + decoderCounters.droppedInputBufferCount += droppedInputBufferCount; + int totalDroppedBufferCount = droppedInputBufferCount + droppedDecoderBufferCount; + decoderCounters.droppedBufferCount += totalDroppedBufferCount; + droppedFrames += totalDroppedBufferCount; + consecutiveDroppedFrameCount += totalDroppedBufferCount; decoderCounters.maxConsecutiveDroppedBufferCount = max(consecutiveDroppedFrameCount, decoderCounters.maxConsecutiveDroppedBufferCount); if (maxDroppedFramesToNotify > 0 && droppedFrames >= maxDroppedFramesToNotify) { diff --git a/playbacktests/src/androidTest/java/com/google/android/exoplayer2/playbacktests/gts/DashTestRunner.java b/playbacktests/src/androidTest/java/com/google/android/exoplayer2/playbacktests/gts/DashTestRunner.java index d110476911..181412c51c 100644 --- a/playbacktests/src/androidTest/java/com/google/android/exoplayer2/playbacktests/gts/DashTestRunner.java +++ b/playbacktests/src/androidTest/java/com/google/android/exoplayer2/playbacktests/gts/DashTestRunner.java @@ -361,18 +361,8 @@ import java.util.List; tag + AUDIO_TAG_SUFFIX, audioCounters, 0); DecoderCountersUtil.assertSkippedOutputBufferCount( tag + VIDEO_TAG_SUFFIX, videoCounters, 0); - // We allow one fewer output buffer due to the way that MediaCodecRenderer and the - // underlying decoders handle the end of stream. This should be tightened up in the future. - DecoderCountersUtil.assertTotalBufferCount( - tag + AUDIO_TAG_SUFFIX, - audioCounters, - audioCounters.inputBufferCount - 1, - audioCounters.inputBufferCount); - DecoderCountersUtil.assertTotalBufferCount( - tag + VIDEO_TAG_SUFFIX, - videoCounters, - videoCounters.inputBufferCount - 1, - videoCounters.inputBufferCount); + DecoderCountersUtil.assertTotalBufferCount(tag + AUDIO_TAG_SUFFIX, audioCounters); + DecoderCountersUtil.assertTotalBufferCount(tag + VIDEO_TAG_SUFFIX, videoCounters); } try { if (!shouldSkipDroppedOutputBufferPerformanceAssertions()) { diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java index 19102d9fef..a600fc8120 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java @@ -32,7 +32,8 @@ public final class DecoderCountersUtil { */ public static int getTotalBufferCount(DecoderCounters counters) { counters.ensureUpdated(); - return counters.skippedOutputBufferCount + return counters.skippedInputBufferCount + + counters.skippedOutputBufferCount + counters.droppedBufferCount + counters.renderedOutputBufferCount; } @@ -47,6 +48,21 @@ public final class DecoderCountersUtil { .isEqualTo(expected); } + /** Asserts that the input and output values in {@code counters} are self-consistent. */ + public static void assertTotalBufferCount(String name, DecoderCounters counters) { + // We allow one fewer output buffer due to the way that MediaCodecRenderer and the + // underlying decoders handle the end of stream. This should be tightened up in the future. + int totalInputBufferCount = + counters.skippedInputBufferCount + + counters.droppedInputBufferCount + + counters.inputBufferCount; + assertTotalBufferCount( + name, + counters, + /* minCount= */ totalInputBufferCount - 1, + /* maxCount= */ totalInputBufferCount); + } + public static void assertTotalBufferCount( String name, DecoderCounters counters, int minCount, int maxCount) { int actual = getTotalBufferCount(counters); From 97206b9c72ebbe88243d9df4e23f3ee7fab5e0b8 Mon Sep 17 00:00:00 2001 From: krocard Date: Tue, 7 Dec 2021 14:06:05 +0000 Subject: [PATCH 10/27] Add a builder to `DefaultAudioSink` `DefaultAudioSink` already has 3 telescoping constructors and an other one would be have been needed to add a buffer size tuning option. PiperOrigin-RevId: 414703366 --- RELEASENOTES.md | 3 + .../ext/ffmpeg/FfmpegAudioRenderer.java | 2 +- .../exoplayer2/ext/flac/FlacPlaybackTest.java | 5 +- library/core/build.gradle | 1 + .../exoplayer2/DefaultRenderersFactory.java | 18 +- .../google/android/exoplayer2/ExoPlayer.java | 4 +- .../audio/DecoderAudioRenderer.java | 11 +- .../exoplayer2/audio/DefaultAudioSink.java | 225 +++++++++++++----- .../audio/MediaCodecAudioRenderer.java | 5 +- .../audio/DefaultAudioSinkTest.java | 46 ++-- .../testutil/CapturingRenderersFactory.java | 5 +- 11 files changed, 220 insertions(+), 105 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 0d27c53384..b9ddba81e5 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -25,6 +25,9 @@ targeting Android 12, and will crash with an `IllegalArgumentException` when creating `PendingIntent`s ([#9528](https://github.com/google/ExoPlayer/issues/9528)). +* Audio: + * Add a `Builder` to `DefaultAudioSink` and deprecate the existing + constructors. * Extractors: * Fix inconsistency with spec in H.265 SPS nal units parsing ((#9719)[https://github.com/google/ExoPlayer/issues/9719]). diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java index a124c679c4..d605706551 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java @@ -64,7 +64,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer *
  • Audio offload rendering is enabled in {@link * DefaultRenderersFactory#setEnableAudioOffload} or the equivalent option passed to {@link - * DefaultAudioSink#DefaultAudioSink(AudioCapabilities, - * DefaultAudioSink.AudioProcessorChain, boolean, boolean, int)}. + * DefaultAudioSink.Builder#setOffloadMode}. *
  • An audio track is playing in a format that the device supports offloading (for example, * MP3 or AAC). *
  • The {@link AudioSink} is playing with an offload {@link AudioTrack}. diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java index 3ec0f661d9..c00129b37a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java @@ -54,6 +54,7 @@ import com.google.android.exoplayer2.util.MediaClock; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.TraceUtil; import com.google.android.exoplayer2.util.Util; +import com.google.common.base.MoreObjects; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -171,7 +172,15 @@ public abstract class DecoderAudioRenderer< @Nullable AudioRendererEventListener eventListener, @Nullable AudioCapabilities audioCapabilities, AudioProcessor... audioProcessors) { - this(eventHandler, eventListener, new DefaultAudioSink(audioCapabilities, audioProcessors)); + this( + eventHandler, + eventListener, + new DefaultAudioSink.Builder() + .setAudioCapabilities( + MoreObjects.firstNonNull( + audioCapabilities, AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES)) + .setAudioProcessors(audioProcessors) + .build()); } /** diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java index 6faf620755..615a700fd7 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DefaultAudioSink.java @@ -15,6 +15,7 @@ */ package com.google.android.exoplayer2.audio; +import static com.google.android.exoplayer2.util.Assertions.checkNotNull; import static java.lang.Math.max; import static java.lang.Math.min; @@ -41,6 +42,8 @@ import com.google.android.exoplayer2.util.Assertions; import com.google.android.exoplayer2.util.Log; import com.google.android.exoplayer2.util.MimeTypes; import com.google.android.exoplayer2.util.Util; +import com.google.errorprone.annotations.InlineMe; +import com.google.errorprone.annotations.InlineMeValidationDisabled; import java.lang.annotation.Documented; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -50,6 +53,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collections; import org.checkerframework.checker.nullness.qual.MonotonicNonNull; +import org.checkerframework.checker.nullness.qual.RequiresNonNull; /** * Plays audio data. The implementation delegates to an {@link AudioTrack} and handles playback @@ -203,6 +207,106 @@ public final class DefaultAudioSink implements AudioSink { } } + /** A builder to create {@link DefaultAudioSink} instances. */ + public static final class Builder { + + @Nullable private AudioCapabilities audioCapabilities; + @Nullable private AudioProcessorChain audioProcessorChain; + private boolean enableFloatOutput; + private boolean enableAudioTrackPlaybackParams; + private int offloadMode; + + /** Creates a new builder. */ + public Builder() { + offloadMode = OFFLOAD_MODE_DISABLED; + } + + /** + * Sets audio capabilities for playback on this device. May be {@code null} if the default + * capabilities (no encoded audio passthrough support) should be assumed. + * + *

    The default value is {@code null}. + */ + public Builder setAudioCapabilities(@Nullable AudioCapabilities audioCapabilities) { + this.audioCapabilities = audioCapabilities; + return this; + } + + /** + * Sets an array of {@link AudioProcessor AudioProcessors}s that will process PCM audio before + * output. May be empty. Equivalent of {@code setAudioProcessorChain(new + * DefaultAudioProcessorChain(audioProcessors)}. + * + *

    The default value is an empty array. + */ + public Builder setAudioProcessors(AudioProcessor[] audioProcessors) { + checkNotNull(audioProcessors); + return setAudioProcessorChain(new DefaultAudioProcessorChain(audioProcessors)); + } + + /** + * Sets the {@link AudioProcessorChain} to process audio before playback. The instance passed in + * must not be reused in other sinks. Processing chains are only supported for PCM playback (not + * passthrough or offload). + * + *

    By default, no processing will be applied. + */ + public Builder setAudioProcessorChain(AudioProcessorChain audioProcessorChain) { + checkNotNull(audioProcessorChain); + this.audioProcessorChain = audioProcessorChain; + return this; + } + + /** + * Sets whether to enable 32-bit float output or integer output. Where possible, 32-bit float + * output will be used if the input is 32-bit float, and also if the input is high resolution + * (24-bit or 32-bit) integer PCM. Float output is supported from API level 21. Audio processing + * (for example, speed adjustment) will not be available when float output is in use. + * + *

    The default value is {@code false}. + */ + public Builder setEnableFloatOutput(boolean enableFloatOutput) { + this.enableFloatOutput = enableFloatOutput; + return this; + } + + /** + * Sets whether to control the playback speed using the platform implementation (see {@link + * AudioTrack#setPlaybackParams(PlaybackParams)}), if supported. If set to {@code false}, speed + * up/down of the audio will be done by ExoPlayer (see {@link SonicAudioProcessor}). Platform + * speed adjustment is lower latency, but less reliable. + * + *

    The default value is {@code false}. + */ + public Builder setEnableAudioTrackPlaybackParams(boolean enableAudioTrackPlaybackParams) { + this.enableAudioTrackPlaybackParams = enableAudioTrackPlaybackParams; + return this; + } + + /** + * Sets the offload mode. If an audio format can be both played with offload and encoded audio + * passthrough, it will be played in offload. Audio offload is supported from API level 29. Most + * Android devices can only support one offload {@link AudioTrack} at a time and can invalidate + * it at any time. Thus an app can never be guaranteed that it will be able to play in offload. + * Audio processing (for example, speed adjustment) will not be available when offload is in + * use. + * + *

    The default value is {@link #OFFLOAD_MODE_DISABLED}. + */ + public Builder setOffloadMode(@OffloadMode int offloadMode) { + this.offloadMode = offloadMode; + return this; + } + + /** Builds the {@link DefaultAudioSink}. Must only be called once per Builder instance. */ + public DefaultAudioSink build() { + if (audioProcessorChain == null) { + audioProcessorChain = new DefaultAudioProcessorChain(); + } + return new DefaultAudioSink(this); + } + } + /** The default playback speed. */ public static final float DEFAULT_PLAYBACK_SPEED = 1f; /** The minimum allowed playback speed. Lower values will be constrained to fall in range. */ @@ -370,76 +474,78 @@ public final class DefaultAudioSink implements AudioSink { private boolean offloadDisabledUntilNextConfiguration; private boolean isWaitingForOffloadEndOfStreamHandled; - /** - * Creates a new default audio sink. - * - * @param audioCapabilities The audio capabilities for playback on this device. May be null if the - * default capabilities (no encoded audio passthrough support) should be assumed. - * @param audioProcessors An array of {@link AudioProcessor}s that will process PCM audio before - * output. May be empty. - */ + /** @deprecated Use {@link Builder}. */ + @Deprecated + @InlineMeValidationDisabled("Migrate constructor to Builder") + @InlineMe( + replacement = + "new DefaultAudioSink.Builder()" + + ".setAudioCapabilities(audioCapabilities)" + + ".setAudioProcessors(audioProcessors)" + + ".build()", + imports = "com.google.android.exoplayer2.audio.DefaultAudioSink") public DefaultAudioSink( @Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors) { - this(audioCapabilities, audioProcessors, /* enableFloatOutput= */ false); + this(new Builder().setAudioCapabilities(audioCapabilities).setAudioProcessors(audioProcessors)); } - /** - * Creates a new default audio sink, optionally using float output for high resolution PCM. - * - * @param audioCapabilities The audio capabilities for playback on this device. May be null if the - * default capabilities (no encoded audio passthrough support) should be assumed. - * @param audioProcessors An array of {@link AudioProcessor}s that will process PCM audio before - * output. May be empty. - * @param enableFloatOutput Whether to enable 32-bit float output. Where possible, 32-bit float - * output will be used if the input is 32-bit float, and also if the input is high resolution - * (24-bit or 32-bit) integer PCM. Audio processing (for example, speed adjustment) will not - * be available when float output is in use. - */ + /** @deprecated Use {@link Builder}. */ + @Deprecated + @InlineMeValidationDisabled("Migrate constructor to Builder") + @InlineMe( + replacement = + "new DefaultAudioSink.Builder()" + + ".setAudioCapabilities(audioCapabilities)" + + ".setAudioProcessors(audioProcessors)" + + ".setEnableFloatOutput(enableFloatOutput)" + + ".build()", + imports = "com.google.android.exoplayer2.audio.DefaultAudioSink") public DefaultAudioSink( @Nullable AudioCapabilities audioCapabilities, AudioProcessor[] audioProcessors, boolean enableFloatOutput) { this( - audioCapabilities, - new DefaultAudioProcessorChain(audioProcessors), - enableFloatOutput, - /* enableAudioTrackPlaybackParams= */ false, - OFFLOAD_MODE_DISABLED); + new Builder() + .setAudioCapabilities(audioCapabilities) + .setAudioProcessors(audioProcessors) + .setEnableFloatOutput(enableFloatOutput)); } - /** - * Creates a new default audio sink, optionally using float output for high resolution PCM and - * with the specified {@code audioProcessorChain}. - * - * @param audioCapabilities The audio capabilities for playback on this device. May be null if the - * default capabilities (no encoded audio passthrough support) should be assumed. - * @param audioProcessorChain An {@link AudioProcessorChain} which is used to apply playback - * parameters adjustments. The instance passed in must not be reused in other sinks. - * @param enableFloatOutput Whether to enable 32-bit float output. Where possible, 32-bit float - * output will be used if the input is 32-bit float, and also if the input is high resolution - * (24-bit or 32-bit) integer PCM. Float output is supported from API level 21. Audio - * processing (for example, speed adjustment) will not be available when float output is in - * use. - * @param enableAudioTrackPlaybackParams Whether to enable setting playback speed using {@link - * android.media.AudioTrack#setPlaybackParams(PlaybackParams)}, if supported. - * @param offloadMode Audio offload configuration. If an audio format can be both played with - * offload and encoded audio passthrough, it will be played in offload. Audio offload is - * supported from API level 29. Most Android devices can only support one offload {@link - * android.media.AudioTrack} at a time and can invalidate it at any time. Thus an app can - * never be guaranteed that it will be able to play in offload. Audio processing (for example, - * speed adjustment) will not be available when offload is in use. - */ + /** @deprecated Use {@link Builder}. */ + @Deprecated + @InlineMeValidationDisabled("Migrate constructor to Builder") + @InlineMe( + replacement = + "new DefaultAudioSink.Builder()" + + ".setAudioCapabilities(audioCapabilities)" + + ".setAudioProcessorChain(audioProcessorChain)" + + ".setEnableFloatOutput(enableFloatOutput)" + + ".setEnableAudioTrackPlaybackParams(enableAudioTrackPlaybackParams)" + + ".setOffloadMode(offloadMode)" + + ".build()", + imports = "com.google.android.exoplayer2.audio.DefaultAudioSink") public DefaultAudioSink( @Nullable AudioCapabilities audioCapabilities, AudioProcessorChain audioProcessorChain, boolean enableFloatOutput, boolean enableAudioTrackPlaybackParams, @OffloadMode int offloadMode) { - this.audioCapabilities = audioCapabilities; - this.audioProcessorChain = Assertions.checkNotNull(audioProcessorChain); - this.enableFloatOutput = Util.SDK_INT >= 21 && enableFloatOutput; - this.enableAudioTrackPlaybackParams = Util.SDK_INT >= 23 && enableAudioTrackPlaybackParams; - this.offloadMode = Util.SDK_INT >= 29 ? offloadMode : OFFLOAD_MODE_DISABLED; + this( + new Builder() + .setAudioCapabilities(audioCapabilities) + .setAudioProcessorChain(audioProcessorChain) + .setEnableFloatOutput(enableFloatOutput) + .setEnableAudioTrackPlaybackParams(enableAudioTrackPlaybackParams) + .setOffloadMode(offloadMode)); + } + + @RequiresNonNull("#1.audioProcessorChain") + private DefaultAudioSink(Builder builder) { + audioCapabilities = builder.audioCapabilities; + audioProcessorChain = builder.audioProcessorChain; + enableFloatOutput = Util.SDK_INT >= 21 && builder.enableFloatOutput; + enableAudioTrackPlaybackParams = Util.SDK_INT >= 23 && builder.enableAudioTrackPlaybackParams; + offloadMode = Util.SDK_INT >= 29 ? builder.offloadMode : OFFLOAD_MODE_DISABLED; releasingConditionVariable = new ConditionVariable(true); audioTrackPositionTracker = new AudioTrackPositionTracker(new PositionTrackerListener()); channelMappingAudioProcessor = new ChannelMappingAudioProcessor(); @@ -587,8 +693,7 @@ public final class DefaultAudioSink implements AudioSink { if (useOffloadedPlayback(inputFormat, audioAttributes)) { outputMode = OUTPUT_MODE_OFFLOAD; outputEncoding = - MimeTypes.getEncoding( - Assertions.checkNotNull(inputFormat.sampleMimeType), inputFormat.codecs); + MimeTypes.getEncoding(checkNotNull(inputFormat.sampleMimeType), inputFormat.codecs); outputChannelConfig = Util.getAudioTrackChannelConfig(inputFormat.channelCount); } else { outputMode = OUTPUT_MODE_PASSTHROUGH; @@ -862,7 +967,7 @@ public final class DefaultAudioSink implements AudioSink { private AudioTrack buildAudioTrack() throws InitializationException { try { - return Assertions.checkNotNull(configuration) + return checkNotNull(configuration) .buildAudioTrack(tunneling, audioAttributes, audioSessionId); } catch (InitializationException e) { maybeDisableOffload(); @@ -1207,7 +1312,7 @@ public final class DefaultAudioSink implements AudioSink { audioTrack.pause(); } if (isOffloadedPlayback(audioTrack)) { - Assertions.checkNotNull(offloadStreamEventCallbackV29).unregister(audioTrack); + checkNotNull(offloadStreamEventCallbackV29).unregister(audioTrack); } // AudioTrack.release can take some time, so we call it on a background thread. final AudioTrack toRelease = audioTrack; @@ -1509,8 +1614,7 @@ public final class DefaultAudioSink implements AudioSink { } @C.Encoding - int encoding = - MimeTypes.getEncoding(Assertions.checkNotNull(format.sampleMimeType), format.codecs); + int encoding = MimeTypes.getEncoding(checkNotNull(format.sampleMimeType), format.codecs); // Check for encodings that are known to work for passthrough with the implementation in this // class. This avoids trying to use passthrough with an encoding where the device/app reports // it's capable but it is untested or known to be broken (for example AAC-LC). @@ -1621,8 +1725,7 @@ public final class DefaultAudioSink implements AudioSink { return false; } @C.Encoding - int encoding = - MimeTypes.getEncoding(Assertions.checkNotNull(format.sampleMimeType), format.codecs); + int encoding = MimeTypes.getEncoding(checkNotNull(format.sampleMimeType), format.codecs); if (encoding == C.ENCODING_INVALID) { return false; } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index 40943aafbf..3c27163466 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -155,7 +155,10 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media mediaCodecSelector, eventHandler, eventListener, - new DefaultAudioSink(audioCapabilities, audioProcessors)); + new DefaultAudioSink.Builder() + .setAudioCapabilities(audioCapabilities) + .setAudioProcessors(audioProcessors) + .build()); } /** diff --git a/library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java b/library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java index ae36f7edad..264cd67a68 100644 --- a/library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java +++ b/library/core/src/test/java/com/google/android/exoplayer2/audio/DefaultAudioSinkTest.java @@ -15,6 +15,7 @@ */ package com.google.android.exoplayer2.audio; +import static com.google.android.exoplayer2.audio.AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES; import static com.google.android.exoplayer2.audio.AudioSink.CURRENT_POSITION_NOT_SET; import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY; import static com.google.android.exoplayer2.audio.AudioSink.SINK_FORMAT_SUPPORTED_WITH_TRANSCODING; @@ -24,6 +25,7 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.Format; import com.google.android.exoplayer2.PlaybackParameters; +import com.google.android.exoplayer2.audio.DefaultAudioSink.DefaultAudioProcessorChain; import com.google.android.exoplayer2.util.MimeTypes; import java.nio.ByteBuffer; import java.nio.ByteOrder; @@ -37,7 +39,6 @@ import org.robolectric.annotation.Config; /** Unit tests for {@link DefaultAudioSink}. */ @RunWith(AndroidJUnit4.class) public final class DefaultAudioSinkTest { - private static final int CHANNEL_COUNT_MONO = 1; private static final int CHANNEL_COUNT_STEREO = 2; private static final int BYTES_PER_FRAME_16_BIT = 2; @@ -59,19 +60,20 @@ public final class DefaultAudioSinkTest { arrayAudioBufferSink = new ArrayAudioBufferSink(); TeeAudioProcessor teeAudioProcessor = new TeeAudioProcessor(arrayAudioBufferSink); defaultAudioSink = - new DefaultAudioSink( - AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES, - new DefaultAudioSink.DefaultAudioProcessorChain(teeAudioProcessor), - /* enableFloatOutput= */ false, - /* enableAudioTrackPlaybackParams= */ false, - DefaultAudioSink.OFFLOAD_MODE_DISABLED); + new DefaultAudioSink.Builder() + .setAudioCapabilities(DEFAULT_AUDIO_CAPABILITIES) + .setAudioProcessorChain(new DefaultAudioProcessorChain(teeAudioProcessor)) + .setOffloadMode(DefaultAudioSink.OFFLOAD_MODE_DISABLED) + .build(); } @Test public void handlesSpecializedAudioProcessorArray() { defaultAudioSink = - new DefaultAudioSink( - AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES, new TeeAudioProcessor[0]); + new DefaultAudioSink.Builder() + .setAudioCapabilities(DEFAULT_AUDIO_CAPABILITIES) + .setAudioProcessors(new TeeAudioProcessor[0]) + .build(); } @Test @@ -203,10 +205,7 @@ public final class DefaultAudioSinkTest { @Test public void floatPcmNeedsTranscodingIfFloatOutputDisabled() { defaultAudioSink = - new DefaultAudioSink( - AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES, - new AudioProcessor[0], - /* enableFloatOutput= */ false); + new DefaultAudioSink.Builder().setAudioCapabilities(DEFAULT_AUDIO_CAPABILITIES).build(); Format floatFormat = STEREO_44_1_FORMAT .buildUpon() @@ -221,10 +220,10 @@ public final class DefaultAudioSinkTest { @Test public void floatPcmNeedsTranscodingIfFloatOutputEnabledBeforeApi21() { defaultAudioSink = - new DefaultAudioSink( - AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES, - new AudioProcessor[0], - /* enableFloatOutput= */ true); + new DefaultAudioSink.Builder() + .setAudioCapabilities(DEFAULT_AUDIO_CAPABILITIES) + .setEnableFloatOutput(true) + .build(); Format floatFormat = STEREO_44_1_FORMAT .buildUpon() @@ -239,10 +238,10 @@ public final class DefaultAudioSinkTest { @Test public void floatOutputSupportedIfFloatOutputEnabledFromApi21() { defaultAudioSink = - new DefaultAudioSink( - AudioCapabilities.DEFAULT_AUDIO_CAPABILITIES, - new AudioProcessor[0], - /* enableFloatOutput= */ true); + new DefaultAudioSink.Builder() + .setAudioCapabilities(DEFAULT_AUDIO_CAPABILITIES) + .setEnableFloatOutput(true) + .build(); Format floatFormat = STEREO_44_1_FORMAT .buildUpon() @@ -267,8 +266,9 @@ public final class DefaultAudioSinkTest { @Test public void audioSinkWithAacAudioCapabilitiesWithoutOffload_doesNotSupportAac() { DefaultAudioSink defaultAudioSink = - new DefaultAudioSink( - new AudioCapabilities(new int[] {C.ENCODING_AAC_LC}, 2), new AudioProcessor[0]); + new DefaultAudioSink.Builder() + .setAudioCapabilities(new AudioCapabilities(new int[] {C.ENCODING_AAC_LC}, 2)) + .build(); Format aacLcFormat = STEREO_44_1_FORMAT .buildUpon() diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/CapturingRenderersFactory.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/CapturingRenderersFactory.java index edb07348f3..5540eab1da 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/CapturingRenderersFactory.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/CapturingRenderersFactory.java @@ -31,7 +31,6 @@ import com.google.android.exoplayer2.DefaultRenderersFactory; import com.google.android.exoplayer2.Renderer; import com.google.android.exoplayer2.RenderersFactory; import com.google.android.exoplayer2.audio.AudioCapabilities; -import com.google.android.exoplayer2.audio.AudioProcessor; import com.google.android.exoplayer2.audio.AudioRendererEventListener; import com.google.android.exoplayer2.audio.DefaultAudioSink; import com.google.android.exoplayer2.audio.MediaCodecAudioRenderer; @@ -94,7 +93,9 @@ public class CapturingRenderersFactory implements RenderersFactory, Dumper.Dumpa /* enableDecoderFallback= */ false, eventHandler, audioRendererEventListener, - new DefaultAudioSink(AudioCapabilities.getCapabilities(context), new AudioProcessor[0])), + new DefaultAudioSink.Builder() + .setAudioCapabilities(AudioCapabilities.getCapabilities(context)) + .build()), new TextRenderer(textRendererOutput, eventHandler.getLooper()), new MetadataRenderer(metadataRendererOutput, eventHandler.getLooper()) }; From e5c598468e546f6603c20697cb56d5b4cf62c444 Mon Sep 17 00:00:00 2001 From: christosts Date: Tue, 7 Dec 2021 17:42:27 +0000 Subject: [PATCH 11/27] Configure MediaCodec in API 32+ to always output 99 channels Configure MediaCodec in API 32+ to always output 99 channels so that we use the audio is spatialized, if the platform can apply spatialization to it. In a follow-up change, the output channel count will be set based on the device's spatialization capabilities. PiperOrigin-RevId: 414751543 --- .../android/exoplayer2/audio/MediaCodecAudioRenderer.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java index 3c27163466..c65f46b837 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/MediaCodecAudioRenderer.java @@ -791,6 +791,14 @@ public class MediaCodecAudioRenderer extends MediaCodecRenderer implements Media == AudioSink.SINK_FORMAT_SUPPORTED_DIRECTLY) { mediaFormat.setInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_FLOAT); } + + if (Util.SDK_INT >= 32) { + // Disable down-mixing in the decoder (for decoders that read the max-output-channel-count + // key). + // TODO[b/190759307]: Update key to use MediaFormat.KEY_MAX_OUTPUT_CHANNEL_COUNT once the + // compile SDK target is set to 32. + mediaFormat.setInteger("max-output-channel-count", 99); + } return mediaFormat; } From eaa4ab59a9cffc81a6e49ac0f429701cc987409e Mon Sep 17 00:00:00 2001 From: ibaker Date: Tue, 7 Dec 2021 18:20:54 +0000 Subject: [PATCH 12/27] Rename `DecoderCounters#inputBufferCount` to `queuedInputBufferCount` This more accurately reflects the value stored in this field. PiperOrigin-RevId: 414762892 --- RELEASENOTES.md | 1 + .../android/exoplayer2/audio/DecoderAudioRenderer.java | 2 +- .../google/android/exoplayer2/decoder/DecoderCounters.java | 6 +++--- .../android/exoplayer2/mediacodec/MediaCodecRenderer.java | 2 +- .../android/exoplayer2/video/DecoderVideoRenderer.java | 2 +- .../android/exoplayer2/testutil/DecoderCountersUtil.java | 2 +- 6 files changed, 8 insertions(+), 7 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index b9ddba81e5..b440689d2d 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -36,6 +36,7 @@ the `Format#id` field of the subtitle track created from the configuration ((#9673)[https://github.com/google/ExoPlayer/issues/9673]). + * Rename `DecoderCounters#inputBufferCount` to `queuedInputBufferCount`. * DRM: * Remove `playbackLooper` from `DrmSessionManager.(pre)acquireSession`. When a `DrmSessionManager` is used by an app in a custom `MediaSource`, diff --git a/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java index c00129b37a..5dfcbf0653 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/audio/DecoderAudioRenderer.java @@ -472,7 +472,7 @@ public abstract class DecoderAudioRenderer< onQueueInputBuffer(inputBuffer); decoder.queueInputBuffer(inputBuffer); decoderReceivedBuffers = true; - decoderCounters.inputBufferCount++; + decoderCounters.queuedInputBufferCount++; inputBuffer = null; return true; default: diff --git a/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java b/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java index 8e3d8d9956..48cddf2d50 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/decoder/DecoderCounters.java @@ -31,7 +31,7 @@ public final class DecoderCounters { /** The number of times a decoder has been released. */ public int decoderReleaseCount; /** The number of input buffers queued to the decoder. */ - public int inputBufferCount; + public int queuedInputBufferCount; /** * The number of skipped input buffers. * @@ -68,7 +68,7 @@ public final class DecoderCounters { /** * The maximum number of dropped buffers without an interleaving rendered output buffer. * - *

    Skipped output buffers are ignored for the purposes of calculating this value. + *

    Skipped buffers are ignored for the purposes of calculating this value. */ public int maxConsecutiveDroppedBufferCount; /** @@ -125,7 +125,7 @@ public final class DecoderCounters { public void merge(DecoderCounters other) { decoderInitCount += other.decoderInitCount; decoderReleaseCount += other.decoderReleaseCount; - inputBufferCount += other.inputBufferCount; + queuedInputBufferCount += other.queuedInputBufferCount; skippedInputBufferCount += other.skippedInputBufferCount; renderedOutputBufferCount += other.renderedOutputBufferCount; skippedOutputBufferCount += other.skippedOutputBufferCount; diff --git a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java index 72497d2771..ebebd0aee6 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/mediacodec/MediaCodecRenderer.java @@ -1342,7 +1342,7 @@ public abstract class MediaCodecRenderer extends BaseRenderer { resetInputBuffer(); codecReceivedBuffers = true; codecReconfigurationState = RECONFIGURATION_STATE_NONE; - decoderCounters.inputBufferCount++; + decoderCounters.queuedInputBufferCount++; return true; } diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java index 7ccc1ecbe6..d1444eda15 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/DecoderVideoRenderer.java @@ -757,7 +757,7 @@ public abstract class DecoderVideoRenderer extends BaseRenderer { decoder.queueInputBuffer(inputBuffer); buffersInCodecCount++; decoderReceivedBuffers = true; - decoderCounters.inputBufferCount++; + decoderCounters.queuedInputBufferCount++; inputBuffer = null; return true; default: diff --git a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java index a600fc8120..2e6934a8c8 100644 --- a/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java +++ b/testutils/src/main/java/com/google/android/exoplayer2/testutil/DecoderCountersUtil.java @@ -55,7 +55,7 @@ public final class DecoderCountersUtil { int totalInputBufferCount = counters.skippedInputBufferCount + counters.droppedInputBufferCount - + counters.inputBufferCount; + + counters.queuedInputBufferCount; assertTotalBufferCount( name, counters, From a92e48e5f88360deb0f331326ff0f75ecf7d78d2 Mon Sep 17 00:00:00 2001 From: bachinger Date: Tue, 7 Dec 2021 21:02:24 +0000 Subject: [PATCH 13/27] Support IMA DAI streams for HLS PiperOrigin-RevId: 414804513 --- .../android/exoplayer2/ext/ima/ImaUtil.java | 36 +++++++++---------- .../google/android/exoplayer2/util/Util.java | 12 +++++++ .../source/ads/ServerSideAdInsertionUtil.java | 29 +++++++++------ 3 files changed, 48 insertions(+), 29 deletions(-) diff --git a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaUtil.java b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaUtil.java index 9a40ba74de..b750df3625 100644 --- a/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaUtil.java +++ b/extensions/ima/src/main/java/com/google/android/exoplayer2/ext/ima/ImaUtil.java @@ -37,6 +37,7 @@ import com.google.ads.interactivemedia.v3.api.player.VideoAdPlayer; import com.google.ads.interactivemedia.v3.api.player.VideoProgressUpdate; import com.google.android.exoplayer2.C; import com.google.android.exoplayer2.ui.AdOverlayInfo; +import com.google.android.exoplayer2.ui.AdViewProvider; import com.google.android.exoplayer2.upstream.DataSchemeDataSource; import com.google.android.exoplayer2.upstream.DataSourceUtil; import com.google.android.exoplayer2.upstream.DataSpec; @@ -135,31 +136,28 @@ import java.util.Set; } } - /** Stores configuration for DAI ad playback. */ - static final class DaiConfiguration { + /** Stores configuration for playing server side ad insertion content. */ + public static final class ServerSideAdInsertionConfiguration { - public final AdErrorEvent.AdErrorListener applicationAdErrorListener; + public final AdViewProvider adViewProvider; + public final ImaSdkSettings imaSdkSettings; + @Nullable public final AdEvent.AdEventListener applicationAdEventListener; + @Nullable public final AdErrorEvent.AdErrorListener applicationAdErrorListener; + public final ImmutableList companionAdSlots; public final boolean debugModeEnabled; - @Nullable public final List companionAdSlots; - @Nullable public final AdEvent.AdEventListener applicationAdEventListener; - @Nullable public final VideoAdPlayer.VideoAdPlayerCallback applicationVideoAdPlayerCallback; - @Nullable public final ImaSdkSettings imaSdkSettings; - - public DaiConfiguration( - AdErrorEvent.AdErrorListener applicationAdErrorListener, - @Nullable List companionAdSlots, + public ServerSideAdInsertionConfiguration( + AdViewProvider adViewProvider, + ImaSdkSettings imaSdkSettings, @Nullable AdEvent.AdEventListener applicationAdEventListener, - @Nullable VideoAdPlayer.VideoAdPlayerCallback applicationVideoAdPlayerCallback, - @Nullable ImaSdkSettings imaSdkSettings, + @Nullable AdErrorEvent.AdErrorListener applicationAdErrorListener, + List companionAdSlots, boolean debugModeEnabled) { - - this.applicationAdErrorListener = applicationAdErrorListener; - this.companionAdSlots = - companionAdSlots != null ? ImmutableList.copyOf(companionAdSlots) : null; - this.applicationAdEventListener = applicationAdEventListener; - this.applicationVideoAdPlayerCallback = applicationVideoAdPlayerCallback; this.imaSdkSettings = imaSdkSettings; + this.adViewProvider = adViewProvider; + this.applicationAdEventListener = applicationAdEventListener; + this.applicationAdErrorListener = applicationAdErrorListener; + this.companionAdSlots = ImmutableList.copyOf(companionAdSlots); this.debugModeEnabled = debugModeEnabled; } } diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/Util.java b/library/common/src/main/java/com/google/android/exoplayer2/util/Util.java index b7678d35e0..9dad181b91 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/Util.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/Util.java @@ -1141,6 +1141,18 @@ public final class Util { return (timeMs == C.TIME_UNSET || timeMs == C.TIME_END_OF_SOURCE) ? timeMs : (timeMs * 1000); } + /** + * Converts a time in seconds to the corresponding time in microseconds. + * + * @param timeSec The time in seconds. + * @return The corresponding time in microseconds. + */ + public static long secToUs(double timeSec) { + return BigDecimal.valueOf(timeSec) + .multiply(BigDecimal.valueOf(C.MICROS_PER_SECOND)) + .longValue(); + } + /** * Parses an xs:duration attribute value, returning the parsed duration in milliseconds. * diff --git a/library/core/src/main/java/com/google/android/exoplayer2/source/ads/ServerSideAdInsertionUtil.java b/library/core/src/main/java/com/google/android/exoplayer2/source/ads/ServerSideAdInsertionUtil.java index 9b4d4710f9..d7609848af 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/source/ads/ServerSideAdInsertionUtil.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/source/ads/ServerSideAdInsertionUtil.java @@ -67,16 +67,8 @@ public final class ServerSideAdInsertionUtil { .withAdCount(insertionIndex, /* adCount= */ 1) .withAdDurationsUs(insertionIndex, adDurationUs) .withContentResumeOffsetUs(insertionIndex, contentResumeOffsetUs); - long followingAdGroupTimeUsOffset = -adDurationUs + contentResumeOffsetUs; - for (int i = insertionIndex + 1; i < adPlaybackState.adGroupCount; i++) { - long adGroupTimeUs = adPlaybackState.getAdGroup(i).timeUs; - if (adGroupTimeUs != C.TIME_END_OF_SOURCE) { - adPlaybackState = - adPlaybackState.withAdGroupTimeUs( - /* adGroupIndex= */ i, adGroupTimeUs + followingAdGroupTimeUsOffset); - } - } - return adPlaybackState; + return correctFollowingAdGroupTimes( + adPlaybackState, insertionIndex, adDurationUs, contentResumeOffsetUs); } /** @@ -314,4 +306,21 @@ public final class ServerSideAdInsertionUtil { AdPlaybackState.AdGroup adGroup = adPlaybackState.getAdGroup(adGroupIndex); return adGroup.count == C.LENGTH_UNSET ? 0 : adGroup.count; } + + private static AdPlaybackState correctFollowingAdGroupTimes( + AdPlaybackState adPlaybackState, + int adGroupInsertionIndex, + long insertedAdDurationUs, + long addedContentResumeOffsetUs) { + long followingAdGroupTimeUsOffset = -insertedAdDurationUs + addedContentResumeOffsetUs; + for (int i = adGroupInsertionIndex + 1; i < adPlaybackState.adGroupCount; i++) { + long adGroupTimeUs = adPlaybackState.getAdGroup(i).timeUs; + if (adGroupTimeUs != C.TIME_END_OF_SOURCE) { + adPlaybackState = + adPlaybackState.withAdGroupTimeUs( + /* adGroupIndex= */ i, adGroupTimeUs + followingAdGroupTimeUsOffset); + } + } + return adPlaybackState; + } } From dff04b343eba81aa06ad287baec95c540490cc4c Mon Sep 17 00:00:00 2001 From: olly Date: Wed, 8 Dec 2021 12:40:10 +0000 Subject: [PATCH 14/27] Fix FFWD/RWND color in night mode The color set via textAppearance is overridden by any non-null textColor set directly on the style. We always want the specific properties the textAppearance specifies, so set them directly to prevent them from being overridden. #minor-release Issue: google/ExoPlayer#9765 PiperOrigin-RevId: 414967143 --- RELEASENOTES.md | 11 ++++++++--- library/ui/src/main/res/values/styles.xml | 16 +++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/RELEASENOTES.md b/RELEASENOTES.md index b440689d2d..c3d63ba58e 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -59,11 +59,16 @@ ([#2882](https://github.com/google/ExoPlayer/issues/2882)). * Correctly populate `Format.label` for audio only HLS streams ([#9608](https://github.com/google/ExoPlayer/issues/9608)). -* Transformer: increase required min API version to 21. -* MediaSession extension +* UI: + * Fix the color of the numbers in `StyledPlayerView` rewind and + fastforward buttons when using certain themes + ([#9765](https://github.com/google/ExoPlayer/issues/9765)). +* Transformer: + * Increase required min API version to 21. +* MediaSession extension: * Remove deprecated call to `onStop(/* reset= */ true)` and provide an opt-out flag for apps that don't want to clear the playlist on stop. -* RTSP +* RTSP: * Provide a client API to override the `SocketFactory` used for any server connection ([#9606](https://github.com/google/ExoPlayer/pull/9606)). diff --git a/library/ui/src/main/res/values/styles.xml b/library/ui/src/main/res/values/styles.xml index 84f2db543d..66efb3eee6 100644 --- a/library/ui/src/main/res/values/styles.xml +++ b/library/ui/src/main/res/values/styles.xml @@ -92,9 +92,11 @@ @drawable/exo_styled_controls_fastforward center|bottom @dimen/exo_icon_padding_bottom - @style/ExoStyledControls.ButtonText + bold + @dimen/exo_icon_text_size + @color/exo_white - @android:color/white + @color/exo_white 0dp @@ -102,16 +104,12 @@ @drawable/exo_styled_controls_rewind center|bottom @dimen/exo_icon_padding_bottom - @style/ExoStyledControls.ButtonText - - @android:color/white - 0dp - - -