From ec26539aebff278b5c26e872d58c2e320f97005d Mon Sep 17 00:00:00 2001 From: Dustin Date: Sun, 23 Jan 2022 11:29:56 -0700 Subject: [PATCH] BitmapFactoryVideoRenderer improvements --- .../android/exoplayer2/util/MimeTypes.java | 3 +- .../video/BitmapFactoryVideoRenderer.java | 55 ++++++++++++------- .../extractor/avi/StreamHeaderBox.java | 2 +- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java b/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java index c9bc1f58cf..85ed1d3df4 100644 --- a/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java +++ b/library/common/src/main/java/com/google/android/exoplayer2/util/MimeTypes.java @@ -55,7 +55,8 @@ public final class MimeTypes { public static final String VIDEO_DOLBY_VISION = BASE_TYPE_VIDEO + "/dolby-vision"; public static final String VIDEO_OGG = BASE_TYPE_VIDEO + "/ogg"; public static final String VIDEO_AVI = BASE_TYPE_VIDEO + "/x-msvideo"; - public static final String VIDEO_JPEG = BASE_TYPE_VIDEO + "/JPEG"; //RFC 3555 + //This exists on Nvidia Shield + public static final String VIDEO_MJPEG = BASE_TYPE_VIDEO + "/mjpeg"; public static final String VIDEO_UNKNOWN = BASE_TYPE_VIDEO + "/x-unknown"; // audio/ MIME types diff --git a/library/core/src/main/java/com/google/android/exoplayer2/video/BitmapFactoryVideoRenderer.java b/library/core/src/main/java/com/google/android/exoplayer2/video/BitmapFactoryVideoRenderer.java index fe571b1e9b..ea0eb5712a 100644 --- a/library/core/src/main/java/com/google/android/exoplayer2/video/BitmapFactoryVideoRenderer.java +++ b/library/core/src/main/java/com/google/android/exoplayer2/video/BitmapFactoryVideoRenderer.java @@ -17,6 +17,7 @@ import com.google.android.exoplayer2.FormatHolder; import com.google.android.exoplayer2.RendererCapabilities; import com.google.android.exoplayer2.decoder.DecoderCounters; import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.source.SampleStream; import com.google.android.exoplayer2.util.MimeTypes; import java.nio.ByteBuffer; import java.util.concurrent.ArrayBlockingQueue; @@ -38,7 +39,7 @@ public class BitmapFactoryVideoRenderer extends BaseRenderer { private Thread thread; private long currentTimeUs; private long nextFrameUs; - private long frameUs; + private long frameUs = Long.MIN_VALUE; private boolean ended; private DecoderCounters decoderCounters; @@ -69,18 +70,11 @@ public class BitmapFactoryVideoRenderer extends BaseRenderer { eventDispatcher.disabled(decoderCounters); } - @Override - protected void onStreamChanged(Format[] formats, long startPositionUs, long offsetUs) - throws ExoPlaybackException { - nextFrameUs = startPositionUs; - for (final Format format : formats) { - @NonNull final FormatHolder formatHolder = getFormatHolder(); - @Nullable final Format currentFormat = formatHolder.format; - if (formatHolder.format == null || !currentFormat.equals(format)) { - getFormatHolder().format = format; - eventDispatcher.inputFormatChanged(format, null); - frameUs = (long)(1_000_000L / format.frameRate); - } + private void onFormatChanged(@NonNull FormatHolder formatHolder) { + @Nullable final Format format = formatHolder.format; + if (format != null) { + eventDispatcher.inputFormatChanged(format, null); + frameUs = (long)(1_000_000L / format.frameRate); } } @@ -91,6 +85,7 @@ public class BitmapFactoryVideoRenderer extends BaseRenderer { eventDispatcher.notify(); } if (renderExecutor.getActiveCount() > 0) { + //Handle decoder overrun if (positionUs > nextFrameUs) { long us = (positionUs - nextFrameUs) + frameUs; long dropped = us / frameUs; @@ -100,13 +95,20 @@ public class BitmapFactoryVideoRenderer extends BaseRenderer { return; } final FormatHolder formatHolder = getFormatHolder(); - final DecoderInputBuffer decoderInputBuffer = new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); - int result = readSource(formatHolder, decoderInputBuffer, 0); + final DecoderInputBuffer decoderInputBuffer = + new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_NORMAL); + final int result = readSource(formatHolder, decoderInputBuffer, + frameUs == Long.MIN_VALUE ? SampleStream.FLAG_REQUIRE_FORMAT : 0); + if (result == C.RESULT_BUFFER_READ) { renderExecutor.execute(new RenderRunnable(decoderInputBuffer, nextFrameUs)); - nextFrameUs += frameUs; - } else if (result == C.RESULT_END_OF_INPUT) { - ended = true; + if (decoderInputBuffer.isEndOfStream()) { + ended = true; + } else { + nextFrameUs += frameUs; + } + } else if (result == C.RESULT_FORMAT_READ) { + onFormatChanged(formatHolder); } } @@ -145,13 +147,14 @@ public class BitmapFactoryVideoRenderer extends BaseRenderer { @Override public int supportsFormat(Format format) throws ExoPlaybackException { //Technically could support any format BitmapFactory supports - if (MimeTypes.VIDEO_JPEG.equals(format.sampleMimeType)) { + if (MimeTypes.VIDEO_MJPEG.equals(format.sampleMimeType)) { return RendererCapabilities.create(C.FORMAT_HANDLED); } return RendererCapabilities.create(C.FORMAT_UNSUPPORTED_TYPE); } class RenderRunnable implements Runnable { + @Nullable private DecoderInputBuffer decoderInputBuffer; private final long renderUs; @@ -160,7 +163,18 @@ public class BitmapFactoryVideoRenderer extends BaseRenderer { this.renderUs = renderUs; } + private boolean maybeDropFrame(long frameUs) { + if (Math.abs(frameUs - currentTimeUs) > frameUs) { + eventDispatcher.droppedFrames(1, frameUs); + return true; + } + return false; + } + public void run() { + if (maybeDropFrame(renderUs)) { + return; + } @Nullable final ByteBuffer byteBuffer = decoderInputBuffer.data; @Nullable @@ -192,6 +206,9 @@ public class BitmapFactoryVideoRenderer extends BaseRenderer { } } } + if (maybeDropFrame(renderUs)) { + return; + } //Log.d(TAG, "Drawing: " + bitmap.getWidth() + "x" + bitmap.getHeight()); final Canvas canvas = surface.lockCanvas(null); diff --git a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/StreamHeaderBox.java b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/StreamHeaderBox.java index 6b44a3de5e..5aecb8ea62 100644 --- a/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/StreamHeaderBox.java +++ b/library/extractor/src/main/java/com/google/android/exoplayer2/extractor/avi/StreamHeaderBox.java @@ -37,7 +37,7 @@ public class StreamHeaderBox extends ResidentBox { STREAM_MAP.put(XVID, mimeType); STREAM_MAP.put('D' | ('X' << 8) | ('5' << 16) | ('0' << 24), mimeType); - STREAM_MAP.put('m' | ('j' << 8) | ('p' << 16) | ('g' << 24), MimeTypes.VIDEO_JPEG); + STREAM_MAP.put('m' | ('j' << 8) | ('p' << 16) | ('g' << 24), MimeTypes.VIDEO_MJPEG); } StreamHeaderBox(int type, int size, ByteBuffer byteBuffer) {